From b11baf4ba4455d2876e84ac5c5dc43b09bf09b0e Mon Sep 17 00:00:00 2001 From: Jan Malakhovski Date: Fri, 1 Mar 2024 13:26:27 +0000 Subject: [PATCH 1/2] lib, treewide: introduce `repoRevToName`, use it in most `fetch*` functions This patch adds `lib.repoRevToName` function that generalizes away most of the code used for derivation name generation by `fetch*` functions (`fetchzip`, `fetchFromGitHub`, etc, except those which are delayed until latter commits for mass-rebuild reasons). It's first argument controls how the resulting name will look (see below). Since `lib` has no equivalent of Nixpkgs' `config`, this patch adds `config.nameSourcesPrettily` option to Nixpkgs and then re-exposes `lib.repoRevToName config.nameSourcesPrettily` expression as `pkgs.repoRevToNameMaybe` which is then used in `fetch*` derivations. The result is that different values of `config.nameSourcesPrettily` now control how the `src` derivations produced by `fetch*` functions are to be named, e.g.: - `nameSourcesPrettily = false` (the default): ``` $ nix-instantiate -A fuse.src /nix/store/-source.drv ``` - `nameSourcesPrettily = true`: ``` $ nix-instantiate -A fuse.src /nix/store/-libfuse-2.9.9-source.drv ``` - `nameSourcesPrettily = "full"`: ``` $ nix-instantiate -A fuse.src /nix/store/-libfuse-2.9.9-github-source.drv ``` See documentation of `config.nameSourcesPrettily` for more info. --- lib/default.nix | 2 +- lib/sources.nix | 60 +++++++++++++++++++ pkgs/build-support/fetchbitbucket/default.nix | 5 +- pkgs/build-support/fetchgit/default.nix | 22 ++++--- pkgs/build-support/fetchgithub/default.nix | 5 +- pkgs/build-support/fetchgitiles/default.nix | 7 ++- pkgs/build-support/fetchgitlab/default.nix | 5 +- pkgs/build-support/fetchrepoorcz/default.nix | 5 +- pkgs/build-support/fetchsavannah/default.nix | 5 +- pkgs/build-support/fetchsourcehut/default.nix | 4 +- pkgs/build-support/fetchzip/default.nix | 12 ++-- pkgs/top-level/all-packages.nix | 3 + pkgs/top-level/config.nix | 33 ++++++++++ 13 files changed, 137 insertions(+), 31 deletions(-) diff --git a/lib/default.nix b/lib/default.nix index d5d47defb8e645b..0725a1c9ba3f1e2 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -125,7 +125,7 @@ let inherit (self.sources) cleanSourceFilter cleanSource sourceByRegex sourceFilesBySuffices commitIdFromGitRepo cleanSourceWith pathHasContext - canCleanSource pathIsGitRepo; + canCleanSource pathIsGitRepo repoRevToName; inherit (self.modules) evalModules setDefaultModuleLocation unifyModuleSyntax applyModuleArgsIfFunction mergeModules mergeModules' mergeOptionDecls mergeDefinitions diff --git a/lib/sources.nix b/lib/sources.nix index f61ea306aec562d..3f3036e19d6aee0 100644 --- a/lib/sources.nix +++ b/lib/sources.nix @@ -254,6 +254,62 @@ let outPath = builtins.path { inherit filter name; path = origSrc; }; }; + # urlToName : (URL | Path | String) -> String + # + # Transform a URL (or path, or string) into a clean package name. + urlToName = url: + let + inherit (builtins) stringLength; + base = baseNameOf (lib.removeSuffix "/" (lib.last (lib.splitString ":" (toString url)))); + # chop away one git or archive-related extension + removeExt = name: let + matchExt = builtins.match "(.*)\\.(git|tar|zip|gz|tgz|bz|tbz|bz2|tbz2|lzma|txz|xz|zstd)" name; + in if matchExt != null then lib.head matchExt else name; + # apply function f to string x while the result shrinks + shrink = f: x: let v = f x; in if stringLength v < stringLength x then shrink f v else x; + in shrink removeExt base; + + # shortRev : (String | Integer) -> String + # + # Given a package revision (like "refs/tags/v12.0"), produce a short revision ("12.0"). + shortRev = rev: + let + baseRev = baseNameOf (toString rev); + matchHash = builtins.match "[a-f0-9]+" baseRev; + matchVer = builtins.match "([A-Za-z]+[-_. ]?)*(v)?([0-9.]+.*)" baseRev; + in + if matchHash != null then builtins.substring 0 7 baseRev + else if matchVer != null then lib.last matchVer + else baseRev; + + # repoRevToNameFull : (URL | Path | String) -> (String | Integer | null) -> (String | null) -> String + # + # See `repoRevToName` below. + repoRevToNameFull = repo_: rev_: suffix_: + let + repo = urlToName repo_; + rev = if rev_ != null then "-${shortRev rev_}" else ""; + suffix = if suffix_ != null then "-${suffix_}" else ""; + in "${repo}${rev}${suffix}-source"; + + # repoRevToName : (Bool | String) -> (URL | Path | String) -> (String | Integer | null) -> String -> String + # + # Produce derivation.name attribute for a given repository URL/path/name and (optionally) its revision/version tag. + # + # This is used by fetch(zip|git|FromGitHub|hg|svn|etc) to generate discoverable + # /nix/store paths. + # + # This uses a different implementation depending on the `pretty` argument: + # false -> name everything as "source" + # true -> name everything as "${repo}-${rev}-source" + # "full" -> name everything as "${repo}-${rev}-${fetcher}-source" + repoRevToName = pretty: + # match on `pretty` first to minimize the thunk + if pretty == false then (repo: rev: suffix: "source") + else if pretty == true then (repo: rev: suffix: repoRevToNameFull repo rev null) + else if pretty == "full" then repoRevToNameFull + else throw "invalid value of `pretty'"; + in { pathType = lib.warnIf (lib.isInOldestRelease 2305) @@ -278,6 +334,10 @@ in { pathHasContext canCleanSource + urlToName + shortRev + repoRevToName + sourceByRegex sourceFilesBySuffices diff --git a/pkgs/build-support/fetchbitbucket/default.nix b/pkgs/build-support/fetchbitbucket/default.nix index 2f9103f2bb3e0e0..ceb06d5683c8bec 100644 --- a/pkgs/build-support/fetchbitbucket/default.nix +++ b/pkgs/build-support/fetchbitbucket/default.nix @@ -1,7 +1,8 @@ -{ fetchzip, lib }: +{ lib, repoRevToNameMaybe, fetchzip }: lib.makeOverridable ( -{ owner, repo, rev, name ? "source" +{ owner, repo, rev +, name ? repoRevToNameMaybe repo rev "bitbucket" , ... # For hash agility }@args: fetchzip ({ inherit name; diff --git a/pkgs/build-support/fetchgit/default.nix b/pkgs/build-support/fetchgit/default.nix index 66bb3b7c09ffcf4..da753438c933bb0 100644 --- a/pkgs/build-support/fetchgit/default.nix +++ b/pkgs/build-support/fetchgit/default.nix @@ -1,22 +1,20 @@ -{lib, stdenvNoCC, git, git-lfs, cacert}: let - urlToName = url: rev: let - inherit (lib) removeSuffix splitString last; - base = last (splitString ":" (baseNameOf (removeSuffix "/" url))); - - matched = builtins.match "(.*)\\.git" base; +{lib, stdenvNoCC, git, git-lfs, cacert}: - short = builtins.substring 0 7 rev; - - appendShort = lib.optionalString ((builtins.match "[a-f0-9]*" rev) != null) "-${short}"; - in "${if matched == null then base else builtins.head matched}${appendShort}"; +let + urlToName = url: rev: let + shortRev = lib.sources.shortRev rev; + appendShort = lib.optionalString ((builtins.match "[a-f0-9]*" rev) != null) "-${shortRev}"; + in "${lib.sources.urlToName url}${appendShort}"; in + lib.makeOverridable ( -{ url, rev ? "HEAD", sha256 ? "", hash ? "", leaveDotGit ? deepClone +{ url, rev ? "HEAD" +, name ? urlToName url rev +, sha256 ? "", hash ? "", leaveDotGit ? deepClone , fetchSubmodules ? true, deepClone ? false , branchName ? null , sparseCheckout ? [] , nonConeMode ? false -, name ? urlToName url rev , # Shell code executed after the file has been fetched # successfully. This can do things like check or transform the file. postFetch ? "" diff --git a/pkgs/build-support/fetchgithub/default.nix b/pkgs/build-support/fetchgithub/default.nix index 4ce3c6e84d768c5..b260f306546c661 100644 --- a/pkgs/build-support/fetchgithub/default.nix +++ b/pkgs/build-support/fetchgithub/default.nix @@ -1,7 +1,8 @@ -{ lib, fetchgit, fetchzip }: +{ lib, repoRevToNameMaybe, fetchgit, fetchzip }: lib.makeOverridable ( -{ owner, repo, rev, name ? "source" +{ owner, repo, rev +, name ? repoRevToNameMaybe repo rev "github" , fetchSubmodules ? false, leaveDotGit ? null , deepClone ? false, private ? false, forceFetchGit ? false , sparseCheckout ? [] diff --git a/pkgs/build-support/fetchgitiles/default.nix b/pkgs/build-support/fetchgitiles/default.nix index be81c6e8a4c2170..87c1d2662bc6c8c 100644 --- a/pkgs/build-support/fetchgitiles/default.nix +++ b/pkgs/build-support/fetchgitiles/default.nix @@ -1,7 +1,10 @@ -{ fetchzip, lib }: +{ fetchzip, repoRevToNameMaybe, lib }: lib.makeOverridable ( -{ url, rev, name ? "source", ... } @ args: +{ url, rev +, name ? repoRevToNameMaybe url rev "gitiles" +, ... +} @ args: fetchzip ({ inherit name; diff --git a/pkgs/build-support/fetchgitlab/default.nix b/pkgs/build-support/fetchgitlab/default.nix index 749883f2365eb64..b4daacf705f94a2 100644 --- a/pkgs/build-support/fetchgitlab/default.nix +++ b/pkgs/build-support/fetchgitlab/default.nix @@ -1,8 +1,9 @@ -{ lib, fetchgit, fetchzip }: +{ lib, repoRevToNameMaybe, fetchgit, fetchzip }: lib.makeOverridable ( # gitlab example -{ owner, repo, rev, protocol ? "https", domain ? "gitlab.com", name ? "source", group ? null +{ owner, repo, rev, protocol ? "https", domain ? "gitlab.com", group ? null +, name ? repoRevToNameMaybe repo rev "gitlab" , fetchSubmodules ? false, leaveDotGit ? false , deepClone ? false, forceFetchGit ? false , sparseCheckout ? [] diff --git a/pkgs/build-support/fetchrepoorcz/default.nix b/pkgs/build-support/fetchrepoorcz/default.nix index 3ac7cace0dcf707..650c930c707ec90 100644 --- a/pkgs/build-support/fetchrepoorcz/default.nix +++ b/pkgs/build-support/fetchrepoorcz/default.nix @@ -1,7 +1,8 @@ -{ fetchzip }: +{ lib, repoRevToNameMaybe, fetchzip }: # gitweb example, snapshot support is optional in gitweb -{ repo, rev, name ? "source" +{ repo, rev +, name ? repoRevToNameMaybe repo rev "repoorcz" , ... # For hash agility }@args: fetchzip ({ inherit name; diff --git a/pkgs/build-support/fetchsavannah/default.nix b/pkgs/build-support/fetchsavannah/default.nix index e75e25fc1e70b64..3d0a23f87179cae 100644 --- a/pkgs/build-support/fetchsavannah/default.nix +++ b/pkgs/build-support/fetchsavannah/default.nix @@ -1,8 +1,9 @@ -{ fetchzip, lib }: +{ lib, repoRevToNameMaybe, fetchzip }: lib.makeOverridable ( # cgit example, snapshot support is optional in cgit -{ repo, rev, name ? "source" +{ repo, rev +, name ? repoRevToNameMaybe repo rev "savannah" , ... # For hash agility }@args: fetchzip ({ inherit name; diff --git a/pkgs/build-support/fetchsourcehut/default.nix b/pkgs/build-support/fetchsourcehut/default.nix index 42d437b3555e6ac..5df72ac50351f86 100644 --- a/pkgs/build-support/fetchsourcehut/default.nix +++ b/pkgs/build-support/fetchsourcehut/default.nix @@ -1,4 +1,4 @@ -{ fetchgit, fetchhg, fetchzip, lib }: +{ lib, repoRevToNameMaybe, fetchgit, fetchhg, fetchzip }: let inherit (lib) @@ -13,7 +13,7 @@ makeOverridable ( , repo, rev , domain ? "sr.ht" , vc ? "git" -, name ? "source" +, name ? repoRevToNameMaybe repo rev "sourcehut" , fetchSubmodules ? false , ... # For hash agility } @ args: diff --git a/pkgs/build-support/fetchzip/default.nix b/pkgs/build-support/fetchzip/default.nix index dd04ccb6e0931cf..3cd1abb577ff273 100644 --- a/pkgs/build-support/fetchzip/default.nix +++ b/pkgs/build-support/fetchzip/default.nix @@ -5,11 +5,14 @@ # (e.g. due to minor changes in the compression algorithm, or changes # in timestamps). -{ lib, fetchurl, withUnzip ? true, unzip, glibcLocalesUtf8 }: +{ lib, repoRevToNameMaybe, fetchurl +, withUnzip ? true, unzip +, glibcLocalesUtf8 +}: -{ name ? "source" -, url ? "" +{ url ? "" , urls ? [] +, name ? repoRevToNameMaybe (if url != "" then url else builtins.head urls) null "unpacked" , nativeBuildInputs ? [] , postFetch ? "" , extraPostFetch ? "" @@ -22,7 +25,8 @@ , extension ? null # the rest are given to fetchurl as is -, ... } @ args: +, ... +} @ args: assert (extraPostFetch != "") -> lib.warn "use 'postFetch' instead of 'extraPostFetch' with 'fetchzip' and 'fetchFromGitHub' or 'fetchFromGitLab'." true; diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index c2719dfa4558b56..5b87d8b85490043 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -1090,6 +1090,9 @@ with pkgs; broadlink-cli = callPackage ../tools/misc/broadlink-cli { }; + # this is used by most `fetch*` functions + repoRevToNameMaybe = lib.repoRevToName config.nameSourcesPrettily; + fetchpatch = callPackage ../build-support/fetchpatch { # 0.3.4 would change hashes: https://github.com/NixOS/nixpkgs/issues/25154 patchutils = buildPackages.patchutils_0_3_3; diff --git a/pkgs/top-level/config.nix b/pkgs/top-level/config.nix index 67a9a60dbaeaa34..f239c968686ee1c 100644 --- a/pkgs/top-level/config.nix +++ b/pkgs/top-level/config.nix @@ -50,6 +50,39 @@ let default = false; }; + nameSourcesPrettily = mkOption { + type = types.uniq (types.either types.bool (types.enum ["full"])); + default = false; + description = lib.mdDoc '' + This controls the default derivation name attribute generated by the + `fetch*` (like `fetchzip`, `fetchFromGitHub`, etc) functions. + + Possible values and the resulting `.name`: + + - `false` -> `"source"` + - `true` -> `"''${repo}-''${rev}-source"` + - `"full"` -> `"''${repo}-''${rev}-''${fetcherName}-source"` + + The default `false` is the best choice for minimal rebuilds, it will + ignore any non-hash changes (like branches being renamed, source URLs + changing, etc). + + Setting this to `true` greatly helps with discoverability of sources + in `/nix/store` at the cost of a single mass-rebuild for all `src` + derivations (but not their referrers, as all `src`s are fixed-output) + and occasional rebuild when a source changes some of its non-hash + attributes. + + Setting this to `"full"` is similar to setting it to `true`, but the + use of `fetcherName` in the derivation name will force a rebuild when + `src` switches between `fetch*` functions, thus forcing `nix` to check + new derivation's `outputHash`, which is useful for debugging. + + Also, `"full"` is useful for easy collection and tracking of + statistics of where the packages you use are hosted. + ''; + }; + doCheckByDefault = mkMassRebuild { feature = "run `checkPhase` by default"; }; From a09a1e74e37f0335a764c9ddb74c0dfae9ef69a3 Mon Sep 17 00:00:00 2001 From: Jan Malakhovski Date: Sun, 2 Jun 2024 12:20:51 +0000 Subject: [PATCH 2/2] fetchsvn: move the name generator outside of the thunk This is slightly more efficient. --- pkgs/build-support/fetchsvn/default.nix | 49 ++++++++++++------------- 1 file changed, 23 insertions(+), 26 deletions(-) diff --git a/pkgs/build-support/fetchsvn/default.nix b/pkgs/build-support/fetchsvn/default.nix index 41752eb55a7a810..70bbcc3fe659118 100644 --- a/pkgs/build-support/fetchsvn/default.nix +++ b/pkgs/build-support/fetchsvn/default.nix @@ -2,41 +2,38 @@ , subversion, glibcLocales, sshSupport ? true, openssh ? null }: -{ url, rev ? "HEAD", sha256 ? "", hash ? "" -, ignoreExternals ? false, ignoreKeywords ? false, name ? null +let + repoToName = url: rev: + let + inherit (lib) removeSuffix splitString reverseList head last elemAt; + base = removeSuffix "/" (last (splitString ":" url)); + path = reverseList (splitString "/" base); + repoName = + # ../repo/trunk -> repo + if head path == "trunk" then elemAt path 1 + # ../repo/branches/branch -> repo-branch + else if elemAt path 1 == "branches" then "${elemAt path 2}-${head path}" + # ../repo/tags/tag -> repo-tag + else if elemAt path 1 == "tags" then "${elemAt path 2}-${head path}" + # ../repo (no trunk) -> repo + else head path; + in "${repoName}-r${toString rev}"; +in + +{ url, rev ? "HEAD" +, name ? repoToName url rev +, sha256 ? "", hash ? "" +, ignoreExternals ? false, ignoreKeywords ? false , preferLocalBuild ? true }: assert sshSupport -> openssh != null; -let - repoName = with lib; - let - fst = head; - snd = l: head (tail l); - trd = l: head (tail (tail l)); - path_ = - (p: if head p == "" then tail p else p) # ~ drop final slash if any - (reverseList (splitString "/" url)); - path = [ (removeSuffix "/" (head path_)) ] ++ (tail path_); - in - # ../repo/trunk -> repo - if fst path == "trunk" then snd path - # ../repo/branches/branch -> repo-branch - else if snd path == "branches" then "${trd path}-${fst path}" - # ../repo/tags/tag -> repo-tag - else if snd path == "tags" then "${trd path}-${fst path}" - # ../repo (no trunk) -> repo - else fst path; - - name_ = if name == null then "${repoName}-r${toString rev}" else name; -in - if hash != "" && sha256 != "" then throw "Only one of sha256 or hash can be set" else stdenvNoCC.mkDerivation { - name = name_; + inherit name; builder = ./builder.sh; nativeBuildInputs = [ subversion glibcLocales ] ++ lib.optional sshSupport openssh;