From e15783154fb13819d8a149e5df0a0d3412e73f97 Mon Sep 17 00:00:00 2001 From: emilylange Date: Thu, 4 Jan 2024 01:34:15 +0100 Subject: [PATCH 1/3] chromium: improve and move `recompressTarball` Recap: We need that (arguably stupid) helper function/drv because the chromium tarball is big -- and is likely to increase even more in the future. So big, that we eventually exceeded hydra.nixos.org's max-output-limit (3G). Instead of raising global hydra's limit, it was decided that we recompress the tarball after deleting unused vendored files from it. I spent a lot of time on a version/prototype that does everything (downloading, decompression, tar extraction, deleting unused files, reproducible tar recreation and finally recompression) via stdin but eventually had to scratch that. GNU tar does not allow to create a tarball just from stdin, nixpkgs' stdenv isn't built with stdin/stdout/pipes in mind, and things a lot of other things I probably already forgot. Nonetheless, this version improves multiple things: - No more `mv` (used to be multiple, not just ours, since fetchzip had some as well) - No more `rm` to get rid of the extracted files before recompressing. Instead, we simply don't extract them in the first place (thanks to tar's --exlude). - No more "no space left" that happened due to `downloadToTemp = true;`. - Multithreaded xz decompression, since that commit is still in staging-next. We cannot use stdenv's unpackFile() because that does not allow us to specify the needed --exclude (and --strip-components=1 if we don't want to rely on glob matching). The hash changed because we now have a static base directory ("source") in the tarball, instead of whatever upstream provided us with (e.g. "chromium-120.0.6099.129"). --- .../networking/browsers/chromium/README.md | 10 ---- .../networking/browsers/chromium/common.nix | 29 +----------- .../networking/browsers/chromium/default.nix | 1 + .../browsers/chromium/recompress-tarball.nix | 47 +++++++++++++++++++ .../browsers/chromium/upstream-info.nix | 4 +- 5 files changed, 51 insertions(+), 40 deletions(-) create mode 100644 pkgs/applications/networking/browsers/chromium/recompress-tarball.nix diff --git a/pkgs/applications/networking/browsers/chromium/README.md b/pkgs/applications/networking/browsers/chromium/README.md index c5a537147c487..410c47ee3a0ca 100644 --- a/pkgs/applications/networking/browsers/chromium/README.md +++ b/pkgs/applications/networking/browsers/chromium/README.md @@ -39,16 +39,6 @@ update `upstream-info.nix`. After updates it is important to test at least `nixosTests.chromium` (or basic manual testing) and `google-chrome` (which reuses `upstream-info.nix`). -Note: Due to the script downloading many large tarballs it might be -necessary to adjust the available tmpfs size (it defaults to 10% of the -systems memory) - -```nix -services.logind.extraConfig = '' - RuntimeDirectorySize=4G -''; -``` - Note: The source tarball is often only available a few hours after the release was announced. The CI/CD status can be tracked here: - https://ci.chromium.org/p/infra/builders/cron/publish_tarball diff --git a/pkgs/applications/networking/browsers/chromium/common.nix b/pkgs/applications/networking/browsers/chromium/common.nix index ec355dccd193b..0798be9372e90 100644 --- a/pkgs/applications/networking/browsers/chromium/common.nix +++ b/pkgs/applications/networking/browsers/chromium/common.nix @@ -1,5 +1,5 @@ { stdenv, lib, fetchurl, fetchpatch -, fetchzip, zstd +, recompressTarball , buildPackages , pkgsBuildBuild , pkgsBuildTarget @@ -148,33 +148,6 @@ let else throw "no chromium Rosetta Stone entry for os: ${platform.config}"; }; - recompressTarball = { version, hash ? "" }: fetchzip { - name = "chromium-${version}.tar.zstd"; - url = "https://commondatastorage.googleapis.com/chromium-browser-official/chromium-${version}.tar.xz"; - inherit hash; - - nativeBuildInputs = [ zstd ]; - - postFetch = '' - echo removing unused code from tarball to stay under hydra limit - rm -r $out/third_party/{rust-src,llvm} - - echo moving remains out of \$out - mv $out source - - echo recompressing final contents into new tarball - # try to make a deterministic tarball - tar \ - --use-compress-program "zstd -T$NIX_BUILD_CORES" \ - --sort name \ - --mtime 1970-01-01 \ - --owner=root --group=root \ - --numeric-owner --mode=go=rX,u+rw,a-s \ - -cf $out source - ''; - }; - - base = rec { pname = "${lib.optionalString ungoogled "ungoogled-"}${packageName}-unwrapped"; inherit (upstream-info) version; diff --git a/pkgs/applications/networking/browsers/chromium/default.nix b/pkgs/applications/networking/browsers/chromium/default.nix index f592344bdf656..9da0f725ed560 100644 --- a/pkgs/applications/networking/browsers/chromium/default.nix +++ b/pkgs/applications/networking/browsers/chromium/default.nix @@ -59,6 +59,7 @@ let inherit (upstream-info.deps.gn) url rev hash; }; }); + recompressTarball = callPackage ./recompress-tarball.nix { }; }); browser = callPackage ./browser.nix { diff --git a/pkgs/applications/networking/browsers/chromium/recompress-tarball.nix b/pkgs/applications/networking/browsers/chromium/recompress-tarball.nix new file mode 100644 index 0000000000000..0e77dd230f657 --- /dev/null +++ b/pkgs/applications/networking/browsers/chromium/recompress-tarball.nix @@ -0,0 +1,47 @@ +{ zstd +, fetchurl +}: + +{ version +, hash ? "" +, ... +} @ args: + +fetchurl ({ + name = "chromium-${version}.tar.zstd"; + url = "https://commondatastorage.googleapis.com/chromium-browser-official/chromium-${version}.tar.xz"; + inherit hash; + + # chromium xz tarballs are multiple gigabytes big and are sometimes downloaded multiples + # times for different versions as part of our update script. + # We originally inherited fetchzip's default for downloadToTemp (true). + # Given the size of the /run/user tmpfs used defaults to logind's RuntimeDirectorySize=, + # which in turn defaults to 10% of the total amount of physical RAM, this often lead to + # "no space left" errors, eventually resulting in its own section in our chromium + # README.md (for users wanting to run the update script). + # Nowadays, we use fetchurl instead of fetchzip, which defaults to false instead of true. + # We just want to be explicit and provide a place to document the history and reasoning + # behind this. + downloadToTemp = false; + + nativeBuildInputs = [ zstd ]; + + postFetch = '' + cat "$downloadedFile" \ + | xz -d --threads=$NIX_BUILD_CORES \ + | tar xf - \ + --warning=no-timestamp \ + --one-top-level=source \ + --exclude=third_party/llvm \ + --exclude=third_party/rust-src \ + --strip-components=1 + + tar \ + --use-compress-program "zstd -T$NIX_BUILD_CORES" \ + --sort name \ + --mtime "1970-01-01" \ + --owner=root --group=root \ + --numeric-owner --mode=go=rX,u+rw,a-s \ + -cf $out source + ''; +} // removeAttrs args [ "version" ]) diff --git a/pkgs/applications/networking/browsers/chromium/upstream-info.nix b/pkgs/applications/networking/browsers/chromium/upstream-info.nix index eadcefe71bdc1..4c35abaeb194b 100644 --- a/pkgs/applications/networking/browsers/chromium/upstream-info.nix +++ b/pkgs/applications/networking/browsers/chromium/upstream-info.nix @@ -15,7 +15,7 @@ version = "2023-10-23"; }; }; - hash = "sha256-+T2TOLwIwFxVDae7MFDZrjREGF+3Zx2xt/Dlu7uZggc="; + hash = "sha256-6RURdPU1k3GaQAgA1LMQ0NhSGBEpOEJBPvk2QjLdoHo="; hash_deb_amd64 = "sha256-0FB1gTbsjqFRy0ocE0w5ACtD9kSJ5AMnxg+qBxqCulc="; version = "120.0.6099.129"; }; @@ -32,7 +32,7 @@ rev = "120.0.6099.129-1"; }; }; - hash = "sha256-+T2TOLwIwFxVDae7MFDZrjREGF+3Zx2xt/Dlu7uZggc="; + hash = "sha256-6RURdPU1k3GaQAgA1LMQ0NhSGBEpOEJBPvk2QjLdoHo="; hash_deb_amd64 = "sha256-0FB1gTbsjqFRy0ocE0w5ACtD9kSJ5AMnxg+qBxqCulc="; version = "120.0.6099.129"; }; From 5c77fff6097b20c6d11d3dda15d51d25d3144b96 Mon Sep 17 00:00:00 2001 From: emilylange Date: Thu, 4 Jan 2024 01:48:08 +0100 Subject: [PATCH 2/3] chromium: 120.0.6099.129 -> 120.0.6099.199 https://chromereleases.googleblog.com/2024/01/stable-channel-update-for-desktop.html This update includes 6 security fixes. CVEs: CVE-2024-0222 CVE-2024-0223 CVE-2024-0224 CVE-2024-0225 --- .../networking/browsers/chromium/upstream-info.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/applications/networking/browsers/chromium/upstream-info.nix b/pkgs/applications/networking/browsers/chromium/upstream-info.nix index 4c35abaeb194b..ab02a097f55da 100644 --- a/pkgs/applications/networking/browsers/chromium/upstream-info.nix +++ b/pkgs/applications/networking/browsers/chromium/upstream-info.nix @@ -15,9 +15,9 @@ version = "2023-10-23"; }; }; - hash = "sha256-6RURdPU1k3GaQAgA1LMQ0NhSGBEpOEJBPvk2QjLdoHo="; - hash_deb_amd64 = "sha256-0FB1gTbsjqFRy0ocE0w5ACtD9kSJ5AMnxg+qBxqCulc="; - version = "120.0.6099.129"; + hash = "sha256-lT1CCwYj0hT4tCJb689mZwNecUsEwcfn2Ot8r9LBT+M="; + hash_deb_amd64 = "sha256-4BWLn0+gYNWG4DsolbY6WlTvXWl7tZIZrnqXlrGUGjQ="; + version = "120.0.6099.199"; }; ungoogled-chromium = { deps = { From d163ea4133312f04ebf8eb4875ea4b61400b4b5e Mon Sep 17 00:00:00 2001 From: networkException Date: Fri, 5 Jan 2024 17:29:51 +0100 Subject: [PATCH 3/3] ungoogled-chromium: 120.0.6099.129-1 -> 120.0.6099.199-1 https://chromereleases.googleblog.com/2024/01/stable-channel-update-for-desktop.html This update includes 6 security fixes. CVEs: CVE-2024-0222 CVE-2024-0223 CVE-2024-0224 CVE-2024-0225 --- .../networking/browsers/chromium/upstream-info.nix | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkgs/applications/networking/browsers/chromium/upstream-info.nix b/pkgs/applications/networking/browsers/chromium/upstream-info.nix index ab02a097f55da..b428800206531 100644 --- a/pkgs/applications/networking/browsers/chromium/upstream-info.nix +++ b/pkgs/applications/networking/browsers/chromium/upstream-info.nix @@ -28,12 +28,12 @@ version = "2023-10-23"; }; ungoogled-patches = { - hash = "sha256-kVhAa/+RnYEGy7McysqHsb3ysPIILnxGXe6BTLbioQk="; - rev = "120.0.6099.129-1"; + hash = "sha256-B1MNo8BdjMOmTvIr4uu3kg/MO1t+YLQz2S23L4Cye3E="; + rev = "120.0.6099.199-1"; }; }; - hash = "sha256-6RURdPU1k3GaQAgA1LMQ0NhSGBEpOEJBPvk2QjLdoHo="; - hash_deb_amd64 = "sha256-0FB1gTbsjqFRy0ocE0w5ACtD9kSJ5AMnxg+qBxqCulc="; - version = "120.0.6099.129"; + hash = "sha256-lT1CCwYj0hT4tCJb689mZwNecUsEwcfn2Ot8r9LBT+M="; + hash_deb_amd64 = "sha256-4BWLn0+gYNWG4DsolbY6WlTvXWl7tZIZrnqXlrGUGjQ="; + version = "120.0.6099.199"; }; }