Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

minimal-bootstrap: make sources a non-tarballs.nixos.org FOD #238357

Merged
merged 3 commits into from Jun 26, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,34 +1,106 @@
{
}:

rec {
name = "stage0-posix-${version}-${rev}-source";
# Pinned from https://github.com/oriansj/stage0-posix/commit/3189b5f325b7ef8b88e3edec7c1cde4fce73c76c
version = "unstable-2023-05-02";
rev = "3189b5f325b7ef8b88e3edec7c1cde4fce73c76c";
outputHashAlgo = "sha256";

# This 256 byte seed is the only pre-compiled binary in the bootstrap chain.
# While it is included in the stage0-posix source bundle and is synced with
# stage0-posix updates, we have split it out into its own derivation to highlight
# its unique status as a trusted binary seed.
hex0-seed = import <nix/fetchurl.nix> {
name = "hex0-seed-${version}";
url = "https://github.com/oriansj/bootstrap-seeds/raw/b1263ff14a17835f4d12539226208c426ced4fba/POSIX/x86/hex0-seed";
hash = "sha256-QU3RPGy51W7M2xnfFY1IqruKzusrSLU+L190ztN6JW8=";
executable = true;
This conversation was marked as resolved.
Show resolved Hide resolved
};

# Packaged resources required for the first bootstrapping stage.
# Contains source code and 256-byte hex0 binary seed.
#
# We don't have access to utilities such as fetchgit and fetchzip since this
# is this is part of the bootstrap process and would introduce a circular
# dependency. The only tool we have to fetch source trees is `import <nix/fetchurl.nix>`
# with the unpack option, taking a NAR file as input. This requires source
# tarballs to be repackaged.
#
# To build see `make-bootstrap-sources.nix`
src = import <nix/fetchurl.nix> {
inherit name;
url = "https://github.com/emilytrau/bootstrap-tools-nar-mirror/releases/download/2023-05-02/${name}.nar.xz";
hash = "sha256-ZRG0k49MxL1UTZhuMTvPoEprdSpJRNVy8QhLE6k+etg=";
unpack = true;
/*
Since `make-minimal-bootstrap-sources` requires nixpkgs and nix it
will create a circular dependency if it is used in place of the
binary bootstrap-files. To break the circular dependency,
`minimal-bootstrap-sources` extends `make-minimal-bootstrap-sources`
by adding Fixed Output Derivation (FOD) attributes. These cause
the builder to be skipped if the expected output is found (by
its hash) in the store or on a substituter.
# How do I update the hash?
Run the following command:
```
nix hash file $(nix build --print-out-paths -f '<nixpkgs>' make-minimal-bootstrap-sources)
```
# Why do we need this `.nar` archive?
This archive exists only because of a quirk/limitation of Nix: in
restricted mode the builtin fetchers can download only single
files; they have no way to unpack multi-file archives except for
NAR archives:
https://github.com/NixOS/nixpkgs/pull/232576#issuecomment-1592415619
# Why don't we have to upload this to tarballs.nixos.org like the binary bootstrap-files did?
Unlike this archive, the binary bootstrap-files contained binaries,
which meant that we had to:
1. Make sure they came from a trusted builder (Hydra)
2. Keep careful track of exactly what toolchain (i.e. nixpkgs
commit) that builder used to create them.
3. Keep copies of the built binaries, in case the toolchains that
produced them failed to be perfectly deterministic.
The curated archives at tarballs.nixos.org exist in order to
satisfy these requirements.
The second point created a significant burden: since the nixpkgs
toolchain used to build a given copy of the binary bootstrap-files
itself used a *previous* copy of the bootstrap-files, this meant
we had to track the provenance of all bootstrap-files tarballs
ever used, for all eternity. There was no explanation of where
the "original" bootstrap-files came from: turtles all the way
down. In spite of all this effort we still can't be sure of our
ability to reproduce the binary bootstrap-files, since the
compilers that built them don't always produce exactly bit-for-bit
deterministic results.
Since this archive contains no binaries and uses a format (NAR)
specifically designed for bit-exact reproducibility, none of the
requirements above apply to `minimal-bootstrap-sources`.
*/
minimal-bootstrap-sources = derivation {
name = "${name}.nar.xz";
system = builtins.currentSystem;
outputHashMode = "flat";
inherit outputHashAlgo;
outputHash = "sha256-ig988BiRTz92hhZZgKQW1tVPoV4aQ2D69Cq3wHvVgHg=";

# This builder always fails, but fortunately Nix will print the
# "builder", which is really the error message that we want the
# user to see.
builder = ''
#
#
# Neither your store nor your substituters seems to have:
#
# ${name}.nar.xz
#
# Please obtain or create this file, give it exactly the name
# shown above, and then run the following command:
#
# nix-store --add-fixed ${outputHashAlgo} ${name}.nar.xz
#
# You can create this file from an already-bootstrapped nixpkgs
# using the following command:
#
# nix-build '<nixpkgs>' -A make-minimal-bootstrap-sources
#
# Or, if you prefer, you can create this file using only `git`,
# `nix`, and `xz`. For the commands needed in order to do this,
# see `make-bootstrap-sources.nix`.
#
'';
};
}
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,10 @@
}:

lib.makeScope newScope (self: with self; {
inherit (import ./bootstrap-sources.nix) version hex0-seed src;
inherit (self.callPackage ./bootstrap-sources.nix {})
version hex0-seed minimal-bootstrap-sources;

src = minimal-bootstrap-sources;

m2libc = src + "/M2libc";

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ derivationWithMeta {
pname = "hex0";
builder = hex0-seed;
args = [
"${src}/bootstrap-seeds/POSIX/x86/hex0_x86.hex0"
"${src}/x86/hex0_x86.hex0"
(placeholder "out")
];

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ derivationWithMeta {
pname = "kaem-minimal";
builder = hex0;
args = [
"${src}/bootstrap-seeds/POSIX/x86/kaem-minimal.hex0"
"${src}/x86/kaem-minimal.hex0"
(placeholder "out")
];

Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
# Packaged resources required for the first bootstrapping stage.
# Contains source code and 256-byte hex0 binary seed.
# Packaged source files for the first bootstrapping stage.
#
# We don't have access to utilities such as fetchgit and fetchzip since this
# is this is part of the bootstrap process and would introduce a circular
Expand All @@ -9,23 +8,29 @@
#
# To build:
#
# nix-build pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/make-bootstrap-sources.nix
# => ./result/stage0-posix-$version-$rev-source.nar.xz
# nix-build '<nixpkgs>' -o sources.nar.xz -A make-minimal-bootstrap-sources
#

{ pkgs ? import ../../../../.. {} }:
{ lib
, fetchFromGitHub
, runCommand
, nix
, xz
}:
let
inherit (pkgs) callPackage runCommand fetchFromGitHub nix xz;

inherit (import ./bootstrap-sources.nix) name rev;
inherit (import ./bootstrap-sources.nix { }) name rev;

src = fetchFromGitHub {
owner = "oriansj";
repo = "stage0-posix";
inherit rev;
sha256 = "sha256-ZRG0k49MxL1UTZhuMTvPoEprdSpJRNVy8QhLE6k+etg=";
sha256 = "sha256-FpMp7z+B3cR3LkQ+PooH/b1/NlxH8NHVJNWifaPWt4U=";
fetchSubmodules = true;
postFetch = ''
# Seed binaries will be fetched separately
echo "Removing seed binaries"
rm -rf $out/bootstrap-seeds/*
# Remove vendored/duplicate M2libc's
echo "Removing duplicate M2libc"
rm -rf \
Expand All @@ -35,12 +40,20 @@ let
$out/mescc-tools-extra/M2libc
'';
};

in
runCommand name {
runCommand "${name}.nar.xz" {
nativeBuildInputs = [ nix xz ];

passthru = { inherit src; };

meta = with lib; {
description = "Packaged sources for the first bootstrapping stage";
homepage = "https://github.com/oriansj/stage0-posix";
license = licenses.gpl3Plus;
maintainers = teams.minimal-bootstrap.members;
platforms = platforms.all;
};
} ''
mkdir $out
nix-store --dump ${src} | xz -c > "$out/${name}.nar.xz"
nix-store --dump ${src} | xz -c > $out
''
2 changes: 2 additions & 0 deletions pkgs/top-level/all-packages.nix
Original file line number Diff line number Diff line change
Expand Up @@ -27441,6 +27441,8 @@ with pkgs;
};
checkMeta = callPackage ../stdenv/generic/check-meta.nix { };
});
minimal-bootstrap-sources = callPackage ../os-specific/linux/minimal-bootstrap/stage0-posix/bootstrap-sources.nix { };
make-minimal-bootstrap-sources = callPackage ../os-specific/linux/minimal-bootstrap/stage0-posix/make-bootstrap-sources.nix { };

mingetty = callPackage ../os-specific/linux/mingetty { };

Expand Down