fix: use emulated system for everything else except the kernel

ryan4yin · Mar 6, 2024 · fba7b6c · fba7b6c
1 parent d05451e
commit fba7b6c
Show file tree

Hide file tree

Showing 9 changed files with 110 additions and 123 deletions.
diff --git a/README.md b/README.md
@@ -138,9 +138,15 @@ sudo dd if=./xxx-UEFI-xxx.img of=/dev/mtdblk0
 # 2. partition & format the disk via disko
 sudo nix --experimental-features "nix-command flakes" run github:nix-community/disko -- --mode disko ./modules/uefi/disko-filesystem.nix
 
-# install nixos
-# NOTE: the root password you set here will be discarded when reboot
-sudo nixos-install --root /mnt --flake .#orangepi5plus-uefi --no-root-password --show-trace --verbose
+# 3. build the raw efi image
+# as we have to use the emulated system on x64 host to build the raw efi image, it will takes a long time(about 12mins) to finish.
+nom build .#rawEfiImage-opi5plus
+```
+
+If you encounter issues like [`cannot allocate memory` despite free reporting "available"](https://stackoverflow.com/questions/46464785/cannot-allocate-memory-despite-free-reporting-available) when building the raw efi image, check your `dmesg`, and try to fix it via:
+
+```bash
+echo 1 > /proc/sys/vm/compact_memory
 ```
 
 ## Debug via serial port(UART)

diff --git a/demo/flake.nix b/demo/flake.nix
@@ -6,18 +6,28 @@
   };
 
   outputs = {nixos-rk3588, ...}: let
-    system = "x86_64-linux"; # for cross-compilation on x86_64-linux
-    # system = "aarch64-linux";  # for native compilation on aarch64-linux
+    # using the same nixpkgs as nixos-rk3588 to utilize the cross-compilation cache.
+    inherit (nixos-rk3588.inputs) nixpkgs;
+    boardModule = nixos-rk3588.nixosModules.orangepi5;
+
+    # 1. for cross-compilation on x86_64-linux
+    system = "x86_64-linux";
+    # Compile the kernel using a cross-compilation tool chain
+    # Which is faster than emulating the target system.
+    pkgsKernel = import nixpkgs {
+      localSystem = "x86_64-linux";
+      crossSystem = "aarch64-linux";
+    };
+    # 2. for native compilation on aarch64-linux SBCs.
+    # system = "aarch64-linux";
+    # native compilation tool chain for the linux kernel
+    # pkgsKernel = nixpkgs;
   in {
     colmena = {
       meta = {
-        # using the same nixpkgs as nixos-rk3588 to utilize the cross-compilation cache.
-        nixpkgs = import nixos-rk3588.inputs.nixpkgs {inherit system;};
-        specialArgs = {
-          inherit (nixos-rk3588.inputs) nixpkgs;
-
-          # Provide rk3588 inputs as special argument
-          rk3588 = nixos-rk3588.inputs;
+        nixpkgs = import nixpkgs {inherit system;};
+        rk3588 = {
+          inherit nixpkgs pkgsKernel;
         };
       };
 
@@ -27,13 +37,10 @@
         # Allow local deployment with `colmena apply-local`
         # deployment.allowLocalDeployment = true;
 
-        # Use the crossSystem configuration for cross-compilation
-        # Remove this module if you want to compile natively on aarch64-linux!
-        nixpkgs.crossSystem.config = "aarch64-unknown-linux-gnu";
-
         imports = [
           # import the rk3588 module, which contains the configuration for bootloader/kernel/firmware
-          nixos-rk3588.nixosModules.orangepi5
+          boardModule.core
+          boardModule.sd-image
 
           # your custom configuration
           ./configuration.nix

diff --git a/flake.lock b/flake.lock
diff --git a/flake.nix b/flake.nix
@@ -5,11 +5,6 @@
     nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.11";
     flake-utils.url = "github:numtide/flake-utils";
 
-    disko = {
-      url = "github:nix-community/disko";
-      inputs.nixpkgs.follows = "nixpkgs";
-    };
-
     nixos-generators = {
       url = "github:nix-community/nixos-generators";
       inputs.nixpkgs.follows = "nixpkgs";
@@ -35,7 +30,26 @@
     nixos-generators,
     pre-commit-hooks,
     ...
-  }:
+  }: let
+    # Local system's architecture, the host you are running this flake on.
+    localSystem = "x86_64-linux";
+    # The native system of the target SBC.
+    aarch64System = "aarch64-linux";
+    pkgsLocal = import nixpkgs {system = localSystem;};
+    pkgsCross = import nixpkgs {
+      inherit localSystem;
+      crossSystem = aarch64System;
+    };
+
+    specialArgs = {
+      rk3588 = {
+        inherit nixpkgs;
+        # Compile the kernel using a cross-compilation tool chain
+        # Which is faster than emulating the target system.
+        pkgsKernel = pkgsCross;
+      };
+    };
+  in
     {
       nixosModules = {
         # Orange Pi 5 SBC
@@ -57,24 +71,24 @@
         };
       };
 
-      nixosConfigurations = let
-        # Local system's architecture, the host you are running this flake on.
-        system = "x86_64-linux";
-      in
+      nixosConfigurations =
         # sdImage - boot via U-Boot
         (builtins.mapAttrs (name: board:
           nixpkgs.lib.nixosSystem {
-            inherit system;
-            specialArgs.rk3588 = {inherit (inputs) nixpkgs mesa-panfork;};
+            # Use emulated target system here.
+            # NOTE: we use pkgsCross for the kernel build only,
+            # and emulated the target system for everything else,
+            # so that we can use nixos's official binary cache for the rest of the packages.
+            system = aarch64System;
+            inherit specialArgs;
             modules = [
               ./modules/configuration.nix
               board.core
               board.sd-image
 
               {
                 networking.hostName = name;
-                imageBaseName = "${name}-sd-image";
-                nixpkgs.crossSystem.config = "aarch64-unknown-linux-gnu";
+                sdImage.imageBaseName = "${name}-sd-image";
               }
             ];
           })
@@ -85,55 +99,42 @@
             nixpkgs.lib.nameValuePair
             (name + "-uefi")
             (nixpkgs.lib.nixosSystem {
-              inherit system;
-              specialArgs.rk3588 = {inherit (inputs) nixpkgs mesa-panfork;};
+              # Use emulated target system here.
+              system = aarch64System;
+              inherit specialArgs;
               modules = [
                 board.core
                 ./modules/configuration.nix
                 {
                   networking.hostName = name;
-                  nixpkgs.crossSystem.config = "aarch64-unknown-linux-gnu";
 
-                  # define a custom nixos-generators's image format
-                  formatConfigs.aarch64-raw-efi = {
+                  # increase qemu's memory to 4G, to avoid the "Cannot allocate memory" error.
+                  formatConfigs.raw-efi = {
                     config,
                     lib,
                     pkgs,
                     modulesPath,
-                    specialArgs,
                     ...
                   }: {
-                    system.build.raw = import "${toString modulesPath}/../lib/make-disk-image.nix" {
+                    system.build.raw = lib.mkForce (import "${toString modulesPath}/../lib/make-disk-image.nix" {
+                      # https://github.com/NixOS/nixpkgs/blob/nixos-23.11/nixos/lib/make-disk-image.nix
                       inherit lib config pkgs;
+                      partitionTableType = "efi";
                       diskSize = specialArgs.diskSize or "auto";
                       format = "raw";
-                    };
-
-                    formatAttr = "raw";
-                    fileExtension = ".img";
+                      memSize = 8192;
+                    });
                   };
                 }
 
                 nixos-generators.nixosModules.all-formats
-                disko.nixosModules.disko
-                ./modules/uefi/disko-filesystem.nix
               ];
             }))
           self.nixosModules);
     }
     // flake-utils.lib.eachDefaultSystem (system: let
-      pkgsKernel = import nixpkgs {
-        inherit system;
-
-        crossSystem.config = "aarch64-unknown-linux-gnu";
-
-        overlays = [
-          (_self: super: {
-            linuxPackages_rockchip = super.linuxPackagesFor (super.callPackage ./pkgs/kernel/legacy.nix {});
-          })
-        ];
-      };
-      pkgs = import nixpkgs {inherit system;};
+      kernelPackages = pkgsCross.linuxPackagesFor (pkgsCross.callPackage ../../pkgs/kernel/legacy.nix {});
+      pkgs = pkgsLocal;
     in {
       packages = {
         # sdImage
@@ -142,7 +143,9 @@
         sdImage-rock5a = self.nixosConfigurations.rock5a.config.system.build.sdImage;
 
         # UEFI raw image
-        rawEfiImage-opi5plus = self.nixosConfigurations.orangepi5plus-uefi.config.formats.aarch64-raw-efi;
+        rawEfiImage-opi5 = self.nixosConfigurations.orangepi5-uefi.config.formats.raw-efi;
+        rawEfiImage-opi5plus = self.nixosConfigurations.orangepi5plus-uefi.config.formats.raw-efi;
+        rawEfiImage-rock5a = self.nixosConfigurations.rock5a-uefi.config.formats.raw-efi;
 
         # the custom kernel for debugging
         # use `nix develop` to enter the environment with the custom kernel build environment available.
@@ -154,7 +157,7 @@
         # Solution
         #   - unpackPhase, and the use `nix develop .#fhsEnv` to enter the fhs test environment.
         #   - Then use `make menuconfig` to configure the kernel.
-        kernel = pkgsKernel.linuxPackages_rockchip.kernel.dev;
+        kernel = kernelPackages.kernel.dev;
       };
 
       # use `nix develop .#fhsEnv` to enter the fhs test environment defined here.

diff --git a/modules/boards/orangepi5.nix b/modules/boards/orangepi5.nix
@@ -1,14 +1,15 @@
 # =========================================================================
 #      Orange Pi 5 Specific Configuration
 # =========================================================================
-{pkgs, ...}: let
+{rk3588, ...}: let
+  pkgsKernel = rk3588.pkgsKernel;
 in {
   imports = [
     ./base.nix
   ];
 
   boot = {
-    kernelPackages = pkgs.linuxPackagesFor (pkgs.callPackage ../../pkgs/kernel/legacy.nix {});
+    kernelPackages = pkgsKernel.linuxPackagesFor (pkgsKernel.callPackage ../../pkgs/kernel/legacy.nix {});
 
     # kernelParams copy from Armbian's /boot/armbianEnv.txt & /boot/boot.cmd
     kernelParams = [

diff --git a/modules/boards/orangepi5plus.nix b/modules/boards/orangepi5plus.nix
@@ -1,13 +1,15 @@
 # =========================================================================
 #      Orange Pi 5 Plus Specific Configuration
 # =========================================================================
-{pkgs, ...}: {
+{rk3588, ...}: let
+  pkgsKernel = rk3588.pkgsKernel;
+in {
   imports = [
     ./base.nix
   ];
 
   boot = {
-    kernelPackages = pkgs.linuxPackagesFor (pkgs.callPackage ../../pkgs/kernel/legacy.nix {});
+    kernelPackages = pkgsKernel.linuxPackagesFor (pkgsKernel.callPackage ../../pkgs/kernel/legacy.nix {});
 
     # kernelParams copy from Armbian's /boot/armbianEnv.txt & /boot/boot.cmd
     kernelParams = [

diff --git a/modules/boards/rock5a.nix b/modules/boards/rock5a.nix
@@ -1,13 +1,15 @@
 # =========================================================================
 #      Rock 5 Model A Specific Configuration
 # =========================================================================
-{pkgs, ...}: {
+{rk3588, ...}: let
+  pkgsKernel = rk3588.pkgsKernel;
+in {
   imports = [
     ./base.nix
   ];
 
   boot = {
-    kernelPackages = pkgs.linuxPackagesFor (pkgs.callPackage ../../pkgs/kernel/legacy.nix {});
+    kernelPackages = pkgsKernel.linuxPackagesFor (pkgsKernel.callPackage ../../pkgs/kernel/legacy.nix {});
 
     # kernelParams copy from rock5a's official debian image's /boot/extlinux/extlinux.conf
     # https://www.kernel.org/doc/html/latest/admin-guide/kernel-parameters.html

diff --git a/modules/sd-image/orangepi5.nix b/modules/sd-image/orangepi5.nix
@@ -1,13 +1,13 @@
 {
   lib,
   config,
-  nixpkgs,
+  rk3588,
   ...
 }: let
   rootPartitionUUID = "14e19a7b-0ae0-484d-9d54-43bd6fdc20c7";
 in {
   imports = [
-    "${nixpkgs}/nixos/modules/installer/sd-card/sd-image-aarch64.nix"
+    "${rk3588.nixpkgs}/nixos/modules/installer/sd-card/sd-image-aarch64.nix"
   ];
 
   boot = {
@@ -24,7 +24,6 @@ in {
 
   sdImage = {
     inherit rootPartitionUUID;
-
     compressImage = true;
 
     # install firmware into a separate partition: /boot/firmware