Merge pull request #3145 from input-output-hk/fix/ddw-1227-hydra-oom

[DDW-1227] Fix out-of-memory errors on ci.iog.io
input-output-hk · Aug 16, 2023 · 883cbed · 883cbed
2 parents ac43451 + 3effc28
commit 883cbed
Show file tree

Hide file tree

Showing 16 changed files with 270 additions and 273 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,10 @@
 
 ## vNext
 
+### Fixes
+
+- Fixed out-of-memory errors on https://ci.iog.io ([PR 3145](https://github.com/input-output-hk/daedalus/pull/3145))
+
 ## 5.3.0
 
 ### Features

diff --git a/flake.nix b/flake.nix
@@ -25,12 +25,8 @@
       import ./nix/packages.nix { inherit inputs buildSystem; }
     );
 
-    devShells = lib.genAttrs supportedSystems (
-      system: let
-        all = lib.genAttrs inputs.self.internal.installerClusters (
-          cluster: import ./nix/internal/old-shell.nix { inherit inputs system cluster; }
-        );
-      in all // { default = all.mainnet; }
+    devShells = lib.genAttrs supportedSystems (targetSystem:
+      import ./nix/devshells.nix { inherit inputs targetSystem; }
     );
 
     # Compatibility with older Nix:
@@ -39,9 +35,7 @@
 
     hydraJobs = {
       installer = lib.genAttrs (supportedSystems ++ ["x86_64-windows"]) (
-        targetSystem: lib.genAttrs inputs.self.internal.installerClusters (
-          cluster: inputs.self.internal.${targetSystem}.${cluster}.unsignedInstaller
-        )
+        targetSystem: inputs.self.internal.${targetSystem}.unsignedInstaller
       );
       devshell = lib.genAttrs supportedSystems (system: inputs.self.devShells.${system}.default);
       required = inputs.nixpkgs.legacyPackages.x86_64-linux.releaseTools.aggregate {

diff --git a/nix/internal/old-shell.nix → nix/devshells.nix b/nix/internal/old-shell.nix → nix/devshells.nix
@@ -1,45 +1,38 @@
-{ system
-, inputs
-, cluster ? "selfnode"
+{ inputs
+, targetSystem
 }:
 
 let
-  internal = inputs.self.internal.${system}.${cluster};
-  pkgs = internal.newCommon.pkgs;
-
-  daedalusPkgs = import ./old-default.nix {
-    inherit inputs cluster;
-    target = system;
-    devShell = true;
-  };
-
-  regenerateDevCerts = let
-    moddedConfig = pkgs.writeText "launcher-config.yaml" (builtins.toJSON (
-      daedalusPkgs.launcherConfigs.launcherConfig
-      // {
-        daedalusBin = "true";
-      }
-    ));
-  in
-    pkgs.writeShellScriptBin "regenerate-dev-certs" ''
-      ${daedalusPkgs.daedalus-bridge}/bin/cardano-launcher --config ${moddedConfig}
-    '';
-
-  gcRoot = pkgs.runCommandLocal "gc-root" {
-    inherit daedalusShell;
-    cardanoWalletsHaskellNix = daedalusPkgs.walletFlake.outputs.legacyPackages.${system}.roots;
-    daedalusInstallerInputs = with daedalusPkgs.daedalus-installer; buildInputs ++ nativeBuildInputs;
-    # cardano-bridge inputs are GC’d, and rebuilt too often on Apple M1 CI:
-    cardanoBridgeInputs = builtins.map (attr: if daedalusPkgs ? ${attr} && pkgs.lib.isDerivation daedalusPkgs.${attr} then daedalusPkgs.${attr} else null) (builtins.attrNames (builtins.functionArgs (import ./cardano-bridge.nix)));
-  } "export >$out";
-
-  daedalusShell = pkgs.stdenv.mkDerivation (rec {
+  internal = inputs.self.internal.${targetSystem};
+  inherit (internal) common;
+  inherit (common) pkgs;
+
+  devShells = pkgs.lib.genAttrs inputs.self.internal.installerClusters (cluster: let
+
+    launcherConfigs = common.mkLauncherConfigs {
+      inherit cluster;
+      devShell = true;
+    };
+
+    regenerateDevCerts = let
+      moddedConfig = pkgs.writeText "launcher-config.yaml" (builtins.toJSON (
+        launcherConfigs.launcherConfig
+        // {
+          daedalusBin = "true";
+        }
+      ));
+    in
+      pkgs.writeShellScriptBin "regenerate-dev-certs" ''
+        ${common.daedalus-bridge.${cluster}}/bin/cardano-launcher --config ${moddedConfig}
+      '';
+
+  in pkgs.stdenv.mkDerivation (rec {
     buildInputs = [
-      internal.newCommon.nodejs
-      internal.newCommon.yarn
-      daedalusPkgs.daedalus-bridge
-      daedalusPkgs.daedalus-installer
-      daedalusPkgs.mock-token-metadata-server
+      internal.common.nodejs
+      internal.common.yarn
+      common.daedalus-bridge.${cluster}
+      common.daedalus-installer
+      common.mock-token-metadata-server
       regenerateDevCerts
     ] ++ (with pkgs; [
       nix bash binutils coreutils curl gnutar
@@ -61,11 +54,11 @@ let
     name = "daedalus";
     buildCommand = "touch $out";
     LAUNCHER_CONFIG = DAEDALUS_CONFIG + "/launcher-config.yaml";
-    CARDANO_NODE_VERSION = daedalusPkgs.cardanoNodeVersion;
-    CARDANO_WALLET_VERSION = daedalusPkgs.cardanoWalletVersion;
+    CARDANO_NODE_VERSION = common.cardanoNodeVersion;
+    CARDANO_WALLET_VERSION = common.cardanoWalletVersion;
     DAEDALUS_CONFIG = pkgs.runCommand "daedalus-config" {} ''
       mkdir -pv $out
-      cp ${pkgs.writeText "launcher-config.yaml" (builtins.toJSON daedalusPkgs.launcherConfigs.launcherConfig)} $out/launcher-config.yaml
+      cp ${pkgs.writeText "launcher-config.yaml" (builtins.toJSON launcherConfigs.launcherConfig)} $out/launcher-config.yaml
     '';
     DAEDALUS_INSTALL_DIRECTORY = "./";
     DAEDALUS_DIR = DAEDALUS_INSTALL_DIRECTORY;
@@ -89,7 +82,7 @@ let
       source <(cardano-address --bash-completion-script cardano-address)
       [[ $(type -P cardano-wallet) ]] && source <(cardano-wallet --bash-completion-script cardano-wallet)
 
-      cp -f ${daedalusPkgs.launcherConfigs.installerConfig.iconPath.small} $DAEDALUS_INSTALL_DIRECTORY/icon.png
+      cp -f ${launcherConfigs.installerConfig.iconPath.small} $DAEDALUS_INSTALL_DIRECTORY/icon.png
 
       # These links will only occur to binaries that exist for the
       # specific build config
@@ -99,7 +92,7 @@ let
 
       source <(cardano-node --bash-completion-script `type -p cardano-node`)
 
-      export NIX_CFLAGS_COMPILE="$NIX_CFLAGS_COMPILE -I${internal.newCommon.nodejs}/include/node -I${toString ../../.}/node_modules/node-addon-api"
+      export NIX_CFLAGS_COMPILE="$NIX_CFLAGS_COMPILE -I${internal.common.nodejs}/include/node -I${toString ../../.}/node_modules/node-addon-api"
       yarn install
 
       # Rebuild native modules for <https://www.electronjs.org/docs/latest/tutorial/using-native-node-modules>:
@@ -123,7 +116,7 @@ let
 
       echo 'Resolving environment variables to absolute paths…'
       # XXX: they originally contain references to HOME or XDG_DATA_HOME in launcher-config.yaml:
-      export CARDANO_WALLET_TLS_PATH="${daedalusPkgs.launcherConfigs.launcherConfig.tlsPath}"
+      export CARDANO_WALLET_TLS_PATH="${launcherConfigs.launcherConfig.tlsPath}"
 
       echo 'Re-generating dev certificates for ‘cardano-wallet’…'
       mkdir -p "$CARDANO_WALLET_TLS_PATH"
@@ -132,5 +125,6 @@ let
       echo
       echo 'Now, run ‘yarn dev’.'
     '';
-  });
-in daedalusShell // { inherit gcRoot; }
+  }));
+
+in devShells // { default = devShells.mainnet; }
diff --git a/nix/internal.nix b/nix/internal.nix
@@ -5,9 +5,7 @@ let
   inherit (import ./internal/source-lib.nix { inherit inputs; }) installerClusters;
 in
 
-(__mapAttrs (targetSystem: src: lib.genAttrs installerClusters (cluster: import src {
-  inherit inputs targetSystem cluster;
-})) {
+(__mapAttrs (targetSystem: src: import src { inherit inputs targetSystem; }) {
   x86_64-linux = ./internal/x86_64-linux.nix;
   x86_64-windows = ./internal/x86_64-windows.nix;
   x86_64-darwin = ./internal/any-darwin.nix;

diff --git a/nix/internal/any-darwin.nix b/nix/internal/any-darwin.nix
@@ -1,28 +1,30 @@
-{ inputs, targetSystem, cluster }:
+{ inputs, targetSystem }:
 
 assert targetSystem == "x86_64-darwin" || targetSystem == "aarch64-darwin";
 
 let
 
-  newCommon = import ./new-common.nix { inherit inputs targetSystem cluster; };
+  common = import ./common.nix { inherit inputs targetSystem; };
 
-  inherit (newCommon) sourceLib oldCode pkgs;
+  inherit (common) sourceLib pkgs;
   inherit (pkgs) lib;
 
-  inherit (oldCode)
+  inherit (common)
     daedalus-bridge daedalus-installer launcherConfigs mock-token-metadata-server
     cardanoNodeVersion cardanoWalletVersion;
 
-  inherit (newCommon) originalPackageJson electronVersion electronChromedriverVersion commonSources;
+  inherit (common) originalPackageJson electronVersion electronChromedriverVersion commonSources;
 
   archSuffix = if pkgs.system == "aarch64-darwin" then "arm64" else "x64";
   packageVersion = originalPackageJson.version;
-  installerName = "daedalus-${packageVersion}-${toString sourceLib.buildCounter}-${cluster}-${sourceLib.buildRevShort}-${pkgs.system}";
+  installerName = cluster: "daedalus-${packageVersion}-${toString sourceLib.buildCounter}-${cluster}-${sourceLib.buildRevShort}-${pkgs.system}";
+
+  genClusters = lib.genAttrs sourceLib.installerClusters;
 
 in rec {
 
-  inherit newCommon oldCode;
-  inherit (newCommon) nodejs nodePackages yarn yarn2nix offlineCache srcLockfiles srcWithoutNix;
+  inherit common;
+  inherit (common) nodejs nodePackages yarn yarn2nix offlineCache srcLockfiles srcWithoutNix;
 
   # The following is used in all `configurePhase`s:
   darwinSpecificCaches = let
@@ -65,7 +67,7 @@ in rec {
       apple_sdk.frameworks.CoreServices
       apple_sdk.frameworks.AppKit
     ]);
-    configurePhase = newCommon.setupCacheAndGypDirs + darwinSpecificCaches;
+    configurePhase = common.setupCacheAndGypDirs + darwinSpecificCaches;
     buildPhase = ''
       # Do not look up in the registry, but in the offline cache:
       ${yarn2nix.fixup_yarn_lock}/bin/fixup_yarn_lock yarn.lock
@@ -103,7 +105,7 @@ in rec {
 
   darwin-launcher = pkgs.callPackage ./darwin-launcher.nix {};
 
-  package = let
+  package = genClusters (cluster: let
     pname = "daedalus";
   in pkgs.stdenv.mkDerivation {
     name = pname;
@@ -115,7 +117,7 @@ in rec {
       apple_sdk.frameworks.AppKit
       libobjc
     ]) ++ [
-      daedalus-bridge
+      daedalus-bridge.${cluster}
       darwin-launcher
       mock-token-metadata-server
     ];
@@ -125,25 +127,25 @@ in rec {
     BUILD_COUNTER = sourceLib.buildCounter;
     CARDANO_WALLET_VERSION = cardanoWalletVersion;
     CARDANO_NODE_VERSION = cardanoNodeVersion;
-    configurePhase = newCommon.setupCacheAndGypDirs + darwinSpecificCaches + ''
+    configurePhase = common.setupCacheAndGypDirs + darwinSpecificCaches + ''
       # Grab all cached `node_modules` from above:
       cp -r ${node_modules}/. ./
       chmod -R +w .
     '';
     outputs = [ "out" "futureInstaller" ];
     buildPhase = ''
       patchShebangs .
-      sed -r 's#.*patchElectronRebuild.*#${newCommon.patchElectronRebuild}/bin/*#' -i scripts/rebuild-native-modules.sh
+      sed -r 's#.*patchElectronRebuild.*#${common.patchElectronRebuild}/bin/*#' -i scripts/rebuild-native-modules.sh
 
       export DEVX_FIXME_DONT_YARN_INSTALL=1
       (
         cd installers/
-        cp -r ${launcherConfigs.configFiles}/. ./.
+        cp -r ${launcherConfigs.${cluster}.configFiles}/. ./.
 
         # make-installer needs to see `bin/nix-store` to break all references to dylibs inside /nix/store:
         export PATH="${lib.makeBinPath [ pkgs.nixUnstable ]}:$PATH"
 
-        make-installer --cardano ${daedalus-bridge} \
+        make-installer --cardano ${daedalus-bridge.${cluster}} \
           --build-rev-short ${sourceLib.buildRevShort} \
           --build-counter ${toString sourceLib.buildCounter} \
           --cluster ${cluster} \
@@ -153,7 +155,7 @@ in rec {
     '';
     installPhase = ''
       mkdir -p $out/Applications/
-      cp -r release/darwin-${archSuffix}/${lib.escapeShellArg launcherConfigs.installerConfig.spacedName}-darwin-${archSuffix}/${lib.escapeShellArg launcherConfigs.installerConfig.spacedName}.app $out/Applications/
+      cp -r release/darwin-${archSuffix}/${lib.escapeShellArg launcherConfigs.${cluster}.installerConfig.spacedName}-darwin-${archSuffix}/${lib.escapeShellArg launcherConfigs.${cluster}.installerConfig.spacedName}.app $out/Applications/
 
       # XXX: remove redundant native modules, and point bindings.js to Contents/MacOS/*.node instead:
       echo 'Deleting all redundant ‘*.node’ files under to-be-distributed ‘node_modules/’:'
@@ -169,7 +171,7 @@ in rec {
       mkdir -p $out/bin/
       cat >$out/bin/${pname} << EOF
       #!/bin/sh
-      exec $out/Applications/${lib.escapeShellArg launcherConfigs.installerConfig.spacedName}.app/Contents/MacOS/${lib.escapeShellArg launcherConfigs.installerConfig.spacedName}
+      exec $out/Applications/${lib.escapeShellArg launcherConfigs.${cluster}.installerConfig.spacedName}.app/Contents/MacOS/${lib.escapeShellArg launcherConfigs.${cluster}.installerConfig.spacedName}
       EOF
       chmod +x $out/bin/${pname}
 
@@ -181,13 +183,13 @@ in rec {
       ${signAllBinaries} $out
     '';
     dontFixup = true; # TODO: just to shave some seconds, turn back on after everything works
-  };
+  });
 
-  unsignedInstaller = pkgs.stdenv.mkDerivation {
+  unsignedInstaller = genClusters (cluster: pkgs.stdenv.mkDerivation {
     name = "daedalus-unsigned-darwin-installer";
     dontUnpack = true;
     buildPhase = ''
-      ${makeSignedInstaller}/bin/* | tee make-installer.log
+      ${makeSignedInstaller.${cluster}}/bin/* | tee make-installer.log
     '';
     installPhase = ''
       mkdir -p $out
@@ -197,9 +199,9 @@ in rec {
       mkdir -p $out/nix-support
       echo "file binary-dist \"$(echo $out/*.pkg)\"" >$out/nix-support/hydra-build-products
     '';
-  };
+  });
 
-  makeSignedInstaller = pkgs.writeShellScriptBin "make-signed-installer" (let
+  makeSignedInstaller = genClusters (cluster: pkgs.writeShellScriptBin "make-signed-installer" (let
 
     # FIXME: in the future this has to be done better, now let’s reuse the Buildkite legacy:
     credentials = "/var/lib/buildkite-agent/signing.sh";
@@ -230,8 +232,8 @@ in rec {
       eval $(${readConfigs})
 
       workDir=$(mktemp -d)
-      appName=${lib.escapeShellArg launcherConfigs.installerConfig.spacedName}.app
-      appDir=${package}/Applications/"$appName"
+      appName=${lib.escapeShellArg launcherConfigs.${cluster}.installerConfig.spacedName}.app
+      appDir=${package.${cluster}}/Applications/"$appName"
 
       echo "Info: workDir = $workDir"
       cd "$workDir"
@@ -242,20 +244,20 @@ in rec {
 
       if ${shallSignPredicate} ; then
         echo "Signing code…"
-        ${package.futureInstaller}/codesign.sh "$codeSigningIdentity" "$codeSigningKeyChain" \
-          "$appName" ${package.futureInstaller}/entitlements.xml
+        ${package.${cluster}.futureInstaller}/codesign.sh "$codeSigningIdentity" "$codeSigningKeyChain" \
+          "$appName" ${package.${cluster}.futureInstaller}/entitlements.xml
       fi
 
       echo "Making installer…"
       /usr/bin/pkgbuild \
-        --identifier ${lib.escapeShellArg ("org." + launcherConfigs.installerConfig.macPackageName + ".pkg")} \
+        --identifier ${lib.escapeShellArg ("org." + launcherConfigs.${cluster}.installerConfig.macPackageName + ".pkg")} \
         --component "$workDir/$appName" \
         --install-location /Applications \
-        ${lib.escapeShellArg (installerName + ".phase1.pkg")}
+        ${lib.escapeShellArg ((installerName cluster) + ".phase1.pkg")}
       rm -r "$workDir/$appName"
       /usr/bin/productbuild --product ${../../installers/data/plist} \
         --package *.phase1.pkg \
-        ${lib.escapeShellArg (installerName + ".phase2.pkg")}
+        ${lib.escapeShellArg ((installerName cluster) + ".phase2.pkg")}
       rm *.phase1.pkg
 
       if ${shallSignPredicate} ; then
@@ -269,14 +271,14 @@ in rec {
 
         productsign --sign "$signingIdentity" --keychain "$signingKeyChain" \
           *.phase2.pkg \
-          ${lib.escapeShellArg (installerName + ".pkg")}
+          ${lib.escapeShellArg ((installerName cluster) + ".pkg")}
         rm *.phase2.pkg
       else
-        mv *.phase2.pkg ${lib.escapeShellArg (installerName + ".pkg")}
+        mv *.phase2.pkg ${lib.escapeShellArg ((installerName cluster) + ".pkg")}
       fi
 
       echo "Done, you can submit it for notarization now:"
-      echo "$workDir"/${lib.escapeShellArg (installerName + ".pkg")}
+      echo "$workDir"/${lib.escapeShellArg ((installerName cluster) + ".pkg")}
     '';
 
   in ''
@@ -287,7 +289,7 @@ in rec {
     else
       exec ${packAndSign}
     fi
-  '');
+  ''));
 
   darwinSources = {
     electron = pkgs.fetchurl {