diff --git a/nix/workbench/backend/nomad-job.nix b/nix/workbench/backend/nomad-job.nix index 7d2a0d57968..57bcdaed097 100644 --- a/nix/workbench/backend/nomad-job.nix +++ b/nix/workbench/backend/nomad-job.nix @@ -7,8 +7,8 @@ , lib , stateDir , profileData -, containerSpecs , generatorTaskName +, containerPkgs , oneTracerPerNode ? false , withSsh ? false }: @@ -73,8 +73,8 @@ let entrypoint = let - coreutils = containerSpecs.containerPkgs.coreutils.nix-store-path; - supervisor = containerSpecs.containerPkgs.supervisor.nix-store-path; + coreutils = containerPkgs.coreutils.nix-store-path; + supervisor = containerPkgs.supervisor.nix-store-path; in escapeTemplate '' # Store entrypoint's envars and "uname" in a file for debugging purposes. @@ -508,7 +508,7 @@ let # ERROR: cannot verify iog-cardano-perf.s3.eu-central-1.amazonaws.com's certificate, issued by 'CN=Amazon RSA 2048 M01,O=Amazon,C=US': # Unable to locally verify the issuer's authority. # To connect to iog-cardano-perf.s3.eu-central-1.amazonaws.com insecurely, use `--no-check-certificate'. - SSL_CERT_FILE = "${containerSpecs.containerPkgs.cacert.nix-store-path}/etc/ssl/certs/ca-bundle.crt"; + SSL_CERT_FILE = "${containerPkgs.cacert.nix-store-path}/etc/ssl/certs/ca-bundle.crt"; }; # Sensible defaults. @@ -937,9 +937,9 @@ let ../service/ssh.nix { inherit pkgs; - bashInteractive = containerSpecs.containerPkgs.bashInteractive.nix-store-path; - coreutils = containerSpecs.containerPkgs.coreutils.nix-store-path; - openssh_hacks = containerSpecs.containerPkgs.openssh_hacks.nix-store-path; + bashInteractive = containerPkgs.bashInteractive.nix-store-path; + coreutils = containerPkgs.coreutils.nix-store-path; + sshdExecutable = "${containerPkgs.openssh_hacks.nix-store-path}/bin/sshd"; } ; in [ @@ -993,12 +993,12 @@ let { driver = "exec"; config = { - command = "${containerSpecs.containerPkgs.bashInteractive.nix-store-path}/bin/bash"; + command = "${containerPkgs.bashInteractive.nix-store-path}/bin/bash"; args = ["local/entrypoint.sh"]; nix_installables = (lib.attrsets.mapAttrsToList - (name: attr: attr.nix-store-path) - containerSpecs.containerPkgs + (name: attr: attr.installable) + containerPkgs ) ; }; diff --git a/nix/workbench/backend/nomad.nix b/nix/workbench/backend/nomad.nix index a0cfcaea040..a63db0ec8e2 100644 --- a/nix/workbench/backend/nomad.nix +++ b/nix/workbench/backend/nomad.nix @@ -9,221 +9,271 @@ let # Backend-specific Nix bits: - materialise-profile = - { profileData }: - let - # TODO: Repeated code, add the generator's node name to profile.json - generatorTaskName = if builtins.hasAttr "explorer" profileData.node-specs.value - then "explorer" - else "node-0" + materialise-profile = { profileData }: + # Intermediate / workbench-adhoc container specifications + let containerSpecs = rec { + ########################################################################## + # The actual commit. The one used when entering the workbench. + gitrev = pkgs.gitrev; + # Where to deploy inside the Task, needed to send commands (`nomad exec`). + diretories = rec { + work = "/local"; + state = stateDir; + run = lib.strings.concatStringsSep "/" [ work state ]; + }; + # How to configure supervisor and how to access it from the deployer. + supervisord = { + url = "unix:///tmp/supervisor.sock"; + conf = lib.strings.concatStringsSep "/" + [ diretories.run "supervisor" "supervisord.conf"] ; - # Intermediate / workbench-adhoc container specifications - containerSpecs = rec { - # - diretories = rec { - work = "/local"; - state = stateDir; - run = lib.strings.concatStringsSep "/" [ work state ]; - }; - # The actual commit. - gitrev = pkgs.gitrev; - # Binaries. - containerPkgs = - { - coreutils = rec { - nix-store-path = pkgs.coreutils; - flake-reference = "github:intersectmbo/cardano-node"; - flake-output = "legacyPackages.x86_64-linux.coreutils"; - installable = "${flake-reference}/${gitrev}#${flake-output}"; - }; - bashInteractive = rec { - nix-store-path = pkgs.bashInteractive; - flake-reference = "github:intersectmbo/cardano-node"; - flake-output = "legacyPackages.x86_64-linux.bashInteractive"; - installable = "${flake-reference}/${gitrev}#${flake-output}"; - }; - findutils = rec { - nix-store-path = pkgs.findutils; - flake-reference = "github:intersectmbo/cardano-node"; - flake-output = "legacyPackages.x86_64-linux.findutils"; - installable = "${flake-reference}/${gitrev}#${flake-output}"; - }; - iputils = rec { - nix-store-path = pkgs.iputils; - flake-reference = "github:intersectmbo/cardano-node"; - flake-output = "legacyPackages.x86_64-linux.iputils"; - installable = "${flake-reference}/${gitrev}#${flake-output}"; - }; - gnutar = rec { - nix-store-path = pkgs.gnutar; - flake-reference = "github:intersectmbo/cardano-node"; - flake-output = "legacyPackages.x86_64-linux.gnutar"; - installable = "${flake-reference}/${gitrev}#${flake-output}"; - }; - zstd = rec { - nix-store-path = pkgs.zstd; - flake-reference = "github:intersectmbo/cardano-node"; - flake-output = "legacyPackages.x86_64-linux.zstd"; - installable = "${flake-reference}/${gitrev}#${flake-output}"; - }; - wget = rec { - nix-store-path = pkgs.wget; - flake-reference = "github:intersectmbo/cardano-node"; - flake-output = "legacyPackages.x86_64-linux.wget"; - installable = "${flake-reference}/${gitrev}#${flake-output}"; - }; - cacert = rec { - nix-store-path = pkgs.cacert; - flake-reference = "github:intersectmbo/cardano-node"; - flake-output = "legacyPackages.x86_64-linux.cacert"; - installable = "${flake-reference}/${gitrev}#${flake-output}"; - }; - supervisor = rec { - nix-store-path = pkgs.python3Packages.supervisor; - flake-reference = "github:intersectmbo/cardano-node"; - flake-output = "legacyPackages.x86_64-linux.python3Packages.supervisor"; - installable = "${flake-reference}/${gitrev}#${flake-output}"; - }; - gnugrep = rec { - nix-store-path = pkgs.gnugrep; - flake-reference = "github:intersectmbo/cardano-node"; - flake-output = "legacyPackages.x86_64-linux.gnugrep"; - installable = "${flake-reference}/${gitrev}#${flake-output}"; - }; - jq = rec { - nix-store-path = pkgs.jq; - flake-reference = "github:intersectmbo/cardano-node"; - flake-output = "legacyPackages.x86_64-linux.jq"; - installable = "${flake-reference}/${gitrev}#${flake-output}"; - }; - } - // - # TODO: - cardano-node.passthru.profiled - # - cardano-node.passthru.eventlogged - # - cardano-node.passthru.asserted - # profileData.node-services."node-0".serviceConfig.value.eventlog - # builtins.trace (builtins.attrNames profileData.node-services."node-0".serviceConfig.value.eventlog) XXXX - { - cardano-node = rec { - nix-store-path = with pkgs; - if eventlogged - then cardanoNodePackages.cardano-node.passthru.eventlogged - else cardanoNodePackages.cardano-node - ; - flake-reference = "github:intersectmbo/cardano-node"; - flake-output = - if eventlogged - then "cardanoNodePackages.cardano-node.passthru.eventlogged" - else "cardanoNodePackages.cardano-node" - ; - installable = "${flake-reference}/${gitrev}#${flake-output}"; - }; - cardano-cli = rec { - nix-store-path = pkgs.cardanoNodePackages.cardano-cli; - flake-reference = "github:input-output-hk/cardano-cli"; - flake-output = "cardanoNodePackages.cardano-cli"; - installable = "${flake-reference}/${gitrev}#${flake-output}"; - }; - cardano-tracer = rec { - nix-store-path = pkgs.cardanoNodePackages.cardano-tracer; - flake-reference = "github:intersectmbo/cardano-node"; - flake-output = "cardanoNodePackages.cardano-tracer"; - installable = "${flake-reference}/${gitrev}#${flake-output}"; - }; - tx-generator = rec { - nix-store-path = pkgs.cardanoNodePackages.tx-generator; - flake-reference = "github:intersectmbo/cardano-node"; - flake-output = "cardanoNodePackages.tx-generator"; - installable = "${flake-reference}/${gitrev}#${flake-output}"; - }; - } - // - lib.attrsets.optionalAttrs (subBackendName == "cloud") { - openssh_hacks = rec { - commit = "5d8c5913c70723318acf47496e2abf7d2c99384f"; # OpenSSH version 9.8 (Branch "9.8"); - nix-store-path = (__getFlake "github:fmaste/openssh-portable-hacks/${commit}").packages.x86_64-linux.openssh_hacks; - flake-reference = "github:fmaste/openssh-portable-hacks"; - flake-output = "packages.x86_64-linux.openssh_hacks"; - installable = "${flake-reference}/${commit}#${flake-output}"; - }; - rsync = rec { - nix-store-path = pkgs.rsync; - flake-reference = "github:intersectmbo/cardano-node"; - flake-output = "legacyPackages.x86_64-linux.rsync"; - installable = "${flake-reference}/${gitrev}#${flake-output}"; - }; - } - ; - supervisord = { - url = "unix:///tmp/supervisor.sock"; - conf = lib.strings.concatStringsSep "/" - [ diretories.run "supervisor" "supervisord.conf"] - ; + }; + # Binaries. Flake references to the local nix store or remote repos. + containerPkgs = installables {inherit gitrev;}; + # The Nomad Job description for the requested sub-backend. + nomadJob = nomad-job {inherit profileData containerPkgs;}; + ########################################################################## + }; + in pkgs.runCommand "workbench-backend-output-${profileData.profileName}-nomad" + ({ + containerSpecsJSON = pkgs.writeText "workbench-cluster-container-pkgs.json" + (lib.generators.toJSON {} containerSpecs); + }) + '' + mkdir $out + ln -s $containerSpecsJSON $out/container-specs.json + '' + ; + + # The installables are all the Nix packages that will be avaiable inside the + # Nomad Task. This dependencies are defined inside the Nomad Job as strings + # that can be either a path to the local nix store or a flake reference to + # fetch from some repo. + # In the case of cloud deployment the "installable" must always reference + # a commit accesible from every Nomad client machine and for local / "exec" + # the "nix-store-path" property is used to allow to run with local changes. + # The workbench will by default insert in the Nomad Job description the + # "installable" property as defined here while keeping the extra details as a + # `jq` friendly reference that are used to change it later. + installables = {gitrev}: + ############################################################################ + # The "default" / basic environment where the node will run. ############### + # This pkgs rarely change and are almost always cached. ############### + ############################################################################ + (lib.attrsets.mapAttrs + (name: attr: + # The installable property is always the same. + let flakeReference = attr.flake-reference; + flakeOutput = attr.flake-output; + # The commit must come from `pkgs` because all script are using + # this for the shebang and other basic tools ("coreutils") and are + # also the dependencies used inside the workbench, like `jq`. + commit = pkgs.gitrev; + in attr // {installable="${flakeReference}/${commit}#${flakeOutput}";} + ) + { + coreutils = { + nix-store-path = pkgs.coreutils; + flake-reference = "github:intersectmbo/cardano-node"; + flake-output = "legacyPackages.x86_64-linux.coreutils"; + installable = null; + }; + bashInteractive = { + nix-store-path = pkgs.bashInteractive; + flake-reference = "github:intersectmbo/cardano-node"; + flake-output = "legacyPackages.x86_64-linux.bashInteractive"; + installable = null; + }; + findutils = { + nix-store-path = pkgs.findutils; + flake-reference = "github:intersectmbo/cardano-node"; + flake-output = "legacyPackages.x86_64-linux.findutils"; + installable = null; + }; + iputils = { + nix-store-path = pkgs.iputils; + flake-reference = "github:intersectmbo/cardano-node"; + flake-output = "legacyPackages.x86_64-linux.iputils"; + installable = null; + }; + gnutar = { + nix-store-path = pkgs.gnutar; + flake-reference = "github:intersectmbo/cardano-node"; + flake-output = "legacyPackages.x86_64-linux.gnutar"; + installable = null; + }; + zstd = { + nix-store-path = pkgs.zstd; + flake-reference = "github:intersectmbo/cardano-node"; + flake-output = "legacyPackages.x86_64-linux.zstd"; + installable = null; + }; + wget = { + nix-store-path = pkgs.wget; + flake-reference = "github:intersectmbo/cardano-node"; + flake-output = "legacyPackages.x86_64-linux.wget"; + installable = null; + }; + cacert = { + nix-store-path = pkgs.cacert; + flake-reference = "github:intersectmbo/cardano-node"; + flake-output = "legacyPackages.x86_64-linux.cacert"; + installable = null; + }; + supervisor = { + nix-store-path = pkgs.python3Packages.supervisor; + flake-reference = "github:intersectmbo/cardano-node"; + flake-output = "legacyPackages.x86_64-linux.python3Packages.supervisor"; + installable = null; + }; + gnugrep = { + nix-store-path = pkgs.gnugrep; + flake-reference = "github:intersectmbo/cardano-node"; + flake-output = "legacyPackages.x86_64-linux.gnugrep"; + installable = null; + }; + jq = { + nix-store-path = pkgs.jq; + flake-reference = "github:intersectmbo/cardano-node"; + flake-output = "legacyPackages.x86_64-linux.jq"; + installable = null; + }; + } + ) + // + ############################################################################ + # Optional container enabled OpenSSH to properly fetch ~1TB cloud logs. #### + ############################################################################ + lib.attrsets.optionalAttrs (subBackendName == "cloud") { + openssh_hacks = rec { + commit = "5d8c5913c70723318acf47496e2abf7d2c99384f"; # OpenSSH version 9.8 (Branch "9.8"); + # Not used locally. Needed to create the SSH "start.sh" script. + nix-store-path = (__getFlake "github:fmaste/openssh-portable-hacks/${commit}").packages.x86_64-linux.openssh_hacks; + flake-reference = "github:fmaste/openssh-portable-hacks"; + flake-output = "packages.x86_64-linux.openssh_hacks"; + installable = "${flake-reference}/${commit}#${flake-output}"; + }; + rsync = rec { + nix-store-path = pkgs.rsync; # Not used locally. + flake-reference = "github:intersectmbo/cardano-node"; + flake-output = "legacyPackages.x86_64-linux.rsync"; + # Same commit as the basic packages. + installable = "${flake-reference}/${pkgs.gitrev}#${flake-output}"; + }; + } + // + ############################################################################ + # Binaries from `cardanoNodePackages` (node / tracer / generator). ######### + ############################################################################ + { + # Provided that all the "start.sh" scripts are taking it into account, + # this packages could be configured to come from a different commit than + # the one used to enter the shell ?????????? + cardano-node = rec { + # Local reference only used if not "cloud". + nix-store-path = with pkgs; + # TODO: - cardano-node.passthru.profiled + # - cardano-node.passthru.eventlogged + # - cardano-node.passthru.asserted + # profileData.node-services."node-0".serviceConfig.value.eventlog + # builtins.trace (builtins.attrNames profileData.node-services."node-0".serviceConfig.value.eventlog) XXXX + if eventlogged + then cardanoNodePackages.cardano-node.passthru.eventlogged + else cardanoNodePackages.cardano-node + ; + flake-reference = "github:intersectmbo/cardano-node"; + flake-output = + if eventlogged + then "cardanoNodePackages.cardano-node.passthru.eventlogged" + else "cardanoNodePackages.cardano-node" + ; + installable = "${flake-reference}/${gitrev}#${flake-output}"; + }; + cardano-cli = rec { + # Local reference only used if not "cloud". + nix-store-path = pkgs.cardanoNodePackages.cardano-cli; + flake-reference = "github:input-output-hk/cardano-cli"; + flake-output = "cardanoNodePackages.cardano-cli"; + installable = "${flake-reference}/${gitrev}#${flake-output}"; + }; + cardano-tracer = rec { + # Local reference only used if not "cloud". + nix-store-path = pkgs.cardanoNodePackages.cardano-tracer; + flake-reference = "github:intersectmbo/cardano-node"; + flake-output = "cardanoNodePackages.cardano-tracer"; + installable = "${flake-reference}/${gitrev}#${flake-output}"; + }; + tx-generator = rec { + # Local reference only used if not "cloud". + nix-store-path = pkgs.cardanoNodePackages.tx-generator; + flake-reference = "github:intersectmbo/cardano-node"; + flake-output = "cardanoNodePackages.tx-generator"; + installable = "${flake-reference}/${gitrev}#${flake-output}"; + }; + } + ; + + # The "exec" or "cloud" Nomad Job description. + nomad-job = {profileData, containerPkgs}: + # TODO: Repeated code, add the generator's node name to profile.json + let generatorTaskName = if builtins.hasAttr "explorer" profileData.node-specs.value + then "explorer" + else "node-0" + ; + in + { inherit generatorTaskName; } + // + lib.attrsets.optionalAttrs (subBackendName == "exec") { + exec = { + # TODO: oneTracerPerGroup + oneTracerPerCluster = import ./nomad-job.nix + { inherit pkgs lib stateDir; + inherit profileData; + inherit generatorTaskName; + inherit containerPkgs; + oneTracerPerNode = false; + withSsh = false; + }; + oneTracerPerNode = import ./nomad-job.nix + { inherit pkgs lib stateDir; + inherit profileData; + inherit generatorTaskName; + inherit containerPkgs; + oneTracerPerNode = true; + withSsh = false; + }; + }; + } + // + lib.attrsets.optionalAttrs (subBackendName == "cloud") { + cloud = { + # Always "oneTracerPerNode" + # TODO: oneTracerPerCluster and oneTracerPerGroup + nomadExec = import ./nomad-job.nix + { inherit pkgs lib stateDir; + inherit profileData; + inherit generatorTaskName; + inherit containerPkgs; + oneTracerPerNode = true; + withSsh = false; + }; + ssh = import ./nomad-job.nix + { inherit pkgs lib stateDir; + inherit profileData; + inherit generatorTaskName; + inherit containerPkgs; + oneTracerPerNode = true; + withSsh = true; + }; + # AWS S3 bucket that will be used to deploy the genesis files. + s3 = rec { + bucket = "cardano-perf-deploy"; + region = "eu-central-1"; + host = "s3.${region}.amazonaws.com"; + uri = "https://${bucket}.${host}"; }; - nomadJob = - { inherit generatorTaskName; } - // - lib.attrsets.optionalAttrs (subBackendName == "exec") { - exec = { - # TODO: oneTracerPerGroup - oneTracerPerCluster = import ./nomad-job.nix - { inherit pkgs lib stateDir; - inherit profileData; - inherit containerSpecs; - inherit generatorTaskName; - oneTracerPerNode = false; - withSsh = false; - }; - oneTracerPerNode = import ./nomad-job.nix - { inherit pkgs lib stateDir; - inherit profileData; - inherit containerSpecs; - inherit generatorTaskName; - oneTracerPerNode = true; - withSsh = false; - }; - }; - } - // - lib.attrsets.optionalAttrs (subBackendName == "cloud") { - cloud = { - # Always "oneTracerPerNode" - # TODO: oneTracerPerCluster and oneTracerPerGroup - nomadExec = import ./nomad-job.nix - { inherit pkgs lib stateDir; - inherit profileData; - inherit containerSpecs; - inherit generatorTaskName; - oneTracerPerNode = true; - withSsh = false; - }; - ssh = import ./nomad-job.nix - { inherit pkgs lib stateDir; - inherit profileData; - inherit containerSpecs; - inherit generatorTaskName; - oneTracerPerNode = true; - withSsh = true; - }; - # AWS S3 bucket that will be used to deploy the genesis files. - s3 = rec { - bucket = "cardano-perf-deploy"; - region = "eu-central-1"; - host = "s3.${region}.amazonaws.com"; - uri = "https://${bucket}.${host}"; - }; - }; - } - ; }; - in pkgs.runCommand "workbench-backend-output-${profileData.profileName}-nomad" - ({ - containerSpecsJSON = pkgs.writeText "workbench-cluster-container-pkgs.json" - (lib.generators.toJSON {} containerSpecs); - }) - '' - mkdir $out - ln -s $containerSpecsJSON $out/container-specs.json - ''; + } + ; in { inherit materialise-profile; } diff --git a/nix/workbench/backend/nomad/cloud.nix b/nix/workbench/backend/nomad/cloud.nix index 7b949dea1c8..041d9fb0600 100644 --- a/nix/workbench/backend/nomad/cloud.nix +++ b/nix/workbench/backend/nomad/cloud.nix @@ -26,6 +26,8 @@ let ) # Amazon S3 HTTP to upload/download the genesis tar file. pkgs.awscli + # Used to download the logs. + pkgs.rsync ] ; diff --git a/nix/workbench/backend/nomad/exec.sh b/nix/workbench/backend/nomad/exec.sh index b9139da67ab..38a21875f9e 100644 --- a/nix/workbench/backend/nomad/exec.sh +++ b/nix/workbench/backend/nomad/exec.sh @@ -193,7 +193,7 @@ allocate-run-nomadexec() { ## Empty the global namespace. Local runs ignore "${NOMAD_NAMESPACE:-}" backend_nomad allocate-run-nomad-job-patch-namespace "${dir}" # Will set the /nix/store paths from ".nix-store-path" in container-specs.json -# backend_nomad allocate-run-nomad-job-patch-nix "${dir}" + backend_nomad allocate-run-nomad-job-patch-nix "${dir}" } # Called by `run.sh` without exit trap (unlike `scenario_setup_exit_trap`)! diff --git a/nix/workbench/service/ssh.nix b/nix/workbench/service/ssh.nix index 49f41ff6ea9..798aa8f49a6 100644 --- a/nix/workbench/service/ssh.nix +++ b/nix/workbench/service/ssh.nix @@ -1,7 +1,7 @@ { pkgs , bashInteractive , coreutils -, openssh_hacks +, sshdExecutable }: with pkgs.lib; @@ -46,7 +46,7 @@ in { # Specifies the name of the configuration file. The default is # /etc/ssh/sshd_config. sshd refuses to start if there is no configuration # file. - ${openssh_hacks}/bin/sshd -D -e -f /local/run/current/ssh/sshd_config + ${sshdExecutable} -D -e -f /local/run/current/ssh/sshd_config ''; };