Testing: update E2E to use JIT runners

.github/workflows/e2e.yml

@@ -6,11 +6,29 @@ on:
   workflow_dispatch:
 
 jobs:
+  update_vm:
+    runs-on: e2e-host
+    steps:
+      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # ratchet:actions/checkout@v3
+      - name: Update VM
+        env:
+          GCS_KEY: ${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}
+        run: |
+          export GOOGLE_APPLICATION_CREDENTIALS=/tmp/gcp.json
+          echo "${GCS_KEY}" > ${GOOGLE_APPLICATION_CREDENTIALS}
+          function cleanup {
+            rm /tmp/gcp.json
+          }
+          trap cleanup EXIT
+          python3 Testing/integration/actions/update_vm.py macOS_14.bundle.tar.gz
+
   start_vm:
     runs-on: e2e-host
     steps:
       - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # ratchet:actions/checkout@v3
       - name: Start VM
+        env:
+          RUNNER_REG_TOKEN: ${{ secrets.RUNNER_REG_TOKEN }}
         run: python3 Testing/integration/actions/start_vm.py macOS_14.bundle.tar.gz
 
   integration:

Testing/integration/VM/VMCLI/main.m

@@ -40,8 +40,8 @@ - (void)guestDidStopVirtualMachine:(VZVirtualMachine *)virtualMachine {
 @end
 
 int main(int argc, const char *argv[]) {
-  if (argc < 2) {
-    fprintf(stderr, "Usage: %s bundle_path [usb_disk]", argv[0]);
+  if (argc < 3) {
+    fprintf(stderr, "Usage: %s bundle_path runner_disk [usb_disk]", argv[0]);
     exit(-1);
   }
 
@@ -50,14 +50,16 @@ int main(int argc, const char *argv[]) {
     bundleDir = [bundleDir stringByAppendingString:@"/"];
   }
 
-  NSString *usbDisk;
-  if (argc > 2) {
-    usbDisk = @(argv[2]);
+  NSString *runnerDisk = @(argv[2]);
+
+  NSString *usbDisk = NULL;
+  if (argc > 3) {
+    usbDisk = @(argv[3]);
   }
 
   VZVirtualMachine *vm =
     [MacOSVirtualMachineConfigurationHelper createVirtualMachineWithBundleDir:bundleDir
-                                                                       roDisk:nil
+                                                                       roDisk:runnerDisk
                                                                       usbDisk:usbDisk];
 
   MacOSVirtualMachineDelegate *delegate = [MacOSVirtualMachineDelegate new];

Testing/integration/VM/setup.sh

@@ -33,16 +33,23 @@ fi
 # Install rosetta (for test binaries)
 softwareupdate --install-rosetta --agree-to-license
 
-# Install actions runner
-mkdir ~/actions-runner
-pushd ~/actions-runner
-curl -o actions-runner-osx-arm64-2.296.0.tar.gz -L https://github.com/actions/runner/releases/download/v2.296.0/actions-runner-osx-arm64-2.296.0.tar.gz
-echo 'e358086b924d2e8d8abf50beec57ee7a3bb0c7d412f13abc51380f1b1894d776  actions-runner-osx-arm64-2.296.0.tar.gz' | shasum -a 256 -c
-tar xzf ./actions-runner-osx-arm64-2.296.0.tar.gz
-./config.sh --url https://github.com/google/santa
-./svc.sh install
-./svc.sh start
-popd
+# Add a LaunchAgent to start the mounted runner
+tee ${HOME}/Library/LaunchAgents/runner.plist << EOF
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+    <key>Label</key>
+    <string>com.google.santa.e2erunner</string>
+    <key>ProgramArguments</key>
+    <array>
+        <string>/Volumes/init/run.sh</string>
+    </array>
+    <key>RunAtLoad</key>
+    <true/>
+</dict>
+</plist>
+EOF
 
 # Run sample applescript to grant bash accessibility and automation control
 clang "${SCRIPT_DIR}/disclaim.c" -o /tmp/disclaim

Testing/integration/actions/start_vm.py

@@ -1,121 +1,104 @@
 #!/usr/bin/env python3
 """Download and run the given Santa E2E testing VM image."""
-import datetime
+import argparse
 import json
+import logging
 import os
 import pathlib
-import shutil
 import subprocess
-import sys
 import tempfile
+import urllib.request
 
-from google.cloud import storage
-from google.oauth2 import service_account
-
-PROJECT = "santa-e2e"
-SA_KEY = "/opt/santa-e2e-sa.json"
-BUCKET = "santa-e2e-vms"
-COSIGN = "/opt/bin/cosign"
-PUBKEY = "/opt/santa-e2e-vm-signer.pub"
-VMCLI = "/opt/bin/VMCLI"
 VMS_DIR = pathlib.Path.home() / "VMs"
 TIMEOUT = 15 * 60  # in seconds
 
 if __name__ == "__main__":
-  VMS_DIR.mkdir(exist_ok=True)
-
-  tar_name = sys.argv[1]
-  if not tar_name.endswith(".tar.gz"):
-    print("Image name should be .tar.gz file", file=sys.stderr)
-    sys.exit(1)
+  logging.basicConfig(level=logging.INFO)
 
-  tar_path = VMS_DIR / tar_name
-  extracted_path = pathlib.Path(str(tar_path)[:-len(".tar.gz")])
-
-  with open(SA_KEY, "rb") as key_file:
-    storage_client = storage.Client(
-        project=PROJECT,
-        credentials=service_account.Credentials.from_service_account_info(
-            json.load(key_file)),
-    )
-  bucket = storage_client.bucket(BUCKET)
-  blob = bucket.get_blob(tar_name)
+  parser = argparse.ArgumentParser(description="Start E2E VM")
+  # This is redundant, but kept to keep consistency with update_vm.py
+  parser.add_argument("--vm", help="VM tar.gz. name", required=True)
+  parser.add_argument("--vmcli", help="Path to VMCLI binary", default="/opt/bin/VMCLI")
+  args = parser.parse_args()
 
-  if blob is None:
-    print("Specified image doesn't exist in GCS", file=sys.stderr)
-    sys.exit(1)
+  if not args.vm.endswith(".tar.gz"):
+    logging.fatal("Image name should be .tar.gz file")
 
-  try:
-    local_ctime = os.stat(extracted_path).st_ctime
-  except FileNotFoundError:
-    local_ctime = 0
-
-  if blob.updated > datetime.datetime.fromtimestamp(
-      local_ctime, tz=datetime.timezone.utc):
-    print(f"VM {extracted_path} not present or not up to date, downloading...")
-
-    # Remove the old version of the image if present
-    try:
-      shutil.rmtree(extracted_path)
-    except FileNotFoundError:
-      pass
-
-    blob.download_to_filename(tar_path)
-
-    hash_blob = bucket.get_blob(str(tar_name) + ".sha256")
-    if hash_blob is None:
-      print("Image hash doesn't exist in GCS", file=sys.stderr)
-      sys.exit(1)
-
-    sig_blob = bucket.get_blob(str(tar_name) + ".sha256.sig")
-    if sig_blob is None:
-      print("Image signature doesn't exist in GCS", file=sys.stderr)
-      sys.exit(1)
-
-    hash_path = str(tar_path) + ".sha256"
-    hash_blob.download_to_filename(hash_path)
-    sig_path = str(tar_path) + ".sha256.sig"
-    sig_blob.download_to_filename(sig_path)
-
-    # cosign OOMs trying to sign/verify the tarball itself, so sign/verify
-    # the SHA256 of the tarball.
-    print("Verifying signature...")
-
-    # Verify the signature of the hash file is OK
-    subprocess.check_output([
-        COSIGN,
-        "verify-blob",
-        "--key", PUBKEY,
-        "--signature", sig_path,
-        hash_path,
-    ])
-    # Then verify that the hash matches what we downloaded
-    subprocess.check_output(
-        ["shasum", "-a", "256", "-c", hash_path],
-        cwd=VMS_DIR,
-    )
-
-    print("Extracting...")
-    subprocess.check_output(
-        ["tar", "-C", VMS_DIR, "-x", "-S", "-z", "-f", tar_path]
-    )
-    tar_path.unlink()
+  tar_path = VMS_DIR / args.vm
+  extracted_path = pathlib.Path(str(tar_path)[:-len(".tar.gz")])
 
   with tempfile.TemporaryDirectory() as snapshot_dir:
-    print(f"Snapshot: {snapshot_dir}")
+    logging.info(f"Snapshot: {snapshot_dir}")
     # COW copy the image to this tempdir
     subprocess.check_output(["cp", "-rc", extracted_path, snapshot_dir])
+
+    # Get a JIT runner key
+    github_token = os.environ["RUNNER_REG_TOKEN"]
+    body = json.dumps({
+      "name": os.environ["GITHUB_RUN_ID"] + " inner",
+      "runner_group_id":1,
+      "labels":[
+        "self-hosted",
+        "macOS",
+        "ARM64",
+        "e2e-vm",
+      ],
+      "work_folder":"/tmp/_work",
+    })
+    owner, repo = os.environ["GITHUB_REPOSITORY"].split("/", 1)
+    request = urllib.request.Request(
+      f"https://api.github.com/repos/{owner}/{repo}/actions/runners/generate-jitconfig",
+      headers={
+        "Content-Type": "application/json",
+        "Accept": "application/vnd.github+json",
+        "Authorization": f"Bearer {github_token}",
+        "X-GitHub-Api-Version": "2022-11-28",
+      },
+      data=body.encode("utf-8"),
+    )
+    with urllib.request.urlopen(request) as response:
+      jit_config = json.loads(response.read())["encoded_jit_config"]
+
+    logging.info("Got JIT runner config")
+
+    # Create a disk image to inject startup script
+    init_dmg = pathlib.Path(snapshot_dir) / "init.dmg"
+    subprocess.check_output(["hdiutil", "create", "-attach", "-size", "1G",
+                                "-fs", "APFS", "-volname", "init", init_dmg])
+    init_dmg_mount = pathlib.Path("/Volumes/init/")
+
+    # And populate startup script with runner and JIT key
+    with open(init_dmg_mount / "run.sh", "w") as run_sh:
+      run_sh.write(f"""#!/bin/sh
+set -xeuo pipefail
+
+curl -L -o /tmp/runner.tar.gz 'https://github.com/actions/runner/releases/download/v2.316.0/actions-runner-osx-arm64-2.316.0.tar.gz'
+echo "8442d39e3d91b67807703ec0825cec4384837b583305ea43a495a9867b7222ca  /tmp/runner.tar.gz" | shasum -a 256 -c -
+mkdir /tmp/runner
+cd /tmp/runner
+tar -xzf /tmp/runner.tar.gz
+./run.sh --jitconfig '{jit_config}'
+""")
+    os.chmod(init_dmg_mount / "run.sh", 0o755)
+    subprocess.check_output(["hdiutil", "detach", init_dmg_mount])
+
+    logging.info("Created init.dmg")
+
     # Create a disk image for USB testing
     usb_dmg = pathlib.Path(snapshot_dir) / "usb.dmg"
     subprocess.check_output(["hdiutil", "create", "-size", "100M",
                                 "-fs", "ExFAT", "-volname", "USB", usb_dmg])
+
+    logging.info("Created usb.dmg")
+
     try:
+      logging.info("Starting VM")
       subprocess.check_output(
-          [VMCLI, pathlib.Path(snapshot_dir) / extracted_path.name, usb_dmg],
+          [args.vmcli, pathlib.Path(snapshot_dir) / extracted_path.name, init_dmg, usb_dmg],
           timeout=TIMEOUT,
       )
     except subprocess.TimeoutExpired:
-      print("VM timed out")
+      logging.warning("VM timed out")
 
-  print("VM deleted")
+  logging.info("VM deleted")
 

Testing/integration/actions/update_vm.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python3
+"""Download/update the given Santa E2E testing VM image."""
+import argparse
+import datetime
+import logging
+import os
+import pathlib
+import shutil
+import subprocess
+import sys
+
+from google.cloud import storage
+
+PROJECT = "santa-e2e"
+BUCKET = "santa-e2e-vms"
+COSIGN = "/opt/bin/cosign"
+PUBKEY = "/opt/santa-e2e-vm-signer.pub"
+VMS_DIR = pathlib.Path.home() / "VMs"
+
+if __name__ == "__main__":
+  logging.basicConfig(level=logging.INFO)
+
+  parser = argparse.ArgumentParser(description="Start E2E VM")
+  parser.add_argument("--vm", help="VM tar.gz. name", required=True)
+  args = parser.parse_args()
+
+  VMS_DIR.mkdir(exist_ok=True)
+
+  tar_name = args.vm
+  if not tar_name.endswith(".tar.gz"):
+    logging.fatal("Image name should be .tar.gz file")
+
+  tar_path = VMS_DIR / tar_name
+  extracted_path = pathlib.Path(str(tar_path)[:-len(".tar.gz")])
+
+  if "GOOGLE_APPLICATION_CREDENTIALS" not in os.environ:
+    logging.fatal("Missing GCS credentials file")
+
+  storage_client = storage.Client(project=PROJECT)
+  bucket = storage_client.bucket(BUCKET)
+  blob = bucket.get_blob(tar_name)
+
+  if blob is None:
+    logging.fatal("Specified image doesn't exist in GCS")
+
+  try:
+    local_ctime = os.stat(extracted_path).st_ctime
+  except FileNotFoundError:
+    local_ctime = 0
+
+  if blob.updated > datetime.datetime.fromtimestamp(
+      local_ctime, tz=datetime.timezone.utc):
+    logging.info(f"VM {extracted_path} not present or not up to date, downloading...")
+
+    # Remove the old version of the image if present
+    try:
+      shutil.rmtree(extracted_path)
+    except FileNotFoundError:
+      pass
+
+    blob.download_to_filename(tar_path)
+
+    hash_blob = bucket.get_blob(str(tar_name) + ".sha256")
+    if hash_blob is None:
+      logging.fatal("Image hash doesn't exist in GCS")
+
+    sig_blob = bucket.get_blob(str(tar_name) + ".sha256.sig")
+    if sig_blob is None:
+      logging.fatal("Image signature doesn't exist in GCS")
+
+    hash_path = str(tar_path) + ".sha256"
+    hash_blob.download_to_filename(hash_path)
+    sig_path = str(tar_path) + ".sha256.sig"
+    sig_blob.download_to_filename(sig_path)
+
+    # cosign OOMs trying to sign/verify the tarball itself, so sign/verify
+    # the SHA256 of the tarball.
+    logging.info("Verifying signature...")
+
+    # Verify the signature of the hash file is OK
+    subprocess.check_output([
+        COSIGN,
+        "verify-blob",
+        "--key", PUBKEY,
+        "--signature", sig_path,
+        hash_path,
+    ])
+    # Then verify that the hash matches what we downloaded
+    subprocess.check_output(
+        ["shasum", "-a", "256", "-c", hash_path],
+        cwd=VMS_DIR,
+    )
+
+    logging.info("Extracting...")
+    subprocess.check_output(
+        ["tar", "-C", VMS_DIR, "-x", "-S", "-z", "-f", tar_path]
+    )
+    tar_path.unlink()