Skip to content

Test Fleetctl, Orbit & Preview #1208

Test Fleetctl, Orbit & Preview

Test Fleetctl, Orbit & Preview #1208

Workflow file for this run

# This workflow tests enrolling of agents on the supported platforms,
# using the latest version of fleet, fleetctl and orbit.
#
# It starts the latest release of fleet with the "fleetctl preview" command.
# It generates the installers for the latest version of Orbit with the
# "fleetctl package" command.
name: Test Fleetctl, Orbit & Preview
on:
workflow_dispatch: # Manual
schedule:
- cron: '0 2 * * *' # Nightly 2AM UTC
# This allows a subsequently queued workflow run to interrupt previous runs
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id}}
cancel-in-progress: true
defaults:
run:
# fail-fast using bash -eo pipefail. See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#exit-codes-and-error-action-preference
shell: bash
permissions:
contents: read
jobs:
gen:
runs-on: ubuntu-latest
outputs:
subdomain: ${{ steps.gen.outputs.subdomain }}
address: ${{ steps.gen.outputs.address }}
steps:
- name: Harden Runner
uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0
with:
egress-policy: audit
- id: gen
run: |
UUID=$(uuidgen)
echo "subdomain=fleet-test-$UUID" >> $GITHUB_OUTPUT
echo "address=https://fleet-test-$UUID.fleetuem.com" >> $GITHUB_OUTPUT
run-server:
runs-on: ubuntu-latest
needs: gen
steps:
- name: Harden Runner
uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0
with:
egress-policy: audit
- name: Start tunnel
env:
CERT_PEM: ${{ secrets.CLOUDFLARE_TUNNEL_FLEETUEM_CERT_B64 }}
run: |
# Increase maximum receive buffer size to roughly 2.5 MB.
# Cloudflared uses quic-go. This buffer holds packets that have been received by the kernel,
# but not yet read by the application (quic-go in this case). Once this buffer fills up, the
# kernel will drop any new incoming packet.
# See https://github.com/quic-go/quic-go/wiki/UDP-Receive-Buffer-Size.
sudo sysctl -w net.core.rmem_max=2500000
# Install cloudflared
wget https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64.deb
sudo dpkg -i cloudflared-linux-amd64.deb
# Add secret
echo "$CERT_PEM" | base64 -d > cert.pem
# Start tunnel
cloudflared tunnel --origincert cert.pem --hostname ${{ needs.gen.outputs.subdomain }} --url http://localhost:1337 --name ${{ needs.gen.outputs.subdomain }} --logfile cloudflared.log &
until [[ $(cloudflared tunnel --origincert cert.pem info -o json ${{ needs.gen.outputs.subdomain }} | jq '.conns[0].conns[0].is_pending_reconnect') = false ]]; do
echo "Awaiting tunnel ready..."
sleep 5
done
# Download fleet and fleetctl binaries from last successful build on main
- name: Download binaries
uses: dawidd6/action-download-artifact@5e780fc7bbd0cac69fc73271ed86edf5dcb72d67
with:
workflow: build-binaries.yaml
branch: main
name: build
path: build
check_artifacts: true
- name: Run Fleet server
timeout-minutes: 10
run: |
chmod +x ./build/fleetctl
./build/fleetctl preview --no-hosts
./build/fleetctl config set --address ${{ needs.gen.outputs.address }}
./build/fleetctl get enroll-secret
docker compose -f ~/.fleet/preview/docker-compose.yml logs --follow fleet01 fleet02 &
# Wait for all of the hosts to be enrolled
EXPECTED=3
until [ $(./build/fleetctl get hosts --json | wc -l | tee hostcount) -ge $EXPECTED ]; do
echo -n "Waiting for hosts to enroll: "
cat hostcount | xargs echo -n
echo " / $EXPECTED"
sleep 20
done
echo "Success! $EXPECTED hosts enrolled."
- name: Show enrolled hosts
if: always()
run: |
./build/fleetctl get hosts --json
- name: Slack Notification
if: failure()
uses: slackapi/slack-github-action@e28cf165c92ffef168d23c5c9000cffc8a25e117 # v1.24.0
with:
payload: |
{
"text": "${{ job.status }}\n${{ github.event.pull_request.html_url || github.event.head.html_url }}",
"blocks": [
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "Integration test result: ${{ job.status }}\nhttps://github.com/fleetdm/fleet/actions/runs/${{ github.run_id }}\n${{ github.event.pull_request.html_url || github.event.head.html_url }}"
}
}
]
}
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_G_HELP_ENGINEERING_WEBHOOK_URL }}
SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
- name: Cleanup tunnel
if: always()
run: cloudflared tunnel --origincert cert.pem delete --force ${{ needs.gen.outputs.subdomain }}
- name: Upload cloudflared logs
if: always()
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v2
with:
name: cloudflared.log
path: cloudflared.log
login:
runs-on: ubuntu-latest
needs: gen
outputs:
token: ${{ steps.login.outputs.token }}
steps:
# Download fleet and fleetctl binaries from last successful build on main
- name: Harden Runner
uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0
with:
egress-policy: audit
- name: Download binaries
uses: dawidd6/action-download-artifact@5e780fc7bbd0cac69fc73271ed86edf5dcb72d67
with:
workflow: build-binaries.yaml
branch: main
name: build
path: build
check_artifacts: true
# Login only here and share the token because otherwise we could hit rate limits.
- id: login
name: Attempt login
timeout-minutes: 5
run: |
chmod +x ./build/fleetctl
./build/fleetctl config set --address ${{ needs.gen.outputs.address }}
until ./build/fleetctl login --email admin@example.com --password preview1337#
do
echo "Retrying in 5s..."
sleep 5
done
TOKEN=$(cat ~/.fleet/config| grep token | awk '{ print $2 }')
echo "token=$TOKEN" >> $GITHUB_OUTPUT
orbit-macos:
timeout-minutes: 10
strategy:
matrix:
# To run multiple VMs that have the same UUID we need to implement
# https://github.com/fleetdm/fleet/issues/8021 (otherwise orbit and osqueryd
# in the same host are enrolled as two hosts in Fleet).
# Until then we will just test the `stable` channel in all components.
#
# Alternatively, we can bring back the `edge` channel when we decide to upgrade
# our worker to macOS 13 in the future, as they changed the virtualization
# layer for 13 and now it has random UUIDs (https://github.com/actions/runner-images/issues/7591).
orbit-channel: [ 'stable' ]
osqueryd-channel: [ 'stable' ]
desktop-channel: [ 'stable' ]
runs-on: macos-latest
needs: [gen, login]
steps:
- name: Harden Runner
uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0
with:
egress-policy: audit
- name: Checkout Code
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
- name: Install dependencies
run: |
npm install -g fleetctl
fleetctl config set --address ${{ needs.gen.outputs.address }} --token ${{ needs.login.outputs.token }}
- name: Install Orbit
run: |
sudo hostname macos-orbit-${{ matrix.orbit-channel }}-osqueryd-${{ matrix.osqueryd-channel }}
SECRET_JSON=$(fleetctl get enroll_secret --json --debug)
echo $SECRET_JSON
SECRET=$(echo $SECRET_JSON | jq -r '.spec.secrets[0].secret')
echo "Secret: $SECRET"
echo "Hostname: $(hostname -s)"
fleetctl package --type pkg --fleet-url=${{ needs.gen.outputs.address }} --enroll-secret=$SECRET --orbit-channel=${{ matrix.orbit-channel }} --osqueryd-channel=${{ matrix.osqueryd-channel }} --desktop-channel=${{ matrix.desktop-channel }} --fleet-desktop --debug
sudo installer -pkg fleet-osquery.pkg -target /
until fleetctl get hosts | grep -iF $(hostname -s);
do
echo "Awaiting enrollment..."
sleep 10
done
- name: Collect orbit logs
if: always()
run: |
mkdir orbit-logs
sudo cp /var/log/orbit/* orbit-logs/
- name: Upload Orbit logs
if: always()
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v2
with:
name: orbit-macos-${{ matrix.orbit-channel }}-${{ matrix.osqueryd-channel }}-${{ matrix.desktop-channel }}-logs
path: |
orbit-logs
- name: Uninstall Orbit
run: |
sudo ./orbit/tools/cleanup/cleanup_macos.sh
orbit-ubuntu:
timeout-minutes: 10
strategy:
matrix:
# To run multiple VMs that have the same UUID we need to implement
# https://github.com/fleetdm/fleet/issues/8021 (otherwise orbit and osqueryd
# in the same host are enrolled as two hosts in Fleet).
# Until then we will just test the `stable` channel in all components.
orbit-channel: [ 'stable' ]
osqueryd-channel: [ 'stable' ]
desktop-channel: [ 'stable' ]
runs-on: ubuntu-latest
needs: [gen, login]
steps:
- name: Harden Runner
uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0
with:
egress-policy: audit
- name: Install dependencies
run: |
npm install -g fleetctl
fleetctl config set --address ${{ needs.gen.outputs.address }} --token ${{ needs.login.outputs.token }}
- name: Checkout Code
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
- name: Install Go
uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.1.0
with:
go-version-file: 'go.mod'
- name: Build Fleetctl
run: make fleetctl
- name: Install Orbit
run: |
sudo hostname ubuntu-orbit-${{ matrix.orbit-channel }}-osqueryd-${{ matrix.osqueryd-channel }}
chmod +x ./build/fleetctl
SECRET_JSON=$(fleetctl get enroll_secret --json --debug)
echo $SECRET_JSON
SECRET=$(echo $SECRET_JSON | jq -r '.spec.secrets[0].secret')
echo "Secret: $SECRET"
echo "Hostname: $(hostname -s)"
./build/fleetctl package --type deb --fleet-url=${{ needs.gen.outputs.address }} --enroll-secret=$SECRET --orbit-channel=${{ matrix.orbit-channel }} --osqueryd-channel=${{ matrix.osqueryd-channel }} --desktop-channel=${{ matrix.desktop-channel }} --fleet-desktop --debug
sudo dpkg -i fleet-osquery*
until fleetctl get hosts | grep -iF $(hostname -s);
do
echo "Awaiting enrollment..."
sudo systemctl status orbit.service || true
sleep 10
done
- name: Collect orbit logs
if: always()
run: |
sudo journalctl -u orbit.service > orbit-logs
- name: Upload Orbit logs
if: always()
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v2
with:
name: orbit-ubuntu-${{ matrix.orbit-channel }}-${{ matrix.osqueryd-channel }}-${{ matrix.desktop-channel }}-logs
path: |
orbit-logs
- name: Uninstall Orbit
run: |
sudo apt remove fleet-osquery -y
orbit-windows-build:
timeout-minutes: 10
strategy:
matrix:
# To run multiple VMs that have the same UUID we need to implement
# https://github.com/fleetdm/fleet/issues/8021 (otherwise orbit and osqueryd
# in the same host are enrolled as two hosts in Fleet).
# Until then we will just test the `stable` channel in all components.
orbit-channel: [ 'stable' ]
osqueryd-channel: [ 'stable' ]
desktop-channel: [ 'stable' ]
runs-on: ubuntu-latest
needs: [gen, login]
steps:
- name: Harden Runner
uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0
with:
egress-policy: audit
- name: Install dependencies
run: |
docker pull fleetdm/wix:latest &
npm install -g fleetctl
fleetctl config set --address ${{ needs.gen.outputs.address }} --token ${{ needs.login.outputs.token }}
- name: Build Orbit
run: |
SECRET_JSON=$(fleetctl get enroll_secret --json --debug)
echo $SECRET_JSON
SECRET=$(echo $SECRET_JSON | jq -r '.spec.secrets[0].secret')
echo "Secret: $SECRET"
echo "Hostname: $(hostname -s)"
fleetctl package --type msi --fleet-url=${{ needs.gen.outputs.address }} --enroll-secret=$SECRET --orbit-channel=${{ matrix.orbit-channel }} --osqueryd-channel=${{ matrix.osqueryd-channel }} --desktop-channel=${{ matrix.desktop-channel }} --fleet-desktop --debug
mv fleet-osquery.msi orbit-${{ matrix.orbit-channel }}-osqueryd-${{ matrix.osqueryd-channel }}-desktop-${{ matrix.desktop-channel }}.msi
- name: Upload MSI
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v2
with:
name: orbit-${{ matrix.orbit-channel }}-osqueryd-${{ matrix.osqueryd-channel }}-desktop-${{ matrix.desktop-channel }}.msi
path: orbit-${{ matrix.orbit-channel }}-osqueryd-${{ matrix.osqueryd-channel }}-desktop-${{ matrix.desktop-channel }}.msi
orbit-windows:
timeout-minutes: 10
strategy:
matrix:
# To run multiple VMs that have the same UUID we need to implement
# https://github.com/fleetdm/fleet/issues/8021 (otherwise orbit and osqueryd
# in the same host are enrolled as two hosts in Fleet).
# Until then we will just test the `stable` channel in all components.
orbit-channel: [ 'stable' ]
osqueryd-channel: [ 'stable' ]
desktop-channel: [ 'stable' ]
needs: [gen, login, orbit-windows-build]
runs-on: windows-latest
steps:
- name: Harden Runner
uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0
with:
egress-policy: audit
- name: Install dependencies
shell: bash
run: |
npm install -g fleetctl
fleetctl config set --address ${{ needs.gen.outputs.address }} --token ${{ needs.login.outputs.token }} --tls-skip-verify
- name: Download MSI
id: download
uses: actions/download-artifact@fb598a63ae348fa914e94cd0ff38f362e927b741 # v2
with:
name: orbit-${{ matrix.orbit-channel }}-osqueryd-${{ matrix.osqueryd-channel }}-desktop-${{ matrix.desktop-channel }}.msi
- name: Install Orbit
shell: cmd
run: |
msiexec /i ${{steps.download.outputs.download-path}}\orbit-${{ matrix.orbit-channel }}-osqueryd-${{ matrix.osqueryd-channel }}-desktop-${{ matrix.desktop-channel }}.msi /quiet /passive /lv log.txt
sleep 30
# We can't very accurately check the install on these Windows hosts since the hostnames tend to
# overlap and we can't control the hostnames. Instead we just return and have the run-server job
# wait until the expected number of hosts enroll.
- name: Upload orbit install log
if: always()
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v2
with:
name: msiexec-install-log
path: log.txt
- name: Upload Orbit logs
if: always()
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v2
with:
name: orbit-windows-${{ matrix.orbit-channel }}-${{ matrix.osqueryd-channel }}-${{ matrix.desktop-channel }}-logs
path: C:\Windows\system32\config\systemprofile\AppData\Local\FleetDM\Orbit\Logs\orbit-osquery.log