Merge branch 'main' into noahtalerman-patch-5

.github/pull_request_template.md

@@ -9,10 +9,11 @@ If some of the following don't apply, delete the relevant line.
 - [ ] Input data is properly validated, `SELECT *` is avoided, SQL injection is prevented (using placeholders for values in statements)
 - [ ] Added support on fleet's osquery simulator `cmd/osquery-perf` for new osquery data ingestion features.
 - [ ] Added/updated tests
-- [ ] If database migrations are included, checked table schema to confirm autoupdate 
+- [ ] If database migrations are included, checked table schema to confirm autoupdate
 - For database migrations:
   - [ ] Checked schema for all modified table for columns that will auto-update timestamps during migration.
   - [ ] Confirmed that updating the timestamps is acceptable, and will not cause unwanted side effects.
+  - [ ] Ensured the correct collation is explicitly set for character columns (`COLLATE utf8mb4_unicode_ci`).
 - [ ] Manual QA for all new/changed functionality
   - For Orbit and Fleet Desktop changes:
     - [ ] Manual QA must be performed in the three main OSs, macOS, Windows and Linux.

.github/workflows/check-tuf-timestamps.yml

@@ -38,11 +38,11 @@ jobs:
       run: |
         expires=$(curl -s http://tuf.fleetctl.com/timestamp.json | jq -r '.signed.expires' | cut -c 1-10)
         today=$(date "+%Y-%m-%d")
-        tomorrow=$(date -d "$today + 1 day" "+%Y-%m-%d")
+        warning_at=$(date -d "$today + 2 day" "+%Y-%m-%d")
         expires_sec=$(date -d "$expires" "+%s")
-        tomorrow_sec=$(date -d "$tomorrow" "+%s")
+        warning_at_sec=$(date -d "$warning_at" "+%s")
         
-        if [ "$expires_sec" -le "$tomorrow_sec" ]; then
+        if [ "$expires_sec" -le "$warning_at_sec" ]; then
             exit 1
         else
             exit 0

.github/workflows/config/slack_payload_template.json

@@ -0,0 +1,19 @@
+{
+    "text": "${{ env.JOB_STATUS }}\n${{ env.EVENT_URL }}",
+    "blocks": [
+        {
+            "type": "section",
+            "text": {
+                "type": "mrkdwn",
+                "text": "Go tests result: ${{ env.JOB_STATUS }}\n${{ env.RUN_URL }}"
+            }
+        },
+        {
+            "type": "section",
+            "text": {
+                "type": "mrkdwn",
+                "text": "Summary:\n```${GO_FAIL_SUMMARY}```"
+            }
+        }
+    ]
+}

.github/workflows/dogfood-gitops.yml

@@ -64,3 +64,4 @@ jobs:
           DOGFOOD_SERVERS_ENROLL_SECRET: ${{ secrets.DOGFOOD_SERVERS_ENROLL_SECRET }}
           DOGFOOD_SERVERS_CANARY_ENROLL_SECRET: ${{ secrets.DOGFOOD_SERVERS_CANARY_ENROLL_SECRET }}
           DOGFOOD_EXPLORE_DATA_ENROLL_SECRET: ${{ secrets.DOGFOOD_EXPLORE_DATA_ENROLL_SECRET }}
+          DOGFOOD_CALENDAR_API_KEY: ${{ secrets.DOGFOOD_CALENDAR_API_KEY }}

.github/workflows/generate-desktop-targets.yml

@@ -24,7 +24,7 @@ defaults:
     shell: bash
 
 env:
-  FLEET_DESKTOP_VERSION: 1.22.0
+  FLEET_DESKTOP_VERSION: 1.23.0
 
 permissions:
   contents: read

.github/workflows/generate-osqueryd-targets.yml

@@ -24,7 +24,7 @@ defaults:
     shell: bash
 
 env:
-  OSQUERY_VERSION: 5.12.0
+  OSQUERY_VERSION: 5.12.1
 
 permissions:
   contents: read

.github/workflows/test-go.yaml

@@ -49,7 +49,7 @@ jobs:
 
     env:
       RACE_ENABLED: false
-      GO_TEST_TIMEOUT: 15m
+      GO_TEST_TIMEOUT: 20m
 
     steps:
     - name: Harden Runner
@@ -74,7 +74,7 @@ jobs:
       run: |
         sudo cp tools/smtp4dev/fleet.crt /usr/local/share/ca-certificates/
         sudo update-ca-certificates
-        
+
     # It seems faster not to cache Go dependencies
     - name: Install Go Dependencies
       run: make deps-go
@@ -99,7 +99,6 @@ jobs:
         done
         echo "mysql is ready"
 
-
     - name: Run Go Tests
       run: |
         GO_TEST_EXTRA_FLAGS="-v -race=$RACE_ENABLED -timeout=$GO_TEST_TIMEOUT" \
@@ -119,24 +118,29 @@ jobs:
         files: coverage.txt
         flags: backend
 
+    - name: Generate summary of errors
+      if: github.event.schedule == '0 4 * * *' && failure()
+      run: |
+        c1grep() { grep "$@" || test $? = 1; }
+        c1grep -oP 'FAIL: .*$' /tmp/gotest.log > /tmp/summary.txt
+        c1grep 'test timed out after' /tmp/gotest.log >> /tmp/summary.txt
+        c1grep 'fatal error:' /tmp/gotest.log >> /tmp/summary.txt
+        GO_FAIL_SUMMARY=$(head -n 5 /tmp/summary.txt | sed ':a;N;$!ba;s/\n/\\n/g')
+        echo "GO_FAIL_SUMMARY=$GO_FAIL_SUMMARY"
+        if [[ -z "$GO_FAIL_SUMMARY" ]]; then
+          GO_FAIL_SUMMARY="unknown, please check the build URL"
+        fi
+        GO_FAIL_SUMMARY=$GO_FAIL_SUMMARY envsubst < .github/workflows/config/slack_payload_template.json > ./payload.json
+
     - name: Slack Notification
       if: github.event.schedule == '0 4 * * *' && failure()
       uses: slackapi/slack-github-action@e28cf165c92ffef168d23c5c9000cffc8a25e117 # v1.24.0
       with:
-        payload: |
-          {
-            "text": "${{ job.status }}\n${{ github.event.pull_request.html_url || github.event.head.html_url }}",
-            "blocks": [
-              {
-                "type": "section",
-                "text": {
-                  "type": "mrkdwn",
-                  "text": "Go tests result: ${{ job.status }}\nhttps://github.com/fleetdm/fleet/actions/runs/${{  github.run_id }}\n${{ github.event.pull_request.html_url || github.event.head.html_url }}"
-                }
-              }
-            ]
-          }
+        payload-file-path: ./payload.json
       env:
+        JOB_STATUS: ${{ job.status }}
+        EVENT_URL: ${{ github.event.pull_request.html_url || github.event.head.html_url }}
+        RUN_URL: https://github.com/fleetdm/fleet/actions/runs/${{ github.run_id }}\n${{ github.event.pull_request.html_url || github.event.head.html_url }}
         SLACK_WEBHOOK_URL: ${{ secrets.SLACK_G_HELP_ENGINEERING_WEBHOOK_URL }}
         SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
 
@@ -147,3 +151,10 @@ jobs:
         name: test-log
         path: /tmp/gotest.log
         if-no-files-found: error
+
+    - name: Upload summary test log
+      if: always()
+      uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v2
+      with:
+        name: summary-test-log
+        path: /tmp/summary.txt

.github/workflows/test-packaging.yml

@@ -85,7 +85,7 @@ jobs:
     - name: Install wine and wix
       if: matrix.os == 'macos-latest'
       run: |
-        ./scripts/macos-install-wine.sh
+        ./scripts/macos-install-wine.sh -n
         wget https://github.com/wixtoolset/wix3/releases/download/wix3112rtm/wix311-binaries.zip -nv -O wix.zip
         mkdir wix
         unzip wix.zip -d wix

.trivyignore

@@ -12,3 +12,8 @@ CVE-2020-7753
 # We feel like the risk of DoS using this technique, which requires being logged in, is low probability and low impact, as such we will not update glob-parent only for this CVE
 
 CVE-2020-28469
+
+# 2024/04/04 (github.com/goreleaser/nfpm/v2 should be updated)
+# When packaging linux files, we do not use global permissions. Manually verified that packed fleet-osquery files do not have group/global write permissions.
+
+CVE-2023-32698

.vscode/extensions.json

@@ -7,6 +7,7 @@
         "redhat.vscode-yaml",
         "dbaeumer.vscode-eslint",
         "firefox-devtools.vscode-firefox-debug",
-        "editorconfig.editorconfig"
+        "editorconfig.editorconfig",
+        "timonwong.shellcheck"
     ]
 }

.yarnrc

@@ -0,0 +1 @@
+save-prefix ""

CHANGELOG.md

@@ -1,3 +1,76 @@
+## Fleet 4.48.2 (Apr 09, 2024)
+
+### Bug fixes
+
+* Fixed an issue with the `20240327115617_CreateTableNanoDDMRequests` database migration where it could fail if the database did not default to the `utf8mb4_unicode_ci` collation.
+* Fixed an issue with automatic release of the device after setup when a DDM profile is pending.
+
+## Fleet 4.48.1 (Apr 08, 2024)
+
+### Bug fixes
+
+- Made block_id mismatch errors more informative as 400s instead of 500s
+- Fixed a bug where values were not being rendered in host-specific query reports
+- Fixed potential server panic when events are created with calendar integration, but then global calendar integration is disabled
+
+## Fleet 4.48.0 (Apr 03, 2024)
+
+### Endpoint operations
+- Added integration with Google Calendar.
+  * Fleet admins can enable Google Calendar integration by using a Google service account with domain-wide delegation.
+  * Calendar integration is enabled at the team level for specific team policies.
+  * If the policy is failing, a calendar event will be put on the host user's calendar for the 3rd Tuesday of the month.
+  * During the event, Fleet will fire a webhook. IT admins should use this webhook to trigger a script or MDM command that will remediate the issue.
+- Reduced the number of 'Deadlock found' errors seen by the server when multiple hosts share the same UUID.
+- Removed outdated tooltips from UI.
+- Added hover states to clickable elements.
+- Added cross-platform check for duplicate MDM profiles names in batch set MDM profiles API.
+
+### Device management (MDM)
+- Added Windows MDM support to the `osquery-perf` host-simulation command.
+- Added a missing database index to the MDM Windows enrollments table that will improve performance at scale.
+- Migrate MDM-related endpoints to new paths, deprecating (but still supporting indefinitely) the old endpoints.
+- Adds API functionality for creating DDM declarations, both individually and as a batch.
+- Added DDM activities to the fleet UI.
+- Added the `enable_release_device_manually` configuration setting for a team and no team. **Note** that the macOS automatic enrollment profile cannot set the `await_device_configured` option anymore, this setting is controlled by Fleet via the new `enable_release_device_manually` option.
+- Automatically release a macOS DEP-enrolled device after enrollment commands and profiles have been delivered, unless `enable_release_device_manually` is set to `true`.
+
+### Vulnerability management
+- Added Visual Studio extensions to Fleet's software inventory.
+
+### Bug fixes
+- Fixed a bug where valid MDM enrollments would show up as unmanaged (EnrollmentState 3).
+- Fixed flash message from closing when a modal closes.
+- Fixed a bug where OS version information would not get detected on Windows Server 2019.
+- Fixed issue where getting host details failed when attempting to read the host's bitlocker status from the datastore.
+- Fixed false negative vulnerabilities on macOS Homebrew python packages.
+- Fixed styling of live query disabled warning.
+- Fixed issue where Windows MDM profile processing was skipping `<Add>` commands.
+- Fixed UI's ability to bulk delete hosts when "All teams" is selected.
+- Fixed error state rendering on the global Host status expiry settings page, fix error state alignment for tooltip-wrapper field labels across organization settings.
+- Fixed `GET fleet/os_versions` and `GET fleet/os_versions/[id]` so team users no longer have access to os versions on hosts from other teams.
+- `fleetctl gitops` now batch processes queries and policies.
+- Fixed UI bug to render the query platform correctly for queries imported from the standard query library.
+- Fixed issue where microsoft edge was not reporting vulnerabilities.
+- Fixed a bug where all Windows MDM enrollments were detected as automatic.
+- Fixed a bug where `null` or excluded `smtp_settings` caused a UI 500.
+- Fixed query reports so they reset when there is a change to the selected platform or selected minimum osquery version.
+- Fixed live query sort of sql result sort for both string and numerical columns.
+
+## Fleet 4.47.3 (Mar 26, 2024)
+
+### Bug fixes
+
+* Fixed a bug where valid Windows MDM enrollments would show up as unmanaged (EnrollmentState 3).
+
+## Fleet 4.47.2 (Mar 22, 2024)
+
+### Bug fixes
+
+* Fixed false negative vulnerabilities on macOS Homebrew Python packages.
+* Fixed policies to check "disable guest user".
+* Resolved the issue where Microsoft Edge was not reporting vulnerabilities.
+
 ## Fleet 4.47.1 (Mar 18, 2024)
 
 ### Bug fixes

CODEOWNERS

@@ -74,7 +74,7 @@ go.mod @fleetdm/go
 #
 # (see website/config/custom.js for DRIs of other paths not listed here)
 ##############################################################################################
-/handbook/company/pricing-features-table.yml          @mikermcneil # « CEO is current DRI for features table
+/handbook/company/pricing-features-table.yml          @noahtalerman # « Head of Product Design is current DRI for features table
 
 ##############################################################################################
 # 🦿 Repo automation and change control settings

Makefile

@@ -318,7 +318,9 @@ changelog:
 	sh -c "git rm changes/*"
 
 changelog-orbit:
-	sh -c "find orbit/changes -type file | grep -v .keep | xargs -I {} sh -c 'grep \"\S\" {}; echo' > new-CHANGELOG.md"
+	$(eval TODAY_DATE := $(shell date "+%b %d, %Y"))
+	@echo -e "## Orbit $(version) ($(TODAY_DATE))\n" > new-CHANGELOG.md
+	sh -c "find orbit/changes -type file | grep -v .keep | xargs -I {} sh -c 'grep \"\S\" {} | sed -E "s/^-/*/"; echo' >> new-CHANGELOG.md"
 	sh -c "cat new-CHANGELOG.md orbit/CHANGELOG.md > tmp-CHANGELOG.md && rm new-CHANGELOG.md && mv tmp-CHANGELOG.md orbit/CHANGELOG.md"
 	sh -c "git rm orbit/changes/*"
 
@@ -394,7 +396,7 @@ ifneq ($(shell uname), Darwin)
 	@exit 1
 endif
 	# locking the version of swiftDialog to 2.2.1-4591 as newer versions
-	# migth have layout issues.
+	# might have layout issues.
 ifneq ($(version), 2.2.1)
 	@echo "Version is locked at 2.1.0, see comments in Makefile target for details"
 	@exit 1

README.md

@@ -71,7 +71,7 @@ The Fleet community is full of [kind and helpful people](https://fleetdm.com/han
 
 The landscape of cybersecurity and IT is too complex.  Let's open it up.
 
-Contributions are welcome, whether you answer questions on [Slack](#chat) / [GitHub](https://github.com/fleetdm/fleet/issues) / [StackOverflow](https://stackoverflow.com/search?q=osquery) / [LinkedIn](https://linkedin.com/company/fleetdm) / [Twitter](https://twitter.com/fleetctl), improve the documentation or [website](./website), write a tutorial, give a talk at a conference or local meetup, give an [interview on a podcast](https://fleetdm.com/podcasts), troubleshoot reported issues, or [submit a patch](https://fleetdm.com/docs/contributing/contributing).  The Fleet code of conduct is [on GitHub](https://github.com/fleetdm/fleet/blob/main/CODE_OF_CONDUCT.md).
+Contributions are welcome, whether you answer questions on [Slack](https://fleetdm.com/slack) / [GitHub](https://github.com/fleetdm/fleet/issues) / [StackOverflow](https://stackoverflow.com/search?q=osquery) / [LinkedIn](https://linkedin.com/company/fleetdm) / [Twitter](https://twitter.com/fleetctl), improve the documentation or [website](./website), write a tutorial, give a talk at a conference or local meetup, give an [interview on a podcast](https://fleetdm.com/podcasts), troubleshoot reported issues, or [submit a patch](https://fleetdm.com/docs/contributing/contributing).  The Fleet code of conduct is [on GitHub](https://github.com/fleetdm/fleet/blob/main/CODE_OF_CONDUCT.md).
 
 <!-- - Great contributions are motivated by real-world use cases or learning.
 - Some of the most valuable contributions might not touch any code at all.
@@ -81,7 +81,7 @@ Contributions are welcome, whether you answer questions on [Slack](#chat) / [Git
 To see what Fleet can do, head over to [fleetdm.com](https://fleetdm.com) and try it out for yourself, grab time with one of the maintainers to discuss, or visit the docs and roll it out to your organization.
 
 #### Production deployment
-Fleet is simple enough to [spin up for yourself](https://fleetdm.com/docs/using-fleet/learn-how-to-use-fleet).  Or you can have us [host it for you](https://fleetdm.com/pricing).  Premium features are [available](https://fleetdm.com/pricing) either way.
+Fleet is simple enough to [spin up for yourself](https://fleetdm.com/docs/get-started/tutorials-and-guides).  Or you can have us [host it for you](https://fleetdm.com/pricing).  Premium features are [available](https://fleetdm.com/pricing) either way.
 
 #### Documentation
 Complete documentation for Fleet can be found at [https://fleetdm.com/docs](https://fleetdm.com/docs).