diff --git a/index.js b/index.js
index 3830063fae..70f9a73d9d 100644
--- a/index.js
+++ b/index.js
@@ -20,8 +20,15 @@ module.exports = (robot, _, Settings = require('./lib/settings')) => {
     }
 
     const settingsModified = payload.commits.find(commit => {
-      return commit.added.includes(Settings.FILE_NAME) ||
+      // If run from a GitHub action, commits lack these properties, see
+      // https://github.blog/changelog/2019-10-16-changes-in-github-actions-push-event-payload/
+      // So act like settings have changed if we run from an action
+      if ('added' in commit || 'modified' in commit) {
+        return commit.added.includes(Settings.FILE_NAME) ||
         commit.modified.includes(Settings.FILE_NAME)
+      } else {
+        return true
+      }
     })
 
     if (!settingsModified) {
diff --git a/package-lock.json b/package-lock.json
index 10b63cc957..4393eb10f9 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -4054,7 +4054,8 @@
       "version": "1.3.0",
       "resolved": "https://registry.npmjs.org/growly/-/growly-1.3.0.tgz",
       "integrity": "sha1-8QdIy+dq+WS3yWyTxrzCivEgwIE=",
-      "dev": true
+      "dev": true,
+      "optional": true
     },
     "handlebars": {
       "version": "4.7.6",
@@ -4634,6 +4635,7 @@
       "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz",
       "integrity": "sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww==",
       "dev": true,
+      "optional": true,
       "requires": {
         "is-docker": "^2.0.0"
       }
@@ -6695,6 +6697,7 @@
       "resolved": "https://registry.npmjs.org/node-notifier/-/node-notifier-8.0.1.tgz",
       "integrity": "sha512-BvEXF+UmsnAfYfoapKM9nGxnP+Wn7P91YfXmrKnfcYCx6VBeoN5Ez5Ogck6I8Bi5k4RlpqRYaw75pAwzX9OphA==",
       "dev": true,
+      "optional": true,
       "requires": {
         "growly": "^1.3.0",
         "is-wsl": "^2.2.0",
@@ -6709,6 +6712,7 @@
           "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
           "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==",
           "dev": true,
+          "optional": true,
           "requires": {
             "isexe": "^2.0.0"
           }
@@ -8384,7 +8388,8 @@
       "version": "0.1.1",
       "resolved": "https://registry.npmjs.org/shellwords/-/shellwords-0.1.1.tgz",
       "integrity": "sha512-vFwSUfQvqybiICwZY5+DAWIPLKsWO31Q91JSKl3UYv+K5c2QRPzn0qzec6QPu1Qc9eHYItiP3NdJqNVqetYAww==",
-      "dev": true
+      "dev": true,
+      "optional": true
     },
     "side-channel": {
       "version": "1.0.3",