diff --git a/.gatsby/plugins/robots-txt.js b/.gatsby/plugins/robots-txt.js new file mode 100644 index 00000000..8a7d4d02 --- /dev/null +++ b/.gatsby/plugins/robots-txt.js @@ -0,0 +1,20 @@ +/* + * Copyright (C) 2018-present Arctic Ice Studio + * Copyright (C) 2018-present Sven Greb + * + * Project: Nord Docs + * Repository: https://github.com/arcticicestudio/nord-docs + * License: MIT + */ + +/** + * @file The configuration for the Gatsby plugin `gatsby-plugin-robots-txt`. + * @author Arctic Ice Studio + * @author Sven Greb + * @see https://github.com/mdreizin/gatsby-plugin-robots-txt + * @since 0.4.0 + */ + +module.exports = { + policy: [{ userAgent: "*", allow: "/" }] +}; diff --git a/gatsby-config.js b/gatsby-config.js index acfce3d5..77eb70fe 100644 --- a/gatsby-config.js +++ b/gatsby-config.js @@ -26,6 +26,7 @@ const { const { BASE_PUBLIC_URL } = require("./src/config/routes/constants"); const gatsbyPluginGoogleGtagConfig = require("./.gatsby/plugins/google/gtag"); const gatsbyPluginManifestConfig = require("./.gatsby/plugins/manifest"); +const gatsbyPluginRobotsTxt = require("./.gatsby/plugins/robots-txt"); module.exports = { siteMetadata: { @@ -87,6 +88,10 @@ module.exports = { resolve: "gatsby-plugin-google-gtag", options: gatsbyPluginGoogleGtagConfig }, + { + resolve: "gatsby-plugin-robots-txt", + options: gatsbyPluginRobotsTxt + }, /* NOTE: The following plugins rely on the order in this array and must be placed at last in order work properly! */ { resolve: "gatsby-plugin-manifest", diff --git a/package-lock.json b/package-lock.json index 14c57658..c160445f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1701,8 +1701,7 @@ "arrify": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz", - "integrity": "sha1-iYUI2iIm84DfkEcoRWhJwVAaSw0=", - "dev": true + "integrity": "sha1-iYUI2iIm84DfkEcoRWhJwVAaSw0=" }, "asap": { "version": "2.0.6", @@ -4422,6 +4421,15 @@ "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=" }, + "decamelize-keys": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/decamelize-keys/-/decamelize-keys-1.1.0.tgz", + "integrity": "sha1-0XGoeTMlKAfrPLYdwcFEXQeN8tk=", + "requires": { + "decamelize": "^1.1.0", + "map-obj": "^1.0.0" + } + }, "decode-uri-component": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.0.tgz", @@ -7397,6 +7405,15 @@ "@babel/runtime": "^7.0.0" } }, + "gatsby-plugin-robots-txt": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/gatsby-plugin-robots-txt/-/gatsby-plugin-robots-txt-1.3.0.tgz", + "integrity": "sha512-vZT0oCDcF07bYf4pM4ISLZDopv0f0MFi1kRP3+PLPkTC0W6kScvif1ubSnehK/O5MILFpyaCWUDDW2zX+0D8LA==", + "requires": { + "babel-runtime": "^6.26.0", + "generate-robotstxt": "^5.0.5" + } + }, "gatsby-plugin-sitemap": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/gatsby-plugin-sitemap/-/gatsby-plugin-sitemap-2.0.3.tgz", @@ -7661,6 +7678,145 @@ } } }, + "generate-robotstxt": { + "version": "5.0.7", + "resolved": "https://registry.npmjs.org/generate-robotstxt/-/generate-robotstxt-5.0.7.tgz", + "integrity": "sha512-fSqO9dTnFMZKrnkVInjaMn1uODH2NWq2L1jlY41kLI2AgCwK4gHwAuZi5979kkWKmvf2khbx+Pldl+9vwh2EXg==", + "requires": { + "cosmiconfig": "^5.0.0", + "fs-extra": "^6.0.0", + "ip-address": "^5.8.6", + "is-absolute-url": "^2.1.0", + "meow": "^5.0.0", + "resolve-from": "^4.0.0" + }, + "dependencies": { + "camelcase": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-4.1.0.tgz", + "integrity": "sha1-1UVjW+HjPFQmScaRc+Xeas+uNN0=" + }, + "camelcase-keys": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/camelcase-keys/-/camelcase-keys-4.2.0.tgz", + "integrity": "sha1-oqpfsa9oh1glnDLBQUJteJI7m3c=", + "requires": { + "camelcase": "^4.1.0", + "map-obj": "^2.0.0", + "quick-lru": "^1.0.0" + } + }, + "find-up": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz", + "integrity": "sha1-RdG35QbHF93UgndaK3eSCjwMV6c=", + "requires": { + "locate-path": "^2.0.0" + } + }, + "fs-extra": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-6.0.1.tgz", + "integrity": "sha512-GnyIkKhhzXZUWFCaJzvyDLEEgDkPfb4/TPvJCJVuS8MWZgoSsErf++QpiAlDnKFcqhRlm+tIOcencCjyJE6ZCA==", + "requires": { + "graceful-fs": "^4.1.2", + "jsonfile": "^4.0.0", + "universalify": "^0.1.0" + } + }, + "load-json-file": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-4.0.0.tgz", + "integrity": "sha1-L19Fq5HjMhYjT9U62rZo607AmTs=", + "requires": { + "graceful-fs": "^4.1.2", + "parse-json": "^4.0.0", + "pify": "^3.0.0", + "strip-bom": "^3.0.0" + } + }, + "map-obj": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/map-obj/-/map-obj-2.0.0.tgz", + "integrity": "sha1-plzSkIepJZi4eRJXpSPgISIqwfk=" + }, + "meow": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/meow/-/meow-5.0.0.tgz", + "integrity": "sha512-CbTqYU17ABaLefO8vCU153ZZlprKYWDljcndKKDCFcYQITzWCXZAVk4QMFZPgvzrnUQ3uItnIE/LoUOwrT15Ig==", + "requires": { + "camelcase-keys": "^4.0.0", + "decamelize-keys": "^1.0.0", + "loud-rejection": "^1.0.0", + "minimist-options": "^3.0.1", + "normalize-package-data": "^2.3.4", + "read-pkg-up": "^3.0.0", + "redent": "^2.0.0", + "trim-newlines": "^2.0.0", + "yargs-parser": "^10.0.0" + } + }, + "path-type": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-3.0.0.tgz", + "integrity": "sha512-T2ZUsdZFHgA3u4e5PfPbjd7HDDpxPnQb5jN0SrDsjNSuVXHJqtwTnWqG0B1jZrgmJ/7lj1EmVIByWt1gxGkWvg==", + "requires": { + "pify": "^3.0.0" + } + }, + "pify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", + "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=" + }, + "read-pkg": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-3.0.0.tgz", + "integrity": "sha1-nLxoaXj+5l0WwA4rGcI3/Pbjg4k=", + "requires": { + "load-json-file": "^4.0.0", + "normalize-package-data": "^2.3.2", + "path-type": "^3.0.0" + } + }, + "read-pkg-up": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-3.0.0.tgz", + "integrity": "sha1-PtSWaF26D4/hGNBpHcUfSh/5bwc=", + "requires": { + "find-up": "^2.0.0", + "read-pkg": "^3.0.0" + } + }, + "redent": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/redent/-/redent-2.0.0.tgz", + "integrity": "sha1-wbIAe0LVfrE4kHmzyDM2OdXhzKo=", + "requires": { + "indent-string": "^3.0.0", + "strip-indent": "^2.0.0" + } + }, + "strip-indent": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/strip-indent/-/strip-indent-2.0.0.tgz", + "integrity": "sha1-XvjbKV0B5u1sv3qrlpmNeCJSe2g=" + }, + "trim-newlines": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/trim-newlines/-/trim-newlines-2.0.0.tgz", + "integrity": "sha1-tAPQuRvlDDMd/EuC7s6yLD3hbSA=" + }, + "yargs-parser": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-10.1.0.tgz", + "integrity": "sha512-VCIyR1wJoEBZUqk5PA+oOBF6ypbwh5aNB3I50guxAL/quggdfs4TtNHQrSazFA3fYZ+tEqfs0zIGlv0c/rgjbQ==", + "requires": { + "camelcase": "^4.1.0" + } + } + } + }, "get-caller-file": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-1.0.3.tgz", @@ -8683,8 +8839,7 @@ "indent-string": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-3.2.0.tgz", - "integrity": "sha1-Sl/W0nzDMvN+VBmlBNu4NxBckok=", - "dev": true + "integrity": "sha1-Sl/W0nzDMvN+VBmlBNu4NxBckok=" }, "indexes-of": { "version": "1.0.1", @@ -8768,6 +8923,32 @@ "resolved": "https://registry.npmjs.org/ip/-/ip-1.1.5.tgz", "integrity": "sha1-vd7XARQpCCjAoDnnLvJfWq7ENUo=" }, + "ip-address": { + "version": "5.8.9", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-5.8.9.tgz", + "integrity": "sha512-7ay355oMN34iXhET1BmCJVsHjOTSItEEIIpOs38qUC23AIhOy+xIPnkrTuEFjeLMrTJ7m8KMXWgWfy/2Vn9sDw==", + "requires": { + "jsbn": "1.1.0", + "lodash.find": "^4.6.0", + "lodash.max": "^4.0.1", + "lodash.merge": "^4.6.0", + "lodash.padstart": "^4.6.1", + "lodash.repeat": "^4.1.0", + "sprintf-js": "1.1.0" + }, + "dependencies": { + "jsbn": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-1.1.0.tgz", + "integrity": "sha1-sBMHyym2GKHtJux56RH4A8TaAEA=" + }, + "sprintf-js": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.0.tgz", + "integrity": "sha1-z/yvcC2vZeo5u04PorKZzsGhvkY=" + } + } + }, "ip-regex": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/ip-regex/-/ip-regex-2.1.0.tgz", @@ -11056,6 +11237,11 @@ "resolved": "https://registry.npmjs.org/lodash.every/-/lodash.every-4.6.0.tgz", "integrity": "sha1-64mYS+vENkJ5uzrvu9HKGb+mxqc=" }, + "lodash.find": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.find/-/lodash.find-4.6.0.tgz", + "integrity": "sha1-ywcE1Hq3F4n/oN6Ll92Sb7iLE7E=" + }, "lodash.flattendeep": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/lodash.flattendeep/-/lodash.flattendeep-4.4.0.tgz", @@ -11086,6 +11272,11 @@ "resolved": "https://registry.npmjs.org/lodash.map/-/lodash.map-4.6.0.tgz", "integrity": "sha1-dx7Hg540c9nEzeKLGTlMNWL09tM=" }, + "lodash.max": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/lodash.max/-/lodash.max-4.0.1.tgz", + "integrity": "sha1-hzVWbGGLNan3YFILSHrnllivE2o=" + }, "lodash.maxby": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/lodash.maxby/-/lodash.maxby-4.6.0.tgz", @@ -11096,6 +11287,21 @@ "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz", "integrity": "sha1-vMbEmkKihA7Zl/Mj6tpezRguC/4=" }, + "lodash.merge": { + "version": "4.6.1", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.1.tgz", + "integrity": "sha512-AOYza4+Hf5z1/0Hztxpm2/xiPZgi/cjMqdnKTUWTBSKchJlxXXuUSxCCl8rJlf4g6yww/j6mA8nC8Hw/EZWxKQ==" + }, + "lodash.padstart": { + "version": "4.6.1", + "resolved": "https://registry.npmjs.org/lodash.padstart/-/lodash.padstart-4.6.1.tgz", + "integrity": "sha1-0uPuv/DZ05rVD1y9G1KnvOa7YRs=" + }, + "lodash.repeat": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/lodash.repeat/-/lodash.repeat-4.1.0.tgz", + "integrity": "sha1-/H3oEx2MisB+S0n3T/6CnR8r7EQ=" + }, "lodash.sortby": { "version": "4.7.0", "resolved": "https://registry.npmjs.org/lodash.sortby/-/lodash.sortby-4.7.0.tgz", @@ -11248,8 +11454,7 @@ "map-obj": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/map-obj/-/map-obj-1.0.1.tgz", - "integrity": "sha1-2TPOuSBdgr3PSIb2dCvcK03qFG0=", - "dev": true + "integrity": "sha1-2TPOuSBdgr3PSIb2dCvcK03qFG0=" }, "map-visit": { "version": "1.0.0", @@ -11646,6 +11851,15 @@ "resolved": "http://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=" }, + "minimist-options": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/minimist-options/-/minimist-options-3.0.2.tgz", + "integrity": "sha512-FyBrT/d0d4+uiZRbqznPXqw3IpZZG3gl3wKWiX784FycUKVwBt0uLBFkQrtE4tZOrgo78nZp2jnKz3L65T5LdQ==", + "requires": { + "arrify": "^1.0.1", + "is-plain-obj": "^1.1.0" + } + }, "minipass": { "version": "2.3.5", "resolved": "https://registry.npmjs.org/minipass/-/minipass-2.3.5.tgz", @@ -14006,6 +14220,11 @@ "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.1.0.tgz", "integrity": "sha512-sluvZZ1YiTLD5jsqZcDmFyV2EwToyXZBfpoVOmktMmW+VEnhgakFHnasVph65fOjGPTWN0Nw3+XQaSeMayr0kg==" }, + "quick-lru": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-1.1.0.tgz", + "integrity": "sha1-Q2CxfGETatOAeDl/8RQW4Ybc+7g=" + }, "raf": { "version": "3.4.1", "resolved": "https://registry.npmjs.org/raf/-/raf-3.4.1.tgz", diff --git a/package.json b/package.json index d445b64a..baaf166f 100644 --- a/package.json +++ b/package.json @@ -94,6 +94,7 @@ "gatsby-plugin-offline": "2.0.20", "gatsby-plugin-react-helmet": "3.0.2", "gatsby-plugin-remove-trailing-slashes": "2.0.5", + "gatsby-plugin-robots-txt": "1.3.0", "gatsby-plugin-sitemap": "2.0.3", "gatsby-plugin-styled-components": "3.0.4", "gatsby-plugin-svgr": "2.0.1",