Merge pull request #99 from arcticicestudio/feature/gh-98-gatsby-plug…

…in-robots-txt

Gatsby Plugin "robots.txt"

.gatsby/plugins/robots-txt.js

@@ -0,0 +1,20 @@
+/*
+ * Copyright (C) 2018-present Arctic Ice Studio <development@arcticicestudio.com>
+ * Copyright (C) 2018-present Sven Greb <development@svengreb.de>
+ *
+ * Project:    Nord Docs
+ * Repository: https://github.com/arcticicestudio/nord-docs
+ * License:    MIT
+ */
+
+/**
+ * @file The configuration for the Gatsby plugin `gatsby-plugin-robots-txt`.
+ * @author Arctic Ice Studio <development@arcticicestudio.com>
+ * @author Sven Greb <development@svengreb.de>
+ * @see https://github.com/mdreizin/gatsby-plugin-robots-txt
+ * @since 0.4.0
+ */
+
+module.exports = {
+  policy: [{ userAgent: "*", allow: "/" }]
+};

gatsby-config.js

@@ -26,6 +26,7 @@ const {
 const { BASE_PUBLIC_URL } = require("./src/config/routes/constants");
 const gatsbyPluginGoogleGtagConfig = require("./.gatsby/plugins/google/gtag");
 const gatsbyPluginManifestConfig = require("./.gatsby/plugins/manifest");
+const gatsbyPluginRobotsTxt = require("./.gatsby/plugins/robots-txt");
 
 module.exports = {
   siteMetadata: {
@@ -87,6 +88,10 @@ module.exports = {
       resolve: "gatsby-plugin-google-gtag",
       options: gatsbyPluginGoogleGtagConfig
     },
+    {
+      resolve: "gatsby-plugin-robots-txt",
+      options: gatsbyPluginRobotsTxt
+    },
     /* NOTE: The following plugins rely on the order in this array and must be placed at last in order work properly! */
     {
       resolve: "gatsby-plugin-manifest",

package.json

@@ -94,6 +94,7 @@
     "gatsby-plugin-offline": "2.0.20",
     "gatsby-plugin-react-helmet": "3.0.2",
     "gatsby-plugin-remove-trailing-slashes": "2.0.5",
+    "gatsby-plugin-robots-txt": "1.3.0",
     "gatsby-plugin-sitemap": "2.0.3",
     "gatsby-plugin-styled-components": "3.0.4",
     "gatsby-plugin-svgr": "2.0.1",