add: robots middleware (#627)

Closes #610

handler/handler.go

@@ -18,6 +18,7 @@ import (
 	"github.com/apex/up/http/poweredby"
 	"github.com/apex/up/http/redirects"
 	"github.com/apex/up/http/relay"
+	"github.com/apex/up/http/robots"
 	"github.com/apex/up/http/static"
 )
 
@@ -36,6 +37,7 @@ func FromConfig(c *up.Config) (http.Handler, error) {
 // New handler complete with all Up middleware.
 func New(c *up.Config, h http.Handler) (http.Handler, error) {
 	h = poweredby.New("up", h)
+	h = robots.New(c, h)
 	h = static.NewDynamic(c, h)
 
 	h, err := headers.New(c, h)

handler/handler_test.go

@@ -40,6 +40,7 @@ func TestNode(t *testing.T) {
 
 	header := make(http.Header)
 	header.Add("X-Powered-By", "up")
+	header.Add("X-Robots-Tag", "none")
 	header.Add("X-Foo", "bar")
 	header.Add("Content-Length", "11")
 	header.Add("Content-Type", "text/plain; charset=utf-8")
@@ -67,6 +68,7 @@ func TestStatic(t *testing.T) {
 
 	header := make(http.Header)
 	header.Add("X-Powered-By", "up")
+	header.Add("X-Robots-Tag", "none")
 	header.Add("Content-Length", "12")
 	header.Add("Content-Type", "text/html; charset=utf-8")
 	header.Add("Accept-Ranges", "bytes")

http/robots/robots.go

@@ -0,0 +1,21 @@
+// Package robots provides a way of dealing with robots exclusion protocol
+package robots
+
+import (
+	"net/http"
+	"os"
+
+	"github.com/apex/up"
+)
+
+// New robots middleware.
+func New(c *up.Config, next http.Handler) http.Handler {
+	if os.Getenv("UP_STAGE") == "production" {
+		return next
+	}
+
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("X-Robots-Tag", "none")
+		next.ServeHTTP(w, r)
+	})
+}

http/robots/robots_test.go

@@ -0,0 +1,52 @@
+package robots
+
+import (
+	"net/http/httptest"
+	"os"
+	"testing"
+
+	"github.com/apex/up"
+	"github.com/tj/assert"
+
+	"github.com/apex/up/config"
+	"github.com/apex/up/http/static"
+)
+
+func TestRobots(t *testing.T) {
+	c := &up.Config{
+		Static: config.Static{
+			Dir: "testdata",
+		},
+	}
+
+	t.Run("should set X-Robots-Tag", func(t *testing.T) {
+		h := New(c, static.New(c))
+
+		res := httptest.NewRecorder()
+		req := httptest.NewRequest("GET", "/", nil)
+
+		h.ServeHTTP(res, req)
+
+		assert.Equal(t, 200, res.Code)
+		assert.Equal(t, "none", res.Header().Get("X-Robots-Tag"))
+		assert.Equal(t, "text/html; charset=utf-8", res.Header().Get("Content-Type"))
+		assert.Equal(t, "Index HTML\n", res.Body.String())
+	})
+
+	t.Run("should not set X-Robots-Tag for production stage", func(t *testing.T) {
+		os.Setenv("UP_STAGE", "production")
+		defer os.Setenv("UP_STAGE", "")
+
+		h := New(c, static.New(c))
+
+		res := httptest.NewRecorder()
+		req := httptest.NewRequest("GET", "/", nil)
+
+		h.ServeHTTP(res, req)
+
+		assert.Equal(t, 200, res.Code)
+		assert.Equal(t, "", res.Header().Get("X-Robots-Tag"))
+		assert.Equal(t, "text/html; charset=utf-8", res.Header().Get("Content-Type"))
+		assert.Equal(t, "Index HTML\n", res.Body.String())
+	})
+}

http/robots/testdata/index.html

@@ -0,0 +1 @@
+Index HTML

http/robots/testdata/up.json

@@ -0,0 +1,3 @@
+{
+  "name": "app"
+}