Serve robots.txt to prevent crawlers from accessing UI

CHANGELOG.md

@@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+- The UI now serves a `/robots.txt` that instructs crawlers to not crawl any part an installation. (You should still use an authentication layer though.) [PR #97](https://github.com/riverqueue/riverui/pull/97).
+
 ## [0.2.0] - 2024-07-02
 
 ### Added

Dockerfile

@@ -30,6 +30,7 @@ RUN go mod download
 COPY *.go internal docs/README.md LICENSE ./
 COPY cmd/ cmd/
 COPY internal/ internal/
+COPY public/ public/
 COPY ui/*.go ./ui/
 COPY --from=build-ui /app/dist ./ui/dist
 

handler.go

@@ -2,6 +2,7 @@ package riverui
 
 import (
 	"context"
+	"embed"
 	"errors"
 	"fmt"
 	"io/fs"
@@ -99,6 +100,11 @@ func NewHandler(opts *HandlerOpts) (http.Handler, error) {
 	apiendpoint.Mount(mux, opts.Logger, &queueResumeEndpoint{apiBundle: apiBundle})
 	apiendpoint.Mount(mux, opts.Logger, &stateAndCountGetEndpoint{apiBundle: apiBundle})
 	apiendpoint.Mount(mux, opts.Logger, &workflowGetEndpoint{apiBundle: apiBundle})
+
+	if err := mountStaticFiles(opts.Logger, mux); err != nil {
+		return nil, err
+	}
+
 	mux.HandleFunc("/api", http.NotFound)
 	mux.Handle("/", intercept404(fileServer, serveIndex))
 
@@ -111,6 +117,51 @@ func NewHandler(opts *HandlerOpts) (http.Handler, error) {
 	return middlewareStack.Mount(mux), nil
 }
 
+//go:embed public
+var publicFS embed.FS
+
+const publicPrefix = "public/"
+
+// Walks the embedded filesystem in publicFS and mounts each file as a route on
+// the given serve mux. Content type is determined by `http.DetectContentType`.
+func mountStaticFiles(logger *slog.Logger, mux *http.ServeMux) error {
+	return fs.WalkDir(publicFS, ".", func(path string, dirEntry fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if dirEntry.IsDir() {
+			return nil
+		}
+
+		servePath := strings.TrimPrefix(path, publicPrefix)
+
+		mux.HandleFunc("GET /"+servePath, func(w http.ResponseWriter, r *http.Request) {
+			runWithError := func() error {
+				data, err := publicFS.ReadFile(path)
+				if err != nil {
+					return err
+				}
+
+				contentType := http.DetectContentType(data)
+				w.Header().Add("Content-Type", contentType)
+
+				if _, err := w.Write(data); err != nil {
+					return err
+				}
+
+				return nil
+			}
+
+			if err := runWithError(); err != nil {
+				logger.ErrorContext(r.Context(), "Error writing static file", "err", err)
+			}
+		})
+
+		return nil
+	})
+}
+
 // Go's http.StripPrefix can sometimes result in an empty path. For example,
 // when removing a prefix like "/foo" from path "/foo", the result is "".  This
 // does not get handled by the ServeMux correctly (it results in a redirect to

handler_test.go

@@ -112,4 +112,34 @@ func TestNewHandlerIntegration(t *testing.T) {
 	makeAPICall(t, "QueueResume", http.MethodPut, makeURL("/api/queues/%s/resume", queuePaused.Name), nil)
 	makeAPICall(t, "StateAndCountGet", http.MethodGet, makeURL("/api/states"), nil)
 	makeAPICall(t, "WorkflowGet", http.MethodGet, makeURL("/api/workflows/%s", workflowID), nil)
+
+	//
+	// Static files
+	//
+
+	makeAPICall(t, "RobotsTxt", http.MethodGet, makeURL("/robots.txt"), nil)
+}
+
+func TestMountStaticFiles(t *testing.T) {
+	t.Parallel()
+
+	var (
+		logger = riverinternaltest.Logger(t)
+		mux    = http.NewServeMux()
+	)
+
+	require.NoError(t, mountStaticFiles(logger, mux))
+
+	var (
+		recorder = httptest.NewRecorder()
+		req      = httptest.NewRequest(http.MethodGet, "/robots.txt", nil)
+	)
+
+	mux.ServeHTTP(recorder, req)
+
+	status := recorder.Result().StatusCode //nolint:bodyclose
+	require.Equal(t, http.StatusOK, status)
+
+	require.Equal(t, "text/plain; charset=utf-8", recorder.Header().Get("Content-Type"))
+	require.Contains(t, recorder.Body.String(), "User-Agent")
 }

internal/apiendpoint/api_endpoint.go

@@ -133,6 +133,10 @@ func executeAPIEndpoint[TReq any, TResp any](w http.ResponseWriter, r *http.Requ
 			return err
 		}
 
+		if rawExtractor, ok := any(resp).(RawResponder); ok {
+			return rawExtractor.RespondRaw(w)
+		}
+
 		respData, err := json.Marshal(resp)
 		if err != nil {
 			return fmt.Errorf("error marshaling response JSON: %w", err)
@@ -189,6 +193,13 @@ type RawExtractor interface {
 	ExtractRaw(r *http.Request) error
 }
 
+// RawResponder is an interface that can be implemented by response structs that
+// allow them to respond directly to a ResponseWriter instead of emitting the
+// normal JSON format.
+type RawResponder interface {
+	RespondRaw(w http.ResponseWriter) error
+}
+
 // Make some broad categories of internal error back into something public
 // facing because in some cases they can be a vast help for debugging.
 func maybeInterpretInternalError(err error) error {

public/robots.txt

@@ -0,0 +1,2 @@
+User-Agent: *
+Disallow: /