cmd/godoc: simplify dev and prod environment for App Engine

Remove all of the code generation and the concept of "APPDIR" - just
generate godoc.zip and index files in the app directory.

Simplify generation of the zip - use a symlink so that every file in
godoc.zip is under the "goroot" directory, regardless of the
environment. Previously, the prefix would be dependent on the location
of the user's GOROOT.

Running the setup script is now optional - it's now possible to run
dev_appserver.py on a regular checkout of cmd/godoc without godoc.zip
and search index files. Use environment variables to switch whether the
zip file is used vs reading GOROOT from the filesystem.

Change-Id: I1ce95c891717fe2da975f979778fd775b23f18c8
Reviewed-on: https://go-review.googlesource.com/46725
Reviewed-by: Andrew Bonventre <andybons@golang.org>

cmd/godoc/.gitignore

@@ -0,0 +1,3 @@
+index.split.*
+godoc.index
+godoc.zip

cmd/godoc/README.godoc-app

@@ -1,56 +1,41 @@
-godoc on appengine
-------------------
+godoc on Google App Engine
+==========================
 
 Prerequisites
 -------------
 
-* Go appengine SDK
-  https://developers.google.com/appengine/downloads#Google_App_Engine_SDK_for_Go
+* Google Cloud SDK
+  https://cloud.google.com/sdk/
 
-* Go sources at tip under $GOROOT
+* Go sources under $GOROOT
 
-* Godoc sources at tip inside $GOPATH
+* Godoc sources inside $GOPATH
   (go get -d golang.org/x/tools/cmd/godoc)
 
 
-Directory structure
--------------------
+Running in dev_appserver.py
+---------------------------
 
-* Let $APPDIR be the directory containing the app engine files.
-  (e.g., $APPDIR=$HOME/godoc-app)
+Use dev_appserver.py to run the server in development mode:
 
-* $APPDIR contains the following entries (this may change depending on
-  app-engine release and version of godoc):
+	dev_appserver.py app.dev.yaml
 
-	app.yaml
-	golang.org/x/tools/cmd/godoc
-	godoc.zip
-	index.split.*
+To run the server with generated zip file and search index:
 
-* The app.yaml file is set up per app engine documentation.
-  For instance:
+	./generate-index.bash
+	dev_appserver.py app.prod.yaml
 
-	application: godoc-app
-	version: 1
-	runtime: go
-	api_version: go1
+godoc should come up at http://localhost:8080
+Use the --host and --port flags to listen on a different address.
 
-	handlers:
-	- url: /.*
-	  script: _go_app
+To clean up the index files, use git:
 
+	git clean -xn # n is dry run, replace with f
 
-Configuring and running godoc
------------------------------
 
-To configure godoc, run
+Troubleshooting
+---------------
 
-	bash setup-godoc-app.bash
-
-to prepare an $APPDIR as described above. See the script for details on usage.
-
-To run godoc locally, using the App Engine development server, run
-
-	<path to go_appengine>/dev_appserver.py $APPDIR
-
-godoc should come up at http://localhost:8080 .
+Ensure the Cloud SDK is on your PATH and you have the app-engine-go component
+installed (gcloud components install app-engine-go) and your components are
+up-to-date (gcloud components update)

cmd/godoc/app.dev.yaml

@@ -0,0 +1,13 @@
+runtime: go
+api_version: go1
+instance_class: F4_1G
+
+handlers:
+- url: /s
+  script: _go_app
+  login: admin
+- url: /dl/init
+  script: _go_app
+  login: admin
+- url: /.*
+  script: _go_app

cmd/godoc/app.prod.yaml

@@ -0,0 +1,18 @@
+runtime: go
+api_version: go1
+instance_class: F4_1G
+
+handlers:
+- url: /s
+  script: _go_app
+  login: admin
+- url: /dl/init
+  script: _go_app
+  login: admin
+- url: /.*
+  script: _go_app
+
+env_variables:
+  GODOC_ZIP: godoc.zip
+  GODOC_ZIP_PREFIX: goroot
+  GODOC_INDEX_GLOB: 'index.split.*'

cmd/godoc/appinit.go

@@ -13,21 +13,36 @@ import (
 	"archive/zip"
 	"log"
 	"net/http"
+	"os"
 	"path"
 	"regexp"
+	"runtime"
 
 	"golang.org/x/tools/godoc"
 	"golang.org/x/tools/godoc/dl"
 	"golang.org/x/tools/godoc/proxy"
 	"golang.org/x/tools/godoc/short"
 	"golang.org/x/tools/godoc/static"
 	"golang.org/x/tools/godoc/vfs"
+	"golang.org/x/tools/godoc/vfs/gatefs"
 	"golang.org/x/tools/godoc/vfs/mapfs"
 	"golang.org/x/tools/godoc/vfs/zipfs"
 	"google.golang.org/appengine"
 )
 
 func init() {
+	var (
+		// .zip filename
+		zipFilename = os.Getenv("GODOC_ZIP")
+
+		// goroot directory in .zip file
+		zipGoroot = os.Getenv("GODOC_ZIP_PREFIX")
+
+		// glob pattern describing search index files
+		// (if empty, the index is built at run-time)
+		indexFilenames = os.Getenv("GODOC_INDEX_GLOB")
+	)
+
 	enforceHosts = !appengine.IsDevAppServer()
 	playEnabled = true
 
@@ -36,16 +51,20 @@ func init() {
 	log.Printf(".zip GOROOT = %s", zipGoroot)
 	log.Printf("index files = %s", indexFilenames)
 
-	goroot := path.Join("/", zipGoroot) // fsHttp paths are relative to '/'
-
-	// read .zip file and set up file systems
-	const zipfile = zipFilename
-	rc, err := zip.OpenReader(zipfile)
-	if err != nil {
-		log.Fatalf("%s: %s\n", zipfile, err)
+	if zipFilename != "" {
+		goroot := path.Join("/", zipGoroot) // fsHttp paths are relative to '/'
+		// read .zip file and set up file systems
+		rc, err := zip.OpenReader(zipFilename)
+		if err != nil {
+			log.Fatalf("%s: %s\n", zipFilename, err)
+		}
+		// rc is never closed (app running forever)
+		fs.Bind("/", zipfs.New(rc, zipFilename), goroot, vfs.BindReplace)
+	} else {
+		rootfs := gatefs.New(vfs.OS(runtime.GOROOT()), make(chan bool, 20))
+		fs.Bind("/", rootfs, "/", vfs.BindReplace)
 	}
-	// rc is never closed (app running forever)
-	fs.Bind("/", zipfs.New(rc, zipFilename), goroot, vfs.BindReplace)
+
 	fs.Bind("/lib/godoc", mapfs.New(static.Files), "/", vfs.BindReplace)
 
 	corpus := godoc.NewCorpus(fs)

cmd/godoc/generate-index.bash

@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+
+# Copyright 2011 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+# This script creates a .zip file representing the $GOROOT file system
+# and computes the corresponding search index files.
+#
+# These are used in production (see app.prod.yaml)
+
+set -e -u -x
+
+ZIPFILE=godoc.zip
+INDEXFILE=godoc.index
+SPLITFILES=index.split.
+
+error() {
+	echo "error: $1"
+	exit 2
+}
+
+install() {
+	go install
+}
+
+getArgs() {
+	if [ ! -v GOROOT ]; then
+		GOROOT="$(go env GOROOT)"
+		echo "GOROOT not set explicitly, using go env value instead"
+	fi
+
+	# safety checks
+	if [ ! -d "$GOROOT" ]; then
+		error "$GOROOT is not a directory"
+	fi
+
+	# reporting
+	echo "GOROOT = $GOROOT"
+}
+
+makeZipfile() {
+	echo "*** make $ZIPFILE"
+	rm -f $ZIPFILE goroot
+	ln -s "$GOROOT" goroot
+	zip -q -r $ZIPFILE goroot/* # glob to ignore dotfiles (like .git)
+	rm goroot
+}
+
+makeIndexfile() {
+	echo "*** make $INDEXFILE"
+	godoc=$(go env GOPATH)/bin/godoc
+	# NOTE: run godoc without GOPATH set. Otherwise third-party packages will end up in the index.
+	GOPATH= $godoc -write_index -goroot goroot -index_files=$INDEXFILE -zip=$ZIPFILE
+}
+
+splitIndexfile() {
+	echo "*** split $INDEXFILE"
+	rm -f $SPLITFILES*
+	split -b8m $INDEXFILE $SPLITFILES
+}
+
+cd $(dirname $0)
+
+install
+getArgs "$@"
+makeZipfile
+makeIndexfile
+splitIndexfile
+rm $INDEXFILE
+
+echo "*** setup complete"