Share HTML template renderers and create a watcher framework

The recovery, API, Web and package frameworks all create their own HTML Renderers. This increases the memory requirements of Gitea unnecessarily with duplicate templates being kept in memory. Further the reloading framework in dev mode for these involves locking and recompiling all of the templates on each load. This will potentially hide concurrency issues and it is inefficient. This PR stores the templates renderer in the context and stores this context in the NormalRoutes, it then creates a fsnotify.Watcher framework to watch files. The watching framework is then extended to the mailer templates which were previously not being reloaded in dev. Then the locales are simplified to a similar structure. Fix go-gitea#20210, go-gitea#20211, go-gitea#20217 Replace go-gitea#20159 Signed-off-by: Andrew Thornton <art27@cantab.net>
zeripath · Jul 3, 2022 · 1409281 · 1409281
1 parent 9d9bf66
commit 1409281
Show file tree

Hide file tree

Showing 36 changed files with 837 additions and 496 deletions.
diff --git a/cmd/embedded.go b/cmd/embedded.go
@@ -123,7 +123,7 @@ func initEmbeddedExtractor(c *cli.Context) error {
 
 	sections["public"] = &section{Path: "public", Names: public.AssetNames, IsDir: public.AssetIsDir, Asset: public.Asset}
 	sections["options"] = &section{Path: "options", Names: options.AssetNames, IsDir: options.AssetIsDir, Asset: options.Asset}
-	sections["templates"] = &section{Path: "templates", Names: templates.AssetNames, IsDir: templates.AssetIsDir, Asset: templates.Asset}
+	sections["templates"] = &section{Path: "templates", Names: templates.BuiltinAssetNames, IsDir: templates.BuiltinAssetIsDir, Asset: templates.BuiltinAsset}
 
 	for _, sec := range sections {
 		assets = append(assets, buildAssetList(sec, pats, c)...)

diff --git a/cmd/web.go b/cmd/web.go
@@ -126,8 +126,10 @@ func runWeb(ctx *cli.Context) error {
 				return err
 			}
 		}
-		c := install.Routes()
+		installCtx, cancel := context.WithCancel(graceful.GetManager().HammerContext())
+		c := install.Routes(installCtx)
 		err := listen(c, false)
+		cancel()
 		if err != nil {
 			log.Critical("Unable to open listener for installer. Is Gitea already running?")
 			graceful.GetManager().DoGracefulShutdown()
@@ -174,7 +176,7 @@ func runWeb(ctx *cli.Context) error {
 	}
 
 	// Set up Chi routes
-	c := routers.NormalRoutes()
+	c := routers.NormalRoutes(graceful.GetManager().HammerContext())
 	err := listen(c, true)
 	<-graceful.GetManager().Done()
 	log.Info("PID: %d Gitea Web Finished", os.Getpid())

diff --git a/contrib/pr/checkout.go b/contrib/pr/checkout.go
@@ -27,6 +27,7 @@ import (
 	"code.gitea.io/gitea/models/db"
 	"code.gitea.io/gitea/models/unittest"
 	gitea_git "code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/graceful"
 	"code.gitea.io/gitea/modules/markup"
 	"code.gitea.io/gitea/modules/markup/external"
 	repo_module "code.gitea.io/gitea/modules/repository"
@@ -118,7 +119,7 @@ func runPR() {
 	// routers.GlobalInit()
 	external.RegisterRenderers()
 	markup.Init()
-	c := routers.NormalRoutes()
+	c := routers.NormalRoutes(graceful.GetManager().HammerContext())
 
 	log.Printf("[PR] Ready for testing !\n")
 	log.Printf("[PR] Login with user1, user2, user3, ... with pass: password\n")

diff --git a/go.mod b/go.mod
@@ -27,6 +27,7 @@ require (
 	github.com/emirpasic/gods v1.18.1
 	github.com/ethantkoenig/rupture v1.0.1
 	github.com/felixge/fgprof v0.9.2
+	github.com/fsnotify/fsnotify v1.5.4
 	github.com/gliderlabs/ssh v0.3.4
 	github.com/go-ap/activitypub v0.0.0-20220615144428-48208c70483b
 	github.com/go-ap/jsonld v0.0.0-20220615144122-1d862b15410d
@@ -160,7 +161,6 @@ require (
 	github.com/envoyproxy/protoc-gen-validate v0.6.2 // indirect
 	github.com/felixge/httpsnoop v1.0.2 // indirect
 	github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect
-	github.com/fsnotify/fsnotify v1.5.4 // indirect
 	github.com/fullstorydev/grpcurl v1.8.1 // indirect
 	github.com/fxamacker/cbor/v2 v2.4.0 // indirect
 	github.com/go-ap/errors v0.0.0-20220615144307-e8bc4a40ae9f // indirect

diff --git a/integrations/api_activitypub_person_test.go b/integrations/api_activitypub_person_test.go
@@ -23,10 +23,10 @@ import (
 
 func TestActivityPubPerson(t *testing.T) {
 	setting.Federation.Enabled = true
-	c = routers.NormalRoutes()
+	c = routers.NormalRoutes(context.TODO())
 	defer func() {
 		setting.Federation.Enabled = false
-		c = routers.NormalRoutes()
+		c = routers.NormalRoutes(context.TODO())
 	}()
 
 	onGiteaRun(t, func(*testing.T, *url.URL) {
@@ -60,10 +60,10 @@ func TestActivityPubPerson(t *testing.T) {
 
 func TestActivityPubMissingPerson(t *testing.T) {
 	setting.Federation.Enabled = true
-	c = routers.NormalRoutes()
+	c = routers.NormalRoutes(context.Background())
 	defer func() {
 		setting.Federation.Enabled = false
-		c = routers.NormalRoutes()
+		c = routers.NormalRoutes(context.Background())
 	}()
 
 	onGiteaRun(t, func(*testing.T, *url.URL) {
@@ -75,10 +75,10 @@ func TestActivityPubMissingPerson(t *testing.T) {
 
 func TestActivityPubPersonInbox(t *testing.T) {
 	setting.Federation.Enabled = true
-	c = routers.NormalRoutes()
+	c = routers.NormalRoutes(context.Background())
 	defer func() {
 		setting.Federation.Enabled = false
-		c = routers.NormalRoutes()
+		c = routers.NormalRoutes(context.Background())
 	}()
 
 	srv := httptest.NewServer(c)

diff --git a/integrations/api_nodeinfo_test.go b/integrations/api_nodeinfo_test.go
@@ -5,6 +5,7 @@
 package integrations
 
 import (
+	"context"
 	"net/http"
 	"net/url"
 	"testing"
@@ -18,10 +19,10 @@ import (
 
 func TestNodeinfo(t *testing.T) {
 	setting.Federation.Enabled = true
-	c = routers.NormalRoutes()
+	c = routers.NormalRoutes(context.TODO())
 	defer func() {
 		setting.Federation.Enabled = false
-		c = routers.NormalRoutes()
+		c = routers.NormalRoutes(context.TODO())
 	}()
 
 	onGiteaRun(t, func(*testing.T, *url.URL) {

diff --git a/integrations/create_no_session_test.go b/integrations/create_no_session_test.go
@@ -5,6 +5,7 @@
 package integrations
 
 import (
+	"context"
 	"net/http"
 	"net/http/httptest"
 	"os"
@@ -57,7 +58,7 @@ func TestSessionFileCreation(t *testing.T) {
 	oldSessionConfig := setting.SessionConfig.ProviderConfig
 	defer func() {
 		setting.SessionConfig.ProviderConfig = oldSessionConfig
-		c = routers.NormalRoutes()
+		c = routers.NormalRoutes(context.TODO())
 	}()
 
 	var config session.Options
@@ -82,7 +83,7 @@ func TestSessionFileCreation(t *testing.T) {
 
 	setting.SessionConfig.ProviderConfig = string(newConfigBytes)
 
-	c = routers.NormalRoutes()
+	c = routers.NormalRoutes(context.TODO())
 
 	t.Run("NoSessionOnViewIssue", func(t *testing.T) {
 		defer PrintCurrentTest(t)()

diff --git a/integrations/integration_test.go b/integrations/integration_test.go
@@ -89,7 +89,7 @@ func TestMain(m *testing.M) {
 	defer cancel()
 
 	initIntegrationTest()
-	c = routers.NormalRoutes()
+	c = routers.NormalRoutes(context.TODO())
 
 	// integration test settings...
 	if setting.Cfg != nil {

diff --git a/modules/context/context.go b/modules/context/context.go
@@ -673,8 +673,8 @@ func Auth(authMethod auth.Method) func(*Context) {
 }
 
 // Contexter initializes a classic context for a request.
-func Contexter() func(next http.Handler) http.Handler {
-	rnd := templates.HTMLRenderer()
+func Contexter(ctx context.Context) func(next http.Handler) http.Handler {
+	_, rnd := templates.HTMLRenderer(ctx)
 	csrfOpts := getCsrfOpts()
 	if !setting.IsProd {
 		CsrfTokenRegenerationInterval = 5 * time.Second // in dev, re-generate the tokens more aggressively for debug purpose

diff --git a/modules/context/package.go b/modules/context/package.go
@@ -5,6 +5,7 @@
 package context
 
 import (
+	gocontext "context"
 	"fmt"
 	"net/http"
 
@@ -13,6 +14,7 @@ import (
 	"code.gitea.io/gitea/models/perm"
 	user_model "code.gitea.io/gitea/models/user"
 	"code.gitea.io/gitea/modules/structs"
+	"code.gitea.io/gitea/modules/templates"
 )
 
 // Package contains owner, access mode and optional the package descriptor
@@ -101,12 +103,14 @@ func packageAssignment(ctx *Context, errCb func(int, string, interface{})) {
 }
 
 // PackageContexter initializes a package context for a request.
-func PackageContexter() func(next http.Handler) http.Handler {
+func PackageContexter(ctx gocontext.Context) func(next http.Handler) http.Handler {
+	_, rnd := templates.HTMLRenderer(ctx)
 	return func(next http.Handler) http.Handler {
 		return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
 			ctx := Context{
-				Resp: NewResponse(resp),
-				Data: map[string]interface{}{},
+				Resp:   NewResponse(resp),
+				Data:   map[string]interface{}{},
+				Render: rnd,
 			}
 			defer ctx.Close()
 

diff --git a/modules/options/base.go b/modules/options/base.go
@@ -0,0 +1,34 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package options
+
+import (
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+)
+
+func walkAssetDir(root string, callback func(path string, name string, d fs.DirEntry, err error) error) error {
+	if err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
+		name := path[len(root):]
+		if len(name) > 0 && name[0] == '/' {
+			name = name[1:]
+		}
+		if err != nil {
+			if os.IsNotExist(err) {
+				return callback(path, name, d, err)
+			}
+			return err
+		}
+		if d.Name() == ".DS_Store" && d.IsDir() { // Because Macs...
+			return fs.SkipDir
+		}
+		return callback(path, name, d, err)
+	}); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("unable to get files for assets in %s: %w", root, err)
+	}
+	return nil
+}
diff --git a/modules/options/dynamic.go b/modules/options/dynamic.go
@@ -8,8 +8,10 @@ package options
 
 import (
 	"fmt"
+	"io/fs"
 	"os"
 	"path"
+	"path/filepath"
 
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
@@ -45,7 +47,7 @@ func Dir(name string) ([]string, error) {
 
 	isDir, err = util.IsDir(staticDir)
 	if err != nil {
-		return []string{}, fmt.Errorf("Unabe to check if static directory %s is a directory. %v", staticDir, err)
+		return []string{}, fmt.Errorf("unable to check if static directory %s is a directory. %v", staticDir, err)
 	}
 	if isDir {
 		files, err := util.StatDir(staticDir, true)
@@ -64,6 +66,18 @@ func Locale(name string) ([]byte, error) {
 	return fileFromDir(path.Join("locale", name))
 }
 
+// WalkLocales reads the content of a specific locale from static or custom path.
+func WalkLocales(callback func(path string, name string, d fs.DirEntry, err error) error) error {
+	if err := walkAssetDir(filepath.Join(setting.StaticRootPath, "options", "locale"), callback); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("failed to walk locales. Error: %w", err)
+	}
+
+	if err := walkAssetDir(filepath.Join(setting.CustomPath, "options", "locale"), callback); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("failed to walk locales. Error: %w", err)
+	}
+	return nil
+}
+
 // Readme reads the content of a specific readme from static or custom path.
 func Readme(name string) ([]byte, error) {
 	return fileFromDir(path.Join("readme", name))

diff --git a/modules/options/static.go b/modules/options/static.go
@@ -9,8 +9,10 @@ package options
 import (
 	"fmt"
 	"io"
+	"io/fs"
 	"os"
 	"path"
+	"path/filepath"
 
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
@@ -74,6 +76,14 @@ func Locale(name string) ([]byte, error) {
 	return fileFromDir(path.Join("locale", name))
 }
 
+// WalkLocales reads the content of a specific locale from static or custom path.
+func WalkLocales(callback func(path string, name string, d fs.DirEntry, err error) error) error {
+	if err := walkAssetDir(filepath.Join(setting.CustomPath, "options", "locale"), callback); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("failed to walk locales. Error: %w", err)
+	}
+	return nil
+}
+
 // Readme reads the content of a specific readme from bindata or custom path.
 func Readme(name string) ([]byte, error) {
 	return fileFromDir(path.Join("readme", name))