Improve content map, memory cache and dependency resolution

TODO(bep) improve commit message. Hugo has always been a active user of in-memory caches, but before this commit we did nothing to control the memory usage. One failing example would be loading lots of big JSON data files and unmarshal them via `transform.Unmarshal`. This commit consolidates all these caches into one single LRU cache with an eviction strategy that also considers used vs. available memory. Hugo will try to limit its memory usage to 1/4 or total system memory, but this can be controlled with the `HUGO_MEMORYLIMIT` environment variable (a float value representing Gigabytes). A natural next step after this would be to use this cache for `.Content`. Fixes gohugoio#10386 Fixes gohugoio#8307 Fixes gohugoio#8498 Fixes gohugoio#8927 Fixes gohugoio#9192 Fixes gohugoio#9189 Fixes gohugoio#7425 Fixes gohugoio#7437 Fixes gohugoio#7436 Fixes gohugoio#7882 Updates gohugoio#7544 Fixes gohugoio#9224 Fixes gohugoio#9324 Fixes gohugoio#9352 Fixes gohugoio#9343 Fixes gohugoio#9171 Fixes gohugoio#10104 Fixes gohugoio#10380
bep · Nov 25, 2022 · 9a9ea8c · 9a9ea8c
1 parent b8d5c37
commit 9a9ea8c
Show file tree

Hide file tree

Showing 258 changed files with 13,717 additions and 11,182 deletions.
diff --git a/.hugo_build.lock b/.hugo_build.lock
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -0,0 +1,3 @@
+{
+    "autoHide.autoHidePanel": false
+}
diff --git a/bench.sh b/bench.sh
diff --git a/benchSite.sh b/benchSite.sh
diff --git a/benchbep.sh b/benchbep.sh
@@ -1 +1 @@
-gobench -package=./hugolib -bench="BenchmarkSiteNew/Deep_content_tree"
+gobench --package ./hugolib --bench "BenchmarkSiteNew/Regular_Deep" -base v0.89.4
diff --git a/bepdock.sh b/bepdock.sh
diff --git a/cache/filecache/filecache.go b/cache/filecache/filecache.go
@@ -15,6 +15,7 @@ package filecache
 
 import (
 	"bytes"
+	"context"
 	"errors"
 	"io"
 	"io/ioutil"
@@ -163,7 +164,7 @@ func (c *Cache) ReadOrCreate(id string,
 // GetOrCreate tries to get the file with the given id from cache. If not found or expired, create will
 // be invoked and the result cached.
 // This method is protected by a named lock using the given id as identifier.
-func (c *Cache) GetOrCreate(id string, create func() (io.ReadCloser, error)) (ItemInfo, io.ReadCloser, error) {
+func (c *Cache) GetOrCreate(ctx context.Context, id string, create func() (io.ReadCloser, error)) (ItemInfo, io.ReadCloser, error) {
 	id = cleanID(id)
 
 	c.nlocker.Lock(id)

diff --git a/cache/filecache/filecache_test.go b/cache/filecache/filecache_test.go
@@ -14,6 +14,7 @@
 package filecache
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"io"
@@ -134,7 +135,7 @@ dir = ":cacheDir/c"
 
 		for _, ca := range []*Cache{caches.ImageCache(), caches.AssetsCache(), caches.GetJSONCache(), caches.GetCSVCache()} {
 			for i := 0; i < 2; i++ {
-				info, r, err := ca.GetOrCreate("a", rf("abc"))
+				info, r, err := ca.GetOrCreate(context.TODO(), "a", rf("abc"))
 				c.Assert(err, qt.IsNil)
 				c.Assert(r, qt.Not(qt.IsNil))
 				c.Assert(info.Name, qt.Equals, "a")
@@ -152,7 +153,7 @@ dir = ":cacheDir/c"
 				c.Assert(err, qt.IsNil)
 				c.Assert(string(b), qt.Equals, "abc")
 
-				_, r, err = ca.GetOrCreate("a", rf("bcd"))
+				_, r, err = ca.GetOrCreate(context.TODO(), "a", rf("bcd"))
 				c.Assert(err, qt.IsNil)
 				b, _ = ioutil.ReadAll(r)
 				r.Close()
@@ -229,7 +230,7 @@ dir = "/cache/c"
 				ca := caches.Get(cacheName)
 				c.Assert(ca, qt.Not(qt.IsNil))
 				filename, data := filenameData(i)
-				_, r, err := ca.GetOrCreate(filename, func() (io.ReadCloser, error) {
+				_, r, err := ca.GetOrCreate(context.TODO(), filename, func() (io.ReadCloser, error) {
 					return hugio.ToReadCloser(strings.NewReader(data)), nil
 				})
 				c.Assert(err, qt.IsNil)