Speed up GetTerms

```text name old time/op new time/op delta TaxonomiesGetTerms/pages_100-10 5.25ms 5% 5.13ms 4% ~ (p=0.486 n=4+4) TaxonomiesGetTerms/pages_1000-10 30.1ms 1% 26.8ms 1% -11.13% (p=0.029 n=4+4) TaxonomiesGetTerms/pages_10000-10 1.33s 24% 0.29s 2% -78.42% (p=0.029 n=4+4) TaxonomiesGetTerms/pages_20000-10 5.50s 12% 0.83s 28% -84.88% (p=0.029 n=4+4) name old alloc/op new alloc/op delta TaxonomiesGetTerms/pages_100-10 4.08MB 0% 4.06MB 0% -0.59% (p=0.029 n=4+4) TaxonomiesGetTerms/pages_1000-10 25.1MB 0% 24.9MB 0% -0.87% (p=0.029 n=4+4) TaxonomiesGetTerms/pages_10000-10 238MB 2% 233MB 0% -1.94% (p=0.029 n=4+4) TaxonomiesGetTerms/pages_20000-10 469MB 0% 465MB 0% -1.00% (p=0.029 n=4+4) name old allocs/op new allocs/op delta TaxonomiesGetTerms/pages_100-10 49.5k 0% 48.9k 0% -1.17% (p=0.029 n=4+4) TaxonomiesGetTerms/pages_1000-10 304k 0% 298k 0% -1.97% (p=0.029 n=4+4) TaxonomiesGetTerms/pages_10000-10 3.02M 7% 2.81M 0% -7.09% (p=0.029 n=4+4) TaxonomiesGetTerms/pages_20000-10 5.77M 1% 5.59M 0% -3.19% (p=0.029 n=4+4) ``` Note that the numbers above represents a full site build, but GetTerms is a big part of the site in question. Fixes #12610
gohugoio · Jun 19, 2024 · 478a910 · 478a910
1 parent b46d101
commit 478a910
Show file tree

Hide file tree

Showing 2 changed files with 66 additions and 9 deletions.
diff --git a/hugolib/content_map_page.go b/hugolib/content_map_page.go
@@ -98,6 +98,7 @@ type pageMap struct {
 	cachePages1            *dynacache.Partition[string, page.Pages]
 	cachePages2            *dynacache.Partition[string, page.Pages]
 	cacheResources         *dynacache.Partition[string, resource.Resources]
+	cacheGetTerms          *dynacache.Partition[string, map[string]page.Pages]
 	cacheContentRendered   *dynacache.Partition[string, *resources.StaleValue[contentSummary]]
 	cacheContentPlain      *dynacache.Partition[string, *resources.StaleValue[contentPlainPlainWords]]
 	contentTableOfContents *dynacache.Partition[string, *resources.StaleValue[contentTableOfContents]]
@@ -448,32 +449,32 @@ func (m *pageMap) getPagesWithTerm(q pageMapQueryPagesBelowPath) page.Pages {
 func (m *pageMap) getTermsForPageInTaxonomy(path, taxonomy string) page.Pages {
 	prefix := paths.AddLeadingSlash(taxonomy)
 
-	v, err := m.cachePages1.GetOrCreate(prefix+path, func(string) (page.Pages, error) {
-		var pas page.Pages
-
+	termPages, err := m.cacheGetTerms.GetOrCreate(prefix, func(string) (map[string]page.Pages, error) {
+		mm := make(map[string]page.Pages)
 		err := m.treeTaxonomyEntries.WalkPrefix(
 			doctree.LockTypeNone,
 			paths.AddTrailingSlash(prefix),
 			func(s string, n *weightedContentNode) (bool, error) {
-				if strings.HasSuffix(s, path) {
-					pas = append(pas, n.term)
-				}
+				mm[n.n.Path()] = append(mm[n.n.Path()], n.term)
 				return false, nil
 			},
 		)
 		if err != nil {
 			return nil, err
 		}
 
-		page.SortByDefault(pas)
+		// Sort the terms.
+		for _, v := range mm {
+			page.SortByDefault(v)
+		}
 
-		return pas, nil
+		return mm, nil
 	})
 	if err != nil {
 		panic(err)
 	}
 
-	return v
+	return termPages[path]
 }
 
 func (m *pageMap) forEachResourceInPage(
@@ -898,6 +899,7 @@ func newPageMap(i int, s *Site, mcache *dynacache.Cache, pageTrees *pageTrees) *
 		pageTrees:              pageTrees.Shape(0, i),
 		cachePages1:            dynacache.GetOrCreatePartition[string, page.Pages](mcache, fmt.Sprintf("/pag1/%d", i), dynacache.OptionsPartition{Weight: 10, ClearWhen: dynacache.ClearOnRebuild}),
 		cachePages2:            dynacache.GetOrCreatePartition[string, page.Pages](mcache, fmt.Sprintf("/pag2/%d", i), dynacache.OptionsPartition{Weight: 10, ClearWhen: dynacache.ClearOnRebuild}),
+		cacheGetTerms:          dynacache.GetOrCreatePartition[string, map[string]page.Pages](mcache, fmt.Sprintf("/gett/%d", i), dynacache.OptionsPartition{Weight: 5, ClearWhen: dynacache.ClearOnRebuild}),
 		cacheResources:         dynacache.GetOrCreatePartition[string, resource.Resources](mcache, fmt.Sprintf("/ress/%d", i), dynacache.OptionsPartition{Weight: 60, ClearWhen: dynacache.ClearOnRebuild}),
 		cacheContentRendered:   dynacache.GetOrCreatePartition[string, *resources.StaleValue[contentSummary]](mcache, fmt.Sprintf("/cont/ren/%d", i), dynacache.OptionsPartition{Weight: 70, ClearWhen: dynacache.ClearOnChange}),
 		cacheContentPlain:      dynacache.GetOrCreatePartition[string, *resources.StaleValue[contentPlainPlainWords]](mcache, fmt.Sprintf("/cont/pla/%d", i), dynacache.OptionsPartition{Weight: 70, ClearWhen: dynacache.ClearOnChange}),

diff --git a/hugolib/taxonomy_test.go b/hugolib/taxonomy_test.go
@@ -970,3 +970,58 @@ title: p1
 	b.AssertFileExists("public/ja/s1/index.html", false) // failing test
 	b.AssertFileExists("public/ja/s1/category/index.html", true)
 }
+
+func BenchmarkTaxonomiesGetTerms(b *testing.B) {
+	createBuilders := func(b *testing.B, numPages int) []*IntegrationTestBuilder {
+		files := `
+-- hugo.toml --
+baseURL = "https://example.com"
+disableKinds = ["RSS", "sitemap", "section"]
+[taxononomies]
+tag = "tags"
+-- layouts/_default/list.html --
+List.
+-- layouts/_default/single.html --
+GetTerms.tags: {{ range .GetTerms "tags" }}{{ .Title }}|{{ end }}
+-- content/_index.md --
+`
+
+		tagsVariants := []string{
+			"tags: ['a']",
+			"tags: ['a', 'b']",
+			"tags: ['a', 'b', 'c']",
+			"tags: ['a', 'b', 'c', 'd']",
+			"tags: ['a', 'b',  'd', 'e']",
+			"tags: ['a', 'b', 'c', 'd', 'e']",
+			"tags: ['a', 'd']",
+			"tags: ['a',  'f']",
+		}
+
+		for i := 1; i < numPages; i++ {
+			tags := tagsVariants[i%len(tagsVariants)]
+			files += fmt.Sprintf("\n-- content/posts/p%d.md --\n---\n%s\n---", i+1, tags)
+		}
+		cfg := IntegrationTestConfig{
+			T:           b,
+			TxtarString: files,
+		}
+		builders := make([]*IntegrationTestBuilder, b.N)
+
+		for i := range builders {
+			builders[i] = NewIntegrationTestBuilder(cfg)
+		}
+
+		b.ResetTimer()
+
+		return builders
+	}
+
+	for _, numPages := range []int{100, 1000, 10000, 20000} {
+		b.Run(fmt.Sprintf("pages_%d", numPages), func(b *testing.B) {
+			builders := createBuilders(b, numPages)
+			for i := 0; i < b.N; i++ {
+				builders[i].Build()
+			}
+		})
+	}
+}