From 478a9107a68aeb5dac9ea0cec0a347fadb708b64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Wed, 19 Jun 2024 12:52:10 +0200 Subject: [PATCH] Speed up GetTerms ```text name old time/op new time/op delta TaxonomiesGetTerms/pages_100-10 5.25ms 5% 5.13ms 4% ~ (p=0.486 n=4+4) TaxonomiesGetTerms/pages_1000-10 30.1ms 1% 26.8ms 1% -11.13% (p=0.029 n=4+4) TaxonomiesGetTerms/pages_10000-10 1.33s 24% 0.29s 2% -78.42% (p=0.029 n=4+4) TaxonomiesGetTerms/pages_20000-10 5.50s 12% 0.83s 28% -84.88% (p=0.029 n=4+4) name old alloc/op new alloc/op delta TaxonomiesGetTerms/pages_100-10 4.08MB 0% 4.06MB 0% -0.59% (p=0.029 n=4+4) TaxonomiesGetTerms/pages_1000-10 25.1MB 0% 24.9MB 0% -0.87% (p=0.029 n=4+4) TaxonomiesGetTerms/pages_10000-10 238MB 2% 233MB 0% -1.94% (p=0.029 n=4+4) TaxonomiesGetTerms/pages_20000-10 469MB 0% 465MB 0% -1.00% (p=0.029 n=4+4) name old allocs/op new allocs/op delta TaxonomiesGetTerms/pages_100-10 49.5k 0% 48.9k 0% -1.17% (p=0.029 n=4+4) TaxonomiesGetTerms/pages_1000-10 304k 0% 298k 0% -1.97% (p=0.029 n=4+4) TaxonomiesGetTerms/pages_10000-10 3.02M 7% 2.81M 0% -7.09% (p=0.029 n=4+4) TaxonomiesGetTerms/pages_20000-10 5.77M 1% 5.59M 0% -3.19% (p=0.029 n=4+4) ``` Note that the numbers above represents a full site build, but GetTerms is a big part of the site in question. Fixes #12610 --- hugolib/content_map_page.go | 20 ++++++++------ hugolib/taxonomy_test.go | 55 +++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 9 deletions(-) diff --git a/hugolib/content_map_page.go b/hugolib/content_map_page.go index 0ce43ea6803..f9709df15ea 100644 --- a/hugolib/content_map_page.go +++ b/hugolib/content_map_page.go @@ -98,6 +98,7 @@ type pageMap struct { cachePages1 *dynacache.Partition[string, page.Pages] cachePages2 *dynacache.Partition[string, page.Pages] cacheResources *dynacache.Partition[string, resource.Resources] + cacheGetTerms *dynacache.Partition[string, map[string]page.Pages] cacheContentRendered *dynacache.Partition[string, *resources.StaleValue[contentSummary]] cacheContentPlain *dynacache.Partition[string, *resources.StaleValue[contentPlainPlainWords]] contentTableOfContents *dynacache.Partition[string, *resources.StaleValue[contentTableOfContents]] @@ -448,16 +449,13 @@ func (m *pageMap) getPagesWithTerm(q pageMapQueryPagesBelowPath) page.Pages { func (m *pageMap) getTermsForPageInTaxonomy(path, taxonomy string) page.Pages { prefix := paths.AddLeadingSlash(taxonomy) - v, err := m.cachePages1.GetOrCreate(prefix+path, func(string) (page.Pages, error) { - var pas page.Pages - + termPages, err := m.cacheGetTerms.GetOrCreate(prefix, func(string) (map[string]page.Pages, error) { + mm := make(map[string]page.Pages) err := m.treeTaxonomyEntries.WalkPrefix( doctree.LockTypeNone, paths.AddTrailingSlash(prefix), func(s string, n *weightedContentNode) (bool, error) { - if strings.HasSuffix(s, path) { - pas = append(pas, n.term) - } + mm[n.n.Path()] = append(mm[n.n.Path()], n.term) return false, nil }, ) @@ -465,15 +463,18 @@ func (m *pageMap) getTermsForPageInTaxonomy(path, taxonomy string) page.Pages { return nil, err } - page.SortByDefault(pas) + // Sort the terms. + for _, v := range mm { + page.SortByDefault(v) + } - return pas, nil + return mm, nil }) if err != nil { panic(err) } - return v + return termPages[path] } func (m *pageMap) forEachResourceInPage( @@ -898,6 +899,7 @@ func newPageMap(i int, s *Site, mcache *dynacache.Cache, pageTrees *pageTrees) * pageTrees: pageTrees.Shape(0, i), cachePages1: dynacache.GetOrCreatePartition[string, page.Pages](mcache, fmt.Sprintf("/pag1/%d", i), dynacache.OptionsPartition{Weight: 10, ClearWhen: dynacache.ClearOnRebuild}), cachePages2: dynacache.GetOrCreatePartition[string, page.Pages](mcache, fmt.Sprintf("/pag2/%d", i), dynacache.OptionsPartition{Weight: 10, ClearWhen: dynacache.ClearOnRebuild}), + cacheGetTerms: dynacache.GetOrCreatePartition[string, map[string]page.Pages](mcache, fmt.Sprintf("/gett/%d", i), dynacache.OptionsPartition{Weight: 5, ClearWhen: dynacache.ClearOnRebuild}), cacheResources: dynacache.GetOrCreatePartition[string, resource.Resources](mcache, fmt.Sprintf("/ress/%d", i), dynacache.OptionsPartition{Weight: 60, ClearWhen: dynacache.ClearOnRebuild}), cacheContentRendered: dynacache.GetOrCreatePartition[string, *resources.StaleValue[contentSummary]](mcache, fmt.Sprintf("/cont/ren/%d", i), dynacache.OptionsPartition{Weight: 70, ClearWhen: dynacache.ClearOnChange}), cacheContentPlain: dynacache.GetOrCreatePartition[string, *resources.StaleValue[contentPlainPlainWords]](mcache, fmt.Sprintf("/cont/pla/%d", i), dynacache.OptionsPartition{Weight: 70, ClearWhen: dynacache.ClearOnChange}), diff --git a/hugolib/taxonomy_test.go b/hugolib/taxonomy_test.go index bfdfd8dfdc2..e96b82d3977 100644 --- a/hugolib/taxonomy_test.go +++ b/hugolib/taxonomy_test.go @@ -970,3 +970,58 @@ title: p1 b.AssertFileExists("public/ja/s1/index.html", false) // failing test b.AssertFileExists("public/ja/s1/category/index.html", true) } + +func BenchmarkTaxonomiesGetTerms(b *testing.B) { + createBuilders := func(b *testing.B, numPages int) []*IntegrationTestBuilder { + files := ` +-- hugo.toml -- +baseURL = "https://example.com" +disableKinds = ["RSS", "sitemap", "section"] +[taxononomies] +tag = "tags" +-- layouts/_default/list.html -- +List. +-- layouts/_default/single.html -- +GetTerms.tags: {{ range .GetTerms "tags" }}{{ .Title }}|{{ end }} +-- content/_index.md -- +` + + tagsVariants := []string{ + "tags: ['a']", + "tags: ['a', 'b']", + "tags: ['a', 'b', 'c']", + "tags: ['a', 'b', 'c', 'd']", + "tags: ['a', 'b', 'd', 'e']", + "tags: ['a', 'b', 'c', 'd', 'e']", + "tags: ['a', 'd']", + "tags: ['a', 'f']", + } + + for i := 1; i < numPages; i++ { + tags := tagsVariants[i%len(tagsVariants)] + files += fmt.Sprintf("\n-- content/posts/p%d.md --\n---\n%s\n---", i+1, tags) + } + cfg := IntegrationTestConfig{ + T: b, + TxtarString: files, + } + builders := make([]*IntegrationTestBuilder, b.N) + + for i := range builders { + builders[i] = NewIntegrationTestBuilder(cfg) + } + + b.ResetTimer() + + return builders + } + + for _, numPages := range []int{100, 1000, 10000, 20000} { + b.Run(fmt.Sprintf("pages_%d", numPages), func(b *testing.B) { + builders := createBuilders(b, numPages) + for i := 0; i < b.N; i++ { + builders[i].Build() + } + }) + } +}