diff --git a/cache/httpcache/httpcache.go b/cache/httpcache/httpcache.go index ff360001f6a..98f7fedd4c5 100644 --- a/cache/httpcache/httpcache.go +++ b/cache/httpcache/httpcache.go @@ -83,7 +83,6 @@ func (c *Config) Compile() (ConfigCompiled, error) { } // PollConfig holds the configuration for polling remote resources to detect changes in watch mode. -// TODO1 make sure this enabled only in watch mode. type PollConfig struct { // What remote resources to apply this configuration to. For GlobMatcher diff --git a/common/herrors/errors.go b/common/herrors/errors.go index 7c389c1aeaf..98d2f41f830 100644 --- a/common/herrors/errors.go +++ b/common/herrors/errors.go @@ -131,6 +131,9 @@ func ImproveIfNilPointer(inErr error) (outErr error) { call := m[1] field := m[2] parts := strings.Split(call, ".") + if len(parts) < 2 { + return + } receiverName := parts[len(parts)-2] receiver := strings.Join(parts[:len(parts)-1], ".") s := fmt.Sprintf("– %s is nil; wrap it in if or with: {{ with %s }}{{ .%s }}{{ end }}", receiverName, receiver, field) diff --git a/common/hugio/hasBytesWriter.go b/common/hugio/hasBytesWriter.go index 5148c82f96a..d2bcd1bb408 100644 --- a/common/hugio/hasBytesWriter.go +++ b/common/hugio/hasBytesWriter.go @@ -17,24 +17,35 @@ import ( "bytes" ) -// HasBytesWriter is a writer that will set Match to true if the given pattern -// is found in the stream. +// HasBytesWriter is a writer will match against a slice of patterns. type HasBytesWriter struct { - Match bool - Pattern []byte + Patterns []*HasBytesPattern i int done bool buff []byte } +type HasBytesPattern struct { + Match bool + Pattern []byte +} + +func (h *HasBytesWriter) patternLen() int { + l := 0 + for _, p := range h.Patterns { + l += len(p.Pattern) + } + return l +} + func (h *HasBytesWriter) Write(p []byte) (n int, err error) { if h.done { return len(p), nil } if len(h.buff) == 0 { - h.buff = make([]byte, len(h.Pattern)*2) + h.buff = make([]byte, h.patternLen()*2) } for i := range p { @@ -46,11 +57,23 @@ func (h *HasBytesWriter) Write(p []byte) (n int, err error) { h.i = len(h.buff) / 2 } - if bytes.Contains(h.buff, h.Pattern) { - h.Match = true - h.done = true - return len(p), nil + for _, pp := range h.Patterns { + if bytes.Contains(h.buff, pp.Pattern) { + pp.Match = true + done := true + for _, ppp := range h.Patterns { + if !ppp.Match { + done = false + break + } + } + if done { + h.done = true + } + return len(p), nil + } } + } return len(p), nil diff --git a/common/hugio/hasBytesWriter_test.go b/common/hugio/hasBytesWriter_test.go index af53fa5dd49..49487ab0bb7 100644 --- a/common/hugio/hasBytesWriter_test.go +++ b/common/hugio/hasBytesWriter_test.go @@ -34,8 +34,11 @@ func TestHasBytesWriter(t *testing.T) { var b bytes.Buffer h := &HasBytesWriter{ - Pattern: []byte("__foo"), + Patterns: []*HasBytesPattern{ + {Pattern: []byte("__foo")}, + }, } + return h, io.MultiWriter(&b, h) } @@ -46,19 +49,19 @@ func TestHasBytesWriter(t *testing.T) { for i := 0; i < 22; i++ { h, w := neww() fmt.Fprintf(w, rndStr()+"abc __foobar"+rndStr()) - c.Assert(h.Match, qt.Equals, true) + c.Assert(h.Patterns[0].Match, qt.Equals, true) h, w = neww() fmt.Fprintf(w, rndStr()+"abc __f") fmt.Fprintf(w, "oo bar"+rndStr()) - c.Assert(h.Match, qt.Equals, true) + c.Assert(h.Patterns[0].Match, qt.Equals, true) h, w = neww() fmt.Fprintf(w, rndStr()+"abc __moo bar") - c.Assert(h.Match, qt.Equals, false) + c.Assert(h.Patterns[0].Match, qt.Equals, false) } h, w := neww() fmt.Fprintf(w, "__foo") - c.Assert(h.Match, qt.Equals, true) + c.Assert(h.Patterns[0].Match, qt.Equals, true) } diff --git a/common/maps/cache.go b/common/maps/cache.go index 3723d318e45..7cd7410c216 100644 --- a/common/maps/cache.go +++ b/common/maps/cache.go @@ -74,6 +74,26 @@ func (c *Cache[K, T]) ForEeach(f func(K, T)) { } } +func (c *Cache[K, T]) Drain() map[K]T { + c.Lock() + m := c.m + c.m = make(map[K]T) + c.Unlock() + return m +} + +func (c *Cache[K, T]) Len() int { + c.RLock() + defer c.RUnlock() + return len(c.m) +} + +func (c *Cache[K, T]) Reset() { + c.Lock() + c.m = make(map[K]T) + c.Unlock() +} + // SliceCache is a simple thread safe cache backed by a map. type SliceCache[T any] struct { m map[string][]T diff --git a/deps/deps.go b/deps/deps.go index 678f8a2fccf..9200f2b12d2 100644 --- a/deps/deps.go +++ b/deps/deps.go @@ -15,6 +15,7 @@ import ( "github.com/gohugoio/hugo/cache/filecache" "github.com/gohugoio/hugo/common/hexec" "github.com/gohugoio/hugo/common/loggers" + "github.com/gohugoio/hugo/common/maps" "github.com/gohugoio/hugo/common/types" "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/config/allconfig" @@ -161,20 +162,29 @@ func (d *Deps) Init() error { } if d.PathSpec == nil { - hashBytesReceiverFunc := func(name string, match bool) { - if !match { - return + hashBytesReceiverFunc := func(name string, match []byte) { + s := string(match) + switch s { + case postpub.PostProcessPrefix: + d.BuildState.AddFilenameWithPostPrefix(name) + case tpl.HugoDeferredTemplatePrefix: + d.BuildState.DeferredExecutions.FilenamesWithPostPrefix.Set(name, true) } - d.BuildState.AddFilenameWithPostPrefix(name) } // Skip binary files. mediaTypes := d.Conf.GetConfigSection("mediaTypes").(media.Types) - hashBytesSHouldCheck := func(name string) bool { + hashBytesShouldCheck := func(name string) bool { ext := strings.TrimPrefix(filepath.Ext(name), ".") return mediaTypes.IsTextSuffix(ext) } - d.Fs.PublishDir = hugofs.NewHasBytesReceiver(d.Fs.PublishDir, hashBytesSHouldCheck, hashBytesReceiverFunc, []byte(postpub.PostProcessPrefix)) + d.Fs.PublishDir = hugofs.NewHasBytesReceiver( + d.Fs.PublishDir, + hashBytesShouldCheck, + hashBytesReceiverFunc, + []byte(tpl.HugoDeferredTemplatePrefix), + []byte(postpub.PostProcessPrefix)) + pathSpec, err := helpers.NewPathSpec(d.Fs, d.Conf, d.Log) if err != nil { return err @@ -371,10 +381,41 @@ type BuildState struct { // A set of filenames in /public that // contains a post-processing prefix. filenamesWithPostPrefix map[string]bool + + filenamesWithDefferedExecution map[string]bool + + DeferredExecutions *DeferredExecutions + + // Deferred executions grouped by rendering context. + DeferredExecutionsGroupedByRenderingContext map[tpl.RenderingContext]*DeferredExecutions +} + +type DeferredExecutions struct { + // A set of filenames in /public that + // contains a post-processing prefix. + FilenamesWithPostPrefix *maps.Cache[string, bool] + + // Maps a placeholder to a deferred execution. + Executions *maps.Cache[string, *tpl.DeferredExecution] } var _ identity.SignalRebuilder = (*BuildState)(nil) +func (b *BuildState) StartRenderingStage(r tpl.RenderingContext) { + if b.DeferredExecutionsGroupedByRenderingContext == nil { + b.DeferredExecutionsGroupedByRenderingContext = make(map[tpl.RenderingContext]*DeferredExecutions) + } + b.DeferredExecutions = &DeferredExecutions{ + Executions: maps.NewCache[string, *tpl.DeferredExecution](), + FilenamesWithPostPrefix: maps.NewCache[string, bool](), + } +} + +func (b *BuildState) StopRenderingStage(r tpl.RenderingContext) { + b.DeferredExecutionsGroupedByRenderingContext[r] = b.DeferredExecutions + b.DeferredExecutions = nil +} + func (b *BuildState) SignalRebuild(ids ...identity.Identity) { b.OnSignalRebuild(ids...) } @@ -399,6 +440,12 @@ func (b *BuildState) GetFilenamesWithPostPrefix() []string { return filenames } +func (b *BuildState) AddDeferredExecution(id string, d *tpl.DeferredExecution) { + b.mu.Lock() + defer b.mu.Unlock() + b.DeferredExecutions.Executions.Set(id, d) +} + func (b *BuildState) Incr() int { return int(atomic.AddUint64(&b.counter, uint64(1))) } diff --git a/hugofs/hasbytes_fs.go b/hugofs/hasbytes_fs.go index 238fbc9c475..ac9e881ef6e 100644 --- a/hugofs/hasbytes_fs.go +++ b/hugofs/hasbytes_fs.go @@ -28,12 +28,12 @@ var ( type hasBytesFs struct { afero.Fs shouldCheck func(name string) bool - hasBytesCallback func(name string, match bool) - pattern []byte + hasBytesCallback func(name string, match []byte) + patterns [][]byte } -func NewHasBytesReceiver(delegate afero.Fs, shouldCheck func(name string) bool, hasBytesCallback func(name string, match bool), pattern []byte) afero.Fs { - return &hasBytesFs{Fs: delegate, shouldCheck: shouldCheck, hasBytesCallback: hasBytesCallback, pattern: pattern} +func NewHasBytesReceiver(delegate afero.Fs, shouldCheck func(name string) bool, hasBytesCallback func(name string, match []byte), patterns ...[]byte) afero.Fs { + return &hasBytesFs{Fs: delegate, shouldCheck: shouldCheck, hasBytesCallback: hasBytesCallback, patterns: patterns} } func (fs *hasBytesFs) UnwrapFilesystem() afero.Fs { @@ -60,10 +60,15 @@ func (fs *hasBytesFs) wrapFile(f afero.File) afero.File { if !fs.shouldCheck(f.Name()) { return f } + patterns := make([]*hugio.HasBytesPattern, len(fs.patterns)) + for i, p := range fs.patterns { + patterns[i] = &hugio.HasBytesPattern{Pattern: p} + } + return &hasBytesFile{ File: f, hbw: &hugio.HasBytesWriter{ - Pattern: fs.pattern, + Patterns: patterns, }, hasBytesCallback: fs.hasBytesCallback, } @@ -74,7 +79,7 @@ func (fs *hasBytesFs) Name() string { } type hasBytesFile struct { - hasBytesCallback func(name string, match bool) + hasBytesCallback func(name string, match []byte) hbw *hugio.HasBytesWriter afero.File } @@ -88,6 +93,10 @@ func (h *hasBytesFile) Write(p []byte) (n int, err error) { } func (h *hasBytesFile) Close() error { - h.hasBytesCallback(h.Name(), h.hbw.Match) + for _, p := range h.hbw.Patterns { + if p.Match { + h.hasBytesCallback(h.Name(), p.Pattern) + } + } return h.File.Close() } diff --git a/hugolib/hugo_sites_build.go b/hugolib/hugo_sites_build.go index 12eb6a5f8d7..8e2962712c7 100644 --- a/hugolib/hugo_sites_build.go +++ b/hugolib/hugo_sites_build.go @@ -26,6 +26,7 @@ import ( "time" "github.com/bep/logg" + "github.com/gohugoio/hugo/bufferpool" "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/hugofs/files" @@ -173,6 +174,16 @@ func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error { h.SendError(fmt.Errorf("postRenderOnce: %w", err)) } + // Make sure to write any build stats to disk first so it's available + // to the post processors. + if err := h.writeBuildStats(); err != nil { + return err + } + + if err := h.renderDeferred(infol); err != nil { + h.SendError(fmt.Errorf("renderDeferred: %w", err)) + } + if err := h.postProcess(infol); err != nil { h.SendError(fmt.Errorf("postProcess: %w", err)) } @@ -352,47 +363,172 @@ func (h *HugoSites) render(l logg.LevelLogger, config *BuildCfg) error { continue } - siteRenderContext.outIdx = siteOutIdx - siteRenderContext.sitesOutIdx = i - i++ - - select { - case <-h.Done(): - return nil - default: - for _, s2 := range h.Sites { - // We render site by site, but since the content is lazily rendered - // and a site can "borrow" content from other sites, every site - // needs this set. - s2.rc = &siteRenderingContext{Format: renderFormat} - - if err := s2.preparePagesForRender(s == s2, siteRenderContext.sitesOutIdx); err != nil { - return err + if err := func() error { + rc := tpl.RenderingContext{Site: s, SiteOutIdx: siteOutIdx} + h.BuildState.StartRenderingStage(rc) + defer h.BuildState.StopRenderingStage(rc) + + siteRenderContext.outIdx = siteOutIdx + siteRenderContext.sitesOutIdx = i + i++ + + select { + case <-h.Done(): + return nil + default: + for _, s2 := range h.Sites { + if err := s2.preparePagesForRender(s == s2, siteRenderContext.sitesOutIdx); err != nil { + return err + } + } + if !config.SkipRender { + ll := l.WithField("substep", "pages"). + WithField("site", s.language.Lang). + WithField("outputFormat", renderFormat.Name) + + start := time.Now() + + if config.PartialReRender { + if err := s.renderPages(siteRenderContext); err != nil { + return err + } + } else { + if err := s.render(siteRenderContext); err != nil { + return err + } + } + loggers.TimeTrackf(ll, start, nil, "") } } - if !config.SkipRender { - ll := l.WithField("substep", "pages"). - WithField("site", s.language.Lang). - WithField("outputFormat", renderFormat.Name) + return nil + }(); err != nil { + return err + } - start := time.Now() + } + } - if config.PartialReRender { - if err := s.renderPages(siteRenderContext); err != nil { - return err - } - } else { - if err := s.render(siteRenderContext); err != nil { + return nil +} + +func (h *HugoSites) renderDeferred(l logg.LevelLogger) error { + l = l.WithField("step", "render deferred") + start := time.Now() + + var deferredCount int + + for rc, de := range h.Deps.BuildState.DeferredExecutionsGroupedByRenderingContext { + if de.FilenamesWithPostPrefix.Len() == 0 { + continue + } + + deferredCount += de.FilenamesWithPostPrefix.Len() + + s := rc.Site.(*Site) + for _, s2 := range h.Sites { + if err := s2.preparePagesForRender(s == s2, rc.SiteOutIdx); err != nil { + return err + } + } + if err := s.executeDeferredTemplates(de); err != nil { + return err + } + } + + loggers.TimeTrackf(l, start, logg.Fields{ + logg.Field{Name: "count", Value: deferredCount}, + }, "") + + return nil +} + +func (s *Site) executeDeferredTemplates(de *deps.DeferredExecutions) error { + handleFile := func(filename string) error { + content, err := afero.ReadFile(s.BaseFs.PublishFs, filename) + if err != nil { + return err + } + + k := 0 + changed := false + + for { + if k >= len(content) { + break + } + l := bytes.Index(content[k:], []byte(tpl.HugoDeferredTemplatePrefix)) + if l == -1 { + break + } + m := bytes.Index(content[k+l:], []byte(tpl.HugoDeferredTemplateSuffix)) + len(tpl.HugoDeferredTemplateSuffix) + + low, high := k+l, k+l+m + + forward := l + m + id := string(content[low:high]) + + if err := func() error { + deferred, found := de.Executions.Get(id) + if !found { + panic(fmt.Sprintf("deferred execution with id %q not found", id)) + } + deferred.Mu.Lock() + defer deferred.Mu.Unlock() + + if !deferred.Executed { + tmpl := s.Deps.Tmpl() + templ, found := tmpl.Lookup(deferred.TemplateName) + if !found { + panic(fmt.Sprintf("template %q not found", deferred.TemplateName)) + } + + if err := func() error { + buf := bufferpool.GetBuffer() + defer bufferpool.PutBuffer(buf) + + err = tmpl.ExecuteWithContext(deferred.Ctx, templ, buf, deferred.Data) + if err != nil { return err } + deferred.Result = buf.String() + deferred.Executed = true + + return nil + }(); err != nil { + return err } - loggers.TimeTrackf(ll, start, nil, "") } + + content = append(content[:low], append([]byte(deferred.Result), content[high:]...)...) + changed = true + + return nil + }(); err != nil { + return err } + + k += forward } + + if changed { + return afero.WriteFile(s.BaseFs.PublishFs, filename, content, 0o666) + } + + return nil } - return nil + g := rungroup.Run[string](context.Background(), rungroup.Config[string]{ + NumWorkers: s.h.numWorkers, + Handle: func(ctx context.Context, filename string) error { + return handleFile(filename) + }, + }) + + de.FilenamesWithPostPrefix.ForEeach(func(filename string, _ bool) { + g.Enqueue(filename) + }) + + return g.Wait() } // / postRenderOnce runs some post processing that only needs to be done once, e.g. printing of unused templates. @@ -428,12 +564,6 @@ func (h *HugoSites) postProcess(l logg.LevelLogger) error { l = l.WithField("step", "postProcess") defer loggers.TimeTrackf(l, time.Now(), nil, "") - // Make sure to write any build stats to disk first so it's available - // to the post processors. - if err := h.writeBuildStats(); err != nil { - return err - } - // This will only be set when js.Build have been triggered with // imports that resolves to the project or a module. // Write a jsconfig.json file to the project's /asset directory diff --git a/hugolib/site.go b/hugolib/site.go index b4b89975d84..442de940adc 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -200,10 +200,6 @@ func (s *Site) prepareInits() { }) } -type siteRenderingContext struct { - output.Format -} - func (s *Site) Menus() navigation.Menus { s.checkReady() s.init.menus.Do(context.Background()) diff --git a/hugolib/site_new.go b/hugolib/site_new.go index 2ba5ef2fb3b..d5ca02044ee 100644 --- a/hugolib/site_new.go +++ b/hugolib/site_new.go @@ -88,10 +88,6 @@ type Site struct { publisher publisher.Publisher frontmatterHandler pagemeta.FrontMatterHandler - // We render each site for all the relevant output formats in serial with - // this rendering context pointing to the current one. - rc *siteRenderingContext - // The output formats that we need to render this site in. This slice // will be fixed once set. // This will be the union of Site.Pages' outputFormats. diff --git a/hugolib/site_render.go b/hugolib/site_render.go index a7ecf89af2b..47ee8658c76 100644 --- a/hugolib/site_render.go +++ b/hugolib/site_render.go @@ -226,7 +226,7 @@ func (s *Site) renderPaginator(p *pageState, templ tpl.Template) error { paginatePath := s.Conf.Pagination().Path d := p.targetPathDescriptor - f := p.s.rc.Format + f := p.outputFormat() d.Type = f if p.paginator.current == nil || p.paginator.current != p.paginator.current.First() { diff --git a/tpl/template.go b/tpl/template.go index 0ab1abf2f93..7e42e2aa136 100644 --- a/tpl/template.go +++ b/tpl/template.go @@ -20,11 +20,13 @@ import ( "reflect" "regexp" "strings" + "sync" "unicode" bp "github.com/gohugoio/hugo/bufferpool" "github.com/gohugoio/hugo/common/hcontext" "github.com/gohugoio/hugo/identity" + "github.com/gohugoio/hugo/langs" "github.com/gohugoio/hugo/output/layouts" "github.com/gohugoio/hugo/output" @@ -160,6 +162,11 @@ type TemplateFuncGetter interface { GetFunc(name string) (reflect.Value, bool) } +type RenderingContext struct { + Site site + SiteOutIdx int +} + type contextKey string // Context manages values passed in the context to templates. @@ -191,6 +198,15 @@ type page interface { IsNode() bool } +type site interface { + Language() *langs.Language +} + +const ( + HugoDeferredTemplatePrefix = "___hdeferred/" + HugoDeferredTemplateSuffix = "__d=" +) + const hugoNewLinePlaceholder = "___hugonl_" var stripHTMLReplacerPre = strings.NewReplacer("\n", " ", "
", hugoNewLinePlaceholder, "