Skip to content

Commit

Permalink
Add js.Batch
Browse files Browse the repository at this point in the history
  • Loading branch information
bep committed Nov 30, 2024
1 parent 487bb96 commit b624bb6
Show file tree
Hide file tree
Showing 55 changed files with 4,064 additions and 951 deletions.
1 change: 1 addition & 0 deletions commands/commandeer.go
Original file line number Diff line number Diff line change
Expand Up @@ -452,6 +452,7 @@ func (r *rootCommand) PreRun(cd, runner *simplecobra.Commandeer) error {
r.hugoSites = lazycache.New(lazycache.Options[configKey, *hugolib.HugoSites]{
MaxEntries: 1,
OnEvict: func(key configKey, value *hugolib.HugoSites) {
fmt.Println("Evicting HugoSites", key) // TODO1 remove me.
value.Close()
runtime.GC()
},
Expand Down
6 changes: 5 additions & 1 deletion commands/hugobuilder.go
Original file line number Diff line number Diff line change
Expand Up @@ -920,7 +920,11 @@ func (c *hugoBuilder) handleEvents(watcher *watcher.Batcher,

changed := c.changeDetector.changed()
if c.changeDetector != nil {
lrl.Logf("build changed %d files", len(changed))
if len(changed) >= 10 {
lrl.Logf("build changed %d files", len(changed))
} else {
lrl.Logf("build changed %d files: %q", len(changed), changed)
}
if len(changed) == 0 {
// Nothing has changed.
return
Expand Down
6 changes: 4 additions & 2 deletions commands/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import (
"path"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"sync"
Expand Down Expand Up @@ -210,16 +211,17 @@ func (f *fileChangeDetector) changed() []string {
}
}

return f.filterIrrelevant(c)
return f.filterIrrelevantAndSort(c)
}

func (f *fileChangeDetector) filterIrrelevant(in []string) []string {
func (f *fileChangeDetector) filterIrrelevantAndSort(in []string) []string {
var filtered []string
for _, v := range in {
if !f.irrelevantRe.MatchString(v) {
filtered = append(filtered, v)
}
}
sort.Strings(filtered)
return filtered
}

Expand Down
15 changes: 15 additions & 0 deletions common/herrors/errors.go
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,21 @@ func IsNotExist(err error) bool {
return false
}

// IsExist returns true if the error is a file exists error.
// Unlike os.IsExist, this also considers wrapped errors.
func IsExist(err error) bool {
if os.IsExist(err) {
return true
}

// os.IsExist does not consider wrapped errors.
if os.IsExist(errors.Unwrap(err)) {
return true
}

return false
}

var nilPointerErrRe = regexp.MustCompile(`at <(.*)>: error calling (.*?): runtime error: invalid memory address or nil pointer dereference`)

const deferredPrefix = "__hdeferred/"
Expand Down
21 changes: 21 additions & 0 deletions common/hreflect/helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,27 @@ func AsTime(v reflect.Value, loc *time.Location) (time.Time, bool) {
return time.Time{}, false
}

// ToSliceAny converts the given value to a slice of any if possible.
func ToSliceAny(v any) ([]any, bool) {
if v == nil {
return nil, false
}
switch vv := v.(type) {
case []any:
return vv, true
default:
vvv := reflect.ValueOf(v)
if vvv.Kind() == reflect.Slice {
out := make([]any, vvv.Len())
for i := 0; i < vvv.Len(); i++ {
out[i] = vvv.Index(i).Interface()
}
return out, true
}
}
return nil, false
}

func CallMethodByName(cxt context.Context, name string, v reflect.Value) []reflect.Value {
fn := v.MethodByName(name)
var args []reflect.Value
Expand Down
13 changes: 13 additions & 0 deletions common/hreflect/helpers_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,19 @@ func TestIsContextType(t *testing.T) {
c.Assert(IsContextType(reflect.TypeOf(valueCtx)), qt.IsTrue)
}

func TestToSliceAny(t *testing.T) {
c := qt.New(t)

checkOK := func(in any, expected []any) {
out, ok := ToSliceAny(in)
c.Assert(ok, qt.Equals, true)
c.Assert(out, qt.DeepEquals, expected)
}

checkOK([]any{1, 2, 3}, []any{1, 2, 3})
checkOK([]int{1, 2, 3}, []any{1, 2, 3})
}

func BenchmarkIsContextType(b *testing.B) {
type k string
b.Run("value", func(b *testing.B) {
Expand Down
7 changes: 5 additions & 2 deletions common/maps/cache.go
Original file line number Diff line number Diff line change
Expand Up @@ -113,11 +113,14 @@ func (c *Cache[K, T]) set(key K, value T) {
}

// ForEeach calls the given function for each key/value pair in the cache.
func (c *Cache[K, T]) ForEeach(f func(K, T)) {
// If the function returns false, the iteration stops.
func (c *Cache[K, T]) ForEeach(f func(K, T) bool) {
c.RLock()
defer c.RUnlock()
for k, v := range c.m {
f(k, v)
if !f(k, v) {
return
}
}
}

Expand Down
7 changes: 7 additions & 0 deletions common/types/closer.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,13 @@ type Closer interface {
Close() error
}

// CloserFunc is a convenience type to create a Closer from a function.
type CloserFunc func() error

func (f CloserFunc) Close() error {
return f()
}

type CloseAdder interface {
Add(Closer)
}
Expand Down
12 changes: 10 additions & 2 deletions config/allconfig/configlanguage.go
Original file line number Diff line number Diff line change
Expand Up @@ -137,11 +137,19 @@ func (c ConfigLanguage) Watching() bool {
return c.m.Base.Internal.Watch
}

func (c ConfigLanguage) NewIdentityManager(name string) identity.Manager {
func (c ConfigLanguage) NewIdentityManager(name string, opts ...identity.ManagerOption) identity.Manager {
if !c.Watching() {
return identity.NopManager
}
return identity.NewManager(name)
// TODO1 remove.
opts = append(opts, identity.WithOnAddIdentity(func(id identity.Identity) {
if id.IdentifierBase() == "instance_mybatch_p1_p1scriptjs_0" {
// panic("where?")
}
// fmt.Println("===> ===>identity added", id.IdentifierBase())
}))

return identity.NewManager(name, opts...)
}

func (c ConfigLanguage) ContentTypes() config.ContentTypesProvider {
Expand Down
2 changes: 1 addition & 1 deletion config/configProvider.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ type AllProvider interface {
BuildDrafts() bool
Running() bool
Watching() bool
NewIdentityManager(name string) identity.Manager
NewIdentityManager(name string, opts ...identity.ManagerOption) identity.Manager
FastRenderMode() bool
PrintUnusedTemplates() bool
EnableMissingTranslationPlaceholders() bool
Expand Down
45 changes: 36 additions & 9 deletions deps/deps.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"fmt"
"io"
"os"
"path/filepath"
"sort"
"strings"
Expand Down Expand Up @@ -83,10 +84,13 @@ type Deps struct {
Metrics metrics.Provider

// BuildStartListeners will be notified before a build starts.
BuildStartListeners *Listeners
BuildStartListeners *Listeners[any]

// BuildEndListeners will be notified after a build finishes.
BuildEndListeners *Listeners
BuildEndListeners *Listeners[any]

// OnChangeListeners will be notified when something changes.
OnChangeListeners *Listeners[identity.Identity]

// Resources that gets closed when the build is done or the server shuts down.
BuildClosers *types.Closers
Expand Down Expand Up @@ -154,17 +158,21 @@ func (d *Deps) Init() error {
}

if d.BuildStartListeners == nil {
d.BuildStartListeners = &Listeners{}
d.BuildStartListeners = &Listeners[any]{}
}

if d.BuildEndListeners == nil {
d.BuildEndListeners = &Listeners{}
d.BuildEndListeners = &Listeners[any]{}
}

if d.BuildClosers == nil {
d.BuildClosers = &types.Closers{}
}

if d.OnChangeListeners == nil {
d.OnChangeListeners = &Listeners[identity.Identity]{}
}

if d.Metrics == nil && d.Conf.TemplateMetrics() {
d.Metrics = metrics.NewProvider(d.Conf.TemplateMetricsHints())
}
Expand Down Expand Up @@ -268,6 +276,20 @@ func (d *Deps) Compile(prototype *Deps) error {
return nil
}

// MkdirTemp returns a temporary directory path that will be cleaned up on exit.
func (d Deps) MkdirTemp(pattern string) (string, error) {
filename, err := os.MkdirTemp("", pattern)
if err != nil {
return "", err
}
d.BuildClosers.Add(types.CloserFunc(
func() error {
return os.RemoveAll(filename)
}))

return filename, nil
}

type globalErrHandler struct {
logger loggers.Logger

Expand Down Expand Up @@ -306,15 +328,16 @@ func (e *globalErrHandler) StopErrorCollector() {
}

// Listeners represents an event listener.
type Listeners struct {
type Listeners[T any] struct {
sync.Mutex

// A list of funcs to be notified about an event.
listeners []func()
// If the return value is true, the listener will be removed.
listeners []func(...T) bool
}

// Add adds a function to a Listeners instance.
func (b *Listeners) Add(f func()) {
func (b *Listeners[T]) Add(f func(...T) bool) {
if b == nil {
return
}
Expand All @@ -324,12 +347,16 @@ func (b *Listeners) Add(f func()) {
}

// Notify executes all listener functions.
func (b *Listeners) Notify() {
func (b *Listeners[T]) Notify(vs ...T) {
b.Lock()
defer b.Unlock()
temp := b.listeners[:0]
for _, notify := range b.listeners {
notify()
if !notify(vs...) {
temp = append(temp, notify)
}
}
b.listeners = temp
}

// ResourceProvider is used to create and refresh, and clone resources needed.
Expand Down
4 changes: 4 additions & 0 deletions hugolib/hugo_sites.go
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,10 @@ func (h *HugoSites) ShouldSkipFileChangeEvent(ev fsnotify.Event) bool {
return h.skipRebuildForFilenames[ev.Name]
}

func (h *HugoSites) Close() error {
return h.Deps.Close()
}

func (h *HugoSites) isRebuild() bool {
return h.buildCounter.Load() > 0
}
Expand Down
5 changes: 4 additions & 1 deletion hugolib/hugo_sites_build.go
Original file line number Diff line number Diff line change
Expand Up @@ -520,8 +520,9 @@ func (s *Site) executeDeferredTemplates(de *deps.DeferredExecutions) error {
},
})

de.FilenamesWithPostPrefix.ForEeach(func(filename string, _ bool) {
de.FilenamesWithPostPrefix.ForEeach(func(filename string, _ bool) bool {
g.Enqueue(filename)
return true
})

return g.Wait()
Expand Down Expand Up @@ -1058,6 +1059,8 @@ func (h *HugoSites) processPartialFileEvents(ctx context.Context, l logg.LevelLo
}
}

h.Deps.OnChangeListeners.Notify(changed.Changes()...)

if err := h.resolveAndClearStateForIdentities(ctx, l, cacheBusterOr, changed.Drain()); err != nil {
return err
}
Expand Down
21 changes: 18 additions & 3 deletions hugolib/integrationtest_builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,13 @@ func TestOptWithNFDOnDarwin() TestOpt {
}
}

// TestOptWithOSFs enables the real file system.
func TestOptWithOSFs() TestOpt {
return func(c *IntegrationTestConfig) {
c.NeedsOsFS = true
}
}

// TestOptWithWorkingDir allows setting any config optiona as a function al option.
func TestOptWithConfig(fn func(c *IntegrationTestConfig)) TestOpt {
return func(c *IntegrationTestConfig) {
Expand Down Expand Up @@ -284,8 +291,9 @@ func (s *IntegrationTestBuilder) negate(match string) (string, bool) {
func (s *IntegrationTestBuilder) AssertFileContent(filename string, matches ...string) {
s.Helper()
content := strings.TrimSpace(s.FileContent(filename))

for _, m := range matches {
cm := qt.Commentf("File: %s Match %s", filename, m)
cm := qt.Commentf("File: %s Match %s\nContent:\n%s", filename, m, content)
lines := strings.Split(m, "\n")
for _, match := range lines {
match = strings.TrimSpace(match)
Expand All @@ -295,7 +303,8 @@ func (s *IntegrationTestBuilder) AssertFileContent(filename string, matches ...s
var negate bool
match, negate = s.negate(match)
if negate {
s.Assert(content, qt.Not(qt.Contains), match, cm)
if !s.Assert(content, qt.Not(qt.Contains), match, cm) {
}
continue
}
s.Assert(content, qt.Contains, match, cm)
Expand All @@ -313,7 +322,8 @@ func (s *IntegrationTestBuilder) AssertFileContentExact(filename string, matches
s.Helper()
content := s.FileContent(filename)
for _, m := range matches {
s.Assert(content, qt.Contains, m, qt.Commentf(m))
cm := qt.Commentf("File: %s Match %s\nContent:\n%s", filename, m, content)
s.Assert(content, qt.Contains, m, cm)
}
}

Expand Down Expand Up @@ -450,6 +460,11 @@ func (s *IntegrationTestBuilder) Build() *IntegrationTestBuilder {
return s
}

func (s *IntegrationTestBuilder) Close() {
s.Helper()
s.Assert(s.H.Close(), qt.IsNil)
}

func (s *IntegrationTestBuilder) LogString() string {
return s.lastBuildLog
}
Expand Down
4 changes: 4 additions & 0 deletions hugolib/page.go
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,10 @@ func (p *pageState) GetDependencyManagerForScope(scope int) identity.Manager {
}
}

func (p *pageState) GetDependencyManagerForScopesAll() []identity.Manager {
return []identity.Manager{p.dependencyManager, p.dependencyManagerOutput}
}

func (p *pageState) Key() string {
return "page-" + strconv.FormatUint(p.pid, 10)
}
Expand Down
Loading

0 comments on commit b624bb6

Please sign in to comment.