Skip to content

Commit

Permalink
Misc resource fixes/improvements
Browse files Browse the repository at this point in the history
* Add --pprof flag to server to enable profile debugging.
* Don't cache the resource content, it seem to eat memory on bigger sites.
* Keep --printMemoryUsag running in server

Fixes #11974
  • Loading branch information
bep committed Feb 2, 2024
1 parent d0788b9 commit 2873324
Show file tree
Hide file tree
Showing 7 changed files with 58 additions and 65 deletions.
10 changes: 8 additions & 2 deletions commands/commandeer.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"os"
"os/signal"
"path/filepath"
"runtime"
"strings"
"sync"
"sync/atomic"
Expand Down Expand Up @@ -341,8 +342,12 @@ func (r *rootCommand) Run(ctx context.Context, cd *simplecobra.Commandeer, args
if r.buildWatch {
defer r.timeTrack(time.Now(), "Built")
}
err := b.build()
return err
close, err := b.build()
if err != nil {
return err
}
close()
return nil
}()
if err != nil {
return err
Expand Down Expand Up @@ -411,6 +416,7 @@ func (r *rootCommand) PreRun(cd, runner *simplecobra.Commandeer) error {
MaxEntries: 1,
OnEvict: func(key int32, value *hugolib.HugoSites) {
value.Close()
runtime.GC()
},
})

Expand Down
20 changes: 9 additions & 11 deletions commands/hugobuilder.go
Original file line number Diff line number Diff line change
Expand Up @@ -361,34 +361,32 @@ func (c *hugoBuilder) newWatcher(pollIntervalStr string, dirList ...string) (*wa
return watcher, nil
}

func (c *hugoBuilder) build() error {
func (c *hugoBuilder) build() (func(), error) {
stopProfiling, err := c.initProfiling()
if err != nil {
return err
return nil, err
}

defer func() {
if stopProfiling != nil {
stopProfiling()
}
}()

if err := c.fullBuild(false); err != nil {
return err
return nil, err
}

if !c.r.quiet {
c.r.Println()
h, err := c.hugo()
if err != nil {
return err
return nil, err
}

h.PrintProcessingStats(os.Stdout)
c.r.Println()
}

return nil
return func() {
if stopProfiling != nil {
stopProfiling()
}
}, nil
}

func (c *hugoBuilder) buildSites(noBuildLock bool) (err error) {
Expand Down
14 changes: 13 additions & 1 deletion commands/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"io"
"net"
"net/http"
_ "net/http/pprof"
"net/url"
"os"
"os/signal"
Expand Down Expand Up @@ -451,6 +452,7 @@ type serverCommand struct {
tlsCertFile string
tlsKeyFile string
tlsAuto bool
pprof bool
serverPort int
liveReloadPort int
serverWatch bool
Expand All @@ -465,6 +467,11 @@ func (c *serverCommand) Name() string {
}

func (c *serverCommand) Run(ctx context.Context, cd *simplecobra.Commandeer, args []string) error {
if c.pprof {
go func() {
http.ListenAndServe("localhost:8080", nil)
}()
}
// Watch runs its own server as part of the routine
if c.serverWatch {

Expand All @@ -487,15 +494,19 @@ func (c *serverCommand) Run(ctx context.Context, cd *simplecobra.Commandeer, arg

}

var close func()
err := func() error {
defer c.r.timeTrack(time.Now(), "Built")
err := c.build()
var err error
close, err = c.build()
return err
}()
if err != nil {
return err
}

defer close()

return c.serve()
}

Expand All @@ -520,6 +531,7 @@ of a second, you will be able to save and see your changes nearly instantly.`
cmd.Flags().StringVarP(&c.tlsCertFile, "tlsCertFile", "", "", "path to TLS certificate file")
cmd.Flags().StringVarP(&c.tlsKeyFile, "tlsKeyFile", "", "", "path to TLS key file")
cmd.Flags().BoolVar(&c.tlsAuto, "tlsAuto", false, "generate and use locally-trusted certificates.")
cmd.Flags().BoolVar(&c.pprof, "pprof", false, "enable the pprof server (port 8080)")
cmd.Flags().BoolVarP(&c.serverWatch, "watch", "w", true, "watch filesystem for changes and recreate as needed")
cmd.Flags().BoolVar(&c.noHTTPCache, "noHTTPCache", false, "prevent HTTP caching")
cmd.Flags().BoolVarP(&c.serverAppend, "appendPort", "", true, "append port to baseURL")
Expand Down
2 changes: 2 additions & 0 deletions hugolib/hugo_sites_build.go
Original file line number Diff line number Diff line change
Expand Up @@ -720,9 +720,11 @@ func (h *HugoSites) processPartial(ctx context.Context, l logg.LevelLogger, conf
h.pageTrees.treeTaxonomyEntries.DeletePrefix("")

if delete {

_, ok := h.pageTrees.treePages.LongestPrefixAll(pathInfo.Base())
if ok {
h.pageTrees.treePages.DeleteAll(pathInfo.Base())
h.pageTrees.resourceTrees.DeleteAll(pathInfo.Base())
if pathInfo.IsBundle() {
// Assume directory removed.
h.pageTrees.treePages.DeletePrefixAll(pathInfo.Base() + "/")
Expand Down
4 changes: 2 additions & 2 deletions hugolib/rebuild_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -101,10 +101,10 @@ func TestRebuildEditTextFileInBranchBundle(t *testing.T) {

func TestRebuildRenameTextFileInLeafBundle(t *testing.T) {
b := TestRunning(t, rebuildFilesSimple)
b.AssertFileContent("public/mysection/mysectionbundle/index.html", "My Section Bundle Text 2 Content.")
b.AssertFileContent("public/mysection/mysectionbundle/index.html", "My Section Bundle Text 2 Content.", "Len Resources: 2|")

b.RenameFile("content/mysection/mysectionbundle/mysectionbundletext.txt", "content/mysection/mysectionbundle/mysectionbundletext2.txt").Build()
b.AssertFileContent("public/mysection/mysectionbundle/index.html", "mysectionbundletext2", "My Section Bundle Text 2 Content.")
b.AssertFileContent("public/mysection/mysectionbundle/index.html", "mysectionbundletext2", "My Section Bundle Text 2 Content.", "Len Resources: 2|")
b.AssertRenderCountPage(3)
b.AssertRenderCountContent(3)
}
Expand Down
55 changes: 15 additions & 40 deletions resources/resource.go
Original file line number Diff line number Diff line change
Expand Up @@ -343,7 +343,7 @@ func GetTestInfoForResource(r resource.Resource) GenericResourceTestInfo {

// genericResource represents a generic linkable resource.
type genericResource struct {
*resourceContent
publishInit *sync.Once

sd ResourceSourceDescriptor
paths internal.ResourcePaths
Expand Down Expand Up @@ -412,11 +412,18 @@ func (l *genericResource) cloneTo(targetPath string) resource.Resource {
}

func (l *genericResource) Content(context.Context) (any, error) {
if err := l.initContent(); err != nil {
return nil, err
r, err := l.ReadSeekCloser()
if err != nil {
return "", err
}
defer r.Close()

return l.content, nil
var b []byte
b, err = io.ReadAll(r)
if err != nil {
return "", err
}
return string(b), nil
}

func (r *genericResource) Err() resource.ResourceError {
Expand Down Expand Up @@ -527,28 +534,6 @@ func (l *genericResource) Title() string {
return l.title
}

func (l *genericResource) initContent() error {
var err error
l.contentInit.Do(func() {
var r hugio.ReadSeekCloser
r, err = l.ReadSeekCloser()
if err != nil {
return
}
defer r.Close()

var b []byte
b, err = io.ReadAll(r)
if err != nil {
return
}

l.content = string(b)
})

return err
}

func (l *genericResource) getSpec() *Spec {
return l.spec
}
Expand Down Expand Up @@ -588,12 +573,9 @@ func (rc *genericResource) cloneWithUpdates(u *transformationUpdate) (baseResour
r := rc.clone()

if u.content != nil {
r.contentInit.Do(func() {
r.content = *u.content
r.sd.OpenReadSeekCloser = func() (hugio.ReadSeekCloser, error) {
return hugio.NewReadSeekerNoOpCloserFromString(r.content), nil
}
})
r.sd.OpenReadSeekCloser = func() (hugio.ReadSeekCloser, error) {
return hugio.NewReadSeekerNoOpCloserFromString(*u.content), nil
}
}

r.sd.MediaType = u.mediaType
Expand All @@ -620,7 +602,7 @@ func (rc *genericResource) cloneWithUpdates(u *transformationUpdate) (baseResour
}

func (l genericResource) clone() *genericResource {
l.resourceContent = &resourceContent{}
l.publishInit = &sync.Once{}
return &l
}

Expand All @@ -633,13 +615,6 @@ type targetPather interface {
TargetPath() string
}

type resourceContent struct {
content string
contentInit sync.Once

publishInit sync.Once
}

type resourceHash struct {
value string
size int64
Expand Down
18 changes: 9 additions & 9 deletions resources/resource_spec.go
Original file line number Diff line number Diff line change
Expand Up @@ -163,15 +163,15 @@ func (r *Spec) NewResource(rd ResourceSourceDescriptor) (resource.Resource, erro
}

gr := &genericResource{
Staler: &AtomicStaler{},
h: &resourceHash{},
paths: rp,
spec: r,
sd: rd,
params: make(map[string]any),
name: rd.Name,
title: rd.Name,
resourceContent: &resourceContent{},
Staler: &AtomicStaler{},
h: &resourceHash{},
publishInit: &sync.Once{},
paths: rp,
spec: r,
sd: rd,
params: make(map[string]any),
name: rd.Name,
title: rd.Name,
}

if rd.MediaType.MainType == "image" {
Expand Down

0 comments on commit 2873324

Please sign in to comment.