Skip to content

Commit

Permalink
Add a HTTP cache for remote resources.
Browse files Browse the repository at this point in the history
  • Loading branch information
bep committed May 21, 2024
1 parent 3d40aba commit a3b076e
Show file tree
Hide file tree
Showing 19 changed files with 579 additions and 133 deletions.
51 changes: 51 additions & 0 deletions cache/filecache/filecache.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
"sync"
"time"

"github.com/gohugoio/httpcache"
"github.com/gohugoio/hugo/common/hugio"
"github.com/gohugoio/hugo/hugofs"

Expand Down Expand Up @@ -182,6 +183,15 @@ func (c *Cache) ReadOrCreate(id string,
return
}

// NamedLock locks the given id. The lock is released when the returned function is called.
func (c *Cache) NamedLock(id string) func() {
id = cleanID(id)
c.nlocker.Lock(id)
return func() {
c.nlocker.Unlock(id)
}
}

// GetOrCreate tries to get the file with the given id from cache. If not found or expired, create will
// be invoked and the result cached.
// This method is protected by a named lock using the given id as identifier.
Expand Down Expand Up @@ -398,3 +408,44 @@ func NewCaches(p *helpers.PathSpec) (Caches, error) {
func cleanID(name string) string {
return strings.TrimPrefix(filepath.Clean(name), helpers.FilePathSeparator)
}

// AsHTTPCache returns an httpcache.Cache implementation for this file cache.
// Note that none of the methods are protected by named locks, so you need to make sure
// to do that in your own code.
func (c *Cache) AsHTTPCache() httpcache.Cache {
return &httpCache{c: c}
}

type httpCache struct {
c *Cache
}

func (h *httpCache) Get(id string) (resp []byte, ok bool) {
id = cleanID(id)
if r := h.c.getOrRemove(id); r != nil {
defer r.Close()
b, err := io.ReadAll(r)
if err != nil {
panic(err)
}
return b, true
}
return nil, false
}

func (h *httpCache) Set(id string, resp []byte) {
if h.c.maxAge == 0 {
return
}


id = cleanID(id)

if err := afero.WriteReader(h.c.Fs, id, bytes.NewReader(resp)); err != nil {
panic(err)
}
}

func (h *httpCache) Delete(key string) {
h.c.Fs.Remove(key)
}
14 changes: 12 additions & 2 deletions commands/commandeer.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ import (
"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/hugofs"
"github.com/gohugoio/hugo/hugolib"
"github.com/gohugoio/hugo/identity"
"github.com/gohugoio/hugo/resources/kinds"
"github.com/spf13/afero"
"github.com/spf13/cobra"
Expand Down Expand Up @@ -103,6 +104,9 @@ type rootCommand struct {
commonConfigs *lazycache.Cache[int32, *commonConfig]
hugoSites *lazycache.Cache[int32, *hugolib.HugoSites]

// changesFromBuild received from Hugo in watch mode.
changesFromBuild chan []identity.Identity

commands []simplecobra.Commander

// Flags
Expand Down Expand Up @@ -304,7 +308,7 @@ func (r *rootCommand) ConfigFromProvider(key int32, cfg config.Provider) (*commo

func (r *rootCommand) HugFromConfig(conf *commonConfig) (*hugolib.HugoSites, error) {
h, _, err := r.hugoSites.GetOrCreate(r.configVersionID.Load(), func(key int32) (*hugolib.HugoSites, error) {
depsCfg := deps.DepsCfg{Configs: conf.configs, Fs: conf.fs, LogOut: r.logger.Out(), LogLevel: r.logger.Level()}
depsCfg := r.newDepsConfig(conf)
return hugolib.NewHugoSites(depsCfg)
})
return h, err
Expand All @@ -316,12 +320,16 @@ func (r *rootCommand) Hugo(cfg config.Provider) (*hugolib.HugoSites, error) {
if err != nil {
return nil, err
}
depsCfg := deps.DepsCfg{Configs: conf.configs, Fs: conf.fs, LogOut: r.logger.Out(), LogLevel: r.logger.Level()}
depsCfg := r.newDepsConfig(conf)
return hugolib.NewHugoSites(depsCfg)
})
return h, err
}

func (r *rootCommand) newDepsConfig(conf *commonConfig) deps.DepsCfg {
return deps.DepsCfg{Configs: conf.configs, Fs: conf.fs, LogOut: r.logger.Out(), LogLevel: r.logger.Level(), ChangesFromBuild: r.changesFromBuild}
}

func (r *rootCommand) Name() string {
return "hugo"
}
Expand Down Expand Up @@ -408,6 +416,8 @@ func (r *rootCommand) PreRun(cd, runner *simplecobra.Commandeer) error {
return err
}

r.changesFromBuild = make(chan []identity.Identity, 10)

r.commonConfigs = lazycache.New(lazycache.Options[int32, *commonConfig]{MaxEntries: 5})
// We don't want to keep stale HugoSites in memory longer than needed.
r.hugoSites = lazycache.New(lazycache.Options[int32, *hugolib.HugoSites]{
Expand Down
29 changes: 29 additions & 0 deletions commands/hugobuilder.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ import (
"github.com/gohugoio/hugo/hugofs"
"github.com/gohugoio/hugo/hugolib"
"github.com/gohugoio/hugo/hugolib/filesystems"
"github.com/gohugoio/hugo/identity"
"github.com/gohugoio/hugo/livereload"
"github.com/gohugoio/hugo/resources/page"
"github.com/gohugoio/hugo/watcher"
Expand Down Expand Up @@ -343,6 +344,21 @@ func (c *hugoBuilder) newWatcher(pollIntervalStr string, dirList ...string) (*wa
go func() {
for {
select {
case changes := <-c.r.changesFromBuild:
unlock, err := h.LockBuild()
if err != nil {
c.r.logger.Errorln("Failed to acquire a build lock: %s", err)
return
}
err = c.rebuildSitesForChanges(changes)
if err != nil {
c.r.logger.Errorln("Error while watching:", err)
}
if c.s != nil && c.s.doLiveReload {
livereload.ForceRefresh()
}
unlock()

case evs := <-watcher.Events:
unlock, err := h.LockBuild()
if err != nil {
Expand Down Expand Up @@ -1019,6 +1035,19 @@ func (c *hugoBuilder) rebuildSites(events []fsnotify.Event) error {
return h.Build(hugolib.BuildCfg{NoBuildLock: true, RecentlyVisited: c.visitedURLs, ErrRecovery: c.errState.wasErr()}, events...)
}

func (c *hugoBuilder) rebuildSitesForChanges(ids []identity.Identity) error {
c.errState.setBuildErr(nil)
h, err := c.hugo()
if err != nil {
return err
}
whatChanged := &hugolib.WhatChanged{}
whatChanged.Add(ids...)
err = h.Build(hugolib.BuildCfg{NoBuildLock: true, WhatChanged: whatChanged, RecentlyVisited: c.visitedURLs, ErrRecovery: c.errState.wasErr()})
c.errState.setBuildErr(err)
return err
}

func (c *hugoBuilder) reloadConfig() error {
c.r.Reset()
c.r.configVersionID.Add(1)
Expand Down
149 changes: 149 additions & 0 deletions common/tasks/tasks.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tasks

import (
"sync"
"time"
)

// RunEvery runs a function at a regular interval.
// Functions can be added and removed while running.
type RunEvery struct {
// The shortest interval between each run.
IntervalLow time.Duration

// The longest interval between each run.
IntervalHigh time.Duration

// Any error returned from the function will be passed to this function.
HandleError func(string, error)

// If set, the function will be run immediately.
RunImmediately bool

// The named functions to run.
funcs map[string]*fn

mu sync.Mutex
started bool
closed bool
quit chan struct{}
}

type fn struct {
interval time.Duration
last time.Time
f func(interval time.Duration) (time.Duration, error)
}

func (r *RunEvery) Start() error {
if r.started {
return nil
}
if r.IntervalLow == 0 {
r.IntervalLow = 1 * time.Second
}

if r.IntervalHigh < r.IntervalLow {
r.IntervalHigh = r.IntervalLow * 30
}

r.started = true
r.quit = make(chan struct{})

go func() {
if r.RunImmediately {
r.run()
}
ticker := time.NewTicker(500 * time.Millisecond)
defer ticker.Stop()
for {
select {
case <-r.quit:
return
case <-ticker.C:
r.run()
}
}
}()

return nil
}

// Close stops the RunEvery from running.
func (r *RunEvery) Close() error {
if r.closed {
return nil
}
r.closed = true
if r.quit != nil {
close(r.quit)
}
return nil
}

// Add adds a function to the RunEvery.
func (r *RunEvery) Add(name string, f func(time.Duration) (time.Duration, error)) {
r.mu.Lock()
defer r.mu.Unlock()
if r.funcs == nil {
r.funcs = make(map[string]*fn)
}
start := r.IntervalHigh / 3
if start < r.IntervalLow {
start = r.IntervalLow
}
r.funcs[name] = &fn{f: f, interval: start, last: time.Now()}
}

// Remove removes a function from the RunEvery.
func (r *RunEvery) Remove(name string) {
r.mu.Lock()
defer r.mu.Unlock()
delete(r.funcs, name)
}

// Has returns whether the RunEvery has a function with the given name.
func (r *RunEvery) Has(name string) bool {
r.mu.Lock()
defer r.mu.Unlock()
_, found := r.funcs[name]
return found
}

func (r *RunEvery) run() {
r.mu.Lock()
defer r.mu.Unlock()
for name, f := range r.funcs {
if time.Now().Before(f.last.Add(f.interval)) {
continue
}
f.last = time.Now()
interval, err := f.f(f.interval)
if err != nil && r.HandleError != nil {
r.HandleError(name, err)
}

if interval < r.IntervalLow {
interval = r.IntervalLow
}

if interval > r.IntervalHigh {
interval = r.IntervalHigh
}

f.interval = interval
}
}
47 changes: 47 additions & 0 deletions common/types/closer.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package types

import "sync"

type Closer interface {
Close() error
}

type CloseAdder interface {
Add(Closer)
}

type Closers struct {
mu sync.Mutex
cs []Closer
}

func (cs *Closers) Add(c Closer) {
cs.mu.Lock()
defer cs.mu.Unlock()
cs.cs = append(cs.cs, c)
}

func (cs *Closers) Close() error {
cs.mu.Lock()
defer cs.mu.Unlock()
for _, c := range cs.cs {
c.Close()
}

cs.cs = cs.cs[:0]

return nil
}
Loading

0 comments on commit a3b076e

Please sign in to comment.