From 8b9f6b92b7f49fb9346d68bf9a6e848aa19ffeb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Thu, 27 Jun 2024 16:22:35 +0200 Subject: [PATCH] Add js.Bundle Fixes #12626 --- common/maps/scratch.go | 14 + resources/resource_transformers/js/build.go | 117 ++-- resources/resource_transformers/js/options.go | 111 +++- .../resource_transformers/js/options_test.go | 102 +++- .../resource_transformers/js/transform.go | 69 +++ .../go_templates/texttemplate/exec.go | 5 +- .../texttemplate/hugo_template.go | 34 +- tpl/js/batch-esm-callback.gotmpl | 15 + tpl/js/batch.go | 542 ++++++++++++++++++ tpl/js/batch_integration_test.go | 145 +++++ tpl/js/js.go | 15 +- 11 files changed, 1036 insertions(+), 133 deletions(-) create mode 100644 resources/resource_transformers/js/transform.go create mode 100644 tpl/js/batch-esm-callback.gotmpl create mode 100644 tpl/js/batch.go create mode 100644 tpl/js/batch_integration_test.go diff --git a/common/maps/scratch.go b/common/maps/scratch.go index e9f412540b2..3bb160ae037 100644 --- a/common/maps/scratch.go +++ b/common/maps/scratch.go @@ -107,6 +107,20 @@ func (c *Scratch) Get(key string) any { return val } +// GetOrCreate returns the value for the given key if it exists, or creates it +// using the given func and stores that value in the map. +// For internal use. +func (c *Scratch) GetOrCreate(key string, create func() any) any { + c.mu.Lock() + defer c.mu.Unlock() + if val, found := c.values[key]; found { + return val + } + val := create() + c.values[key] = val + return val +} + // Values returns the raw backing map. Note that you should just use // this method on the locally scoped Scratch instances you obtain via newScratch, not // .Page.Scratch etc., as that will lead to concurrency issues. diff --git a/resources/resource_transformers/js/build.go b/resources/resource_transformers/js/build.go index cc68d225335..39b232225fc 100644 --- a/resources/resource_transformers/js/build.go +++ b/resources/resource_transformers/js/build.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -16,25 +16,20 @@ package js import ( "errors" "fmt" - "io" "os" "path" "path/filepath" "regexp" "strings" - "github.com/spf13/afero" - - "github.com/gohugoio/hugo/hugofs" - "github.com/gohugoio/hugo/media" - + "github.com/evanw/esbuild/pkg/api" "github.com/gohugoio/hugo/common/herrors" "github.com/gohugoio/hugo/common/text" - + "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/hugolib/filesystems" - "github.com/gohugoio/hugo/resources/internal" + "github.com/gohugoio/hugo/identity" + "github.com/spf13/afero" - "github.com/evanw/esbuild/pkg/api" "github.com/gohugoio/hugo/resources" "github.com/gohugoio/hugo/resources/resource" ) @@ -53,46 +48,47 @@ func New(fs *filesystems.SourceFilesystem, rs *resources.Spec) *Client { } } -type buildTransformation struct { - optsm map[string]any - c *Client +// ProcessExernal processes a resource with the user provided options. +func (c *Client) ProcessExernal(res resources.ResourceTransformer, opts map[string]any) (resource.Resource, error) { + return res.Transform( + &buildTransformation{c: c, optsm: opts}, + ) } -func (t *buildTransformation) Key() internal.ResourceTransformationKey { - return internal.NewResourceTransformationKey("jsbuild", t.optsm) +// ProcessExernal processes a resource with the given options. +func (c *Client) ProcessInternal(res resources.ResourceTransformer, opts Options) (resource.Resource, error) { + return res.Transform( + &buildTransformation{c: c, opts: opts}, + ) } -func (t *buildTransformation) Transform(ctx *resources.ResourceTransformationCtx) error { - ctx.OutMediaType = media.Builtin.JavascriptType +func (c *Client) BuildBundle(opts Options) error { + return c.build(opts, nil) +} - opts, err := decodeOptions(t.optsm) - if err != nil { - return err +// Note that transformCtx may be nil. +func (c *Client) build(opts Options, transformCtx *resources.ResourceTransformationCtx) error { + dependencyManager := opts.DependencyManager + if transformCtx != nil { + dependencyManager = transformCtx.DependencyManager // TODO1 } - - if opts.TargetPath != "" { - ctx.OutPath = opts.TargetPath - } else { - ctx.ReplaceOutPathExtension(".js") + if dependencyManager == nil { + dependencyManager = identity.NopManager } - src, err := io.ReadAll(ctx.From) - if err != nil { + opts.ResolveDir = c.rs.Cfg.BaseConfig().WorkingDir // where node_modules gets resolved + opts.TsConfig = c.rs.ResolveJSConfigFile("tsconfig.json") + + if err := opts.validate(); err != nil { return err } - opts.sourceDir = filepath.FromSlash(path.Dir(ctx.SourcePath)) - opts.resolveDir = t.c.rs.Cfg.BaseConfig().WorkingDir // where node_modules gets resolved - opts.contents = string(src) - opts.mediaType = ctx.InMediaType - opts.tsConfig = t.c.rs.ResolveJSConfigFile("tsconfig.json") - buildOptions, err := toBuildOptions(opts) if err != nil { return err } - buildOptions.Plugins, err = createBuildPlugins(ctx.DependencyManager, t.c, opts) + buildOptions.Plugins, err = createBuildPlugins(c, dependencyManager, opts) if err != nil { return err } @@ -113,7 +109,7 @@ func (t *buildTransformation) Transform(ctx *resources.ResourceTransformationCtx return fmt.Errorf("inject: absolute paths not supported, must be relative to /assets") } - m := resolveComponentInAssets(t.c.rs.Assets.Fs, impPath) + m := resolveComponentInAssets(c.rs.Assets.Fs, impPath) if m == nil { return fmt.Errorf("inject: file %q not found", ext) @@ -138,7 +134,7 @@ func (t *buildTransformation) Transform(ctx *resources.ResourceTransformationCtx } path := loc.File if path == stdinImporter { - path = ctx.SourcePath + path = transformCtx.SourcePath } errorMessage := msg.Text @@ -154,7 +150,7 @@ func (t *buildTransformation) Transform(ctx *resources.ResourceTransformationCtx f, err = hugofs.Os.Open(path) } else { var fi os.FileInfo - fi, err = t.c.sfs.Fs.Stat(path) + fi, err = c.sfs.Fs.Stat(path) if err == nil { m := fi.(hugofs.FileMetaInfo).Meta() path = m.Filename @@ -185,38 +181,37 @@ func (t *buildTransformation) Transform(ctx *resources.ResourceTransformationCtx // Return 1, log the rest. for i, err := range errors { if i > 0 { - t.c.rs.Logger.Errorf("js.Build failed: %s", err) + c.rs.Logger.Errorf("js.Build failed: %s", err) } } return errors[0] } - if buildOptions.Sourcemap == api.SourceMapExternal { - content := string(result.OutputFiles[1].Contents) - symPath := path.Base(ctx.OutPath) + ".map" - re := regexp.MustCompile(`//# sourceMappingURL=.*\n?`) - content = re.ReplaceAllString(content, "//# sourceMappingURL="+symPath+"\n") + if transformCtx != nil { + if buildOptions.Sourcemap == api.SourceMapExternal { + content := string(result.OutputFiles[1].Contents) + symPath := path.Base(transformCtx.OutPath) + ".map" + re := regexp.MustCompile(`//# sourceMappingURL=.*\n?`) + content = re.ReplaceAllString(content, "//# sourceMappingURL="+symPath+"\n") - if err = ctx.PublishSourceMap(string(result.OutputFiles[0].Contents)); err != nil { - return err - } - _, err := ctx.To.Write([]byte(content)) - if err != nil { - return err - } - } else { - _, err := ctx.To.Write(result.OutputFiles[0].Contents) - if err != nil { - return err + if err = transformCtx.PublishSourceMap(string(result.OutputFiles[0].Contents)); err != nil { + return err + } + _, err := transformCtx.To.Write([]byte(content)) + if err != nil { + return err + } + } else { + _, err := transformCtx.To.Write(result.OutputFiles[0].Contents) + if err != nil { + return err + } } + + return nil } - return nil -} -// Process process esbuild transform -func (c *Client) Process(res resources.ResourceTransformer, opts map[string]any) (resource.Resource, error) { - return res.Transform( - &buildTransformation{c: c, optsm: opts}, - ) + // TODO1 + return nil } diff --git a/resources/resource_transformers/js/options.go b/resources/resource_transformers/js/options.go index 8c271d032d7..7a8e83aeed7 100644 --- a/resources/resource_transformers/js/options.go +++ b/resources/resource_transformers/js/options.go @@ -39,8 +39,44 @@ const ( stdinImporter = "" ) -// Options esbuild configuration type Options struct { + ExternalOptions + InternalOptions +} + +func (opts *Options) validate() error { + if opts.ImportOnResolveFunc != nil && opts.ImportOnLoadFunc == nil { + return fmt.Errorf("ImportOnLoadFunc must be set if ImportOnResolveFunc is set") + } + if opts.ImportOnResolveFunc == nil && opts.ImportOnLoadFunc != nil { + return fmt.Errorf("ImportOnResolveFunc must be set if ImportOnLoadFunc is set") + } + return nil +} + +// InternalOptions holds internal options for the js.Build template function. +type InternalOptions struct { + MediaType media.Type + OutDir string + Contents string + SourceDir string + ResolveDir string + + DependencyManager identity.Manager + + // TODO1 + Write bool + AllowOverwrite bool + Splitting bool + TsConfig string + EntryPoints []string + ImportOnResolveFunc func(string) string + ImportOnLoadFunc func(string) string + Stdin bool +} + +// ExternalOptions holds user facing options for the js.Build template function. +type ExternalOptions struct { // If not set, the source path will be used as the base target path. // Note that the target path's extension may change if the target MIME type // is different, e.g. when the source is TypeScript. @@ -105,17 +141,10 @@ type Options struct { // Deprecated: This no longer have any effect and will be removed. // TODO(bep) remove. See https://github.com/evanw/esbuild/commit/869e8117b499ca1dbfc5b3021938a53ffe934dba AvoidTDZ bool - - mediaType media.Type - outDir string - contents string - sourceDir string - resolveDir string - tsConfig string } -func decodeOptions(m map[string]any) (Options, error) { - var opts Options +func decodeOptions(m map[string]any) (ExternalOptions, error) { + var opts ExternalOptions if err := mapstructure.WeakDecode(m, &opts); err != nil { return opts, err @@ -212,7 +241,7 @@ func resolveComponentInAssets(fs afero.Fs, impPath string) *hugofs.FileMeta { return m } -func createBuildPlugins(depsManager identity.Manager, c *Client, opts Options) ([]api.Plugin, error) { +func createBuildPlugins(c *Client, depsManager identity.Manager, opts Options) ([]api.Plugin, error) { fs := c.rs.Assets resolveImport := func(args api.OnResolveArgs) (api.OnResolveResult, error) { @@ -223,6 +252,13 @@ func createBuildPlugins(depsManager identity.Manager, c *Client, opts Options) ( impPath = override } } + + if opts.ImportOnResolveFunc != nil { + if s := opts.ImportOnResolveFunc(impPath); s != "" { + return api.OnResolveResult{Path: s, Namespace: nsImportHugo}, nil + } + } + isStdin := args.Importer == stdinImporter var relDir string if !isStdin { @@ -236,7 +272,7 @@ func createBuildPlugins(depsManager identity.Manager, c *Client, opts Options) ( relDir = filepath.Dir(rel) } else { - relDir = opts.sourceDir + relDir = opts.SourceDir } // Imports not starting with a "." is assumed to live relative to /assets. @@ -272,16 +308,26 @@ func createBuildPlugins(depsManager identity.Manager, c *Client, opts Options) ( }) build.OnLoad(api.OnLoadOptions{Filter: `.*`, Namespace: nsImportHugo}, func(args api.OnLoadArgs) (api.OnLoadResult, error) { - b, err := os.ReadFile(args.Path) - if err != nil { - return api.OnLoadResult{}, fmt.Errorf("failed to read %q: %w", args.Path, err) + var c string + if opts.ImportOnLoadFunc != nil { + if s := opts.ImportOnLoadFunc(args.Path); s != "" { + c = s + } + } + + if c == "" { + b, err := os.ReadFile(args.Path) + if err != nil { + return api.OnLoadResult{}, fmt.Errorf("failed to read %q: %w", args.Path, err) + } + c = string(b) } - c := string(b) + return api.OnLoadResult{ // See https://github.com/evanw/esbuild/issues/502 // This allows all modules to resolve dependencies // in the main project's node_modules. - ResolveDir: opts.resolveDir, + ResolveDir: opts.ResolveDir, Contents: &c, Loader: loaderFromFilename(args.Path), }, nil @@ -353,7 +399,7 @@ func toBuildOptions(opts Options) (buildOptions api.BuildOptions, err error) { return } - mediaType := opts.mediaType + mediaType := opts.MediaType if mediaType.IsZero() { mediaType = media.Builtin.JavascriptType } @@ -371,7 +417,7 @@ func toBuildOptions(opts Options) (buildOptions api.BuildOptions, err error) { case media.Builtin.JSXType.SubType: loader = api.LoaderJSX default: - err = fmt.Errorf("unsupported Media Type: %q", opts.mediaType) + err = fmt.Errorf("unsupported Media Type: %q", opts.MediaType) return } @@ -408,7 +454,7 @@ func toBuildOptions(opts Options) (buildOptions api.BuildOptions, err error) { } // By default we only need to specify outDir and no outFile - outDir := opts.outDir + outDir := opts.OutDir outFile := "" var sourceMap api.SourceMap switch opts.SourceMap { @@ -435,9 +481,12 @@ func toBuildOptions(opts Options) (buildOptions api.BuildOptions, err error) { MinifyIdentifiers: opts.Minify, MinifySyntax: opts.Minify, - Outdir: outDir, - Define: defines, + Outdir: outDir, + Write: opts.Write, + AllowOverwrite: opts.AllowOverwrite, + Splitting: opts.Splitting, + Define: defines, External: opts.Externals, JSXFactory: opts.JSXFactory, @@ -446,16 +495,18 @@ func toBuildOptions(opts Options) (buildOptions api.BuildOptions, err error) { JSX: jsx, JSXImportSource: opts.JSXImportSource, - Tsconfig: opts.tsConfig, + Tsconfig: opts.TsConfig, + + EntryPoints: opts.EntryPoints, + } - // Note: We're not passing Sourcefile to ESBuild. - // This makes ESBuild pass `stdin` as the Importer to the import - // resolver, which is what we need/expect. - Stdin: &api.StdinOptions{ - Contents: opts.contents, - ResolveDir: opts.resolveDir, + if opts.Stdin { + // This makes ESBuild pass `stdin` as the Importer to the import. + buildOptions.Stdin = &api.StdinOptions{ + Contents: opts.Contents, + ResolveDir: opts.ResolveDir, Loader: loader, - }, + } } return } diff --git a/resources/resource_transformers/js/options_test.go b/resources/resource_transformers/js/options_test.go index de20cbd05de..c210ee6a59e 100644 --- a/resources/resource_transformers/js/options_test.go +++ b/resources/resource_transformers/js/options_test.go @@ -50,7 +50,11 @@ func TestOptionKey(t *testing.T) { func TestToBuildOptions(t *testing.T) { c := qt.New(t) - opts, err := toBuildOptions(Options{mediaType: media.Builtin.JavascriptType}) + opts, err := toBuildOptions(Options{ + InternalOptions: InternalOptions{ + MediaType: media.Builtin.JavascriptType, + }, + }) c.Assert(err, qt.IsNil) c.Assert(opts, qt.DeepEquals, api.BuildOptions{ @@ -62,13 +66,19 @@ func TestToBuildOptions(t *testing.T) { }, }) - opts, err = toBuildOptions(Options{ - Target: "es2018", - Format: "cjs", - Minify: true, - mediaType: media.Builtin.JavascriptType, - AvoidTDZ: true, - }) + opts, err = toBuildOptions( + Options{ + ExternalOptions: ExternalOptions{ + Target: "es2018", + Format: "cjs", + Minify: true, + AvoidTDZ: true, + }, + InternalOptions: InternalOptions{ + MediaType: media.Builtin.JavascriptType, + }, + }, + ) c.Assert(err, qt.IsNil) c.Assert(opts, qt.DeepEquals, api.BuildOptions{ Bundle: true, @@ -82,10 +92,17 @@ func TestToBuildOptions(t *testing.T) { }, }) - opts, err = toBuildOptions(Options{ - Target: "es2018", Format: "cjs", Minify: true, mediaType: media.Builtin.JavascriptType, - SourceMap: "inline", - }) + opts, err = toBuildOptions( + Options{ + ExternalOptions: ExternalOptions{ + Target: "es2018", Format: "cjs", Minify: true, + SourceMap: "inline", + }, + InternalOptions: InternalOptions{ + MediaType: media.Builtin.JavascriptType, + }, + }, + ) c.Assert(err, qt.IsNil) c.Assert(opts, qt.DeepEquals, api.BuildOptions{ Bundle: true, @@ -100,10 +117,17 @@ func TestToBuildOptions(t *testing.T) { }, }) - opts, err = toBuildOptions(Options{ - Target: "es2018", Format: "cjs", Minify: true, mediaType: media.Builtin.JavascriptType, - SourceMap: "inline", - }) + opts, err = toBuildOptions( + Options{ + ExternalOptions: ExternalOptions{ + Target: "es2018", Format: "cjs", Minify: true, + SourceMap: "inline", + }, + InternalOptions: InternalOptions{ + MediaType: media.Builtin.JavascriptType, + }, + }, + ) c.Assert(err, qt.IsNil) c.Assert(opts, qt.DeepEquals, api.BuildOptions{ Bundle: true, @@ -118,10 +142,18 @@ func TestToBuildOptions(t *testing.T) { }, }) - opts, err = toBuildOptions(Options{ - Target: "es2018", Format: "cjs", Minify: true, mediaType: media.Builtin.JavascriptType, - SourceMap: "external", - }) + opts, err = toBuildOptions( + Options{ + ExternalOptions: ExternalOptions{ + Target: "es2018", Format: "cjs", Minify: true, + SourceMap: "external", + }, + InternalOptions: InternalOptions{ + MediaType: media.Builtin.JavascriptType, + }, + }, + ) + c.Assert(err, qt.IsNil) c.Assert(opts, qt.DeepEquals, api.BuildOptions{ Bundle: true, @@ -136,10 +168,17 @@ func TestToBuildOptions(t *testing.T) { }, }) - opts, err = toBuildOptions(Options{ - mediaType: media.Builtin.JavascriptType, - JSX: "automatic", JSXImportSource: "preact", - }) + opts, err = toBuildOptions( + Options{ + ExternalOptions: ExternalOptions{ + JSX: "automatic", JSXImportSource: "preact", + }, + InternalOptions: InternalOptions{ + MediaType: media.Builtin.JavascriptType, + }, + }, + ) + c.Assert(err, qt.IsNil) c.Assert(opts, qt.DeepEquals, api.BuildOptions{ Bundle: true, @@ -173,10 +212,17 @@ func TestToBuildOptionsTarget(t *testing.T) { {"esnext", api.ESNext}, } { c.Run(test.target, func(c *qt.C) { - opts, err := toBuildOptions(Options{ - Target: test.target, - mediaType: media.Builtin.JavascriptType, - }) + opts, err := toBuildOptions( + Options{ + ExternalOptions: ExternalOptions{ + Target: test.target, + }, + InternalOptions: InternalOptions{ + MediaType: media.Builtin.JavascriptType, + }, + }, + ) + c.Assert(err, qt.IsNil) c.Assert(opts.Target, qt.Equals, test.expect) }) diff --git a/resources/resource_transformers/js/transform.go b/resources/resource_transformers/js/transform.go new file mode 100644 index 00000000000..339d01b82d6 --- /dev/null +++ b/resources/resource_transformers/js/transform.go @@ -0,0 +1,69 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package js + +import ( + "io" + "path" + "path/filepath" + + "github.com/gohugoio/hugo/media" + "github.com/gohugoio/hugo/resources" + "github.com/gohugoio/hugo/resources/internal" +) + +type buildTransformation struct { + optsm map[string]any + opts Options + c *Client +} + +func (t *buildTransformation) Key() internal.ResourceTransformationKey { + // Pick the most stable key source. + var v any = t.optsm + if v == nil { + v = t.opts + } + return internal.NewResourceTransformationKey("jsbuild", v) +} + +func (t *buildTransformation) Transform(ctx *resources.ResourceTransformationCtx) error { + ctx.OutMediaType = media.Builtin.JavascriptType + + if t.optsm != nil { + optsExt, err := decodeOptions(t.optsm) + if err != nil { + return err + } + t.opts.ExternalOptions = optsExt + } + + if t.opts.TargetPath != "" { + ctx.OutPath = t.opts.TargetPath + } else { + ctx.ReplaceOutPathExtension(".js") + } + + src, err := io.ReadAll(ctx.From) + if err != nil { + return err + } + + t.opts.SourceDir = filepath.FromSlash(path.Dir(ctx.SourcePath)) + t.opts.Contents = string(src) + t.opts.MediaType = ctx.InMediaType + t.opts.Stdin = true + + return t.c.build(t.opts, ctx) +} diff --git a/tpl/internal/go_templates/texttemplate/exec.go b/tpl/internal/go_templates/texttemplate/exec.go index 73153c7648d..8305457a580 100644 --- a/tpl/internal/go_templates/texttemplate/exec.go +++ b/tpl/internal/go_templates/texttemplate/exec.go @@ -306,7 +306,10 @@ func (s *state) walkIfOrWith(typ parse.NodeType, dot reflect.Value, pipe *parse. } if truth { if typ == parse.NodeWith { - s.walk(val, list) + func() { + defer s.pushWithValue(val)() + s.walk(val, list) + }() } else { s.walk(dot, list) } diff --git a/tpl/internal/go_templates/texttemplate/hugo_template.go b/tpl/internal/go_templates/texttemplate/hugo_template.go index 276367a7c38..15ac1a4bb63 100644 --- a/tpl/internal/go_templates/texttemplate/hugo_template.go +++ b/tpl/internal/go_templates/texttemplate/hugo_template.go @@ -19,6 +19,7 @@ import ( "reflect" "github.com/gohugoio/hugo/common/hreflect" + "github.com/gohugoio/hugo/common/types" "github.com/gohugoio/hugo/tpl/internal/go_templates/texttemplate/parse" ) @@ -110,14 +111,31 @@ func (t *Template) executeWithState(state *state, value reflect.Value) (err erro // template so that multiple executions of the same template // can execute in parallel. type state struct { - tmpl *Template - ctx context.Context // Added for Hugo. The original data context. - prep Preparer // Added for Hugo. - helper ExecHelper // Added for Hugo. - wr io.Writer - node parse.Node // current node, for errors - vars []variable // push-down stack of variable values. - depth int // the height of the stack of executing templates. + tmpl *Template + ctx context.Context // Added for Hugo. The original data context. + prep Preparer // Added for Hugo. + helper ExecHelper // Added for Hugo. + withValues []reflect.Value // Added for Hugo. Push-down stack of values. + + wr io.Writer + node parse.Node // current node, for errors + vars []variable // push-down stack of variable values. + depth int // the height of the stack of executing templates. +} + +func (s *state) pushWithValue(value reflect.Value) func() { + s.withValues = append(s.withValues, value) + return func() { + // TODO1 integrate with GO 1.23. + v, _ := indirect(s.withValues[len(s.withValues)-1]) + if hreflect.IsValid(v) { + if closer, ok := v.Interface().(types.Closer); ok { + closer.Close() + } + } + + s.withValues = s.withValues[:len(s.withValues)-1] + } } func (s *state) evalFunction(dot reflect.Value, node *parse.IdentifierNode, cmd parse.Node, args []parse.Node, final reflect.Value) reflect.Value { diff --git a/tpl/js/batch-esm-callback.gotmpl b/tpl/js/batch-esm-callback.gotmpl new file mode 100644 index 00000000000..aa198719a8b --- /dev/null +++ b/tpl/js/batch-esm-callback.gotmpl @@ -0,0 +1,15 @@ +{{ range $i, $e := .Modules -}} + import { default as {{ printf "Mod%d" $i }} } from "{{ .ImportPath }}"; +{{ end -}} +{{ with .CallbackImportPath }} + import { default as Callback } from "{{ . }}"; +{{ end }} +{{/* */}} +let mods = []; +{{ range $i, $e := .Modules -}} + mods.push({{ .CallbackJSON $i }}); +{{ end -}} +{{/* */}} +{{ if .CallbackImportPath }} + Callback(mods); +{{ end }} diff --git a/tpl/js/batch.go b/tpl/js/batch.go new file mode 100644 index 00000000000..a73e9173688 --- /dev/null +++ b/tpl/js/batch.go @@ -0,0 +1,542 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package js provides functions for building JavaScript resources +package js + +import ( + "bytes" + "context" + _ "embed" + "encoding/json" + "fmt" + "path" + "path/filepath" + "sort" + "strings" + "sync" + + "github.com/gohugoio/hugo/common/herrors" + "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/common/paths" + "github.com/gohugoio/hugo/common/types" + "github.com/gohugoio/hugo/media" + "github.com/gohugoio/hugo/resources/resource" + "github.com/gohugoio/hugo/resources/resource_transformers/js" + template "github.com/gohugoio/hugo/tpl/internal/go_templates/texttemplate" + "github.com/mitchellh/mapstructure" + "github.com/spf13/cast" +) + +func (ns *Namespace) Bundle(ctx context.Context, id string, store *maps.Scratch) (Bundler, error) { + key := path.Join(nsBatch, id) + b := store.GetOrCreate(key, func() any { + return &bundler{id: id, scriptOnes: make(map[string]*scriptOne), scriptManys: make(map[string]*scriptMany), client: ns} + }) + return b.(*bundler), nil +} + +func (b *bundler) UseScriptOne(id string) BundleScriptOne { + b.mu.Lock() + + one, found := b.scriptOnes[id] + if !found { + one = &scriptOne{id: id, client: b.client} + b.scriptOnes[id] = one + } + + b.mu.Unlock() + one.mu.Lock() + + // This will be auto closed if used in a with statement. + // But the caller may also call Close, so make sure we only do it once. + var closeOnce sync.Once + + return struct { + BundleScriptOneOps + types.Closer + }{ + one, + close(func() error { + closeOnce.Do(func() { + one.mu.Unlock() + }) + return nil + }), + } +} + +func (b *bundler) UseScriptMany(id string) BundleScriptMany { + b.mu.Lock() + + many, found := b.scriptManys[id] + if !found { + many = &scriptMany{id: id, client: b.client, items: make(map[string]*scriptManyItem)} + b.scriptManys[id] = many + } + + b.mu.Unlock() + many.mu.Lock() + + // This will be auto closed if used in a with statement. + // But the caller may also call Close, so make sure we only do it once. + var closeOnce sync.Once + + return struct { + BundleScriptManyOps + types.Closer + }{ + many, + close(func() error { + closeOnce.Do(func() { + many.mu.Unlock() + }) + return nil + }), + } +} + +type Bundler interface { + UseScriptOne(id string) BundleScriptOne + UseScriptMany(id string) BundleScriptMany + Build(opts ...any) (map[string]resource.Resource, error) +} + +type BundleScriptOne interface { + BundleScriptOneOps + types.Closer +} + +type BundleScriptOneOps interface { + ResourceGetSetter + SetInstance(opts any) string +} + +type BundleScriptMany interface { + BundleScriptManyOps + types.Closer +} + +type BundleScriptItem interface { + BundleScriptItemOps + types.Closer +} + +type BundleScriptItemOps interface { + ResourceGetSetter + AddInstance(id string, opts any) string +} + +type BundleScriptManyOps interface { + GetCallback() resource.Resource + SetCallback(r resource.Resource) string + UseItem(id string) BundleScriptItem +} + +type ScriptItem interface{} + +type BundleCommonScriptOps interface { + ResourceGetSetter +} + +type ResourceGetSetter interface { + GetResource() resource.Resource + SetResource(r resource.Resource) string +} + +type close func() error + +func (c close) Close() error { + return c() +} + +var ( + _ Bundler = (*bundler)(nil) + _ BundleScriptOneOps = (*scriptOne)(nil) + _ BundleScriptManyOps = (*scriptMany)(nil) +) + +func (b *scriptOne) SetInstance(opts any) string { + /*if b.r == nil { + panic("resource not set") + } + if id == "" { + panic("id not set") + } + + paramsm := cast.ToStringMap(params) + b.instances[id] = &batchInstance{params: paramsm} + */ // TODO1 + + return "" +} + +func (b *scriptOne) GetResource() resource.Resource { + return b.r +} + +func (b *scriptOne) SetResource(r resource.Resource) string { + b.r = r + return "" +} + +func (b *scriptManyItem) GetResource() resource.Resource { + return b.r +} + +func (b *scriptManyItem) SetResource(r resource.Resource) string { + b.r = r + return "" +} + +func decodeScriptInstance(opts any) scriptInstance { + var inst scriptInstance + if err := mapstructure.WeakDecode(opts, &inst); err != nil { + panic(err) + } + return inst +} + +func (b *scriptManyItem) AddInstance(id string, opts any) string { + b.instances[id] = decodeScriptInstance(opts) + return "" +} + +func (b *scriptMany) GetCallback() resource.Resource { + return b.callback +} + +func (b *scriptMany) SetCallback(r resource.Resource) string { + b.callback = r + return "" +} + +func (b *scriptMany) UseItem(id string) BundleScriptItem { + item, found := b.items[id] + if !found { + item = &scriptManyItem{id: id, instances: make(map[string]scriptInstance), client: b.client} + b.items[id] = item + } + + item.mu.Lock() + + // This will be auto closed if used in a with statement. + // But the caller may also call Close, so make sure we only do it once. + var closeOnce sync.Once + + return struct { + BundleScriptItemOps + types.Closer + }{ + item, + close(func() error { + closeOnce.Do(func() { + item.mu.Unlock() + }) + return nil + }), + } +} + +type batchTemplateContext struct { + keyPath string + ID string + CallbackImportPath string + Modules []batchTemplateExecutionsContext +} + +type batchTemplateExecutionsContext struct { + ID string `json:"id"` + ImportPath string `json:"importPath"` + Instances []batchTemplateExecution `json:"instances"` + + r resource.Resource +} + +func (b batchTemplateExecutionsContext) CallbackJSON(i int) string { + mod := fmt.Sprintf("Mod%d", i) + + v := struct { + Mod string `json:"mod"` + batchTemplateExecutionsContext + }{ + mod, + b, + } + + bb, err := json.Marshal(v) + if err != nil { + panic(err) + } + s := string(bb) + + s = strings.ReplaceAll(s, fmt.Sprintf("%q", mod), mod) + + return s +} + +type batchTemplateExecution struct { + ID string `json:"id"` + Params any `json:"params"` +} + +type batchBuildOpts struct { + Callback resource.Resource + js.ExternalOptions `mapstructure:",squash"` +} + +type scriptsOne struct { + mu sync.Mutex + + id string + batches map[string]*scriptOne + + client *Namespace +} + +type scriptsMany struct { + mu sync.Mutex + + id string + batches map[string]*scriptMany + + client *Namespace +} + +type scriptOne struct { + mu sync.Mutex + id string + r resource.Resource + + client *Namespace +} + +type scriptInstance struct { + Params map[string]any +} + +type scriptManyItem struct { + mu sync.Mutex + id string + + r resource.Resource + instances map[string]scriptInstance + + client *Namespace +} + +type scriptMany struct { + mu sync.Mutex + id string + callback resource.Resource + + items map[string]*scriptManyItem + + client *Namespace +} + +type bundler struct { + mu sync.Mutex + id string + scriptOnes map[string]*scriptOne + scriptManys map[string]*scriptMany + + client *Namespace +} + +func (b *bundler) Build(opts ...any) (map[string]resource.Resource, error) { + defer herrors.Recover() // TODO1 + b.mu.Lock() + defer b.mu.Unlock() + + if len(opts) > 0 { + panic("not implemented") + } + + keyPath := b.id + + idResource := make(map[string]resource.Resource) + impMap := make(map[string]resource.Resource) + var entryPoints []string + addEntryPoint := func(id, s string, r resource.Resource) { + impMap[s] = r + entryPoints = append(entryPoints, s) + idResource[id] = bundleResource(s) + } + + if len(b.scriptOnes) > 0 { + for k, v := range b.scriptOnes { + if v.r == nil { + return nil, fmt.Errorf("resource not set for %q", k) + } + keyPath := path.Join(keyPath, k) + resourcePath := paths.AddLeadingSlash(keyPath + v.r.MediaType().FirstSuffix.FullSuffix) + addEntryPoint(k, resourcePath, v.r) + + } + } + + if len(b.scriptManys) > 0 { + for k, v := range b.scriptManys { + keyPath := path.Join(keyPath, k) + bopts := batchBuildOpts{ + Callback: v.callback, + } + var callbackImpPath string + if bopts.Callback != nil { + callbackImpPath = paths.AddLeadingSlash(keyPath + "_callback" + bopts.Callback.MediaType().FirstSuffix.FullSuffix) + addEntryPoint(k, callbackImpPath, bopts.Callback) + } + + t := &batchTemplateContext{ + keyPath: keyPath, + ID: v.id, + CallbackImportPath: callbackImpPath, + } + + for kk, vv := range v.items { + keyPath := path.Join(keyPath, kk) + bt := batchTemplateExecutionsContext{ + ID: kk, + r: vv.r, + ImportPath: keyPath + vv.r.MediaType().FirstSuffix.FullSuffix, + } + impMap[bt.ImportPath] = vv.r + for kkk, vvv := range vv.instances { + bt.Instances = append(bt.Instances, batchTemplateExecution{ID: kkk, Params: vvv.Params}) + sort.Slice(bt.Instances, func(i, j int) bool { + return bt.Instances[i].ID < bt.Instances[j].ID + }) + } + t.Modules = append(t.Modules, bt) + } + sort.Slice(t.Modules, func(i, j int) bool { + return t.Modules[i].ID < t.Modules[j].ID + }) + + r, s, err := b.client.buildBatch(t) + if err != nil { + return nil, err + } + addEntryPoint(v.id, s, r) + } + } + + target := "es2018" + + outDir := filepath.Join(b.client.d.Paths.AbsPublishDir, "js", "bundles", b.id) + + jopts := js.Options{ + ExternalOptions: js.ExternalOptions{ + Format: "esm", + Target: target, + Defines: map[string]any{ + //"process.env.NODE_ENV": `"development"`, + }, + }, + InternalOptions: js.InternalOptions{ + OutDir: outDir, + // TODO1 maybe not. + Write: true, + AllowOverwrite: true, + Splitting: true, + ImportOnResolveFunc: func(imp string) string { + if _, found := impMap[imp]; found { + return imp + } + return "" + }, + ImportOnLoadFunc: func(imp string) string { + if r, found := impMap[imp]; found { + content, err := r.(resource.ContentProvider).Content(context.Background()) // TODO1 + if err != nil { + panic(err) + } + return cast.ToString(content) + } + + return "" + }, + EntryPoints: entryPoints, + }, + } + + if err := b.client.client.BuildBundle(jopts); err != nil { + return nil, err + } + + return idResource, nil +} + +type bundleResource string + +func (b bundleResource) Name() string { + return path.Base(string(b)) +} + +func (b bundleResource) Title() string { + return b.Name() +} + +func (b bundleResource) RelPermalink() string { + return "/js/bundles" + string(b) +} + +func (b bundleResource) Permalink() string { + panic("not implemented") +} + +func (b bundleResource) ResourceType() string { + panic("not implemented") +} + +func (b bundleResource) MediaType() media.Type { + panic("not implemented") +} + +func (b bundleResource) Data() any { + panic("not implemented") +} + +func (b bundleResource) Err() resource.ResourceError { + return nil +} + +func (b bundleResource) Params() maps.Params { + panic("not implemented") +} + +const nsBatch = "/__hugo-js-batch" + +func (ns *Namespace) buildBatch(t *batchTemplateContext) (resource.Resource, string, error) { + var buf bytes.Buffer + if err := batchEsmCallbackTemplate.Execute(&buf, t); err != nil { + return nil, "", err + } + + s := paths.AddLeadingSlash(t.keyPath + ".js") + r, err := ns.createClient.FromString(s, buf.String()) + if err != nil { + return nil, "", err + } + + return r, s, nil +} + +//go:embed batch-esm-callback.gotmpl +var batchEsmCallbackTemplateString string +var batchEsmCallbackTemplate *template.Template + +func init() { + batchEsmCallbackTemplate = template.Must(template.New("batch-esm-callback").Parse(batchEsmCallbackTemplateString)) +} diff --git a/tpl/js/batch_integration_test.go b/tpl/js/batch_integration_test.go new file mode 100644 index 00000000000..e4b07bb9101 --- /dev/null +++ b/tpl/js/batch_integration_test.go @@ -0,0 +1,145 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package js provides functions for building JavaScript resources +package js_test + +import ( + "testing" + + "github.com/gohugoio/hugo/hugolib" +) + +func TestBatch(t *testing.T) { + files := ` +-- hugo.toml -- +disableKinds = ["taxonomy", "term", "page"] +baseURL = "https://example.com" +-- package.json -- +{ + "devDependencies": { + "react": "^18.3.1", + "react-dom": "^18.3.1" + } +} +-- assets/js/reactcallback.js -- +import * as ReactDOM from 'react-dom/client'; +import * as React from 'react'; + +export default function Callback(modules) { + for (const module of modules) { + for (const instance of module.instances) { + /* This is a convention in this project. */ + let elId = §§${module.id}-${instance.id}§§; + let el = document.getElementById(elId); + if (!el) { + console.warn(§§Element with id ${elId} not found§§); + continue; + } + const root = ReactDOM.createRoot(el); + const reactEl = React.createElement(module.mod, instance.params); + root.render(reactEl); + } + } +} +-- assets/js/react1.jsx -- +import * as React from "react"; + +window.React1 = React; + +let text = 'Click me' + +export default function MyButton() { + return ( + + ) +} +-- assets/js/react2.jsx -- +import * as React from "react"; + +window.React2 = React; + +let text = 'Click me, too!' + +export default function MyOtherButton() { + return ( + + ) +} +-- assets/js/main1.js -- +import * as React from "react"; + +console.log('main1.React', React) + +-- assets/js/main2.js -- +import * as React from "react"; + +console.log('main2.React', React) + +-- layouts/index.html -- +Home. +{{ $bundle := (js.Bundle "mybundle" .Store) }} +{{ with $bundle.UseScriptOne "main1" }} + {{ if not .GetResource }} + {{ .SetResource (resources.Get "js/main1.js") }} + {{ end }} + {{ .SetInstance (dict "title" "Main1 Instance") }} +{{ end }} + {{ with $bundle.UseScriptOne "main2" }} + {{ if not .GetResource }} + {{ .SetResource (resources.Get "js/main2.js") }} + {{ end }} + {{ .SetInstance (dict "title" "Main2 Instance") }} +{{ end }} +{{ with $bundle.UseScriptMany "reactbatch" }} + {{ if not .GetCallback }} + {{ .SetCallback (resources.Get "js/reactcallback.js") }} + {{ end }} + {{ with .UseItem "r1" }} + {{ if not .GetResource }} + {{ .SetResource (resources.Get "js/react1.jsx") }} + {{ end }} + {{ .AddInstance "i1" (dict "title" "Instance 1") }} + {{ .AddInstance "i2" (dict "title" "Instance 2") }} + {{ end }} + {{ with .UseItem "r2" }} + {{ if not .GetResource }} + {{ .SetResource (resources.Get "js/react2.jsx") }} + {{ end }} + {{ .AddInstance "i1" (dict "title" "Instance 2-1") }} + {{ end }} +{{ end }} +{{ range $k, $v := $bundle.Build }} +{{ $k }}: {{ .RelPermalink }} +{{ end }}} +` + + b := hugolib.NewIntegrationTestBuilder( + hugolib.IntegrationTestConfig{ + T: t, + NeedsOsFS: true, + NeedsNpmInstall: true, + TxtarString: files, + // PrintAndKeepTempDir: true, + }).Build() + + b.AssertFileContent("public/index.html", ` +main1: /js/bundles/mybundle/main1.js +main2: /js/bundles/mybundle/main2.js +reactbatch: /js/bundles/mybundle/reactbatch.js + + + `) +} + +// TODO1 make instance into a map with params as only key (for now) diff --git a/tpl/js/js.go b/tpl/js/js.go index c68e0af9272..82332d5fc08 100644 --- a/tpl/js/js.go +++ b/tpl/js/js.go @@ -20,6 +20,7 @@ import ( "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/resources" "github.com/gohugoio/hugo/resources/resource" + "github.com/gohugoio/hugo/resources/resource_factories/create" "github.com/gohugoio/hugo/resources/resource_transformers/babel" "github.com/gohugoio/hugo/resources/resource_transformers/js" "github.com/gohugoio/hugo/tpl/internal/resourcehelpers" @@ -31,15 +32,19 @@ func New(deps *deps.Deps) *Namespace { return &Namespace{} } return &Namespace{ - client: js.New(deps.BaseFs.Assets, deps.ResourceSpec), - babelClient: babel.New(deps.ResourceSpec), + d: deps, + client: js.New(deps.BaseFs.Assets, deps.ResourceSpec), + createClient: create.New(deps.ResourceSpec), + babelClient: babel.New(deps.ResourceSpec), } } // Namespace provides template functions for the "js" namespace. type Namespace struct { - client *js.Client - babelClient *babel.Client + d *deps.Deps + client *js.Client + createClient *create.Client + babelClient *babel.Client } // Build processes the given Resource with ESBuild. @@ -65,7 +70,7 @@ func (ns *Namespace) Build(args ...any) (resource.Resource, error) { m = map[string]any{"targetPath": targetPath} } - return ns.client.Process(r, m) + return ns.client.ProcessExernal(r, m) } // Babel processes the given Resource with Babel.