From 0a744ac08d0619c12eaefefc87bbc575ddd2d35b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Fri, 6 Sep 2019 14:11:55 +0200 Subject: [PATCH] Pages from data Fixes #6310 --- go.mod | 2 + go.sum | 4 ++ hugofs/files/classifier.go | 5 ++ hugolib/page__meta.go | 16 ++--- hugolib/pages_capture.go | 38 +++++++++++ hugolib/pages_from_data_test.go | 36 ++++++++++ plugins/content/eval.go | 113 ++++++++++++++++++++++++++++++++ plugins/content/eval_test.go | 39 +++++++++++ plugins/content/interface.go | 45 +++++++++++++ 9 files changed, 288 insertions(+), 10 deletions(-) create mode 100644 hugolib/pages_from_data_test.go create mode 100644 plugins/content/eval.go create mode 100644 plugins/content/eval_test.go create mode 100644 plugins/content/interface.go diff --git a/go.mod b/go.mod index 55f5c4b7329..92c2655bfca 100644 --- a/go.mod +++ b/go.mod @@ -51,9 +51,11 @@ require ( github.com/spf13/jwalterweatherman v1.1.0 github.com/spf13/pflag v1.0.3 github.com/spf13/viper v1.4.0 + github.com/starlight-go/starlight v0.0.0-20181207205707-b06f321544f3 github.com/tdewolff/minify/v2 v2.3.7 github.com/yosssi/ace v0.0.5 go.opencensus.io v0.22.0 // indirect + go.starlark.net v0.0.0-20190820173200-988906f77f65 gocloud.dev v0.15.0 golang.org/x/image v0.0.0-20190523035834-f03afa92d3ff golang.org/x/net v0.0.0-20190606173856-1492cefac77f // indirect diff --git a/go.sum b/go.sum index 5285442e37e..4a47623e4bb 100644 --- a/go.sum +++ b/go.sum @@ -297,6 +297,8 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/starlight-go/starlight v0.0.0-20181207205707-b06f321544f3 h1:/fBh1Ot84ILt/ociFHO98wJ9LxIMA3UG8B0unUJPFpY= +github.com/starlight-go/starlight v0.0.0-20181207205707-b06f321544f3/go.mod h1:pxOc2ZuBV+CNlQgzq/HJ9Z9G/eoEMHFeuGohOvva4Co= github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -334,6 +336,8 @@ go.opencensus.io v0.21.0 h1:mU6zScU4U1YAFPHEHYk+3JC4SY7JxgkqS10ZOSyksNg= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.starlark.net v0.0.0-20190820173200-988906f77f65 h1:0766L84ADcyJQKl+NsKSJC8JBEuer/2RxL37StWfsx4= +go.starlark.net v0.0.0-20190820173200-988906f77f65/go.mod h1:c1/X6cHgvdXj6pUlmWKMkuqRnW4K8x2vwt6JAaaircg= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= diff --git a/hugofs/files/classifier.go b/hugofs/files/classifier.go index 9aa2476b7bf..a7d0c3cbf6c 100644 --- a/hugofs/files/classifier.go +++ b/hugofs/files/classifier.go @@ -52,11 +52,16 @@ func IsContentExt(ext string) bool { const ( ContentClassLeaf = "leaf" ContentClassBranch = "branch" + ContentClassData = "data" ContentClassFile = "zfile" // Sort below ContentClassContent = "zcontent" ) func ClassifyContentFile(filename string) string { + if strings.HasPrefix(filename, "_content.") { + return ContentClassData + } + if !IsContentFile(filename) { return ContentClassFile } diff --git a/hugolib/page__meta.go b/hugolib/page__meta.go index e8ef13bfde2..14e7f9a350e 100644 --- a/hugolib/page__meta.go +++ b/hugolib/page__meta.go @@ -258,19 +258,15 @@ func (p *pageMeta) Section() string { return "" } - if p.IsNode() { - if len(p.sections) == 0 { - // May be a sitemap or similar. - return "" - } - return p.sections[0] - } - - if !p.File().IsZero() { + if p.IsPage() && !p.File().IsZero() { return p.File().Section() } - panic("invalid page state") + if len(p.sections) == 0 { + // May be a sitemap or similar. + return "" + } + return p.sections[0] } diff --git a/hugolib/pages_capture.go b/hugolib/pages_capture.go index 591b8e31755..d47d3e634bb 100644 --- a/hugolib/pages_capture.go +++ b/hugolib/pages_capture.go @@ -16,11 +16,16 @@ package hugolib import ( "context" "fmt" + "io" "os" pth "path" "path/filepath" "strings" + "github.com/gohugoio/hugo/resources/page" + + yaml "gopkg.in/yaml.v2" + "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/hugofs/files" @@ -631,6 +636,37 @@ func (proc *pagesProcessor) newPageFromBundle(b *fileinfoBundle) (*pageState, er return p, nil } +func (proc *pagesProcessor) newPagesFromData(fim hugofs.FileMetaInfo, send func(p *pageState, err error)) { + + meta := fim.Meta() + f, err := meta.Open() + if err != nil { + send(nil, err) + return + } + defer f.Close() + + s := proc.getSite(meta.Lang()) + + dec := yaml.NewDecoder(f) + for { + m := make(map[string]interface{}) + if err := dec.Decode(m); err != nil { + if err == io.EOF { + break + } + send(nil, err) + return + } + + send(newPageFromMeta(m, &pageMeta{ + kind: page.KindPage, + s: s, + })) + + } +} + func (proc *pagesProcessor) newPageFromFi(fim hugofs.FileMetaInfo, owner *pageState) (*pageState, error) { fi, err := newFileInfo(proc.sp, fim) if err != nil { @@ -753,6 +789,8 @@ func (proc *pagesProcessor) process(item interface{}) error { send(proc.newPageFromFi(v, nil)) case files.ContentClassFile: proc.sendError(proc.copyFile(v)) + case files.ContentClassData: + proc.newPagesFromData(v, send) default: panic(fmt.Sprintf("invalid classifier: %q", classifier)) } diff --git a/hugolib/pages_from_data_test.go b/hugolib/pages_from_data_test.go new file mode 100644 index 00000000000..60f97c3df3f --- /dev/null +++ b/hugolib/pages_from_data_test.go @@ -0,0 +1,36 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "testing" + + qt "github.com/frankban/quicktest" +) + +func TestPagesFromYAML(t *testing.T) { + b := newTestSitesBuilder(t) + + b.WithContent("_content.yaml", ` +title: Yaml Page 1 +--- +title: Yaml Page 2 +`) + + b.Build(BuildCfg{}) + + s := b.H.Sites[0] + + b.Assert(s.RegularPages(), qt.HasLen, 2) +} diff --git a/plugins/content/eval.go b/plugins/content/eval.go new file mode 100644 index 00000000000..0d056ba2c32 --- /dev/null +++ b/plugins/content/eval.go @@ -0,0 +1,113 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package content + +import ( + "io" + + "github.com/spf13/cast" + + "github.com/gohugoio/hugo/helpers" + "go.starlark.net/starlark" + + "github.com/starlight-go/starlight" + "github.com/starlight-go/starlight/convert" +) + +type sourcePluginFiles struct { + FilenamesProvider + SourcePlugin +} + +type sourcePluginStream struct { + StreamProvider + SourcePlugin +} + +type sourcePluginFunc func(item map[string]interface{}) Bundle + +func (f sourcePluginFunc) ToBundle(item map[string]interface{}) Bundle { + return f(item) +} + +type getFilenamesFunc func() []string + +func (f getFilenamesFunc) GetFilenames() []string { + return f() +} + +var globals = make(map[string]interface{}) +var thread = &starlark.Thread{} // TODO1 + +func EvalSourcePlugin(r io.Reader) (SourcePlugin, error) { + out, err := starlight.Eval(helpers.ReaderToBytes(r), globals, nil) + if err != nil { + return nil, err + } + + return toSourcePlugin(out) +} + +func toSourcePlugin(out map[string]interface{}) (SourcePlugin, error) { + if fn, found := out["GetFilenames"]; found { + gf := getFilenamesFunc( + func() []string { + return cast.ToStringSlice(starlarkCall(thread, fn)) + }, + ) + + return sourcePluginFiles{ + FilenamesProvider: gf, + SourcePlugin: getToBundleFunc(out), + }, nil + } + + return nil, nil +} + +func getToBundleFunc(out map[string]interface{}) sourcePluginFunc { + if fn, found := out["ToBundle"]; found { + return func(item map[string]interface{}) Bundle { + return starlarkCall(thread, fn, item).(Bundle) + } + } + + return func(item map[string]interface{}) Bundle { + return Bundle{} + } +} + +func starlarkCall(thread *starlark.Thread, fn interface{}, args ...interface{}) interface{} { + argsv := make(starlark.Tuple, len(args)) + for i, arg := range args { + argv, err := convert.ToValue(arg) + if err != nil { + panic(err) // TODO1 + } + argsv[i] = argv + } + v, err := starlark.Call(thread, fn.(*starlark.Function), argsv, nil) + if err != nil { + panic(err) + } + + return fromValue(v) +} + +func fromValue(v starlark.Value) interface{} { + switch v := v.(type) { + default: + return convert.FromValue(v) + } +} diff --git a/plugins/content/eval_test.go b/plugins/content/eval_test.go new file mode 100644 index 00000000000..9f8770e25b5 --- /dev/null +++ b/plugins/content/eval_test.go @@ -0,0 +1,39 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package content + +import ( + "strings" + "testing" + + qt "github.com/frankban/quicktest" +) + +func TestEvalSourcePlugin(t *testing.T) { + c := qt.New(t) + + pluginFiles := ` + +def GetFilenames(): + return ["file1.json", "file2.json"] + +` + + plugin, err := EvalSourcePlugin(strings.NewReader(pluginFiles)) + c.Assert(err, qt.IsNil) + source, ok := plugin.(SourcePluginFiles) + c.Assert(ok, qt.Equals, true) + c.Assert(source.GetFilenames(), qt.DeepEquals, []string{"file1.json", "file2.json"}) + +} diff --git a/plugins/content/interface.go b/plugins/content/interface.go new file mode 100644 index 00000000000..69965c433e5 --- /dev/null +++ b/plugins/content/interface.go @@ -0,0 +1,45 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package content + +import "io" + +type FilenamesProvider interface { + GetFilenames() []string +} + +type StreamProvider interface { + GetStream() ReadCloserProvider +} + +type SourcePluginFiles interface { + FilenamesProvider + SourcePlugin +} + +type SourcePluginStream interface { + GetStream() ReadCloserProvider + SourcePlugin +} + +type SourcePlugin interface { + ToBundle(item map[string]interface{}) Bundle +} + +type ReadCloserProvider interface { + OpenReadCloser() (io.ReadCloser, error) +} + +type Bundle struct { +}