Skip to content

Commit

Permalink
Pages from data
Browse files Browse the repository at this point in the history
  • Loading branch information
bep committed Sep 9, 2019
1 parent 77b23fe commit 0a744ac
Show file tree
Hide file tree
Showing 9 changed files with 288 additions and 10 deletions.
2 changes: 2 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,11 @@ require (
github.com/spf13/jwalterweatherman v1.1.0
github.com/spf13/pflag v1.0.3
github.com/spf13/viper v1.4.0
github.com/starlight-go/starlight v0.0.0-20181207205707-b06f321544f3
github.com/tdewolff/minify/v2 v2.3.7
github.com/yosssi/ace v0.0.5
go.opencensus.io v0.22.0 // indirect
go.starlark.net v0.0.0-20190820173200-988906f77f65
gocloud.dev v0.15.0
golang.org/x/image v0.0.0-20190523035834-f03afa92d3ff
golang.org/x/net v0.0.0-20190606173856-1492cefac77f // indirect
Expand Down
4 changes: 4 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -297,6 +297,8 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU=
github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
github.com/starlight-go/starlight v0.0.0-20181207205707-b06f321544f3 h1:/fBh1Ot84ILt/ociFHO98wJ9LxIMA3UG8B0unUJPFpY=
github.com/starlight-go/starlight v0.0.0-20181207205707-b06f321544f3/go.mod h1:pxOc2ZuBV+CNlQgzq/HJ9Z9G/eoEMHFeuGohOvva4Co=
github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
Expand Down Expand Up @@ -334,6 +336,8 @@ go.opencensus.io v0.21.0 h1:mU6zScU4U1YAFPHEHYk+3JC4SY7JxgkqS10ZOSyksNg=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4=
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
go.starlark.net v0.0.0-20190820173200-988906f77f65 h1:0766L84ADcyJQKl+NsKSJC8JBEuer/2RxL37StWfsx4=
go.starlark.net v0.0.0-20190820173200-988906f77f65/go.mod h1:c1/X6cHgvdXj6pUlmWKMkuqRnW4K8x2vwt6JAaaircg=
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
Expand Down
5 changes: 5 additions & 0 deletions hugofs/files/classifier.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,11 +52,16 @@ func IsContentExt(ext string) bool {
const (
ContentClassLeaf = "leaf"
ContentClassBranch = "branch"
ContentClassData = "data"
ContentClassFile = "zfile" // Sort below
ContentClassContent = "zcontent"
)

func ClassifyContentFile(filename string) string {
if strings.HasPrefix(filename, "_content.") {
return ContentClassData
}

if !IsContentFile(filename) {
return ContentClassFile
}
Expand Down
16 changes: 6 additions & 10 deletions hugolib/page__meta.go
Original file line number Diff line number Diff line change
Expand Up @@ -258,19 +258,15 @@ func (p *pageMeta) Section() string {
return ""
}

if p.IsNode() {
if len(p.sections) == 0 {
// May be a sitemap or similar.
return ""
}
return p.sections[0]
}

if !p.File().IsZero() {
if p.IsPage() && !p.File().IsZero() {
return p.File().Section()
}

panic("invalid page state")
if len(p.sections) == 0 {
// May be a sitemap or similar.
return ""
}
return p.sections[0]

}

Expand Down
38 changes: 38 additions & 0 deletions hugolib/pages_capture.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,16 @@ package hugolib
import (
"context"
"fmt"
"io"
"os"
pth "path"
"path/filepath"
"strings"

"github.com/gohugoio/hugo/resources/page"

yaml "gopkg.in/yaml.v2"

"github.com/gohugoio/hugo/config"

"github.com/gohugoio/hugo/hugofs/files"
Expand Down Expand Up @@ -631,6 +636,37 @@ func (proc *pagesProcessor) newPageFromBundle(b *fileinfoBundle) (*pageState, er
return p, nil
}

func (proc *pagesProcessor) newPagesFromData(fim hugofs.FileMetaInfo, send func(p *pageState, err error)) {

meta := fim.Meta()
f, err := meta.Open()
if err != nil {
send(nil, err)
return
}
defer f.Close()

s := proc.getSite(meta.Lang())

dec := yaml.NewDecoder(f)
for {
m := make(map[string]interface{})
if err := dec.Decode(m); err != nil {
if err == io.EOF {
break
}
send(nil, err)
return
}

send(newPageFromMeta(m, &pageMeta{
kind: page.KindPage,
s: s,
}))

}
}

func (proc *pagesProcessor) newPageFromFi(fim hugofs.FileMetaInfo, owner *pageState) (*pageState, error) {
fi, err := newFileInfo(proc.sp, fim)
if err != nil {
Expand Down Expand Up @@ -753,6 +789,8 @@ func (proc *pagesProcessor) process(item interface{}) error {
send(proc.newPageFromFi(v, nil))
case files.ContentClassFile:
proc.sendError(proc.copyFile(v))
case files.ContentClassData:
proc.newPagesFromData(v, send)
default:
panic(fmt.Sprintf("invalid classifier: %q", classifier))
}
Expand Down
36 changes: 36 additions & 0 deletions hugolib/pages_from_data_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package hugolib

import (
"testing"

qt "github.com/frankban/quicktest"
)

func TestPagesFromYAML(t *testing.T) {
b := newTestSitesBuilder(t)

b.WithContent("_content.yaml", `
title: Yaml Page 1
---
title: Yaml Page 2
`)

b.Build(BuildCfg{})

s := b.H.Sites[0]

b.Assert(s.RegularPages(), qt.HasLen, 2)
}
113 changes: 113 additions & 0 deletions plugins/content/eval.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package content

import (
"io"

"github.com/spf13/cast"

"github.com/gohugoio/hugo/helpers"
"go.starlark.net/starlark"

"github.com/starlight-go/starlight"
"github.com/starlight-go/starlight/convert"
)

type sourcePluginFiles struct {
FilenamesProvider
SourcePlugin
}

type sourcePluginStream struct {
StreamProvider
SourcePlugin
}

type sourcePluginFunc func(item map[string]interface{}) Bundle

func (f sourcePluginFunc) ToBundle(item map[string]interface{}) Bundle {
return f(item)
}

type getFilenamesFunc func() []string

func (f getFilenamesFunc) GetFilenames() []string {
return f()
}

var globals = make(map[string]interface{})
var thread = &starlark.Thread{} // TODO1

func EvalSourcePlugin(r io.Reader) (SourcePlugin, error) {
out, err := starlight.Eval(helpers.ReaderToBytes(r), globals, nil)
if err != nil {
return nil, err
}

return toSourcePlugin(out)
}

func toSourcePlugin(out map[string]interface{}) (SourcePlugin, error) {
if fn, found := out["GetFilenames"]; found {
gf := getFilenamesFunc(
func() []string {
return cast.ToStringSlice(starlarkCall(thread, fn))
},
)

return sourcePluginFiles{
FilenamesProvider: gf,
SourcePlugin: getToBundleFunc(out),
}, nil
}

return nil, nil
}

func getToBundleFunc(out map[string]interface{}) sourcePluginFunc {
if fn, found := out["ToBundle"]; found {
return func(item map[string]interface{}) Bundle {
return starlarkCall(thread, fn, item).(Bundle)
}
}

return func(item map[string]interface{}) Bundle {
return Bundle{}
}
}

func starlarkCall(thread *starlark.Thread, fn interface{}, args ...interface{}) interface{} {
argsv := make(starlark.Tuple, len(args))
for i, arg := range args {
argv, err := convert.ToValue(arg)
if err != nil {
panic(err) // TODO1
}
argsv[i] = argv
}
v, err := starlark.Call(thread, fn.(*starlark.Function), argsv, nil)
if err != nil {
panic(err)
}

return fromValue(v)
}

func fromValue(v starlark.Value) interface{} {
switch v := v.(type) {
default:
return convert.FromValue(v)
}
}
39 changes: 39 additions & 0 deletions plugins/content/eval_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package content

import (
"strings"
"testing"

qt "github.com/frankban/quicktest"
)

func TestEvalSourcePlugin(t *testing.T) {
c := qt.New(t)

pluginFiles := `
def GetFilenames():
return ["file1.json", "file2.json"]
`

plugin, err := EvalSourcePlugin(strings.NewReader(pluginFiles))
c.Assert(err, qt.IsNil)
source, ok := plugin.(SourcePluginFiles)
c.Assert(ok, qt.Equals, true)
c.Assert(source.GetFilenames(), qt.DeepEquals, []string{"file1.json", "file2.json"})

}
45 changes: 45 additions & 0 deletions plugins/content/interface.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package content

import "io"

type FilenamesProvider interface {
GetFilenames() []string
}

type StreamProvider interface {
GetStream() ReadCloserProvider
}

type SourcePluginFiles interface {
FilenamesProvider
SourcePlugin
}

type SourcePluginStream interface {
GetStream() ReadCloserProvider
SourcePlugin
}

type SourcePlugin interface {
ToBundle(item map[string]interface{}) Bundle
}

type ReadCloserProvider interface {
OpenReadCloser() (io.ReadCloser, error)
}

type Bundle struct {
}

0 comments on commit 0a744ac

Please sign in to comment.