From 0eaaa8fee37068bfc8ecfb760f770ecc9a7af22a Mon Sep 17 00:00:00 2001 From: Paul van Brouwershaven Date: Thu, 2 Dec 2021 17:30:36 +0100 Subject: [PATCH] Implement XML data support Example: ``` {{ with resources.Get "https://example.com/rss.xml" | transform.Unmarshal }} {{ range .channel.item }} {{ .title | plainify | htmlUnescape }}

{{ .description | plainify | htmlUnescape }}

{{ $link := .link | plainify | htmlUnescape }} {{ $link }}
{{ end }} {{ end }} ``` Closes #4470 --- .../en/functions/transform.Unmarshal.md | 31 ++++++++++- docs/content/en/templates/data-templates.md | 10 ++-- go.mod | 1 + go.sum | 2 + hugolib/resource_chain_test.go | 9 ++- parser/frontmatter.go | 9 +++ parser/metadecoders/decoder.go | 20 +++++++ parser/metadecoders/decoder_test.go | 55 +++++++++++++++++++ parser/metadecoders/format.go | 16 ++++-- parser/metadecoders/format_test.go | 3 + tpl/transform/remarshal_test.go | 20 +++++++ tpl/transform/unmarshal_test.go | 3 + 12 files changed, 167 insertions(+), 12 deletions(-) diff --git a/docs/content/en/functions/transform.Unmarshal.md b/docs/content/en/functions/transform.Unmarshal.md index 973779c4c1d..9b380dc578c 100644 --- a/docs/content/en/functions/transform.Unmarshal.md +++ b/docs/content/en/functions/transform.Unmarshal.md @@ -1,6 +1,6 @@ --- title: "transform.Unmarshal" -description: "`transform.Unmarshal` (alias `unmarshal`) parses the input and converts it into a map or an array. Supported formats are JSON, TOML, YAML and CSV." +description: "`transform.Unmarshal` (alias `unmarshal`) parses the input and converts it into a map or an array. Supported formats are JSON, TOML, YAML, XML and CSV." date: 2018-12-23 categories: [functions] menu: @@ -45,3 +45,32 @@ Example: ```go-html-template {{ $csv := "a;b;c" | transform.Unmarshal (dict "delimiter" ";") }} ``` + +## XML data + +As a convenience, Hugo allows you to access XML data in the same way that you access JSON, TOML, and YAML: you do not need to specify the root node when accessing the data. + +To get the contents of `` in the document below, you use `{{ .message.title }}`: + +``` +<root> + <message> + <title>Hugo rocks! + Thanks for using Hugo + + +``` + +The following example lists the items of an RSS feed: + +``` +{{ with resources.Get "https://example.com/rss.xml" | transform.Unmarshal }} + {{ range .channel.item }} + {{ .title | plainify | htmlUnescape }}
+

{{ .description | plainify | htmlUnescape }}

+ {{ $link := .link | plainify | htmlUnescape }} + {{ $link }}
+
+ {{ end }} +{{ end }} +``` diff --git a/docs/content/en/templates/data-templates.md b/docs/content/en/templates/data-templates.md index c36344776be..441cc2f1015 100644 --- a/docs/content/en/templates/data-templates.md +++ b/docs/content/en/templates/data-templates.md @@ -6,7 +6,7 @@ date: 2017-02-01 publishdate: 2017-02-01 lastmod: 2017-03-12 categories: [templates] -keywords: [data,dynamic,csv,json,toml,yaml] +keywords: [data,dynamic,csv,json,toml,yaml,xml] menu: docs: parent: "templates" @@ -20,7 +20,7 @@ toc: true -Hugo supports loading data from YAML, JSON, and TOML files located in the `data` directory in the root of your Hugo project. +Hugo supports loading data from YAML, JSON, XML, and TOML files located in the `data` directory in the root of your Hugo project. {{< youtube FyPgSuwIMWQ >}} @@ -28,7 +28,7 @@ Hugo supports loading data from YAML, JSON, and TOML files located in the `data` The `data` folder is where you can store additional data for Hugo to use when generating your site. Data files aren't used to generate standalone pages; rather, they're meant to be supplemental to content files. This feature can extend the content in case your front matter fields grow out of control. Or perhaps you want to show a larger dataset in a template (see example below). In both cases, it's a good idea to outsource the data in their own files. -These files must be YAML, JSON, or TOML files (using the `.yml`, `.yaml`, `.json`, or `.toml` extension). The data will be accessible as a `map` in the `.Site.Data` variable. +These files must be YAML, JSON, XML, or TOML files (using the `.yml`, `.yaml`, `.json`, `.xml`, or `.toml` extension). The data will be accessible as a `map` in the `.Site.Data` variable. ## Data Files in Themes @@ -95,7 +95,7 @@ Discover a new favorite bass player? Just add another `.toml` file in the same d ## Example: Accessing Named Values in a Data File -Assume you have the following data structure in your `User0123.[yml|toml|json]` data file located directly in `data/`: +Assume you have the following data structure in your `User0123.[yml|toml|xml|json]` data file located directly in `data/`: {{< code-toggle file="User0123" >}} Name: User0123 @@ -232,6 +232,7 @@ If you change any local file and the LiveReload is triggered, Hugo will read the * [YAML Spec][yaml] * [JSON Spec][json] * [CSV Spec][csv] +* [XML Spec][xml] [config]: /getting-started/configuration/ [csv]: https://tools.ietf.org/html/rfc4180 @@ -247,3 +248,4 @@ If you change any local file and the LiveReload is triggered, Hugo will read the [variadic]: https://en.wikipedia.org/wiki/Variadic_function [vars]: /variables/ [yaml]: https://yaml.org/spec/ +[xml]: https://www.w3.org/XML/ diff --git a/go.mod b/go.mod index 0fbade221d3..db14b0100d3 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/bep/golibsass v1.0.0 github.com/bep/gowebp v0.1.0 github.com/bep/tmc v0.5.1 + github.com/clbanning/mxj/v2 v2.5.5 github.com/cli/safeexec v1.0.0 github.com/disintegration/gift v1.2.1 github.com/dustin/go-humanize v1.0.0 diff --git a/go.sum b/go.sum index 4093bf6958f..952d02793aa 100644 --- a/go.sum +++ b/go.sum @@ -144,6 +144,8 @@ github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgk github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/clbanning/mxj/v2 v2.5.5 h1:oT81vUeEiQQ/DcHbzSytRngP6Ky9O+L+0Bw0zSJag9E= +github.com/clbanning/mxj/v2 v2.5.5/go.mod h1:hNiWqW14h+kc+MdF9C6/YoRfjEJoR3ou6tn/Qo+ve2s= github.com/cli/safeexec v1.0.0 h1:0VngyaIyqACHdcMNWfo6+KdUYnqEr2Sg+bSP1pdF+dI= github.com/cli/safeexec v1.0.0/go.mod h1:Z/D4tTN8Vs5gXYHDCbaM1S/anmEDnJb1iW0+EJ5zx3Q= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= diff --git a/hugolib/resource_chain_test.go b/hugolib/resource_chain_test.go index 0a997cda35f..05e8a9d0099 100644 --- a/hugolib/resource_chain_test.go +++ b/hugolib/resource_chain_test.go @@ -591,9 +591,9 @@ func TestResourceChains(t *testing.T) { case "/mydata/xml1.xml": w.Write([]byte(` - - Hugo Rocks! - `)) + + Hugo Rocks! + `)) return case "/mydata/svg1.svg": @@ -872,16 +872,19 @@ Publish 2: {{ $cssPublish2.Permalink }} {{ $toml := "slogan = \"Hugo Rocks!\"" | resources.FromString "slogan.toml" | transform.Unmarshal }} {{ $csv1 := "\"Hugo Rocks\",\"Hugo is Fast!\"" | resources.FromString "slogans.csv" | transform.Unmarshal }} {{ $csv2 := "a;b;c" | transform.Unmarshal (dict "delimiter" ";") }} +{{ $xml := "YouMeReminderDo not forget XML" | transform.Unmarshal }} Slogan: {{ $toml.slogan }} CSV1: {{ $csv1 }} {{ len (index $csv1 0) }} CSV2: {{ $csv2 }} +XML: {{ $xml.body }} `) }, func(b *sitesBuilder) { b.AssertFileContent("public/index.html", `Slogan: Hugo Rocks!`, `[[Hugo Rocks Hugo is Fast!]] 2`, `CSV2: [[a b c]]`, + `XML: Do not forget XML`, ) }}, {"resources.Get", func() bool { return true }, func(b *sitesBuilder) { diff --git a/parser/frontmatter.go b/parser/frontmatter.go index 79701a0fcd2..a998295217e 100644 --- a/parser/frontmatter.go +++ b/parser/frontmatter.go @@ -23,6 +23,8 @@ import ( toml "github.com/pelletier/go-toml/v2" yaml "gopkg.in/yaml.v2" + + xml "github.com/clbanning/mxj/v2" ) const ( @@ -62,7 +64,14 @@ func InterfaceToConfig(in interface{}, format metadecoders.Format, w io.Writer) _, err = w.Write([]byte{'\n'}) return err + case metadecoders.XML: + b, err := xml.AnyXmlIndent(in, "", "\t", "root") + if err != nil { + return err + } + _, err = w.Write(b) + return err default: return errors.New("unsupported Format provided") } diff --git a/parser/metadecoders/decoder.go b/parser/metadecoders/decoder.go index 168c130ed90..f0dcb08560d 100644 --- a/parser/metadecoders/decoder.go +++ b/parser/metadecoders/decoder.go @@ -24,6 +24,7 @@ import ( "github.com/gohugoio/hugo/common/herrors" "github.com/niklasfasching/go-org/org" + xml "github.com/clbanning/mxj/v2" toml "github.com/pelletier/go-toml/v2" "github.com/pkg/errors" "github.com/spf13/afero" @@ -135,6 +136,25 @@ func (d Decoder) UnmarshalTo(data []byte, f Format, v interface{}) error { err = d.unmarshalORG(data, v) case JSON: err = json.Unmarshal(data, v) + case XML: + var xmlRoot xml.Map + xmlRoot, err = xml.NewMapXml(data) + + var xmlValue map[string]interface{} + if err == nil { + xmlRootName, err := xmlRoot.Root() + if err != nil { + return toFileError(f, errors.Wrap(err, "failed to unmarshal XML")) + } + xmlValue = xmlRoot[xmlRootName].(map[string]interface{}) + } + + switch v := v.(type) { + case *map[string]interface{}: + *v = xmlValue + case *interface{}: + *v = xmlValue + } case TOML: err = toml.Unmarshal(data, v) case YAML: diff --git a/parser/metadecoders/decoder_test.go b/parser/metadecoders/decoder_test.go index e0990a5f7a9..8cd5513b2c6 100644 --- a/parser/metadecoders/decoder_test.go +++ b/parser/metadecoders/decoder_test.go @@ -20,6 +20,59 @@ import ( qt "github.com/frankban/quicktest" ) +func TestUnmarshalXML(t *testing.T) { + c := qt.New(t) + + xmlDoc := ` + + + Example feed + https://example.com/ + Example feed + Hugo -- gohugo.io + en-us + Example + Fri, 08 Jan 2021 14:44:10 +0000 + + + Example title + https://example.com/2021/11/30/example-title/ + Tue, 30 Nov 2021 15:00:00 +0000 + https://example.com/2021/11/30/example-title/ + Example description + + + ` + + expect := map[string]interface{}{ + "-atom": "http://www.w3.org/2005/Atom", "-version": "2.0", + "channel": map[string]interface{}{ + "copyright": "Example", + "description": "Example feed", + "generator": "Hugo -- gohugo.io", + "item": map[string]interface{}{ + "description": "Example description", + "guid": "https://example.com/2021/11/30/example-title/", + "link": "https://example.com/2021/11/30/example-title/", + "pubDate": "Tue, 30 Nov 2021 15:00:00 +0000", + "title": "Example title"}, + "language": "en-us", + "lastBuildDate": "Fri, 08 Jan 2021 14:44:10 +0000", + "link": []interface{}{"https://example.com/", map[string]interface{}{ + "-href": "https://example.com/feed.xml", + "-rel": "self", + "-type": "application/rss+xml"}}, + "title": "Example feed", + }} + + d := Default + + m, err := d.Unmarshal([]byte(xmlDoc), XML) + c.Assert(err, qt.IsNil) + c.Assert(m, qt.DeepEquals, expect) + +} func TestUnmarshalToMap(t *testing.T) { c := qt.New(t) @@ -38,6 +91,7 @@ func TestUnmarshalToMap(t *testing.T) { {"a: Easy!\nb:\n c: 2\n d: [3, 4]", YAML, map[string]interface{}{"a": "Easy!", "b": map[string]interface{}{"c": 2, "d": []interface{}{3, 4}}}}, {"a:\n true: 1\n false: 2", YAML, map[string]interface{}{"a": map[string]interface{}{"true": 1, "false": 2}}}, {`{ "a": "b" }`, JSON, expect}, + {`b`, XML, expect}, {`#+a: b`, ORG, expect}, // errors {`a = b`, TOML, false}, @@ -72,6 +126,7 @@ func TestUnmarshalToInterface(t *testing.T) { {`#+DATE: <2020-06-26 Fri>`, ORG, map[string]interface{}{"date": "2020-06-26"}}, {`a = "b"`, TOML, expect}, {`a: "b"`, YAML, expect}, + {`b`, XML, expect}, {`a,b,c`, CSV, [][]string{{"a", "b", "c"}}}, {"a: Easy!\nb:\n c: 2\n d: [3, 4]", YAML, map[string]interface{}{"a": "Easy!", "b": map[string]interface{}{"c": 2, "d": []interface{}{3, 4}}}}, // errors diff --git a/parser/metadecoders/format.go b/parser/metadecoders/format.go index bba89dbea93..d34a261bf10 100644 --- a/parser/metadecoders/format.go +++ b/parser/metadecoders/format.go @@ -30,6 +30,7 @@ const ( TOML Format = "toml" YAML Format = "yaml" CSV Format = "csv" + XML Format = "xml" ) // FormatFromString turns formatStr, typically a file extension without any ".", @@ -51,6 +52,8 @@ func FormatFromString(formatStr string) Format { return ORG case "csv": return CSV + case "xml": + return XML } return "" @@ -68,27 +71,32 @@ func FormatFromMediaType(m media.Type) Format { return "" } -// FormatFromContentString tries to detect the format (JSON, YAML or TOML) +// FormatFromContentString tries to detect the format (JSON, YAML, TOML or XML) // in the given string. // It return an empty string if no format could be detected. func (d Decoder) FormatFromContentString(data string) Format { csvIdx := strings.IndexRune(data, d.Delimiter) jsonIdx := strings.Index(data, "{") yamlIdx := strings.Index(data, ":") + xmlIdx := strings.Index(data, "<") tomlIdx := strings.Index(data, "=") - if isLowerIndexThan(csvIdx, jsonIdx, yamlIdx, tomlIdx) { + if isLowerIndexThan(csvIdx, jsonIdx, yamlIdx, xmlIdx, tomlIdx) { return CSV } - if isLowerIndexThan(jsonIdx, yamlIdx, tomlIdx) { + if isLowerIndexThan(jsonIdx, yamlIdx, xmlIdx, tomlIdx) { return JSON } - if isLowerIndexThan(yamlIdx, tomlIdx) { + if isLowerIndexThan(yamlIdx, xmlIdx, tomlIdx) { return YAML } + if isLowerIndexThan(xmlIdx, tomlIdx) { + return XML + } + if tomlIdx != -1 { return TOML } diff --git a/parser/metadecoders/format_test.go b/parser/metadecoders/format_test.go index 2f625935e07..0d94cf67e87 100644 --- a/parser/metadecoders/format_test.go +++ b/parser/metadecoders/format_test.go @@ -30,6 +30,7 @@ func TestFormatFromString(t *testing.T) { {"json", JSON}, {"yaml", YAML}, {"yml", YAML}, + {"xml", XML}, {"toml", TOML}, {"config.toml", TOML}, {"tOMl", TOML}, @@ -48,6 +49,7 @@ func TestFormatFromMediaType(t *testing.T) { }{ {media.JSONType, JSON}, {media.YAMLType, YAML}, + {media.XMLType, XML}, {media.TOMLType, TOML}, {media.CalendarType, ""}, } { @@ -70,6 +72,7 @@ func TestFormatFromContentString(t *testing.T) { {`foo:"bar"`, YAML}, {`{ "foo": "bar"`, JSON}, {`a,b,c"`, CSV}, + {`bar"`, XML}, {`asdfasdf`, Format("")}, {``, Format("")}, } { diff --git a/tpl/transform/remarshal_test.go b/tpl/transform/remarshal_test.go index 2cb4c3a2fe6..8e94ef6bf14 100644 --- a/tpl/transform/remarshal_test.go +++ b/tpl/transform/remarshal_test.go @@ -82,6 +82,25 @@ title: Test Metadata "title": "Test Metadata" } ` + xmlExample := ` + + + picasso + + **image-4.png + The Fourth Image! + + + my-cool-image-:counter + + bep + + **.png + TOML: The Image #:counter + + Test Metadata + + ` variants := []struct { format string @@ -93,6 +112,7 @@ title: Test Metadata {"TOML", tomlExample}, {"Toml", tomlExample}, {" TOML ", tomlExample}, + {"XML", xmlExample}, } for _, v1 := range variants { diff --git a/tpl/transform/unmarshal_test.go b/tpl/transform/unmarshal_test.go index 85e3610d155..fb0e446c338 100644 --- a/tpl/transform/unmarshal_test.go +++ b/tpl/transform/unmarshal_test.go @@ -111,6 +111,9 @@ func TestUnmarshal(t *testing.T) { {testContentResource{key: "r1", content: `slogan = "Hugo Rocks!"`, mime: media.TOMLType}, nil, func(m map[string]interface{}) { assertSlogan(m) }}, + {testContentResource{key: "r1", content: `Hugo Rocks!"`, mime: media.XMLType}, nil, func(m map[string]interface{}) { + assertSlogan(m) + }}, {testContentResource{key: "r1", content: `1997,Ford,E350,"ac, abs, moon",3000.00 1999,Chevy,"Venture ""Extended Edition""","",4900.00`, mime: media.CSVType}, nil, func(r [][]string) { c.Assert(len(r), qt.Equals, 2)