diff --git a/README.md b/README.md new file mode 100644 index 0000000..780daad --- /dev/null +++ b/README.md @@ -0,0 +1,123 @@ +`clj-commons/clj-yaml` provides [YAML](http://yaml.org) encoding and +decoding for Clojure via the [snakeyaml][] Java library. + +[SnakeYAML]: https://bitbucket.org/snakeyaml/snakeyaml + + +[![Clojars Project](https://img.shields.io/clojars/v/clj-commons/clj-yaml.svg)](https://clojars.org/clj-commons/clj-yaml) +[![cljdoc badge](https://cljdoc.org/badge/clj-commons/clj-yaml)](https://cljdoc.org/d/clj-commons/clj-yaml) +[![CircleCI Status](https://circleci.com/gh/clj-commons/clj-yaml.svg?style=svg)](https://circleci.com/gh/clj-commons/clj-yaml) +[![Slack chat](https://img.shields.io/badge/slack-join_chat-brightgreen.svg)](https://clojurians.slack.com/archives/C042XAQFCCU) + +(This is a maintained fork of [the circleci fork][] which forked from [the original][]) + +[the circleci fork]: https://github.com/CircleCI-Archived/clj-yaml +[the original]: https://github.com/lancepantz/clj-yaml + +## Usage + +```clojure +(require '[clj-yaml.core :as yaml]) + +(yaml/generate-string + [{:name "John Smith", :age 33} + {:name "Mary Smith", :age 27}]) +"- {name: John Smith, age: 33}\n- {name: Mary Smith, age: 27}\n" + +(yaml/parse-string " +- {name: John Smith, age: 33} +- name: Mary Smith + age: 27 +") +=> ({:name "John Smith", :age 33} + {:name "Mary Smith", :age 27}) +``` + +By default, keys are converted to clojure keywords. To prevent this, +add `:keywords false` parameters to the `parse-string` function: + +```clojure +(yaml/parse-string " +- {name: John Smith} +" :keywords false) +``` + +Unknown tags can be handled by passing a handler function via the +:unknown-tag-fn parameter. The handler function is provided a map +which includes `:tag` and `:value` keys. Note that the value passed to +the `unknown-tag-fn` is a string if it's a scalar, regardless of the +quoting (or lack thereof) of the scalar value. + +```clojure +;; drop tags +(yaml/parse-string "!Base12 10" :unknown-tag-fn :value +;; => "10" +(yaml/parse-string "!Base12 10" + :unknown-tag-fn (fn [{:keys [tag value]}] + (if (= "!Base12" tag) + (Integer/parseInt value 12) + value))) +;; => 12 +``` + + +Different flow styles (`:auto`, `:block`, `:flow`) allow customization of how YAML is rendered: + + +```clojure +(yaml/generate-string some-data :dumper-options {:flow-style :block}) +``` + +Use the `:indent` (default: 2) and `:indicator-indent` (default: 0) options to adjust indentation: + +```clojure +(yaml/generate-string some-data :dumper-options {:indent 6 + :indicator-indent 3 + :flow-style :block}) +=> +todo: + - name: Fix issue + responsible: + name: Rita +``` +`:indent` must always be larger than `:indicator-indent`. If only 1 higher, the indicator will be on a separate line: +```clojure +(yaml/generate-string some-data :dumper-options {:indent 2 + :indicator-indent 1 + :flow-style :block}) +=> +todo: + - + name: Fix issue + responsible: + name: Rita +``` + +## Installation + +`clj-commons/clj-yaml` is available as a Maven artifact from [Clojars](http://clojars.org/clj-commons/clj-yaml). + +### Leiningen/Boot + +```clojure +[clj-commons/clj-yaml "0.7.0"] +``` + +### Clojure CLI/`deps.edn` + +```clojure +clj-commons/clj-yaml {:mvn/version "0.7.0"} +``` + +## Development + + $ git clone git://github.com/clj-commons/clj-yaml.git + $ lein deps + $ lein test + $ lein install + +## License + +(c) Lance Bradley - Licensed under the same terms as clojure itself. See LICENCE file for details. + +Portions (c) Owain Lewis as marked. diff --git a/src/clojure/clj_yaml/core.clj b/src/clojure/clj_yaml/core.clj index b6a924c..bf665b9 100644 --- a/src/clojure/clj_yaml/core.clj +++ b/src/clojure/clj_yaml/core.clj @@ -5,6 +5,8 @@ (org.yaml.snakeyaml.constructor Constructor SafeConstructor BaseConstructor) (org.yaml.snakeyaml.representer Representer) (org.yaml.snakeyaml.error Mark) + (clj_yaml MarkedConstructor UnknownTagsConstructor) + (java.util LinkedHashMap) (clj_yaml MarkedConstructor) (java.util LinkedHashMap) (java.io StringReader))) @@ -51,19 +53,24 @@ (defn ^Yaml make-yaml "Make a yaml encoder/decoder with some given options." - [& {:keys [dumper-options unsafe mark max-aliases-for-collections allow-recursive-keys allow-duplicate-keys]}] + [& {:keys [unknown-tag-fn dumper-options unsafe mark max-aliases-for-collections allow-recursive-keys allow-duplicate-keys]}] (let [loader (make-loader-options :max-aliases-for-collections max-aliases-for-collections :allow-recursive-keys allow-recursive-keys :allow-duplicate-keys allow-duplicate-keys) ^BaseConstructor constructor - (if unsafe (Constructor. loader) - (if mark - ;; construct2ndStep isn't implemented by MarkedConstructor, - ;; causing an exception to be thrown before loader options are - ;; used - (MarkedConstructor.) - (SafeConstructor. loader))) - ;; TODO: unsafe marked constructor + (cond + unsafe (Constructor. loader) + + ;; construct2ndStep isn't implemented by MarkedConstructor, + ;; causing an exception to be thrown before loader options + ;; are used + mark (MarkedConstructor.) + + unknown-tag-fn (UnknownTagsConstructor.) + + ;; TODO: unsafe marked constructor + :else (SafeConstructor. loader)) + dumper (make-dumper-options dumper-options)] (Yaml. constructor (Representer.) dumper loader))) @@ -91,11 +98,11 @@ "A protocol for things that can be coerced to and from the types that snakeyaml knows how to encode and decode." (encode [data]) - (decode [data keywords])) + (decode [data keywords unknown-tag-fn])) (extend-protocol YAMLCodec clj_yaml.MarkedConstructor$Marked - (decode [data keywords] + (decode [data keywords unknown-tag-fn] (letfn [(from-Mark [^Mark mark] {:line (.getLine mark) :index (.getIndex mark) @@ -104,7 +111,12 @@ (mark (-> data .start from-Mark) (-> data .end from-Mark) (-> data .marked - (decode keywords))))) + (decode keywords unknown-tag-fn))))) + + clj_yaml.UnknownTagsConstructor$UnknownTag + (decode [data keywords unknown-tag-fn] + (unknown-tag-fn {:tag (str (.tag data)) + :value (-> (.value data) (decode keywords unknown-tag-fn))})) clojure.lang.IPersistentMap (encode [data] @@ -123,7 +135,7 @@ (subs (str data) 1)) java.util.LinkedHashMap - (decode [data keywords] + (decode [data keywords unknown-tag-fn] (letfn [(decode-key [k] (if keywords ;; (keyword k) is nil for numbers etc @@ -131,43 +143,44 @@ k))] (into (ordered-map) (for [[k v] data] - [(-> k (decode keywords) decode-key) (decode v keywords)])))) + [(-> k (decode keywords unknown-tag-fn) decode-key) (decode v keywords unknown-tag-fn)])))) java.util.LinkedHashSet - (decode [data keywords] + (decode [data _keywords _unknown-tag-fn] (into (ordered-set) data)) java.util.ArrayList - (decode [data keywords] - (map #(decode % keywords) data)) + (decode [data keywords unknown-tag-fn] + (map #(decode % keywords unknown-tag-fn) data)) Object (encode [data] data) - (decode [data keywords] data) + (decode [data _keywords _unknown-tag-fn] data) nil (encode [data] data) - (decode [data keywords] data)) + (decode [data _keywords _unknown-tag-fn] data)) (defn generate-string [data & opts] (.dump ^Yaml (apply make-yaml opts) (encode data))) -(defn- load-stream [^Yaml yaml ^java.io.Reader input load-all? keywords] +(defn- load-stream [^Yaml yaml ^java.io.Reader input load-all? keywords unknown-tag-fn] (if load-all? - (map #(decode % keywords) (.loadAll yaml input)) - (decode (.load yaml input) keywords))) + (map #(decode % keywords unknown-tag-fn) (.loadAll yaml input)) + (decode (.load yaml input) keywords unknown-tag-fn))) (defn parse-string - [^String string & {:keys [unsafe mark keywords max-aliases-for-collections allow-recursive-keys allow-duplicate-keys load-all?] :or {keywords true}}] - (load-stream (make-yaml :unsafe unsafe - :mark mark - :max-aliases-for-collections max-aliases-for-collections - :allow-recursive-keys allow-recursive-keys - :allow-duplicate-keys allow-duplicate-keys) - (StringReader. string) - load-all? keywords)) + [^String string & {:keys [unknown-tag-fn unsafe mark keywords max-aliases-for-collections + allow-recursive-keys allow-duplicate-keys load-all?] :or {keywords true}}] + (let [yaml (make-yaml :unsafe unsafe + :mark mark + :unknown-tag-fn unknown-tag-fn + :max-aliases-for-collections max-aliases-for-collections + :allow-recursive-keys allow-recursive-keys + :allow-duplicate-keys allow-duplicate-keys)] + (load-stream yaml (StringReader. string) load-all? keywords unknown-tag-fn))) ;; From https://github.com/metosin/muuntaja/pull/94/files (defn generate-stream @@ -176,7 +189,7 @@ (.dump ^Yaml (apply make-yaml opts) (encode data) writer)) (defn parse-stream - [^java.io.Reader reader & {:keys [keywords load-all?] :or {keywords true} :as opts}] + [^java.io.Reader reader & {:keys [keywords load-all? unknown-tag-fn] :or {keywords true} :as opts}] (load-stream (apply make-yaml (into [] cat opts)) reader - load-all? keywords)) + load-all? keywords unknown-tag-fn)) diff --git a/src/java/clj_yaml/UnknownTagsConstructor.java b/src/java/clj_yaml/UnknownTagsConstructor.java new file mode 100644 index 0000000..a08a1ba --- /dev/null +++ b/src/java/clj_yaml/UnknownTagsConstructor.java @@ -0,0 +1,48 @@ +package clj_yaml; + +import org.yaml.snakeyaml.constructor.Construct; +import org.yaml.snakeyaml.constructor.Constructor; +import org.yaml.snakeyaml.constructor.SafeConstructor; +import org.yaml.snakeyaml.constructor.AbstractConstruct; +import org.yaml.snakeyaml.nodes.Node; +import org.yaml.snakeyaml.nodes.Tag; + +public class UnknownTagsConstructor extends SafeConstructor { + + public UnknownTagsConstructor() { + this.yamlMultiConstructors.put("", new UnknownTagConstruct()); + } + + public class UnknownTagConstruct extends AbstractConstruct { + + public Object construct(Node node) { + Tag unknownTag = node.getTag(); + + Tag newTag = null; + switch (node.getNodeId()) { + case scalar: + newTag = Tag.STR; + break; + case sequence: + newTag = Tag.SEQ; + break; + default: + newTag = Tag.MAP; + break; + } + node.setTag(newTag); + + return new UnknownTag(unknownTag, getConstructor(node).construct(node)); + } + } + + public static class UnknownTag { + public Tag tag; + public Object value; + + public UnknownTag(Tag unknownTag, Object taggedValue) { + this.tag = unknownTag; + this.value = taggedValue; + } + } +} diff --git a/test/clj_yaml/core_test.clj b/test/clj_yaml/core_test.clj index 236754e..52e97cc 100644 --- a/test/clj_yaml/core_test.clj +++ b/test/clj_yaml/core_test.clj @@ -1,16 +1,19 @@ (ns clj-yaml.core-test - (:require [clojure.test :refer (deftest testing is)] - [clojure.string :as string] - [clojure.java.io :as io] - [clj-yaml.core :refer [parse-string unmark generate-string - parse-stream generate-stream]] - [flatland.ordered.map :refer [ordered-map]]) - (:import [java.util Date] - (java.io ByteArrayOutputStream OutputStreamWriter ByteArrayInputStream) - java.nio.charset.StandardCharsets - (org.yaml.snakeyaml.error YAMLException) - (org.yaml.snakeyaml.constructor DuplicateKeyException) - [org.yaml.snakeyaml.composer ComposerException])) + (:require + [clj-yaml.core :as yaml :refer [generate-stream generate-string + parse-stream parse-string unmark]] + [clojure.java.io :as io] + [clojure.string :as string] + [clojure.test :refer (deftest testing is)] + [flatland.ordered.map :refer [ordered-map]]) + (:import + (java.io ByteArrayInputStream ByteArrayOutputStream OutputStreamWriter) + java.nio.charset.StandardCharsets + [java.util Date] + [org.yaml.snakeyaml.composer ComposerException] + (org.yaml.snakeyaml.constructor ConstructorException DuplicateKeyException) + (org.yaml.snakeyaml.constructor DuplicateKeyException) + (org.yaml.snakeyaml.error YAMLException))) (def nested-hash-yaml "root:\n childa: a\n childb: \n grandchild: \n greatgrandchild: bar\n") @@ -330,3 +333,36 @@ lol: yolo") :dumper-options {:indent 5 :indicator-indent 2 :flow-style :block}))))) + +(def yaml-with-unknown-tags "--- +scalar: !CustomScalar some-scalar +mapping: !CustomMapping + x: foo + y: bar +sequence: !CustomSequence + - a + - b + - z +") + +(deftest unknown-tags-test + (testing "Throws with unknown tags and default constructor" + (is (thrown-with-msg? ConstructorException + #"^could not determine a constructor for the tag !CustomScalar" + (parse-string yaml-with-unknown-tags)))) + (testing "Can process unknown tags with strip-unknown-tags? constructor" + (is (= {:scalar "some-scalar" + :mapping {:x "foo" :y "bar"} + :sequence ["a" "b" "z"]} + (parse-string yaml-with-unknown-tags :unknown-tag-fn :value)))) + (testing "Can process unknown tags with :unknown-tag-fn as identity" + (is (= {:scalar {:tag "!CustomScalar" :value "some-scalar"} + :mapping {:tag "!CustomMapping" :value {:x "foo" :y "bar"}} + :sequence {:tag "!CustomSequence" :value ["a" "b" "z"]}} + (parse-string yaml-with-unknown-tags :unknown-tag-fn identity))) + (is (= {:base-12 12 :base-10 "10"} + (parse-string "{base-12: !Base12 10, base-10: !Base10 10}" + :unknown-tag-fn (fn [{:keys [tag value]}] + (if (= "!Base12" tag) + (Integer/parseInt value 12) value))))))) +