From fbc663e4688b0b12ba5f603160dd23c94b880eb0 Mon Sep 17 00:00:00 2001 From: Michael Glaesemann Date: Wed, 14 Sep 2022 22:18:12 -0500 Subject: [PATCH 01/20] Handle unknown tags via PassthroughConstructor. --- src/clojure/clj_yaml/core.clj | 30 +++++++----- src/java/clj_yaml/PassthroughConstructor.java | 46 +++++++++++++++++++ test/clj_yaml/core_test.clj | 29 ++++++++++-- 3 files changed, 91 insertions(+), 14 deletions(-) create mode 100644 src/java/clj_yaml/PassthroughConstructor.java diff --git a/src/clojure/clj_yaml/core.clj b/src/clojure/clj_yaml/core.clj index 6caca03..75f205b 100644 --- a/src/clojure/clj_yaml/core.clj +++ b/src/clojure/clj_yaml/core.clj @@ -5,7 +5,7 @@ (org.yaml.snakeyaml.constructor Constructor SafeConstructor BaseConstructor) (org.yaml.snakeyaml.representer Representer) (org.yaml.snakeyaml.error Mark) - (clj_yaml MarkedConstructor) + (clj_yaml MarkedConstructor PassthroughConstructor) (java.util LinkedHashMap))) (def flow-styles @@ -48,21 +48,28 @@ (.setAllowDuplicateKeys loader allow-duplicate-keys)) loader)) +(def passthrough-constructor (fn [] (PassthroughConstructor.))) + (defn ^Yaml make-yaml "Make a yaml encoder/decoder with some given options." - [& {:keys [dumper-options unsafe mark max-aliases-for-collections allow-recursive-keys allow-duplicate-keys]}] + [& {:keys [constructor dumper-options unsafe mark max-aliases-for-collections allow-recursive-keys allow-duplicate-keys]}] (let [loader (make-loader-options :max-aliases-for-collections max-aliases-for-collections :allow-recursive-keys allow-recursive-keys :allow-duplicate-keys allow-duplicate-keys) ^BaseConstructor constructor - (if unsafe (Constructor. loader) - (if mark - ;; construct2ndStep isn't implemented by MarkedConstructor, - ;; causing an exception to be thrown before loader options are - ;; used - (MarkedConstructor.) - (SafeConstructor. loader))) - ;; TODO: unsafe marked constructor + (cond + unsafe (Constructor. loader) + + ;; construct2ndStep isn't implemented by MarkedConstructor, + ;; causing an exception to be thrown before loader options are + ;; used + mark (MarkedConstructor.) + + constructor (constructor) + + ;; TODO: unsafe marked constructor + :else (SafeConstructor. loader)) + dumper (make-dumper-options dumper-options)] (Yaml. constructor (Representer.) dumper loader))) @@ -154,9 +161,10 @@ (encode data))) (defn parse-string - [^String string & {:keys [unsafe mark keywords max-aliases-for-collections allow-recursive-keys allow-duplicate-keys] :or {keywords true}}] + [^String string & {:keys [constructor unsafe mark keywords max-aliases-for-collections allow-recursive-keys allow-duplicate-keys] :or {keywords true}}] (decode (.load (make-yaml :unsafe unsafe :mark mark + :constructor constructor :max-aliases-for-collections max-aliases-for-collections :allow-recursive-keys allow-recursive-keys :allow-duplicate-keys allow-duplicate-keys) diff --git a/src/java/clj_yaml/PassthroughConstructor.java b/src/java/clj_yaml/PassthroughConstructor.java new file mode 100644 index 0000000..5a8a345 --- /dev/null +++ b/src/java/clj_yaml/PassthroughConstructor.java @@ -0,0 +1,46 @@ +package clj_yaml; + +import org.yaml.snakeyaml.constructor.Construct; +import org.yaml.snakeyaml.constructor.Constructor; +import org.yaml.snakeyaml.constructor.SafeConstructor; +import org.yaml.snakeyaml.constructor.AbstractConstruct; +import org.yaml.snakeyaml.nodes.Node; +import org.yaml.snakeyaml.nodes.Tag; +import org.yaml.snakeyaml.error.Mark; + +/** + * Implementation of Constructor that ignores YAML tags. + * + * This is used as a fallback strategies to use the underlying type instead of + * throwing an exception. + */ +public class PassthroughConstructor extends Constructor { + + private class PassthroughConstruct extends AbstractConstruct { + public Object construct(Node node) { + // reset node to scalar tag type for parsing + Tag tag = null; + switch (node.getNodeId()) { + case scalar: + tag = Tag.STR; + break; + case sequence: + tag = Tag.SEQ; + break; + default: + tag = Tag.MAP; + break; + } + + node.setTag(tag); + return getConstructor(node).construct(node); + } + + public void construct2ndStep(Node node, Object object) {} + } + + public PassthroughConstructor() { + // Add a catch-all to catch any unidentifiable nodes + this.yamlMultiConstructors.put("", new PassthroughConstruct()); + } +} diff --git a/test/clj_yaml/core_test.clj b/test/clj_yaml/core_test.clj index d5b94bd..e564065 100644 --- a/test/clj_yaml/core_test.clj +++ b/test/clj_yaml/core_test.clj @@ -3,12 +3,13 @@ [clojure.string :as string] [clojure.java.io :as io] [clj-yaml.core :refer [parse-string unmark generate-string - parse-stream generate-stream]]) + parse-stream generate-stream + passthrough-constructor]]) (:import [java.util Date] (java.io ByteArrayOutputStream OutputStreamWriter ByteArrayInputStream) java.nio.charset.StandardCharsets (org.yaml.snakeyaml.error YAMLException) - (org.yaml.snakeyaml.constructor DuplicateKeyException))) + (org.yaml.snakeyaml.constructor ConstructorException DuplicateKeyException))) (def nested-hash-yaml "root:\n childa: a\n childb: \n grandchild: \n greatgrandchild: bar\n") @@ -290,4 +291,26 @@ foo/bar: 42 (generate-string (parse-string indented-yaml) :dumper-options {:indent 5 :indicator-indent 2 - :flow-style :block}))))) \ No newline at end of file + :flow-style :block}))))) + +(def yaml-with-unknown-tags "--- +scalar: !CustomScalar some-scalar +mapping: !CustomMapping + x: foo + y: bar +sequence: !CustomSequence + - a + - b + - z +") + +(deftest passthrough-test + (testing "Throws with unknown tags and default constructor" + (is (thrown-with-msg? ConstructorException + #"^could not determine a constructor for the tag !CustomScalar" + (parse-string yaml-with-unknown-tags)))) + (testing "Can process unknown tags with passthrough constructor" + (is (= {:scalar "some-scalar" + :mapping {:x "foo" :y "bar"} + :sequence ["a" "b" "z"]} + (parse-string yaml-with-unknown-tags :constructor passthrough-constructor))))) From 66d6760ad787d6adfcda3f25c4a95dcd4042681d Mon Sep 17 00:00:00 2001 From: Michael Glaesemann Date: Thu, 15 Sep 2022 20:13:37 -0500 Subject: [PATCH 02/20] Add licence details. --- README.md | 6 +++++ src/java/clj_yaml/PassthroughConstructor.java | 22 +++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/README.md b/README.md index 56f8eb3..d763cb4 100644 --- a/README.md +++ b/README.md @@ -95,3 +95,9 @@ clj-commons/clj-yaml {:mvn/version "0.7.0"} $ lein deps $ lein test $ lein install + +## License + +(c) Lance Bradley - Licensed under the same terms as clojure itself. See LICENCE file for details. + +Portions (c) Owain Lewis as marked. diff --git a/src/java/clj_yaml/PassthroughConstructor.java b/src/java/clj_yaml/PassthroughConstructor.java index 5a8a345..54bb5d8 100644 --- a/src/java/clj_yaml/PassthroughConstructor.java +++ b/src/java/clj_yaml/PassthroughConstructor.java @@ -1,3 +1,25 @@ +// MIT License + +// Copyright (c) 2016 Owain Lewis + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + package clj_yaml; import org.yaml.snakeyaml.constructor.Construct; From 60d28e6231715526539dce13a3d16594e14da45d Mon Sep 17 00:00:00 2001 From: Michael Glaesemann Date: Mon, 19 Sep 2022 00:16:32 -0500 Subject: [PATCH 03/20] Add UnknownConstructor. --- src/clojure/clj_yaml/core.clj | 21 ++++++--- src/java/clj_yaml/UnknownTagConstructor.java | 48 ++++++++++++++++++++ test/clj_yaml/core_test.clj | 32 +++++++++++-- 3 files changed, 89 insertions(+), 12 deletions(-) create mode 100644 src/java/clj_yaml/UnknownTagConstructor.java diff --git a/src/clojure/clj_yaml/core.clj b/src/clojure/clj_yaml/core.clj index 75f205b..00835f2 100644 --- a/src/clojure/clj_yaml/core.clj +++ b/src/clojure/clj_yaml/core.clj @@ -5,7 +5,7 @@ (org.yaml.snakeyaml.constructor Constructor SafeConstructor BaseConstructor) (org.yaml.snakeyaml.representer Representer) (org.yaml.snakeyaml.error Mark) - (clj_yaml MarkedConstructor PassthroughConstructor) + (clj_yaml MarkedConstructor PassthroughConstructor UnknownTagConstructor) (java.util LinkedHashMap))) (def flow-styles @@ -52,7 +52,7 @@ (defn ^Yaml make-yaml "Make a yaml encoder/decoder with some given options." - [& {:keys [constructor dumper-options unsafe mark max-aliases-for-collections allow-recursive-keys allow-duplicate-keys]}] + [& {:keys [unknown-tag passthrough dumper-options unsafe mark max-aliases-for-collections allow-recursive-keys allow-duplicate-keys]}] (let [loader (make-loader-options :max-aliases-for-collections max-aliases-for-collections :allow-recursive-keys allow-recursive-keys :allow-duplicate-keys allow-duplicate-keys) @@ -61,11 +61,12 @@ unsafe (Constructor. loader) ;; construct2ndStep isn't implemented by MarkedConstructor, - ;; causing an exception to be thrown before loader options are - ;; used + ;; causing an exception to be thrown before loader options + ;; are used mark (MarkedConstructor.) - constructor (constructor) + unknown-tag (UnknownTagConstructor.) + passthrough (PassthroughConstructor.) ;; TODO: unsafe marked constructor :else (SafeConstructor. loader)) @@ -112,6 +113,11 @@ (-> data .marked (decode keywords))))) + clj_yaml.UnknownTagConstructor$UnknownTag + (decode [data keywords] + {::tag (str (.tag data)) + ::value (-> (.value data) (decode keywords))}) + clojure.lang.IPersistentMap (encode [data] (let [lhm (LinkedHashMap.)] @@ -161,10 +167,11 @@ (encode data))) (defn parse-string - [^String string & {:keys [constructor unsafe mark keywords max-aliases-for-collections allow-recursive-keys allow-duplicate-keys] :or {keywords true}}] + [^String string & {:keys [unknown-tag passthrough unsafe mark keywords max-aliases-for-collections allow-recursive-keys allow-duplicate-keys] :or {keywords true}}] (decode (.load (make-yaml :unsafe unsafe :mark mark - :constructor constructor + :unknown-tag unknown-tag + :passthrough passthrough :max-aliases-for-collections max-aliases-for-collections :allow-recursive-keys allow-recursive-keys :allow-duplicate-keys allow-duplicate-keys) diff --git a/src/java/clj_yaml/UnknownTagConstructor.java b/src/java/clj_yaml/UnknownTagConstructor.java new file mode 100644 index 0000000..db4c78f --- /dev/null +++ b/src/java/clj_yaml/UnknownTagConstructor.java @@ -0,0 +1,48 @@ +package clj_yaml; + +import org.yaml.snakeyaml.constructor.Construct; +import org.yaml.snakeyaml.constructor.Constructor; +import org.yaml.snakeyaml.constructor.SafeConstructor; +import org.yaml.snakeyaml.constructor.AbstractConstruct; +import org.yaml.snakeyaml.nodes.Node; +import org.yaml.snakeyaml.nodes.Tag; + +public class UnknownTagConstructor extends SafeConstructor { + + public UnknownTagConstructor() { + this.yamlMultiConstructors.put("", new UnknownTagConstruct()); + } + + public class UnknownTagConstruct extends AbstractConstruct { + + public Object construct(Node node) { + Tag unknownTag = node.getTag(); + + Tag newTag = null; + switch (node.getNodeId()) { + case scalar: + newTag = Tag.STR; + break; + case sequence: + newTag = Tag.SEQ; + break; + default: + newTag = Tag.MAP; + break; + } + node.setTag(newTag); + + return new UnknownTag(unknownTag, getConstructor(node).construct(node)); + } + } + + public static class UnknownTag { + public Tag tag; + public Object value; + + public UnknownTag(Tag unknownTag, Object taggedValue) { + this.tag = unknownTag; + this.value = taggedValue; + } + } +} diff --git a/test/clj_yaml/core_test.clj b/test/clj_yaml/core_test.clj index e564065..a2627e6 100644 --- a/test/clj_yaml/core_test.clj +++ b/test/clj_yaml/core_test.clj @@ -2,9 +2,8 @@ (:require [clojure.test :refer (deftest testing is)] [clojure.string :as string] [clojure.java.io :as io] - [clj-yaml.core :refer [parse-string unmark generate-string - parse-stream generate-stream - passthrough-constructor]]) + [clj-yaml.core :as yaml :refer [parse-string unmark generate-string + parse-stream generate-stream]]) (:import [java.util Date] (java.io ByteArrayOutputStream OutputStreamWriter ByteArrayInputStream) java.nio.charset.StandardCharsets @@ -304,7 +303,7 @@ sequence: !CustomSequence - z ") -(deftest passthrough-test +(deftest unknown-tags-test (testing "Throws with unknown tags and default constructor" (is (thrown-with-msg? ConstructorException #"^could not determine a constructor for the tag !CustomScalar" @@ -313,4 +312,27 @@ sequence: !CustomSequence (is (= {:scalar "some-scalar" :mapping {:x "foo" :y "bar"} :sequence ["a" "b" "z"]} - (parse-string yaml-with-unknown-tags :constructor passthrough-constructor))))) + (parse-string yaml-with-unknown-tags :passthrough true)))) + (testing "Can process unknown tags with unknown-tag constructor" + (is (= {:scalar {::yaml/tag "!CustomScalar" ::yaml/value "some-scalar"} + :mapping {::yaml/tag "!CustomMapping" ::yaml/value {:x "foo" :y "bar"}} + :sequence {::yaml/tag "!CustomSequence" ::yaml/value ["a" "b" "z"]}} + (parse-string yaml-with-unknown-tags :unknown-tag true))))) + +(comment + + (def dumper-options (doto (org.yaml.snakeyaml.DumperOptions.) + (.setVersion org.yaml.snakeyaml.DumperOptions$Version/V1_1))) + + (def yaml (org.yaml.snakeyaml.Yaml. + (org.yaml.snakeyaml.representer.Representer.) dumper-options)) + + + @(def out (.dump yaml "083")) + ;; => Reflection warning, /Users/grzm/dev/clj-yaml/test/clj_yaml/core_test.clj:327:3 - call to method dump can't be resolved (target class is unknown). + ;; "%YAML 1.1\n--- foo\n" + (print out) + + (parse-string "083" :mark true) + + :end) From 95d7ad057f9cb50454a9aed414f5728779cf2ad3 Mon Sep 17 00:00:00 2001 From: Michael Glaesemann Date: Thu, 22 Sep 2022 21:01:43 -0500 Subject: [PATCH 04/20] Rename unknown-tag-handling constructors. --- src/clojure/clj_yaml/core.clj | 18 +++++------ ...=> PassThroughUnknownTagsConstructor.java} | 4 +-- ....java => StripUnknownTagsConstructor.java} | 8 ++--- test/clj_yaml/core_test.clj | 30 ++++--------------- 4 files changed, 20 insertions(+), 40 deletions(-) rename src/java/clj_yaml/{UnknownTagConstructor.java => PassThroughUnknownTagsConstructor.java} (91%) rename src/java/clj_yaml/{PassthroughConstructor.java => StripUnknownTagsConstructor.java} (90%) diff --git a/src/clojure/clj_yaml/core.clj b/src/clojure/clj_yaml/core.clj index 00835f2..8dcb0fb 100644 --- a/src/clojure/clj_yaml/core.clj +++ b/src/clojure/clj_yaml/core.clj @@ -5,7 +5,7 @@ (org.yaml.snakeyaml.constructor Constructor SafeConstructor BaseConstructor) (org.yaml.snakeyaml.representer Representer) (org.yaml.snakeyaml.error Mark) - (clj_yaml MarkedConstructor PassthroughConstructor UnknownTagConstructor) + (clj_yaml MarkedConstructor PassThroughUnknownTagsConstructor StripUnknownTagsConstructor) (java.util LinkedHashMap))) (def flow-styles @@ -48,11 +48,9 @@ (.setAllowDuplicateKeys loader allow-duplicate-keys)) loader)) -(def passthrough-constructor (fn [] (PassthroughConstructor.))) - (defn ^Yaml make-yaml "Make a yaml encoder/decoder with some given options." - [& {:keys [unknown-tag passthrough dumper-options unsafe mark max-aliases-for-collections allow-recursive-keys allow-duplicate-keys]}] + [& {:keys [pass-through-unknown-tags? strip-unknown-tags? dumper-options unsafe mark max-aliases-for-collections allow-recursive-keys allow-duplicate-keys]}] (let [loader (make-loader-options :max-aliases-for-collections max-aliases-for-collections :allow-recursive-keys allow-recursive-keys :allow-duplicate-keys allow-duplicate-keys) @@ -65,8 +63,8 @@ ;; are used mark (MarkedConstructor.) - unknown-tag (UnknownTagConstructor.) - passthrough (PassthroughConstructor.) + pass-through-unknown-tags? (PassThroughUnknownTagsConstructor.) + strip-unknown-tags? (StripUnknownTagsConstructor.) ;; TODO: unsafe marked constructor :else (SafeConstructor. loader)) @@ -113,7 +111,7 @@ (-> data .marked (decode keywords))))) - clj_yaml.UnknownTagConstructor$UnknownTag + clj_yaml.PassThroughUnknownTagsConstructor$UnknownTag (decode [data keywords] {::tag (str (.tag data)) ::value (-> (.value data) (decode keywords))}) @@ -167,11 +165,11 @@ (encode data))) (defn parse-string - [^String string & {:keys [unknown-tag passthrough unsafe mark keywords max-aliases-for-collections allow-recursive-keys allow-duplicate-keys] :or {keywords true}}] + [^String string & {:keys [pass-through-unknown-tags? strip-unknown-tags? unsafe mark keywords max-aliases-for-collections allow-recursive-keys allow-duplicate-keys] :or {keywords true}}] (decode (.load (make-yaml :unsafe unsafe :mark mark - :unknown-tag unknown-tag - :passthrough passthrough + :pass-through-unknown-tags? pass-through-unknown-tags? + :strip-unknown-tags? strip-unknown-tags? :max-aliases-for-collections max-aliases-for-collections :allow-recursive-keys allow-recursive-keys :allow-duplicate-keys allow-duplicate-keys) diff --git a/src/java/clj_yaml/UnknownTagConstructor.java b/src/java/clj_yaml/PassThroughUnknownTagsConstructor.java similarity index 91% rename from src/java/clj_yaml/UnknownTagConstructor.java rename to src/java/clj_yaml/PassThroughUnknownTagsConstructor.java index db4c78f..195c1c0 100644 --- a/src/java/clj_yaml/UnknownTagConstructor.java +++ b/src/java/clj_yaml/PassThroughUnknownTagsConstructor.java @@ -7,9 +7,9 @@ import org.yaml.snakeyaml.nodes.Node; import org.yaml.snakeyaml.nodes.Tag; -public class UnknownTagConstructor extends SafeConstructor { +public class PassThroughUnknownTagsConstructor extends SafeConstructor { - public UnknownTagConstructor() { + public PassThroughUnknownTagsConstructor() { this.yamlMultiConstructors.put("", new UnknownTagConstruct()); } diff --git a/src/java/clj_yaml/PassthroughConstructor.java b/src/java/clj_yaml/StripUnknownTagsConstructor.java similarity index 90% rename from src/java/clj_yaml/PassthroughConstructor.java rename to src/java/clj_yaml/StripUnknownTagsConstructor.java index 54bb5d8..3e60ed2 100644 --- a/src/java/clj_yaml/PassthroughConstructor.java +++ b/src/java/clj_yaml/StripUnknownTagsConstructor.java @@ -36,9 +36,9 @@ * This is used as a fallback strategies to use the underlying type instead of * throwing an exception. */ -public class PassthroughConstructor extends Constructor { +public class StripUnknownTagsConstructor extends Constructor { - private class PassthroughConstruct extends AbstractConstruct { + private class StripUnknownTagConstruct extends AbstractConstruct { public Object construct(Node node) { // reset node to scalar tag type for parsing Tag tag = null; @@ -61,8 +61,8 @@ public Object construct(Node node) { public void construct2ndStep(Node node, Object object) {} } - public PassthroughConstructor() { + public StripUnknownTagsConstructor() { // Add a catch-all to catch any unidentifiable nodes - this.yamlMultiConstructors.put("", new PassthroughConstruct()); + this.yamlMultiConstructors.put("", new StripUnknownTagConstruct()); } } diff --git a/test/clj_yaml/core_test.clj b/test/clj_yaml/core_test.clj index a2627e6..5ed5e8a 100644 --- a/test/clj_yaml/core_test.clj +++ b/test/clj_yaml/core_test.clj @@ -166,8 +166,8 @@ the-bin: !!binary 0101") ;; This test ensures that generate-string uses the older behavior by default, for the sake ;; of stability, i.e. backwards compatibility. (is - (= "{description: Big-picture diagram showing how our top-level systems and stakeholders interact}\n" - (generate-string data)))))) + (= "{description: Big-picture diagram showing how our top-level systems and stakeholders interact}\n" + (generate-string data)))))) (deftest dump-opts (let [data [{:age 33 :name "jon"} {:age 44 :name "boo"}]] @@ -308,31 +308,13 @@ sequence: !CustomSequence (is (thrown-with-msg? ConstructorException #"^could not determine a constructor for the tag !CustomScalar" (parse-string yaml-with-unknown-tags)))) - (testing "Can process unknown tags with passthrough constructor" + (testing "Can process unknown tags with strip-unknown-tags? constructor" (is (= {:scalar "some-scalar" :mapping {:x "foo" :y "bar"} :sequence ["a" "b" "z"]} - (parse-string yaml-with-unknown-tags :passthrough true)))) - (testing "Can process unknown tags with unknown-tag constructor" + (parse-string yaml-with-unknown-tags :strip-unknown-tags? true)))) + (testing "Can process unknown tags with pass-through-unknown-tags? constructor" (is (= {:scalar {::yaml/tag "!CustomScalar" ::yaml/value "some-scalar"} :mapping {::yaml/tag "!CustomMapping" ::yaml/value {:x "foo" :y "bar"}} :sequence {::yaml/tag "!CustomSequence" ::yaml/value ["a" "b" "z"]}} - (parse-string yaml-with-unknown-tags :unknown-tag true))))) - -(comment - - (def dumper-options (doto (org.yaml.snakeyaml.DumperOptions.) - (.setVersion org.yaml.snakeyaml.DumperOptions$Version/V1_1))) - - (def yaml (org.yaml.snakeyaml.Yaml. - (org.yaml.snakeyaml.representer.Representer.) dumper-options)) - - - @(def out (.dump yaml "083")) - ;; => Reflection warning, /Users/grzm/dev/clj-yaml/test/clj_yaml/core_test.clj:327:3 - call to method dump can't be resolved (target class is unknown). - ;; "%YAML 1.1\n--- foo\n" - (print out) - - (parse-string "083" :mark true) - - :end) + (parse-string yaml-with-unknown-tags :pass-through-unknown-tags? true))))) From d6e83756792995614824f3d0ab323150f368445b Mon Sep 17 00:00:00 2001 From: Michael Glaesemann Date: Fri, 23 Sep 2022 08:17:54 -0500 Subject: [PATCH 05/20] Specify unknown-tag-fn handler rather than specify options or data shapes. --- README.md | 19 ++++++ src/clojure/clj_yaml/core.clj | 57 ++++++++-------- .../clj_yaml/StripUnknownTagsConstructor.java | 68 ------------------- ...uctor.java => UnknownTagsConstructor.java} | 4 +- test/clj_yaml/core_test.clj | 17 +++-- 5 files changed, 59 insertions(+), 106 deletions(-) delete mode 100644 src/java/clj_yaml/StripUnknownTagsConstructor.java rename src/java/clj_yaml/{PassThroughUnknownTagsConstructor.java => UnknownTagsConstructor.java} (91%) diff --git a/README.md b/README.md index d763cb4..1ad3d8e 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,25 @@ add `:keywords false` parameters to the `parse-string` function: " :keywords false) ``` +Unknown tags can be handled by passing a handler function via the +:unknown-tag-fn parameter. The handler function should take two +arguments: the tag and the value. Note that the value passed to the +`unknown-tag-fn` is a string if it's a scalar, regardless of the +quoting (or lack thereof) of the scalar value. + +```clojure +;; drop tags +(yaml/parse-string "!Base12 10" :unknown-tag-fn (fn [tag value] value)) +;; => "10" +(yaml/parse-string "!Base12 10" + :unknown-tag-fn (fn [tag value] + (if (= "!Base12" tag) + (Integer/parseInt value 12) + value))) +;; => 12 +``` + + Different flow styles (`:auto`, `:block`, `:flow`) allow customization of how YAML is rendered: diff --git a/src/clojure/clj_yaml/core.clj b/src/clojure/clj_yaml/core.clj index 8dcb0fb..11287ca 100644 --- a/src/clojure/clj_yaml/core.clj +++ b/src/clojure/clj_yaml/core.clj @@ -5,7 +5,7 @@ (org.yaml.snakeyaml.constructor Constructor SafeConstructor BaseConstructor) (org.yaml.snakeyaml.representer Representer) (org.yaml.snakeyaml.error Mark) - (clj_yaml MarkedConstructor PassThroughUnknownTagsConstructor StripUnknownTagsConstructor) + (clj_yaml MarkedConstructor UnknownTagsConstructor) (java.util LinkedHashMap))) (def flow-styles @@ -50,7 +50,7 @@ (defn ^Yaml make-yaml "Make a yaml encoder/decoder with some given options." - [& {:keys [pass-through-unknown-tags? strip-unknown-tags? dumper-options unsafe mark max-aliases-for-collections allow-recursive-keys allow-duplicate-keys]}] + [& {:keys [unknown-tag-fn dumper-options unsafe mark max-aliases-for-collections allow-recursive-keys allow-duplicate-keys]}] (let [loader (make-loader-options :max-aliases-for-collections max-aliases-for-collections :allow-recursive-keys allow-recursive-keys :allow-duplicate-keys allow-duplicate-keys) @@ -63,8 +63,7 @@ ;; are used mark (MarkedConstructor.) - pass-through-unknown-tags? (PassThroughUnknownTagsConstructor.) - strip-unknown-tags? (StripUnknownTagsConstructor.) + unknown-tag-fn (UnknownTagsConstructor.) ;; TODO: unsafe marked constructor :else (SafeConstructor. loader)) @@ -96,11 +95,11 @@ "A protocol for things that can be coerced to and from the types that snakeyaml knows how to encode and decode." (encode [data]) - (decode [data keywords])) + (decode [data keywords unknown-tag-fn])) (extend-protocol YAMLCodec clj_yaml.MarkedConstructor$Marked - (decode [data keywords] + (decode [data keywords unknown-tag-fn] (letfn [(from-Mark [^Mark mark] {:line (.getLine mark) :index (.getIndex mark) @@ -109,13 +108,12 @@ (mark (-> data .start from-Mark) (-> data .end from-Mark) (-> data .marked - (decode keywords))))) + (decode keywords unknown-tag-fn))))) + + clj_yaml.UnknownTagsConstructor$UnknownTag + (decode [data keywords unknown-tag-fn] + (unknown-tag-fn (str (.tag data)) (-> (.value data) (decode keywords unknown-tag-fn)))) - clj_yaml.PassThroughUnknownTagsConstructor$UnknownTag - (decode [data keywords] - {::tag (str (.tag data)) - ::value (-> (.value data) (decode keywords))}) - clojure.lang.IPersistentMap (encode [data] (let [lhm (LinkedHashMap.)] @@ -133,7 +131,7 @@ (subs (str data) 1)) java.util.LinkedHashMap - (decode [data keywords] + (decode [data keywords unknown-tag-fn] (letfn [(decode-key [k] (if keywords ;; (keyword k) is nil for numbers etc @@ -141,23 +139,23 @@ k))] (into (ordered-map) (for [[k v] data] - [(-> k (decode keywords) decode-key) (decode v keywords)])))) + [(-> k (decode keywords unknown-tag-fn) decode-key) (decode v keywords unknown-tag-fn)])))) java.util.LinkedHashSet - (decode [data keywords] + (decode [data _keywords _unknown-tag-fn] (into (ordered-set) data)) java.util.ArrayList - (decode [data keywords] - (map #(decode % keywords) data)) + (decode [data keywords unknown-tag-fn] + (map #(decode % keywords unknown-tag-fn) data)) Object (encode [data] data) - (decode [data keywords] data) + (decode [data _keywords _unknown-tag-fn] data) nil (encode [data] data) - (decode [data keywords] data)) + (decode [data _keywords _unknown-tag-fn] data)) (defn generate-string [data & opts] @@ -165,15 +163,14 @@ (encode data))) (defn parse-string - [^String string & {:keys [pass-through-unknown-tags? strip-unknown-tags? unsafe mark keywords max-aliases-for-collections allow-recursive-keys allow-duplicate-keys] :or {keywords true}}] - (decode (.load (make-yaml :unsafe unsafe - :mark mark - :pass-through-unknown-tags? pass-through-unknown-tags? - :strip-unknown-tags? strip-unknown-tags? - :max-aliases-for-collections max-aliases-for-collections - :allow-recursive-keys allow-recursive-keys - :allow-duplicate-keys allow-duplicate-keys) - string) keywords)) + [^String string & {:keys [unknown-tag-fn unsafe mark keywords max-aliases-for-collections allow-recursive-keys allow-duplicate-keys] :or {keywords true}}] + (let [yaml (make-yaml :unsafe unsafe + :mark mark + :unknown-tag-fn unknown-tag-fn + :max-aliases-for-collections max-aliases-for-collections + :allow-recursive-keys allow-recursive-keys + :allow-duplicate-keys allow-duplicate-keys)] + (decode (.load yaml string) keywords unknown-tag-fn))) ;; From https://github.com/metosin/muuntaja/pull/94/files (defn generate-stream @@ -182,6 +179,6 @@ (.dump ^Yaml (apply make-yaml opts) (encode data) writer)) (defn parse-stream - [^java.io.Reader reader & {:keys [keywords] :or {keywords true} :as opts}] + [^java.io.Reader reader & {:keys [keywords unknown-tag-fn] :or {keywords true} :as opts}] (decode (.load ^Yaml (apply make-yaml (into [] cat opts)) - reader) keywords)) + reader) keywords unknown-tag-fn)) diff --git a/src/java/clj_yaml/StripUnknownTagsConstructor.java b/src/java/clj_yaml/StripUnknownTagsConstructor.java deleted file mode 100644 index 3e60ed2..0000000 --- a/src/java/clj_yaml/StripUnknownTagsConstructor.java +++ /dev/null @@ -1,68 +0,0 @@ -// MIT License - -// Copyright (c) 2016 Owain Lewis - -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: - -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. - -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -package clj_yaml; - -import org.yaml.snakeyaml.constructor.Construct; -import org.yaml.snakeyaml.constructor.Constructor; -import org.yaml.snakeyaml.constructor.SafeConstructor; -import org.yaml.snakeyaml.constructor.AbstractConstruct; -import org.yaml.snakeyaml.nodes.Node; -import org.yaml.snakeyaml.nodes.Tag; -import org.yaml.snakeyaml.error.Mark; - -/** - * Implementation of Constructor that ignores YAML tags. - * - * This is used as a fallback strategies to use the underlying type instead of - * throwing an exception. - */ -public class StripUnknownTagsConstructor extends Constructor { - - private class StripUnknownTagConstruct extends AbstractConstruct { - public Object construct(Node node) { - // reset node to scalar tag type for parsing - Tag tag = null; - switch (node.getNodeId()) { - case scalar: - tag = Tag.STR; - break; - case sequence: - tag = Tag.SEQ; - break; - default: - tag = Tag.MAP; - break; - } - - node.setTag(tag); - return getConstructor(node).construct(node); - } - - public void construct2ndStep(Node node, Object object) {} - } - - public StripUnknownTagsConstructor() { - // Add a catch-all to catch any unidentifiable nodes - this.yamlMultiConstructors.put("", new StripUnknownTagConstruct()); - } -} diff --git a/src/java/clj_yaml/PassThroughUnknownTagsConstructor.java b/src/java/clj_yaml/UnknownTagsConstructor.java similarity index 91% rename from src/java/clj_yaml/PassThroughUnknownTagsConstructor.java rename to src/java/clj_yaml/UnknownTagsConstructor.java index 195c1c0..a08a1ba 100644 --- a/src/java/clj_yaml/PassThroughUnknownTagsConstructor.java +++ b/src/java/clj_yaml/UnknownTagsConstructor.java @@ -7,9 +7,9 @@ import org.yaml.snakeyaml.nodes.Node; import org.yaml.snakeyaml.nodes.Tag; -public class PassThroughUnknownTagsConstructor extends SafeConstructor { +public class UnknownTagsConstructor extends SafeConstructor { - public PassThroughUnknownTagsConstructor() { + public UnknownTagsConstructor() { this.yamlMultiConstructors.put("", new UnknownTagConstruct()); } diff --git a/test/clj_yaml/core_test.clj b/test/clj_yaml/core_test.clj index 5ed5e8a..0c2bab3 100644 --- a/test/clj_yaml/core_test.clj +++ b/test/clj_yaml/core_test.clj @@ -312,9 +312,14 @@ sequence: !CustomSequence (is (= {:scalar "some-scalar" :mapping {:x "foo" :y "bar"} :sequence ["a" "b" "z"]} - (parse-string yaml-with-unknown-tags :strip-unknown-tags? true)))) - (testing "Can process unknown tags with pass-through-unknown-tags? constructor" - (is (= {:scalar {::yaml/tag "!CustomScalar" ::yaml/value "some-scalar"} - :mapping {::yaml/tag "!CustomMapping" ::yaml/value {:x "foo" :y "bar"}} - :sequence {::yaml/tag "!CustomSequence" ::yaml/value ["a" "b" "z"]}} - (parse-string yaml-with-unknown-tags :pass-through-unknown-tags? true))))) + (parse-string yaml-with-unknown-tags :unknown-tag-fn (fn [_ v] v))))) + (testing "Can process unknown tags with :unknown-tag-fn as identity" + (is (= {:scalar {:tag "!CustomScalar" :value "some-scalar"} + :mapping {:tag "!CustomMapping" :value {:x "foo" :y "bar"}} + :sequence {:tag "!CustomSequence" :value ["a" "b" "z"]}} + (parse-string yaml-with-unknown-tags :unknown-tag-fn (fn [t v] {:tag t :value v})))) + (is (= {:base-12 12 :base-10 "10"} + (parse-string "{base-12: !Base12 10, base-10: !Base10 10}" + :unknown-tag-fn (fn [t v] + (if (= "!Base12" t) + (Integer/parseInt v 12) v))))))) From 4dba666b98bb8c2319564be34e41088fa787b4c2 Mon Sep 17 00:00:00 2001 From: Michael Glaesemann Date: Fri, 23 Sep 2022 11:20:43 -0500 Subject: [PATCH 06/20] Use map instead of positional args for unknown-tag-fn. --- README.md | 10 +++++----- src/clojure/clj_yaml/core.clj | 3 ++- test/clj_yaml/core_test.clj | 10 +++++----- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 1ad3d8e..e1b46ec 100644 --- a/README.md +++ b/README.md @@ -42,17 +42,17 @@ add `:keywords false` parameters to the `parse-string` function: ``` Unknown tags can be handled by passing a handler function via the -:unknown-tag-fn parameter. The handler function should take two -arguments: the tag and the value. Note that the value passed to the -`unknown-tag-fn` is a string if it's a scalar, regardless of the +:unknown-tag-fn parameter. The handler function is provided a map +which includes `:tag` and `:value` keys. Note that the value passed to +the `unknown-tag-fn` is a string if it's a scalar, regardless of the quoting (or lack thereof) of the scalar value. ```clojure ;; drop tags -(yaml/parse-string "!Base12 10" :unknown-tag-fn (fn [tag value] value)) +(yaml/parse-string "!Base12 10" :unknown-tag-fn :value ;; => "10" (yaml/parse-string "!Base12 10" - :unknown-tag-fn (fn [tag value] + :unknown-tag-fn (fn [{:keys [tag value]}] (if (= "!Base12" tag) (Integer/parseInt value 12) value))) diff --git a/src/clojure/clj_yaml/core.clj b/src/clojure/clj_yaml/core.clj index 11287ca..8476efa 100644 --- a/src/clojure/clj_yaml/core.clj +++ b/src/clojure/clj_yaml/core.clj @@ -112,7 +112,8 @@ clj_yaml.UnknownTagsConstructor$UnknownTag (decode [data keywords unknown-tag-fn] - (unknown-tag-fn (str (.tag data)) (-> (.value data) (decode keywords unknown-tag-fn)))) + (unknown-tag-fn {:tag (str (.tag data)) + :value (-> (.value data) (decode keywords unknown-tag-fn))})) clojure.lang.IPersistentMap (encode [data] diff --git a/test/clj_yaml/core_test.clj b/test/clj_yaml/core_test.clj index 0c2bab3..1bfa653 100644 --- a/test/clj_yaml/core_test.clj +++ b/test/clj_yaml/core_test.clj @@ -312,14 +312,14 @@ sequence: !CustomSequence (is (= {:scalar "some-scalar" :mapping {:x "foo" :y "bar"} :sequence ["a" "b" "z"]} - (parse-string yaml-with-unknown-tags :unknown-tag-fn (fn [_ v] v))))) + (parse-string yaml-with-unknown-tags :unknown-tag-fn :value)))) (testing "Can process unknown tags with :unknown-tag-fn as identity" (is (= {:scalar {:tag "!CustomScalar" :value "some-scalar"} :mapping {:tag "!CustomMapping" :value {:x "foo" :y "bar"}} :sequence {:tag "!CustomSequence" :value ["a" "b" "z"]}} - (parse-string yaml-with-unknown-tags :unknown-tag-fn (fn [t v] {:tag t :value v})))) + (parse-string yaml-with-unknown-tags :unknown-tag-fn identity))) (is (= {:base-12 12 :base-10 "10"} (parse-string "{base-12: !Base12 10, base-10: !Base10 10}" - :unknown-tag-fn (fn [t v] - (if (= "!Base12" t) - (Integer/parseInt v 12) v))))))) + :unknown-tag-fn (fn [{:keys [tag value]}] + (if (= "!Base12" tag) + (Integer/parseInt value 12) value))))))) From 3a73ab9880ff3aac25d0326b27a5597ac9fa4af6 Mon Sep 17 00:00:00 2001 From: Michiel Borkent Date: Thu, 15 Sep 2022 12:48:03 +0200 Subject: [PATCH 07/20] Update CODEOWNERS --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 3da2cbf..8008dd4 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1 @@ -* @marcomorain @slipset +* @marcomorain @slipset @borkdude From 474f1fee62bbf00ead55bde054596f21f42312db Mon Sep 17 00:00:00 2001 From: Michiel Borkent Date: Thu, 15 Sep 2022 13:28:01 +0200 Subject: [PATCH 08/20] Update CODEOWNERS --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 8008dd4..631d0b5 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1 @@ -* @marcomorain @slipset @borkdude +* @marcomorain @slipset @borkdude @lread From 44c2f011bd43fdbedbc8adbc05eac80de5e3baac Mon Sep 17 00:00:00 2001 From: Michiel Borkent Date: Thu, 15 Sep 2022 13:30:04 +0200 Subject: [PATCH 09/20] Update CODEOWNERS From 3525643a135e5ce1d0e505181bf08c0e1a46d5b0 Mon Sep 17 00:00:00 2001 From: Michiel Borkent Date: Thu, 15 Sep 2022 13:30:24 +0200 Subject: [PATCH 10/20] Update CODEOWNERS --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 631d0b5..3adf9d8 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1 @@ -* @marcomorain @slipset @borkdude @lread +* @marcomorain @slipset @borkdude @lread From 9bbc88b6aa5a0e76ee49ee2436a939eb72b00146 Mon Sep 17 00:00:00 2001 From: Lee Read Date: Sat, 17 Sep 2022 19:13:29 -0400 Subject: [PATCH 11/20] docs: readme: add slack chat badge [skip ci] --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index e1b46ec..86340ca 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ decoding for Clojure via the [snakeyaml][] Java library. [![Clojars Project](https://img.shields.io/clojars/v/clj-commons/clj-yaml.svg)](https://clojars.org/clj-commons/clj-yaml) [![cljdoc badge](https://cljdoc.org/badge/clj-commons/clj-yaml)](https://cljdoc.org/d/clj-commons/clj-yaml) [![CircleCI Status](https://circleci.com/gh/clj-commons/clj-yaml.svg?style=svg)](https://circleci.com/gh/clj-commons/clj-yaml) +[![Slack chat](https://img.shields.io/badge/slack-join_chat-brightgreen.svg)](https://clojurians.slack.com/archives/C042XAQFCCU) (This is a maintained fork of [the original][]). From fe99d1b3236485515968dfc542993b51cf9c3d51 Mon Sep 17 00:00:00 2001 From: Lee Read Date: Sun, 18 Sep 2022 16:09:20 -0400 Subject: [PATCH 12/20] docs: readme: correct snakeyaml url [skip ci] it moved --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 86340ca..09e1987 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ `clj-commons/clj-yaml` provides [YAML](http://yaml.org) encoding and decoding for Clojure via the [snakeyaml][] Java library. -[SnakeYAML]: https://bitbucket.org/asomov/snakeyaml/ +[SnakeYAML]: https://bitbucket.org/snakeyaml/snakeyaml [![Clojars Project](https://img.shields.io/clojars/v/clj-commons/clj-yaml.svg)](https://clojars.org/clj-commons/clj-yaml) From 4627e70400571ae06cb622215f612b50760a867b Mon Sep 17 00:00:00 2001 From: Frazer Date: Tue, 20 Sep 2022 01:35:39 +1000 Subject: [PATCH 13/20] Feature/load all (#22) A YAML 1.1 stream can contain multiple documents. Currently, clj-yaml will only load the first document. This change adds the ability to optionally load all documents. Enable by specifying `:load-all?` true in `opts` for `parse-string` or `parse-stream`. --- src/clojure/clj_yaml/core.clj | 22 ++++++++---- test/clj_yaml/core_test.clj | 67 ++++++++++++++++++++++++++++------- 2 files changed, 71 insertions(+), 18 deletions(-) diff --git a/src/clojure/clj_yaml/core.clj b/src/clojure/clj_yaml/core.clj index 8476efa..bf665b9 100644 --- a/src/clojure/clj_yaml/core.clj +++ b/src/clojure/clj_yaml/core.clj @@ -6,7 +6,10 @@ (org.yaml.snakeyaml.representer Representer) (org.yaml.snakeyaml.error Mark) (clj_yaml MarkedConstructor UnknownTagsConstructor) - (java.util LinkedHashMap))) + (java.util LinkedHashMap) + (clj_yaml MarkedConstructor) + (java.util LinkedHashMap) + (java.io StringReader))) (def flow-styles {:auto DumperOptions$FlowStyle/AUTO @@ -163,15 +166,21 @@ (.dump ^Yaml (apply make-yaml opts) (encode data))) +(defn- load-stream [^Yaml yaml ^java.io.Reader input load-all? keywords unknown-tag-fn] + (if load-all? + (map #(decode % keywords unknown-tag-fn) (.loadAll yaml input)) + (decode (.load yaml input) keywords unknown-tag-fn))) + (defn parse-string - [^String string & {:keys [unknown-tag-fn unsafe mark keywords max-aliases-for-collections allow-recursive-keys allow-duplicate-keys] :or {keywords true}}] + [^String string & {:keys [unknown-tag-fn unsafe mark keywords max-aliases-for-collections + allow-recursive-keys allow-duplicate-keys load-all?] :or {keywords true}}] (let [yaml (make-yaml :unsafe unsafe :mark mark :unknown-tag-fn unknown-tag-fn :max-aliases-for-collections max-aliases-for-collections :allow-recursive-keys allow-recursive-keys :allow-duplicate-keys allow-duplicate-keys)] - (decode (.load yaml string) keywords unknown-tag-fn))) + (load-stream yaml (StringReader. string) load-all? keywords unknown-tag-fn))) ;; From https://github.com/metosin/muuntaja/pull/94/files (defn generate-stream @@ -180,6 +189,7 @@ (.dump ^Yaml (apply make-yaml opts) (encode data) writer)) (defn parse-stream - [^java.io.Reader reader & {:keys [keywords unknown-tag-fn] :or {keywords true} :as opts}] - (decode (.load ^Yaml (apply make-yaml (into [] cat opts)) - reader) keywords unknown-tag-fn)) + [^java.io.Reader reader & {:keys [keywords load-all? unknown-tag-fn] :or {keywords true} :as opts}] + (load-stream (apply make-yaml (into [] cat opts)) + reader + load-all? keywords unknown-tag-fn)) diff --git a/test/clj_yaml/core_test.clj b/test/clj_yaml/core_test.clj index 1bfa653..52e97cc 100644 --- a/test/clj_yaml/core_test.clj +++ b/test/clj_yaml/core_test.clj @@ -1,14 +1,19 @@ (ns clj-yaml.core-test - (:require [clojure.test :refer (deftest testing is)] - [clojure.string :as string] - [clojure.java.io :as io] - [clj-yaml.core :as yaml :refer [parse-string unmark generate-string - parse-stream generate-stream]]) - (:import [java.util Date] - (java.io ByteArrayOutputStream OutputStreamWriter ByteArrayInputStream) - java.nio.charset.StandardCharsets - (org.yaml.snakeyaml.error YAMLException) - (org.yaml.snakeyaml.constructor ConstructorException DuplicateKeyException))) + (:require + [clj-yaml.core :as yaml :refer [generate-stream generate-string + parse-stream parse-string unmark]] + [clojure.java.io :as io] + [clojure.string :as string] + [clojure.test :refer (deftest testing is)] + [flatland.ordered.map :refer [ordered-map]]) + (:import + (java.io ByteArrayInputStream ByteArrayOutputStream OutputStreamWriter) + java.nio.charset.StandardCharsets + [java.util Date] + [org.yaml.snakeyaml.composer ComposerException] + (org.yaml.snakeyaml.constructor ConstructorException DuplicateKeyException) + (org.yaml.snakeyaml.constructor DuplicateKeyException) + (org.yaml.snakeyaml.error YAMLException))) (def nested-hash-yaml "root:\n childa: a\n childb: \n grandchild: \n greatgrandchild: bar\n") @@ -166,8 +171,8 @@ the-bin: !!binary 0101") ;; This test ensures that generate-string uses the older behavior by default, for the sake ;; of stability, i.e. backwards compatibility. (is - (= "{description: Big-picture diagram showing how our top-level systems and stakeholders interact}\n" - (generate-string data)))))) + (= "{description: Big-picture diagram showing how our top-level systems and stakeholders interact}\n" + (generate-string data)))))) (deftest dump-opts (let [data [{:age 33 :name "jon"} {:age 44 :name "boo"}]] @@ -276,6 +281,43 @@ foo/bar: 42 (is (roundtrip list-yaml)) (is (roundtrip nested-hash-yaml)))) +(defn- ->stream [string] + (io/reader (.getBytes ^String string))) + +(def multi-doc-yaml " +--- +foo: true +--- +bar: false") + +(def single-doc-yaml " +--- +lol: yolo") + +(deftest load-all-test + (testing "Without load-all?" + (is (= (ordered-map {:lol "yolo"}) + (parse-string single-doc-yaml))) + (is (= (ordered-map {:lol "yolo"}) + (parse-stream (->stream single-doc-yaml)))) + (is (thrown-with-msg? ComposerException #"expected a single document in the stream\n" + (parse-stream (->stream multi-doc-yaml)))) + (is (thrown-with-msg? ComposerException #"expected a single document in the stream\n" + (parse-string multi-doc-yaml)))) + + (testing "With load-all?=true on single docs" + (is (= [(ordered-map {:lol "yolo"})] + (parse-string single-doc-yaml :load-all? true))) + (is (= [(ordered-map {:lol "yolo"})] + (parse-stream (->stream single-doc-yaml) :load-all? true)))) + + (testing "With load-all?=true on multi docs" + (is (= [(ordered-map {:foo true}) (ordered-map {:bar false})] + (parse-string multi-doc-yaml :load-all? true))) + (is (= [(ordered-map {:foo true}) (ordered-map {:bar false})] + (parse-stream (->stream multi-doc-yaml) :load-all? true)))) + ) + (def indented-yaml "todo: - name: Fix issue responsible: @@ -323,3 +365,4 @@ sequence: !CustomSequence :unknown-tag-fn (fn [{:keys [tag value]}] (if (= "!Base12" tag) (Integer/parseInt value 12) value))))))) + From 439812cd13a87958f7a0758a579d0d09cf63901b Mon Sep 17 00:00:00 2001 From: Lee Read Date: Mon, 19 Sep 2022 12:11:20 -0400 Subject: [PATCH 14/20] dev: git ignore lein repl history file (#41) [skip ci] --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 20e9490..ce603ef 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,8 @@ classes pom.xml* /target /.lein-failures +/.lein-repl-history /.lsp /.clj-kondo .nrepl-port -/.calva \ No newline at end of file +/.calva From dc85be6246823f0f999126b06ccb688c947f456e Mon Sep 17 00:00:00 2001 From: Lee Read Date: Mon, 19 Sep 2022 12:44:09 -0400 Subject: [PATCH 15/20] github: disable auto PR review for codeowners (#42) [skip ci] --- .github/CODEOWNERS | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 3adf9d8..597f386 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1,3 @@ -* @marcomorain @slipset @borkdude @lread +# Code owners are auto-invited to review PRs including files matching specified pattern(s). +# We opt out of this by only matching the CODEOWNERS file itself. +.github/CODEOWNERS @marcomorain @slipset @borkdude @lread From f39d74d36627a1c86794025dd868da34b9e56027 Mon Sep 17 00:00:00 2001 From: Lee Read Date: Mon, 19 Sep 2022 14:46:16 -0400 Subject: [PATCH 16/20] docs: update changelog, mention circleci fork (#43) Of note: switched changelog from md to adoc. closes #39 --- CHANGELOG.adoc | 85 ++++++++++++++++++++++++++++++++++++++++++++++++++ CHANGELOG.md | 17 ---------- README.md | 4 +-- 3 files changed, 87 insertions(+), 19 deletions(-) create mode 100644 CHANGELOG.adoc delete mode 100644 CHANGELOG.md diff --git a/CHANGELOG.adoc b/CHANGELOG.adoc new file mode 100644 index 0000000..f24c967 --- /dev/null +++ b/CHANGELOG.adoc @@ -0,0 +1,85 @@ += Changelog + +== Unreleased + +* Support loading all YAML docs via new `:load-all?` opt for `parse-string` and `parse-stream` +(https://github.com/clj-commons/clj-yaml/pull/22[#22]) +(https://github.com/clj-commons/clj-yaml/commits?author=clumsyjedi[@clumsyjedi]) + +== v0.7.110 - 2022-09-13 + +* Bump snakeyaml from 1.31 to 1.32 in response to CVE-2022-38752 +(https://github.com/clj-commons/clj-yaml/pull/34[#34]) +(https://github.com/danielcompton[@danielcompton]) + +== v0.7.109 - 2022-09-02 + +* Bump snakeyaml from 1.26 to 1.31 in response to CVE-2022-25857 +(https://github.com/clj-commons/clj-yaml/pull/33[#33]) +(https://github.com/wavejumper[@wavejumper]) + +== v0.7.108 - 2022-02-21 + +* Add indent control via new `:indent` and `:indicator-indent` for `:dumper-options` +(https://github.com/clj-commons/clj-yaml/issues/27[#27]) +(https://github.com/Oddsor[@Oddsor]) + +== v0.7.107 - 2021-07-15 + +* Add `parse-stream` and `generate-stream` +(https://github.com/clj-commons/clj-yaml/issues/3[#3]) +(https://github.com/davidpham87[@davidpham87]) + +== v0.7.106 - 2021-01-28 + +* Support Clojure namespaced keys +(https://github.com/clj-commons/clj-yaml/issues/16[#16]) +(https://github.com/skuro[@skuro]) + +== v0.7.2 - 2020-09-01 + +* Bump snakeyaml from 1.25 to 1.26 in response to CVE-2017-18640 +(https://github.com/clj-commons/clj-yaml/pull/13[#13]) +(https://github.com/erichaberkorn[@erichaberkorn]) +* Expose `:max-aliases-for-collections` and `:allow-recursive-keys` options for `parse-string` +(https://github.com/clj-commons/clj-yaml/pull/13[#13]) +(https://github.com/erichaberkorn[@erichaberkorn]) +* Expose `:allow-duplicate-keys` option for `parse-string` +(https://github.com/clj-commons/clj-yaml/pull/14[#14]) +(https://github.com/erichaberkorn[@erichaberkorn]) + +== v0.7.1 - 2019-04-14 + +* Bump snakeyaml from 1.24 to 1.25, org.flatland/ordered from 1.57 to 1.59, and clojure from 1.7.0 to 1.10.1 (and mark as provided) +(https://github.com/clj-commons/clj-yaml/pull/7[#7]) +(https://github.com/stig[@stig]) +(https://github.com/clj-commons/clj-yaml/pull/10[#10]) +(https://github.com/slipset[@slipset]) + +== v0.7.0 - 2019-03-15 + +* Accept emoji +(https://github.com/clj-commons/clj-yaml/pull/5[#5]) +(https://github.com/gordonsyme[@gordonsyme]) +* Bump snakeyaml from 1.23 to 1.24 +(https://github.com/clj-commons/clj-yaml/pull/5[#5]) +(https://github.com/gordonsyme[@gordonsyme]) + +== v0.6.1 - 2019-02-06 + +* Restore default text-wrapping behavior of prior release +(https://github.com/clj-commons/clj-yaml/pull/2[#2]) +(https://github.com/aviflax[@aviflax]) + +== v0.6.0 - 2019-01-04 + +First release under https://github.com/clj-commons[clj-commons] project! + +* Change org and group-id: `circleci` is now `clj-commons` +(https://github.com/slipset[@slipset]) + +== Older versions + +This project forked from https://github.com/CircleCI-Archived/clj-yaml[circleci/yaml] which forked from https://github.com/lancepantz/clj-yaml[clj-yaml]. + +Neither of these projects maintained a change log. diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index 9e11563..0000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,17 +0,0 @@ -# Changelog -All notable changes to this project will be documented in this file. - -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - -## [0.7.0] - 2019-03-15 -### Changed -- [Accept emoji](https://github.com/clj-commons/clj-yaml/pull/5). -- [Update to SnakeYAML from 1.23 to 1.24](https://github.com/clj-commons/clj-yaml/pull/5). - -## [0.6.1] - 2019-02-06 -### Changed -- [Restore default text-wrapping behavior of prior release](https://github.com/clj-commons/clj-yaml/pull/2). - -## [0.6.0] - 2019-01-04 -### Added -- First release under [clj-commons](https://github.com/clj-commons) project. diff --git a/README.md b/README.md index 09e1987..780daad 100644 --- a/README.md +++ b/README.md @@ -9,11 +9,11 @@ decoding for Clojure via the [snakeyaml][] Java library. [![CircleCI Status](https://circleci.com/gh/clj-commons/clj-yaml.svg?style=svg)](https://circleci.com/gh/clj-commons/clj-yaml) [![Slack chat](https://img.shields.io/badge/slack-join_chat-brightgreen.svg)](https://clojurians.slack.com/archives/C042XAQFCCU) -(This is a maintained fork of [the original][]). +(This is a maintained fork of [the circleci fork][] which forked from [the original][]) +[the circleci fork]: https://github.com/CircleCI-Archived/clj-yaml [the original]: https://github.com/lancepantz/clj-yaml - ## Usage ```clojure From 7e839f48c3574d63e60b5da7dde1ad12a3d6517b Mon Sep 17 00:00:00 2001 From: Lee Read Date: Mon, 19 Sep 2022 16:28:22 -0400 Subject: [PATCH 17/20] docs: update version in README (#45) Of note: switched from md to adoc thinking that a version in a var at the top of the doc might encourage us to keep this up to date? Closes #44 --- README.adoc | 106 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 README.adoc diff --git a/README.adoc b/README.adoc new file mode 100644 index 0000000..63161c5 --- /dev/null +++ b/README.adoc @@ -0,0 +1,106 @@ += `clj-commons/clj-yaml` +:lib-version: 0.7.110 +:project-coords: clj-commons/clj-yaml + +Provides http://yaml.org[YAML] encoding and decoding for Clojure via the https://bitbucket.org/snakeyaml/snakeyaml[snakeyaml] Java library. + +// Badges +https://clojars.org/{project-coords}[image:https://img.shields.io/clojars/v/{project-coords}.svg[Clojars Project]] +https://cljdoc.org/d/{project-coords}[image:https://cljdoc.org/badge/{project-coords}[cljdoc badge]] +https://circleci.com/gh/{project-coords}[image:https://circleci.com/gh/{project-coords}.svg?style=svg[CircleCI Status]] +https://clojurians.slack.com/archives/C042XAQFCCU[image:https://img.shields.io/badge/slack-join_chat-brightgreen.svg[Slack chat]] + +(This is a maintained fork of https://github.com/CircleCI-Archived/clj-yaml[the circleci fork] which forked from https://github.com/lancepantz/clj-yaml[the original]) + +== Usage + +[source,clojure] +---- +(require '[clj-yaml.core :as yaml]) + +(yaml/generate-string + [{:name "John Smith", :age 33} + {:name "Mary Smith", :age 27}]) +"- {name: John Smith, age: 33}\n- {name: Mary Smith, age: 27}\n" + +(yaml/parse-string " +- {name: John Smith, age: 33} +- name: Mary Smith + age: 27 +") +=> ({:name "John Smith", :age 33} + {:name "Mary Smith", :age 27}) +---- + +By default, keys are converted to clojure keywords. To prevent this, add `:keywords false` parameters to the `parse-string` function: + +[source,clojure] +---- +(yaml/parse-string " +- {name: John Smith} +" :keywords false) +---- + +Different flow styles (`:auto`, `:block`, `:flow`) allow customization of how YAML is rendered: + +[source,clojure] +---- +(yaml/generate-string some-data :dumper-options {:flow-style :block}) +---- + +Use the `:indent` (default: `2`) and `:indicator-indent` (default: `0`) options to adjust indentation: + +[source,clojure] +---- +(yaml/generate-string some-data :dumper-options {:indent 6 + :indicator-indent 3 + :flow-style :block}) +=> +todo: + - name: Fix issue + responsible: + name: Rita +---- + +`:indent` must always be larger than `:indicator-indent`. If only 1 higher, the indicator will be on a separate line: + +[source,clojure] +---- +(yaml/generate-string some-data :dumper-options {:indent 2 + :indicator-indent 1 + :flow-style :block}) +=> +todo: + - + name: Fix issue + responsible: + name: Rita +---- + +== Installation + +`clj-commons/clj-yaml` is available as a Maven artifact from http://clojars.org/clj-commons/clj-yaml[Clojars]. + +=== Leiningen/Boot + +[source,clojure,subs="attributes+"] +---- +[clj-commons/clj-yaml "{lib-version}"] +---- + +=== Clojure CLI/`deps.edn` + +[source,clojure,subs="attributes+"] +---- +clj-commons/clj-yaml {:mvn/version "{lib-version}"} +---- + +== Development + +[source,shell] +---- +$ git clone git://github.com/clj-commons/clj-yaml.git +$ lein deps +$ lein test +$ lein install +---- From ebff8a22ab68d004e36d7b56219e1067012d493f Mon Sep 17 00:00:00 2001 From: Lee Read Date: Mon, 19 Sep 2022 17:34:57 -0400 Subject: [PATCH 18/20] ci: trigger build-deploy job only on version tag (#46) Closes #40 --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 594eed0..1a26798 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,6 +12,8 @@ workflows: requires: - build filters: + branches: + ignore: /.*/ tags: only: /Release-.*/ context: From 57b8b3bce3441491d38b55e2aca0128ff412ae74 Mon Sep 17 00:00:00 2001 From: Lee Read Date: Thu, 22 Sep 2022 11:44:02 -0400 Subject: [PATCH 19/20] docs: add babashka built-in badge [skip ci] --- README.adoc | 1 + 1 file changed, 1 insertion(+) diff --git a/README.adoc b/README.adoc index 63161c5..42d63b8 100644 --- a/README.adoc +++ b/README.adoc @@ -8,6 +8,7 @@ Provides http://yaml.org[YAML] encoding and decoding for Clojure via the https:/ https://clojars.org/{project-coords}[image:https://img.shields.io/clojars/v/{project-coords}.svg[Clojars Project]] https://cljdoc.org/d/{project-coords}[image:https://cljdoc.org/badge/{project-coords}[cljdoc badge]] https://circleci.com/gh/{project-coords}[image:https://circleci.com/gh/{project-coords}.svg?style=svg[CircleCI Status]] +https://babashka.org[image:https://raw.githubusercontent.com/babashka/babashka/master/logo/built-in-badge.svg[bb built-in]] https://clojurians.slack.com/archives/C042XAQFCCU[image:https://img.shields.io/badge/slack-join_chat-brightgreen.svg[Slack chat]] (This is a maintained fork of https://github.com/CircleCI-Archived/clj-yaml[the circleci fork] which forked from https://github.com/lancepantz/clj-yaml[the original]) From bdcd5877fdbf5d4f7b33a6dd69c83e536d7cba78 Mon Sep 17 00:00:00 2001 From: Lee Read Date: Fri, 23 Sep 2022 12:45:36 -0400 Subject: [PATCH 20/20] changelog: refer to SnakeYAML changelog [skip ci] --- CHANGELOG.adoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.adoc b/CHANGELOG.adoc index f24c967..3ace303 100644 --- a/CHANGELOG.adoc +++ b/CHANGELOG.adoc @@ -1,5 +1,7 @@ = Changelog +Clj-yaml makes use of SnakeYAML, please also refer to the https://bitbucket.org/snakeyaml/snakeyaml/wiki/Changes[SnakeYAML changelog]. + == Unreleased * Support loading all YAML docs via new `:load-all?` opt for `parse-string` and `parse-stream`