Support :unknown-tag-fn

README.md

@@ -0,0 +1,123 @@
+`clj-commons/clj-yaml` provides [YAML](http://yaml.org) encoding and
+decoding for Clojure via the [snakeyaml][] Java library.
+
+[SnakeYAML]: https://bitbucket.org/snakeyaml/snakeyaml
+
+
+[![Clojars Project](https://img.shields.io/clojars/v/clj-commons/clj-yaml.svg)](https://clojars.org/clj-commons/clj-yaml) 
+[![cljdoc badge](https://cljdoc.org/badge/clj-commons/clj-yaml)](https://cljdoc.org/d/clj-commons/clj-yaml)
+[![CircleCI Status](https://circleci.com/gh/clj-commons/clj-yaml.svg?style=svg)](https://circleci.com/gh/clj-commons/clj-yaml)
+[![Slack chat](https://img.shields.io/badge/slack-join_chat-brightgreen.svg)](https://clojurians.slack.com/archives/C042XAQFCCU)
+
+(This is a maintained fork of [the circleci fork][] which forked from [the original][])
+
+[the circleci fork]: https://github.com/CircleCI-Archived/clj-yaml
+[the original]: https://github.com/lancepantz/clj-yaml
+
+## Usage
+
+```clojure
+(require '[clj-yaml.core :as yaml])
+
+(yaml/generate-string
+  [{:name "John Smith", :age 33}
+   {:name "Mary Smith", :age 27}])
+"- {name: John Smith, age: 33}\n- {name: Mary Smith, age: 27}\n"
+
+(yaml/parse-string "
+- {name: John Smith, age: 33}
+- name: Mary Smith
+  age: 27
+")
+=> ({:name "John Smith", :age 33}
+    {:name "Mary Smith", :age 27})
+```
+
+By default, keys are converted to clojure keywords.  To prevent this,
+add `:keywords false` parameters to the `parse-string` function:
+
+```clojure
+(yaml/parse-string "
+- {name: John Smith}
+" :keywords false)
+```
+
+Unknown tags can be handled by passing a handler function via the
+:unknown-tag-fn parameter. The handler function is provided a map
+which includes `:tag` and `:value` keys. Note that the value passed to
+the `unknown-tag-fn` is a string if it's a scalar, regardless of the
+quoting (or lack thereof) of the scalar value.
+
+```clojure
+;; drop tags
+(yaml/parse-string "!Base12 10" :unknown-tag-fn :value
+;; => "10"
+(yaml/parse-string "!Base12 10"
+                   :unknown-tag-fn (fn [{:keys [tag value]}]
+                                      (if (= "!Base12" tag)
+                                          (Integer/parseInt value 12)
+                                          value)))
+;; => 12
+```
+
+
+Different flow styles (`:auto`, `:block`, `:flow`) allow customization of how YAML is rendered:
+
+
+```clojure
+(yaml/generate-string some-data :dumper-options {:flow-style :block})
+```
+
+Use the `:indent` (default: 2) and `:indicator-indent` (default: 0) options to adjust indentation:
+
+```clojure
+(yaml/generate-string some-data :dumper-options {:indent 6
+                                                 :indicator-indent 3
+                                                 :flow-style :block})
+=>
+todo:
+   -  name: Fix issue
+      responsible:
+            name: Rita
+```
+`:indent` must always be larger than `:indicator-indent`. If only 1 higher, the indicator will be on a separate line:
+```clojure
+(yaml/generate-string some-data :dumper-options {:indent 2
+                                                 :indicator-indent 1
+                                                 :flow-style :block})
+=>
+todo:
+ -
+  name: Fix issue
+  responsible:
+    name: Rita
+```
+
+## Installation
+
+`clj-commons/clj-yaml` is available as a Maven artifact from [Clojars](http://clojars.org/clj-commons/clj-yaml).
+
+### Leiningen/Boot
+
+```clojure
+[clj-commons/clj-yaml "0.7.0"]
+```
+
+### Clojure CLI/`deps.edn`
+
+```clojure
+clj-commons/clj-yaml {:mvn/version "0.7.0"}
+```
+
+## Development
+
+    $ git clone git://github.com/clj-commons/clj-yaml.git
+    $ lein deps
+    $ lein test
+    $ lein install
+
+## License
+
+(c) Lance Bradley - Licensed under the same terms as clojure itself. See LICENCE file for details.
+
+Portions (c) Owain Lewis as marked.

src/clojure/clj_yaml/core.clj

@@ -5,6 +5,8 @@
            (org.yaml.snakeyaml.constructor Constructor SafeConstructor BaseConstructor)
            (org.yaml.snakeyaml.representer Representer)
            (org.yaml.snakeyaml.error Mark)
+           (clj_yaml MarkedConstructor UnknownTagsConstructor)
+           (java.util LinkedHashMap)
            (clj_yaml MarkedConstructor)
            (java.util LinkedHashMap)
            (java.io StringReader)))
@@ -51,19 +53,24 @@
 
 (defn ^Yaml make-yaml
   "Make a yaml encoder/decoder with some given options."
-  [& {:keys [dumper-options unsafe mark max-aliases-for-collections allow-recursive-keys allow-duplicate-keys]}]
+  [& {:keys [unknown-tag-fn dumper-options unsafe mark max-aliases-for-collections allow-recursive-keys allow-duplicate-keys]}]
   (let [loader (make-loader-options :max-aliases-for-collections max-aliases-for-collections
                                     :allow-recursive-keys allow-recursive-keys
                                     :allow-duplicate-keys allow-duplicate-keys)
         ^BaseConstructor constructor
-        (if unsafe (Constructor. loader)
-            (if mark
-              ;; construct2ndStep isn't implemented by MarkedConstructor,
-              ;; causing an exception to be thrown before loader options are
-              ;; used
-              (MarkedConstructor.)
-              (SafeConstructor. loader)))
-        ;; TODO: unsafe marked constructor
+        (cond
+          unsafe (Constructor. loader)
+
+          ;; construct2ndStep isn't implemented by MarkedConstructor,
+          ;; causing an exception to be thrown before loader options
+          ;; are used
+          mark (MarkedConstructor.)
+
+          unknown-tag-fn (UnknownTagsConstructor.)
+
+          ;; TODO: unsafe marked constructor
+          :else (SafeConstructor. loader))
+
         dumper (make-dumper-options dumper-options)]
     (Yaml. constructor (Representer.) dumper loader)))
 
@@ -91,11 +98,11 @@
   "A protocol for things that can be coerced to and from the types
    that snakeyaml knows how to encode and decode."
   (encode [data])
-  (decode [data keywords]))
+  (decode [data keywords unknown-tag-fn]))
 
 (extend-protocol YAMLCodec
   clj_yaml.MarkedConstructor$Marked
-  (decode [data keywords]
+  (decode [data keywords unknown-tag-fn]
     (letfn [(from-Mark [^Mark mark]
               {:line (.getLine mark)
                :index (.getIndex mark)
@@ -104,7 +111,12 @@
       (mark (-> data .start from-Mark)
             (-> data .end from-Mark)
             (-> data .marked
-                (decode keywords)))))
+                (decode keywords unknown-tag-fn)))))
+
+  clj_yaml.UnknownTagsConstructor$UnknownTag
+  (decode [data keywords unknown-tag-fn]
+    (unknown-tag-fn {:tag (str (.tag data))
+                     :value (-> (.value data) (decode keywords unknown-tag-fn))}))
 
   clojure.lang.IPersistentMap
   (encode [data]
@@ -123,51 +135,52 @@
     (subs (str data) 1))
 
   java.util.LinkedHashMap
-  (decode [data keywords]
+  (decode [data keywords unknown-tag-fn]
     (letfn [(decode-key [k]
               (if keywords
                 ;; (keyword k) is nil for numbers etc
                 (or (keyword k) k)
                 k))]
       (into (ordered-map)
             (for [[k v] data]
-              [(-> k (decode keywords) decode-key) (decode v keywords)]))))
+              [(-> k (decode keywords unknown-tag-fn) decode-key) (decode v keywords unknown-tag-fn)]))))
 
   java.util.LinkedHashSet
-  (decode [data keywords]
+  (decode [data _keywords _unknown-tag-fn]
     (into (ordered-set) data))
 
   java.util.ArrayList
-  (decode [data keywords]
-    (map #(decode % keywords) data))
+  (decode [data keywords unknown-tag-fn]
+    (map #(decode % keywords unknown-tag-fn) data))
 
   Object
   (encode [data] data)
-  (decode [data keywords] data)
+  (decode [data _keywords _unknown-tag-fn] data)
 
   nil
   (encode [data] data)
-  (decode [data keywords] data))
+  (decode [data _keywords _unknown-tag-fn] data))
 
 
 (defn generate-string [data & opts]
   (.dump ^Yaml (apply make-yaml opts)
          (encode data)))
 
-(defn- load-stream [^Yaml yaml ^java.io.Reader input load-all? keywords]
+(defn- load-stream [^Yaml yaml ^java.io.Reader input load-all? keywords unknown-tag-fn]
   (if load-all?
-    (map #(decode % keywords) (.loadAll yaml input))
-    (decode (.load yaml input) keywords)))
+    (map #(decode % keywords unknown-tag-fn) (.loadAll yaml input))
+    (decode (.load yaml input) keywords unknown-tag-fn)))
 
 (defn parse-string
-  [^String string & {:keys [unsafe mark keywords max-aliases-for-collections allow-recursive-keys allow-duplicate-keys load-all?] :or {keywords true}}]
-  (load-stream (make-yaml :unsafe unsafe
-                          :mark mark
-                          :max-aliases-for-collections max-aliases-for-collections
-                          :allow-recursive-keys allow-recursive-keys
-                          :allow-duplicate-keys allow-duplicate-keys)
-               (StringReader. string)
-               load-all? keywords))
+  [^String string & {:keys [unknown-tag-fn unsafe mark keywords max-aliases-for-collections
+                            allow-recursive-keys allow-duplicate-keys load-all?] :or {keywords true}}]
+  (let [yaml (make-yaml :unsafe unsafe
+                        :mark mark
+                        :unknown-tag-fn unknown-tag-fn
+                        :max-aliases-for-collections max-aliases-for-collections
+                        :allow-recursive-keys allow-recursive-keys
+                        :allow-duplicate-keys allow-duplicate-keys)]
+    (load-stream yaml (StringReader. string) load-all? keywords unknown-tag-fn)))
 
 ;; From https://github.com/metosin/muuntaja/pull/94/files
 (defn generate-stream
@@ -176,7 +189,7 @@
   (.dump ^Yaml (apply make-yaml opts) (encode data) writer))
 
 (defn parse-stream
-  [^java.io.Reader reader & {:keys [keywords load-all?] :or {keywords true} :as opts}]
+  [^java.io.Reader reader & {:keys [keywords load-all? unknown-tag-fn] :or {keywords true} :as opts}]
   (load-stream (apply make-yaml (into [] cat opts))
                reader
-               load-all? keywords))
+               load-all? keywords unknown-tag-fn))

src/java/clj_yaml/UnknownTagsConstructor.java

@@ -0,0 +1,48 @@
+package clj_yaml;
+
+import org.yaml.snakeyaml.constructor.Construct;
+import org.yaml.snakeyaml.constructor.Constructor;
+import org.yaml.snakeyaml.constructor.SafeConstructor;
+import org.yaml.snakeyaml.constructor.AbstractConstruct;
+import org.yaml.snakeyaml.nodes.Node;
+import org.yaml.snakeyaml.nodes.Tag;
+
+public class UnknownTagsConstructor extends SafeConstructor {
+
+    public UnknownTagsConstructor() {
+        this.yamlMultiConstructors.put("", new UnknownTagConstruct());
+    }
+
+    public class UnknownTagConstruct extends AbstractConstruct {
+
+        public Object construct(Node node) {
+            Tag unknownTag = node.getTag();
+
+            Tag newTag = null;
+            switch (node.getNodeId()) {
+            case scalar:
+                newTag = Tag.STR;
+                break;
+            case sequence:
+                newTag = Tag.SEQ;
+                break;
+            default:
+                newTag = Tag.MAP;
+                break;
+            }
+            node.setTag(newTag);
+
+            return new UnknownTag(unknownTag, getConstructor(node).construct(node));
+        }
+    }
+
+    public static class UnknownTag {
+        public Tag tag;
+        public Object value;
+
+        public UnknownTag(Tag unknownTag, Object taggedValue) {
+            this.tag = unknownTag;
+            this.value = taggedValue;
+        }
+    }
+}

test/clj_yaml/core_test.clj

@@ -1,16 +1,19 @@
 (ns clj-yaml.core-test
-  (:require [clojure.test :refer (deftest testing is)]
-            [clojure.string :as string]
-            [clojure.java.io :as io]
-            [clj-yaml.core :refer [parse-string unmark generate-string
-                                   parse-stream generate-stream]]
-            [flatland.ordered.map :refer [ordered-map]])
-  (:import [java.util Date]
-           (java.io ByteArrayOutputStream OutputStreamWriter ByteArrayInputStream)
-           java.nio.charset.StandardCharsets
-           (org.yaml.snakeyaml.error YAMLException)
-           (org.yaml.snakeyaml.constructor DuplicateKeyException)
-           [org.yaml.snakeyaml.composer ComposerException]))
+  (:require
+   [clj-yaml.core :as yaml :refer [generate-stream generate-string
+                                   parse-stream parse-string unmark]]
+   [clojure.java.io :as io]
+   [clojure.string :as string]
+   [clojure.test :refer (deftest testing is)]
+   [flatland.ordered.map :refer [ordered-map]])
+  (:import
+   (java.io ByteArrayInputStream ByteArrayOutputStream OutputStreamWriter)
+   java.nio.charset.StandardCharsets
+   [java.util Date]
+   [org.yaml.snakeyaml.composer ComposerException]
+   (org.yaml.snakeyaml.constructor ConstructorException DuplicateKeyException)
+   (org.yaml.snakeyaml.constructor DuplicateKeyException)
+   (org.yaml.snakeyaml.error YAMLException)))
 
 (def nested-hash-yaml
   "root:\n  childa: a\n  childb: \n    grandchild: \n      greatgrandchild: bar\n")
@@ -330,3 +333,36 @@ lol: yolo")
                             :dumper-options {:indent 5
                                              :indicator-indent 2
                                              :flow-style :block})))))
+
+(def yaml-with-unknown-tags "---
+scalar: !CustomScalar some-scalar
+mapping: !CustomMapping
+  x: foo
+  y: bar
+sequence: !CustomSequence
+  - a
+  - b
+  - z
+")
+
+(deftest unknown-tags-test
+  (testing "Throws with unknown tags and default constructor"
+    (is (thrown-with-msg? ConstructorException
+                          #"^could not determine a constructor for the tag !CustomScalar"
+                          (parse-string yaml-with-unknown-tags))))
+  (testing "Can process unknown tags with strip-unknown-tags? constructor"
+    (is (= {:scalar "some-scalar"
+            :mapping {:x "foo" :y "bar"}
+            :sequence ["a" "b" "z"]}
+           (parse-string yaml-with-unknown-tags :unknown-tag-fn :value))))
+  (testing "Can process unknown tags with :unknown-tag-fn as identity"
+    (is (= {:scalar {:tag "!CustomScalar" :value "some-scalar"}
+            :mapping {:tag "!CustomMapping" :value {:x "foo" :y "bar"}}
+            :sequence {:tag "!CustomSequence" :value ["a" "b" "z"]}}
+           (parse-string yaml-with-unknown-tags :unknown-tag-fn identity)))
+    (is (= {:base-12 12 :base-10 "10"}
+           (parse-string "{base-12: !Base12 10, base-10: !Base10 10}"
+                         :unknown-tag-fn (fn [{:keys [tag value]}]
+                                           (if (= "!Base12" tag)
+                                             (Integer/parseInt value 12) value)))))))
+