diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn new file mode 100644 index 0000000..8bd021f --- /dev/null +++ b/.clj-kondo/config.edn @@ -0,0 +1 @@ +{:lint-as {me.raynes.conch/let-programs clojure.core/let}} diff --git a/.gitignore b/.gitignore index 69a0f9f..020e839 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ pom.xml.asc .hgignore .hg/ /jet +.clj-kondo/.cache diff --git a/README.md b/README.md index 186c34c..2045e55 100644 --- a/README.md +++ b/README.md @@ -6,27 +6,6 @@ CLI to transform JSON into EDN into Transit and vice versa. -## Usage - -`jet` supports the following options: - - - `--from`: allowed values: `edn`, `transit` or `json` - - `--to`: allowed values: `edn`, `transit` or `json` - - `--keywordize`: if present, keywordizes JSON keys. - - `--pretty`: if present, pretty-prints JSON and EDN output. - - `--version`: if present, prints current version of `jet` and exits. - -Examples: - -``` shellsession -$ echo '{"a": 1}' | jet --from json --to edn -{"a" 1} -$ echo '{"a": 1}' | jet --from json --keywordize --to edn -{:a 1} -$ echo '{"a": 1}' | jet --from json --to transit -["^ ","a",1] -``` - ## Installation Linux and macOS binaries are provided via brew. @@ -39,6 +18,8 @@ Upgrade: brew upgrade jet +You may also download a binary from [Github](https://github.com/borkdude/jet/releases). + This tool can also be used via the JVM. If you use leiningen, you can put the following in your `.lein/profiles`: @@ -55,6 +36,36 @@ $ echo '["^ ","~:a",1]' | lein jet --from transit --to edn {:a 1} ``` +## Usage + +`jet` supports the following options: + + - `--from`: allowed values: `edn`, `transit` or `json` + - `--to`: allowed values: `edn`, `transit` or `json` + - `--keywordize`: if present, keywordizes JSON keys. + - `--pretty`: if present, pretty-prints JSON and EDN output. + - `--query`: if present, applies query to output. See [Query](query.md). + - `--version`: if present, prints current version of `jet` and exits. + +Examples: + +``` shellsession +$ echo '{"a": 1}' | jet --from json --to edn +{"a" 1} + +$ echo '{"a": 1}' | jet --from json --keywordize --to edn +{:a 1} + +$ echo '{"a": 1}' | jet --from json --to transit +["^ ","a",1] + +$ echo '[{:a {:b 1}} {:a {:b 2}}]' \ +| jet --from edn --to edn --query '(filter (= [:a :b] 1))' +[{:a {:b 1}}] +``` + +## [Query](query.md) + ## Test Test the JVM version: diff --git a/doc/cljdoc.edn b/doc/cljdoc.edn new file mode 100644 index 0000000..ea1d935 --- /dev/null +++ b/doc/cljdoc.edn @@ -0,0 +1,3 @@ +{:cljdoc.doc/tree + [["Readme" {:file "README.md"}] + ["Query" {:file "doc/query.md"}]]} diff --git a/doc/query.md b/doc/query.md new file mode 100644 index 0000000..1a90d2e --- /dev/null +++ b/doc/query.md @@ -0,0 +1,148 @@ +# Query + +The `--query` option allows to select or remove specific parts of the output. A +query is written in EDN. + +NOTE: some parts of this query language may change in the coming months after it +has seen more usage (2019-08-04). + +Single values can be selected by using a key: + +``` clojure +echo '{:a 1}' | jet --from edn --to edn --query ':a' +1 +``` + +Multiple values can be selected using a map: + +``` clojure +echo '{:a 1 :b 2 :c 3}' | jet --from edn --to edn --query '{:a true :b true}' +{:a 1, :b 2} +``` + +By default, only keys that have truthy values in the query will be selected from +the output. However, if one of the values has a falsy value, this behavior is +reversed and other keys are left in: + +``` clojure +echo '{:a 1 :b 2 :c 3}' | jet --from edn --to edn --query '{:c false}' +{:a 1, :b 2} +``` + +``` clojure +$ echo '{:a {:a/a 1 :a/b 2} :b 2 :c 3}' \ +| jet --from edn --to edn --query '{:c false :a {:a/b true}}' +{:b 2, :a #:a{:b 2}} +``` + +If the query is applied to a list-like value, the query is applied to all the +elements inside the list-like value: + +``` clojure +echo '[{:a 1 :b 2} {:a 2 :b 3}]' | jet --from edn --to edn --query '{:a true}' +[{:a 1} {:a 2}] +``` + +Nested values can be selected by using a nested query: + +``` clojure +echo '{:a {:a/a 1 :a/b 2} :b 2}' | jet --from edn --to edn --query '{:a {:a/a true}}' +{:a {:a/a 1}} +``` + +Some Clojure-like functions are supported which are mostly intented to operate +on list-like values, except for `keys`, `vals` and `map-vals` which operate on +maps: + +``` clojure +echo '[1 2 3]' | jet --from edn --to edn --query '(first)' +1 +``` + +``` clojure +echo '[1 2 3]' | jet --from edn --to edn --query '(last)' +3 +``` + +``` clojure +echo '[[1 2 3] [4 5 6]]' | jet --from edn --to edn --query '(map last)' +[3 6] +``` + +``` clojure +echo '{:a [1 2 3]}' | jet --from edn --to edn --query '{:a (take 2)}' +{:a [1 2]} +``` + +``` clojure +echo '{:a [1 2 3]}' | jet --from edn --to edn --query '{:a (drop 2)}' +{:a [3]} +``` + +``` clojure +echo '{:a [1 2 3]}' | jet --from edn --to edn --query '{:a (nth 2)}' +{:a 3} +``` + +``` clojure +$ echo '{:a [1 2 3]}' | jet --from edn --to edn --query '{:a (juxt first last)}' +{:a [1 3]} +``` + +``` clojure +$ echo '{:a [1 2 3] :b [4 5 6]}' | jet --from edn --to edn --query '(juxt :a :b)' +[[1 2 3] [4 5 6]] +``` + +``` clojure +$ echo '{:a [1 2 3] :b [4 5 6]}' | jet --from edn --to edn --query '(keys)' +[:a :b] +``` + +``` clojure +$ echo '{:a [1 2 3] :b [4 5 6]}' | jet --from edn --to edn --query '(vals)' +[[1 2 3] [4 5 6]] +``` + +``` clojure +$ echo '{:foo {:a 1 :b 2} :bar {:a 1 :b 2}}' | jet --from edn --to edn --query '(map-vals :a)' +{:foo 1 :bar 2} +``` + +Multiple queries in a vector are applied after one another: + +``` clojure +$ echo '{:a 1 :b 2 :c 3}' | jet --from edn --to edn --query '[{:c false} (vals)]' +[1 2] +``` + +``` clojure +$ echo '{:keys [:a :b :c] :vals [1 2 3]}' \ +| lein jet --from edn --to edn --query '[(juxt :keys :vals) (zipmap)]' +{:a 1, :b 2, :c 3} +``` + +``` clojure +$ curl -s https://jsonplaceholder.typicode.com/todos \ +| lein jet --from json --keywordize --to edn --query '[(filter :completed) (count)]' +90 +``` + +``` clojure +$ curl -s https://jsonplaceholder.typicode.com/todos \ +| lein jet --from json --keywordize --to edn --query '[(remove :completed) (count)]' +110 +``` + +Comparing values can be done with `=`, `>`, `>=`, `<` and `<=`. + +``` clojure +$ echo '[{:a 1} {:a 2} {:a 3}]' | lein jet --from edn --to edn --query '(filter (>= :a 2))' +[{:a 2} {:a 3}] +``` + +``` clojure +echo '[{:a {:b 1}} {:a {:b 2}}]' \ +| jet --from edn --to edn --query '(filter (= [:a :b] 1))' +[{:a {:b 1}}] +``` diff --git a/script/compile b/script/compile index 291f445..55972fb 100755 --- a/script/compile +++ b/script/compile @@ -24,3 +24,5 @@ $GRAALVM_HOME/bin/native-image \ --no-fallback \ --no-server \ "-J-Xmx3g" + +lein clean diff --git a/src/jet/main.clj b/src/jet/main.clj index e43e364..c26af06 100644 --- a/src/jet/main.clj +++ b/src/jet/main.clj @@ -6,6 +6,7 @@ [clojure.edn :as edn] [cognitect.transit :as transit] [clojure.java.io :as io] + [jet.query :as q] [fipp.edn :refer [pprint] :rename {pprint fipp}]) (:gen-class)) @@ -31,17 +32,19 @@ (= "true" (first k)) true :else false)) version (boolean (get opts "--version")) - pretty (boolean (get opts "--pretty"))] + pretty (boolean (get opts "--pretty")) + query (first (get opts "--query"))] {:from from :to to :keywordize keywordize :version version - :pretty pretty})) + :pretty pretty + :query (edn/read-string query)})) (defn -main [& args] (let [{:keys [:from :to :keywordize - :pretty :version]} (parse-opts args)] + :pretty :version :query]} (parse-opts args)] (if version (println (str/trim (slurp (io/resource "JET_VERSION")))) (let [in (slurp *in*) @@ -49,7 +52,9 @@ :edn (edn/read-string in) :json (cheshire/parse-string in keywordize) :transit (transit/read - (transit/reader (io/input-stream (.getBytes in)) :json)))] + (transit/reader (io/input-stream (.getBytes in)) :json))) + input (if query (q/query input query) + input)] (case to :edn (if pretty (fipp input) (prn input)) :json (println (cheshire/generate-string input {:pretty pretty})) diff --git a/src/jet/query.clj b/src/jet/query.clj new file mode 100644 index 0000000..1e314a3 --- /dev/null +++ b/src/jet/query.clj @@ -0,0 +1,87 @@ +(ns jet.query + (:refer-clojure :exclude [comparator])) + +(declare query) + +(defn comparator [[c q v]] + (let [c-f (case c + = = + < < + <= <= + >= >=)] + #(c-f (query % q) v))) + +(defn sexpr-query [x q] + (let [op (first q) + res (case op + take (take (second q) x) + drop (drop (second q) x) + nth (try (nth x (second q)) + (catch Exception _e + (last x))) + keys (vec (keys x)) + vals (vec (vals x)) + first (first x) + last (last x) + map (map #(query % (let [f (second q)] + (if (symbol? f) + (list f) + f))) x) + juxt (vec (for [q (rest q)] + (if (symbol? q) + (sexpr-query x (list q)) + (query x q)))) + map-vals (zipmap (keys x) + (map #(query % (second q)) (vals x))) + zipmap (zipmap (first x) (second x)) + (filter remove) (let [op-f (case op + filter filter + remove remove) + f (second q) + c (if (list? f) + (comparator f) + #(query % f))] + (op-f c x)) + count (count x) + x)] + (if (and (vector? x) (sequential? res)) + (vec res) + res))) + +(defn query + [x q] + (cond + (not query) nil + (vector? q) (if-let [next-op (first q)] + (query (query x next-op) (vec (rest q))) + x) + (list? q) (sexpr-query x q) + (sequential? x) + (mapv #(query % q) x) + (map? q) + (let [default (some #(or (nil? %) (false? %)) (vals q)) + kf (fn [[k v]] + (when-not (contains? q k) + [k (query v default)])) + init (if default (into {} (keep kf x)) {}) + rf (fn [m k v] + (if (and v (contains? x k)) + (assoc m k (query (get x k) v)) + m))] + (reduce-kv rf init q)) + (map? x) (get x q) + :else x)) + +;;;; Scratch + +(comment + (query {:a 1 :b 2} {:a true}) + (query {:a {:a/a 1 :a/b 2} :b 2} {:a {:a/a true}}) + (query {:a [{:a/a 1 :a/b 2}] :b 2} {:a {:a/a true}}) + (query {:a 1 :b 2 :c 3} {:jet.query/all true :b false}) + (query [1 2 3] '(take 1)) + (query [1 2 3] '(drop 1)) + (query {:a [1 2 3]} '{:a (take 1)}) + (query {:a [1 2 3]} '{:a (nth 1)}) + (query {:a [1 2 3]} '{:a (last)}) + ) diff --git a/test/jet/main_test.clj b/test/jet/main_test.clj index e4d5a13..9c292e4 100644 --- a/test/jet/main_test.clj +++ b/test/jet/main_test.clj @@ -1,31 +1,7 @@ (ns jet.main-test (:require [clojure.test :as test :refer [deftest is testing]] - [jet.main :as main] - [me.raynes.conch :refer [programs with-programs let-programs] :as sh])) - -(set! *warn-on-reflection* true) - -(defn jet-jvm [input & args] - (with-out-str - (with-in-str input - (apply main/-main args)))) - -(defn jet-native [input & args] - (let-programs [jet "./jet"] - (binding [sh/*throw* false] - (apply jet (conj (vec args) - {:in input}))))) - -(def jet - (case (System/getenv "JET_TEST_ENV") - "jvm" #'jet-jvm - "native" #'jet-native - #'jet-jvm)) - -(if (= jet #'jet-jvm) - (println "==== Testing JVM version") - (println "==== Testing native version")) + [jet.test-utils :refer [jet]])) (deftest main-test (is (= "{\"a\" 1}\n" @@ -62,4 +38,6 @@ (is (= "{\n \"a\" : [ {\n \"b\" : {\n \"c\" : \"d\"\n }\n } ]\n}\n" (jet "{:a [{:b {:c :d}}]}" "--from" "edn" "--to" "json" "--pretty"))) (is (= "{:a [{:b {:c :d}}\n {:b {:c :d}}\n {:b {:c :d}}\n {:b {:c :d}}\n {:b {:c :d}}\n {:b {:c :d}}\n {:b {:c :d}}]}\n" - (jet "{:a [{:b {:c :d}} {:b {:c :d}} {:b {:c :d}} {:b {:c :d}} {:b {:c :d}} {:b {:c :d}} {:b {:c :d}}]}" "--from" "edn" "--to" "edn" "--pretty"))))) + (jet "{:a [{:b {:c :d}} {:b {:c :d}} {:b {:c :d}} {:b {:c :d}} {:b {:c :d}} {:b {:c :d}} {:b {:c :d}}]}" "--from" "edn" "--to" "edn" "--pretty")))) + (testing "query" + (is (= "{:a 1}\n" (jet "{:a 1 :b 2}" "--from" "edn" "--to" "edn" "--query" "{:a true}"))))) diff --git a/test/jet/query_test.clj b/test/jet/query_test.clj new file mode 100644 index 0000000..8758884 --- /dev/null +++ b/test/jet/query_test.clj @@ -0,0 +1,41 @@ +(ns jet.query-test + (:require + [clojure.test :as test :refer [deftest is]] + [jet.query :refer [query]])) + +(deftest query-test + (is (= '1 (query {:a 1 :b 2} :a))) + (is (= '1 (query {1 1} 1))) + (is (= '1 (query {"1" 1} "1"))) + (is (= '{:a 1} (query {:a 1 :b 2} {:a true}))) + (is (= {:a #:a{:a 1}} (query {:a {:a/a 1 :a/b 2} :b 2} {:a {:a/a true}}))) + (is (= {:a [#:a{:a 1}]} (query {:a [{:a/a 1 :a/b 2}] :b 2} {:a {:a/a true}}))) + (is (= {:a 1, :c 3} (query {:a 1 :b 2 :c 3} {:b false}))) + (is (= [1] (query [1 2 3] '(take 1)))) + (is (= [2 3] (query [1 2 3] '(drop 1)))) + (is (= {:a [1]} (query {:a [1 2 3]} '{:a (take 1)}))) + (is (= {:a 2} (query {:a [1 2 3]} '{:a (nth 1)}))) + (is (= {:a 1} (query {:a [1 2 3]} '{:a (first)}))) + (is (= {:a 3} (query {:a [1 2 3]} '{:a (last)}))) + (is (= {:foo [:bar]} (query {:foo {:bar 2}} '{:foo (keys)}))) + (is (= {:foo [2]} (query {:foo {:bar 2}} '{:foo (vals)}))) + (is (= [3 6] (query [[1 2 3] [4 5 6]] '(map last)))) + (is (= [1 2] (query [{:a 1} {:a 2}] '(map :a)))) + (is (= [1 2] (query [{:a 1} {:a 2}] '(map :a)))) + (is (= [1 2] (query {:a 1 :b 2 :c 3} '[{:c false} (vals)]))) + (is (= {:foo 1 :bar 1} + (query {:foo {:a 1 :b 2} :bar {:a 1 :b 2}} '(map-vals :a)))) + (is (= {:a 1 :b 2 :c 3} + (query {:keys [:a :b :c] :vals [1 2 3]} '[(juxt :keys :vals) (zipmap)]))) + (is (= '[{:name foo :private true}] + (query '[{:name foo :private true} + {:name bar :private false}] '(filter :private)))) + (is (= 1 (query '[{:name foo :private true} + {:name bar :private false}] '[(filter :private) (count)]))) + (is (= '[{:name foo, :private true}] + (query '[{:name foo :private true} + {:name bar :private false}] '(filter (= :name foo))))) + (is (= '[{:a 2} {:a 3}] + (query '[{:a 1} {:a 2} {:a 3}] '(filter (>= :a 2))))) + (is (= '[{:a 1} {:a 2}] + (query '[{:a 1} {:a 2} {:a 3}] '(filter (<= :a 2)))))) diff --git a/test/jet/test_utils.clj b/test/jet/test_utils.clj new file mode 100644 index 0000000..b7f24fc --- /dev/null +++ b/test/jet/test_utils.clj @@ -0,0 +1,29 @@ +(ns jet.test-utils + (:require + [jet.main :as main] + [me.raynes.conch :refer [let-programs] :as sh])) + +(set! *warn-on-reflection* true) + +(defn jet-jvm [input & args] + (with-out-str + (with-in-str input + (apply main/-main args)))) + +(defn jet-native [input & args] + (let-programs [jet "./jet"] + (binding [sh/*throw* false] + (apply jet (conj (vec args) + {:in input}))))) + +(def jet + (case (System/getenv "JET_TEST_ENV") + "jvm" #'jet-jvm + "native" #'jet-native + #'jet-jvm)) + +(if (= jet #'jet-jvm) + (println "==== Testing JVM version") + (println "==== Testing native version")) + +