support queries

.clj-kondo/config.edn

@@ -0,0 +1 @@
+{:lint-as {me.raynes.conch/let-programs clojure.core/let}}

.gitignore

@@ -11,3 +11,4 @@ pom.xml.asc
 .hgignore
 .hg/
 /jet
+.clj-kondo/.cache

README.md

@@ -6,27 +6,6 @@
 
 CLI to transform JSON into EDN into Transit and vice versa.
 
-## Usage
-
-`jet` supports the following options:
-
-   - `--from`: allowed values: `edn`, `transit` or `json`
-   - `--to`: allowed values: `edn`, `transit` or `json`
-   - `--keywordize`: if present, keywordizes JSON keys.
-   - `--pretty`: if present, pretty-prints JSON and EDN output.
-   - `--version`: if present, prints current version of `jet` and exits.
-
-Examples:
-
-``` shellsession
-$ echo '{"a": 1}' | jet --from json --to edn
-{"a" 1}
-$ echo '{"a": 1}' | jet --from json --keywordize --to edn
-{:a 1}
-$ echo '{"a": 1}' | jet --from json --to transit
-["^ ","a",1]
-```
-
 ## Installation
 
 Linux and macOS binaries are provided via brew.
@@ -39,6 +18,8 @@ Upgrade:
 
     brew upgrade jet
 
+You may also download a binary from [Github](https://github.com/borkdude/jet/releases).
+
 This tool can also be used via the JVM. If you use leiningen, you can put the
 following in your `.lein/profiles`:
 
@@ -55,6 +36,36 @@ $ echo '["^ ","~:a",1]' | lein jet --from transit --to edn
 {:a 1}
 ```
 
+## Usage
+
+`jet` supports the following options:
+
+   - `--from`: allowed values: `edn`, `transit` or `json`
+   - `--to`: allowed values: `edn`, `transit` or `json`
+   - `--keywordize`: if present, keywordizes JSON keys.
+   - `--pretty`: if present, pretty-prints JSON and EDN output.
+   - `--query`: if present, applies query to output. See [Query](query.md).
+   - `--version`: if present, prints current version of `jet` and exits.
+
+Examples:
+
+``` shellsession
+$ echo '{"a": 1}' | jet --from json --to edn
+{"a" 1}
+
+$ echo '{"a": 1}' | jet --from json --keywordize --to edn
+{:a 1}
+
+$ echo '{"a": 1}' | jet --from json --to transit
+["^ ","a",1]
+
+$ echo '[{:a {:b 1}} {:a {:b 2}}]' \
+| jet --from edn --to edn --query '(filter (= [:a :b] 1))'
+[{:a {:b 1}}]
+```
+
+## [Query](query.md)
+
 ## Test
 
 Test the JVM version:

doc/cljdoc.edn

@@ -0,0 +1,3 @@
+{:cljdoc.doc/tree
+ [["Readme" {:file "README.md"}]
+  ["Query" {:file "doc/query.md"}]]}

doc/query.md

@@ -0,0 +1,148 @@
+# Query
+
+The `--query` option allows to select or remove specific parts of the output. A
+query is written in EDN.
+
+NOTE: some parts of this query language may change in the coming months after it
+has seen more usage (2019-08-04).
+
+Single values can be selected by using a key:
+
+``` clojure
+echo '{:a 1}' | jet --from edn --to edn --query ':a'
+1
+```
+
+Multiple values can be selected using a map:
+
+``` clojure
+echo '{:a 1 :b 2 :c 3}' | jet --from edn --to edn --query '{:a true :b true}'
+{:a 1, :b 2}
+```
+
+By default, only keys that have truthy values in the query will be selected from
+the output. However, if one of the values has a falsy value, this behavior is
+reversed and other keys are left in:
+
+``` clojure
+echo '{:a 1 :b 2 :c 3}' | jet --from edn --to edn --query '{:c false}'
+{:a 1, :b 2}
+```
+
+``` clojure
+$ echo '{:a {:a/a 1 :a/b 2} :b 2 :c 3}' \
+| jet --from edn --to edn --query '{:c false :a {:a/b true}}'
+{:b 2, :a #:a{:b 2}}
+```
+
+If the query is applied to a list-like value, the query is applied to all the
+elements inside the list-like value:
+
+``` clojure
+echo '[{:a 1 :b 2} {:a 2 :b 3}]' | jet --from edn --to edn --query '{:a true}'
+[{:a 1} {:a 2}]
+```
+
+Nested values can be selected by using a nested query:
+
+``` clojure
+echo '{:a {:a/a 1 :a/b 2} :b 2}' | jet --from edn --to edn --query '{:a {:a/a true}}'
+{:a {:a/a 1}}
+```
+
+Some Clojure-like functions are supported which are mostly intented to operate
+on list-like values, except for `keys`, `vals` and `map-vals` which operate on
+maps:
+
+``` clojure
+echo '[1 2 3]' | jet --from edn --to edn --query '(first)'
+1
+```
+
+``` clojure
+echo '[1 2 3]' | jet --from edn --to edn --query '(last)'
+3
+```
+
+``` clojure
+echo '[[1 2 3] [4 5 6]]' | jet --from edn --to edn --query '(map last)'
+[3 6]
+```
+
+``` clojure
+echo '{:a [1 2 3]}' | jet --from edn --to edn --query '{:a (take 2)}'
+{:a [1 2]}
+```
+
+``` clojure
+echo '{:a [1 2 3]}' | jet --from edn --to edn --query '{:a (drop 2)}'
+{:a [3]}
+```
+
+``` clojure
+echo '{:a [1 2 3]}' | jet --from edn --to edn --query '{:a (nth 2)}'
+{:a 3}
+```
+
+``` clojure
+$ echo '{:a [1 2 3]}' | jet --from edn --to edn --query '{:a (juxt first last)}'
+{:a [1 3]}
+```
+
+``` clojure
+$ echo '{:a [1 2 3] :b [4 5 6]}' | jet --from edn --to edn --query '(juxt :a :b)'
+[[1 2 3] [4 5 6]]
+```
+
+``` clojure
+$ echo '{:a [1 2 3] :b [4 5 6]}' | jet --from edn --to edn --query '(keys)'
+[:a :b]
+```
+
+``` clojure
+$ echo '{:a [1 2 3] :b [4 5 6]}' | jet --from edn --to edn --query '(vals)'
+[[1 2 3] [4 5 6]]
+```
+
+``` clojure
+$ echo '{:foo {:a 1 :b 2} :bar {:a 1 :b 2}}' | jet --from edn --to edn --query '(map-vals :a)'
+{:foo 1 :bar 2}
+```
+
+Multiple queries in a vector are applied after one another:
+
+``` clojure
+$ echo '{:a 1 :b 2 :c 3}' | jet --from edn --to edn --query '[{:c false} (vals)]'
+[1 2]
+```
+
+``` clojure
+$ echo '{:keys [:a :b :c] :vals [1 2 3]}' \
+| lein jet --from edn --to edn --query '[(juxt :keys :vals) (zipmap)]'
+{:a 1, :b 2, :c 3}
+```
+
+``` clojure
+$ curl -s https://jsonplaceholder.typicode.com/todos \
+| lein jet --from json --keywordize --to edn --query '[(filter :completed) (count)]'
+90
+```
+
+``` clojure
+$ curl -s https://jsonplaceholder.typicode.com/todos \
+| lein jet --from json --keywordize --to edn --query '[(remove :completed) (count)]'
+110
+```
+
+Comparing values can be done with `=`, `>`, `>=`, `<` and `<=`.
+
+``` clojure
+$ echo '[{:a 1} {:a 2} {:a 3}]' | lein jet --from edn --to edn --query '(filter (>= :a 2))'
+[{:a 2} {:a 3}]
+```
+
+``` clojure
+echo '[{:a {:b 1}} {:a {:b 2}}]' \
+| jet --from edn --to edn --query '(filter (= [:a :b] 1))'
+[{:a {:b 1}}]
+```

script/compile

@@ -24,3 +24,5 @@ $GRAALVM_HOME/bin/native-image \
   --no-fallback \
   --no-server \
   "-J-Xmx3g"
+
+lein clean

src/jet/main.clj

@@ -6,6 +6,7 @@
    [clojure.edn :as edn]
    [cognitect.transit :as transit]
    [clojure.java.io :as io]
+   [jet.query :as q]
    [fipp.edn :refer [pprint] :rename {pprint fipp}])
   (:gen-class))
 
@@ -31,25 +32,29 @@
                            (= "true" (first k)) true
                            :else false))
         version (boolean (get opts "--version"))
-        pretty (boolean (get opts "--pretty"))]
+        pretty (boolean (get opts "--pretty"))
+        query (first (get opts "--query"))]
     {:from from
      :to to
      :keywordize keywordize
      :version version
-     :pretty pretty}))
+     :pretty pretty
+     :query (edn/read-string query)}))
 
 (defn -main
   [& args]
   (let [{:keys [:from :to :keywordize
-                :pretty :version]} (parse-opts args)]
+                :pretty :version :query]} (parse-opts args)]
     (if version
       (println (str/trim (slurp (io/resource "JET_VERSION"))))
       (let [in (slurp *in*)
             input (case from
                     :edn (edn/read-string in)
                     :json (cheshire/parse-string in keywordize)
                     :transit (transit/read
-                              (transit/reader (io/input-stream (.getBytes in)) :json)))]
+                              (transit/reader (io/input-stream (.getBytes in)) :json)))
+            input (if query (q/query input query)
+                      input)]
         (case to
           :edn (if pretty (fipp input) (prn input))
           :json (println (cheshire/generate-string input {:pretty pretty}))

src/jet/query.clj

@@ -0,0 +1,87 @@
+(ns jet.query
+  (:refer-clojure :exclude [comparator]))
+
+(declare query)
+
+(defn comparator [[c q v]]
+  (let [c-f (case c
+              = =
+              < <
+              <= <=
+              >= >=)]
+    #(c-f (query % q) v)))
+
+(defn sexpr-query [x q]
+  (let [op (first q)
+        res (case op
+              take (take (second q) x)
+              drop (drop (second q) x)
+              nth (try (nth x (second q))
+                       (catch Exception _e
+                         (last x)))
+              keys (vec (keys x))
+              vals (vec (vals x))
+              first (first x)
+              last (last x)
+              map (map #(query % (let [f (second q)]
+                                   (if (symbol? f)
+                                     (list f)
+                                     f))) x)
+              juxt (vec (for [q (rest q)]
+                          (if (symbol? q)
+                            (sexpr-query x (list q))
+                            (query x q))))
+              map-vals (zipmap (keys x)
+                               (map #(query % (second q)) (vals x)))
+              zipmap (zipmap (first x) (second x))
+              (filter remove) (let [op-f (case op
+                                           filter filter
+                                           remove remove)
+                                    f (second q)
+                                    c (if (list? f)
+                                        (comparator f)
+                                        #(query % f))]
+                                (op-f c x))
+              count (count x)
+              x)]
+    (if (and (vector? x) (sequential? res))
+      (vec res)
+      res)))
+
+(defn query
+  [x q]
+  (cond
+    (not query) nil
+    (vector? q) (if-let [next-op (first q)]
+                  (query (query x next-op) (vec (rest q)))
+                  x)
+    (list? q) (sexpr-query x q)
+    (sequential? x)
+    (mapv #(query % q) x)
+    (map? q)
+    (let [default (some #(or (nil? %) (false? %)) (vals q))
+          kf (fn [[k v]]
+               (when-not (contains? q k)
+                 [k (query v default)]))
+          init (if default (into {} (keep kf x)) {})
+          rf (fn [m k v]
+               (if (and v (contains? x k))
+                 (assoc m k (query (get x k) v))
+                 m))]
+      (reduce-kv rf init q))
+    (map? x) (get x q)
+    :else x))
+
+;;;; Scratch
+
+(comment
+  (query {:a 1 :b 2} {:a true})
+  (query {:a {:a/a 1 :a/b 2} :b 2} {:a {:a/a true}})
+  (query {:a [{:a/a 1 :a/b 2}] :b 2} {:a {:a/a true}})
+  (query {:a 1 :b 2 :c 3} {:jet.query/all true :b false})
+  (query [1 2 3] '(take 1))
+  (query [1 2 3] '(drop 1))
+  (query {:a [1 2 3]} '{:a (take 1)})
+  (query {:a [1 2 3]} '{:a (nth 1)})
+  (query {:a [1 2 3]} '{:a (last)})
+  )