Skip to content

Commit

Permalink
refactored
Browse files Browse the repository at this point in the history
  • Loading branch information
behrica committed Nov 3, 2020
1 parent 01a914b commit 1d246a8
Showing 1 changed file with 19 additions and 21 deletions.
40 changes: 19 additions & 21 deletions src/clojure/zero_one/geni/arrow.clj
Original file line number Diff line number Diff line change
@@ -1,19 +1,14 @@
(ns zero-one.geni.arrow
(:import [org.apache.arrow.vector VarCharVector
VectorSchemaRoot BigIntVector Float4Vector Float8Vector IntVector
BitVector ValueVector BaseFixedWidthVector TimeStampMilliVector]
[org.apache.arrow.vector.util Text]
[org.apache.arrow.vector.ipc ArrowStreamWriter]
[org.apache.arrow.memory RootAllocator]
[java.nio.channels Channels]
[org.apache.spark.sql Row]
[org.apache.arrow.vector.types.pojo Schema]
[scala.collection.convert Wrappers$IteratorWrapper]
)
(:require
[clojure.java.io :as io]

))
(:require [clojure.java.io :as io])
(:import java.nio.channels.Channels
org.apache.arrow.memory.RootAllocator
[org.apache.arrow.vector BaseFixedWidthVector BigIntVector BitVector Float4Vector Float8Vector IntVector
TimeStampMilliVector ValueVector VarCharVector VectorSchemaRoot]
org.apache.arrow.vector.ipc.ArrowStreamWriter
org.apache.arrow.vector.types.pojo.Schema
org.apache.arrow.vector.util.Text
org.apache.spark.sql.Row
scala.collection.convert.Wrappers$IteratorWrapper))

;; (set! *warn-on-reflection* true)

Expand Down Expand Up @@ -102,12 +97,15 @@


(defn- schema->clojure [^Schema schema]
(map
#(hash-map
:type %1
:name %2)
(->> schema .fields (map #(keyword (.. % dataType typeName))))
(->> schema .fields (map #(.. % name)))))
(let [fields (.fields schema)
types (map #(keyword (.. % dataType typeName)) fields)
names (map #(.name %) fields)]
(map
(fn [type name]
(hash-map :type type :name name))
types
names
)))

(defn- set-null-or-value [v ^long idx value type]
(if (nil? value)
Expand Down

0 comments on commit 1d246a8

Please sign in to comment.