Skip to content

Commit

Permalink
Merge pull request #260 from yetanalytics/encoding-fix
Browse files Browse the repository at this point in the history 
[SQL-157] Encoding fix
  • Loading branch information
@kelvinqian00
kelvinqian00 authored Dec 12, 2022
2 parents 38a53ee + bf6e228 commit 9e668b0
Show file tree
Hide file tree
Showing 18 changed files with 179 additions and 86 deletions.
2 changes: 1 addition & 1 deletion bin/run_h2.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,4 @@

MACHINE=`bin/machine.sh`

runtimes/$MACHINE/bin/java -server -cp lrsql.jar lrsql.h2.main $@
runtimes/$MACHINE/bin/java -J-Dfile.encoding=UTF-8 -server -cp lrsql.jar lrsql.h2.main $@
2 changes: 1 addition & 1 deletion bin/run_h2_persistent.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,4 @@

MACHINE=`bin/machine.sh`

runtimes/$MACHINE/bin/java -server -cp lrsql.jar lrsql.h2.main --persistent true $@
runtimes/$MACHINE/bin/java -J-Dfile.encoding=UTF-8 -server -cp lrsql.jar lrsql.h2.main --persistent true $@
2 changes: 1 addition & 1 deletion bin/run_postgres.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,4 @@

MACHINE=`bin/machine.sh`

runtimes/$MACHINE/bin/java -server -cp lrsql.jar lrsql.postgres.main $@
runtimes/$MACHINE/bin/java -J-Dfile.encoding=UTF-8 -server -cp lrsql.jar lrsql.postgres.main $@
2 changes: 1 addition & 1 deletion bin/run_sqlite.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,4 @@

MACHINE=`bin/machine.sh`

runtimes/$MACHINE/bin/java -server -cp lrsql.jar lrsql.sqlite.main $@
runtimes/$MACHINE/bin/java -J-Dfile.encoding=UTF-8 -server -cp lrsql.jar lrsql.sqlite.main $@
1 change: 1 addition & 0 deletions exe/config.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,5 +24,6 @@
<maxVersion></maxVersion>
<jdkPreference>preferJre</jdkPreference>
<runtimeBits>64/32</runtimeBits>
<opt>-Dfile.encoding=UTF-8</opt>
</jre>
</launch4jConfig>
1 change: 1 addition & 0 deletions exe/config_pg.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,5 +24,6 @@
<maxVersion></maxVersion>
<jdkPreference>preferJre</jdkPreference>
<runtimeBits>64/32</runtimeBits>
<opt>-Dfile.encoding=UTF-8</opt>
</jre>
</launch4jConfig>
Binary file modified exe/lrsql.exe
View file
Binary file not shown.
Binary file modified exe/lrsql_pg.exe
View file
Binary file not shown.
4 changes: 2 additions & 2 deletions src/bench/lrsql/bench.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@
(when-some [batch (first batches)]
(curl/post endpoint
{:headers headers
:body (String. (u/write-json (vec batch)))
:body (u/write-json-str (vec batch))
:basic-auth [user pass]})
(recur (rest batches))))))

Expand All @@ -171,7 +171,7 @@
stmts (generate-statements inputs size sref-type)
requests (mapv (fn [batch]
{:headers headers
:body (String. (u/write-json (vec batch)))
:body (u/write-json-str (vec batch))
:basic-auth [user pass]})
(partition-all batch-size stmts))]
(perform-async-op! curl/post
Expand Down
2 changes: 1 addition & 1 deletion src/db/postgres/lrsql/postgres/data.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
[jsn]
(doto (PGobject.)
(.setType "json")
(.setValue (String. (u/write-json jsn)))))
(.setValue (u/write-json-str jsn))))

(defn- pg-object->json
[^PGobject pg-obj]
Expand Down
8 changes: 5 additions & 3 deletions src/main/lrsql/spec/auth.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@
[lrsql.backend.protocol :as bp]
[lrsql.spec.common :as c]
[lrsql.spec.admin :as ads]
[lrsql.spec.authority :as ats])
[lrsql.spec.authority :as ats]
[lrsql.util :as u])
(:import [java.util Base64 Base64$Encoder]))

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
Expand All @@ -21,8 +22,9 @@
#(sgen/fmap
(fn [[username password]]
(let [up (str username ":" password)
byts (.encode ^Base64$Encoder (Base64/getEncoder) (.getBytes up))]
(str "Basic " (String. byts))))
byts (.encode ^Base64$Encoder (Base64/getEncoder)
(u/str->bytes up))]
(str "Basic " (u/bytes->str byts))))
(sgen/tuple (sgen/fmap xs/into-str
(sgen/vector (sgen/char-alpha) 3 16))
(sgen/fmap xs/into-str
Expand Down
70 changes: 61 additions & 9 deletions src/main/lrsql/util.clj
View file
Original file line number Diff line number Diff line change
@@ -1,18 +1,20 @@
(ns lrsql.util
(:require [clj-uuid]
[java-time]
[java-time.properties :as jt-props]
[clojure.spec.alpha :as s]
[clojure.java.io :as io]
[cheshire.core :as cjson]
[xapi-schema.spec :as xs]
[java-time.properties :as jt-props]
[clojure.spec.alpha :as s]
[clojure.tools.logging :as log]
[clojure.java.io :as io]
[cheshire.core :as cjson]
[xapi-schema.spec :as xs]
[com.yetanalytics.squuid :as squuid]
[com.yetanalytics.lrs.xapi.document :refer [json-bytes-gen-fn]]
[com.yetanalytics.lrs.xapi.statements.timestamp :refer [normalize]]
[lrsql.spec.common :as cs :refer [instant-spec]])
(:import [java.util UUID]
[java.time Instant]
[java.io StringReader PushbackReader ByteArrayOutputStream]))
[java.io StringReader PushbackReader ByteArrayOutputStream]
[java.nio.charset Charset]))

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Macros
Expand Down Expand Up @@ -161,6 +163,43 @@
[input]
(assoc input :primary-key (generate-squuid)))

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Strings
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

(def utf8-charset
(Charset/forName "UTF-8"))

(def default-charset
(Charset/defaultCharset))

;; Log on compilation if the default charset is not UTF-8 (which can cause
;; errors like in Issue #230). Should not happen in production thanks to
;; setting -J-Dfile.encoding=UTF-8 but it's a fallback, especially for dev.
(when (not= utf8-charset default-charset)
(log/warnf (str "The default charset is set to %s instead of %s, "
"which may cause undefined behavior on Unicode characters.")
default-charset
utf8-charset))

(s/fdef str->bytes
:args (s/cat :s string?)
:ret bytes?)

(defn str->bytes
"Convert `s` into a byte array. Assumes UTF-8 encoding."
[^String s]
(.getBytes s utf8-charset))

(s/fdef bytes->str
:args (s/cat :bytes bytes?)
:ret string?)

(defn bytes->str
"Converts `bytes` into a string. Assumes UTF-8 encoding."
[^"[B" bytes]
(String. bytes utf8-charset))

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; JSON
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
Expand All @@ -171,9 +210,10 @@
;; issues with AOT compliation.

(defn- parse-json*
"Read a JSON string or byte array `data`."
"Read a JSON string or byte array `data`. In the byte array case, it will
be string-encoded using the UTF-8 charset."
[data]
(let [string (if (bytes? data) (String. ^"[B" data) data)]
(let [string (if (bytes? data) (bytes->str data) data)]
(with-open [rdr (PushbackReader. (StringReader. string) 64)]
(doall (cjson/parsed-seq rdr)))))

Expand All @@ -189,7 +229,10 @@
(defn parse-json
"Read a JSON string or byte array `data`. `data` must only consist of one
JSON object, array, or scalar; in addition, `data` must be an object by
default. To parse JSON arrays or scalars, set `:object?` to false."
default. To parse JSON arrays or scalars, set `:object?` to false.
In the byte array case, `data` will be string-encoded using the UTF-8
charset."
[data & {:keys [object?] :or {object? true}}]
(let [[result & ?more] (wrap-parse-fn parse-json* "JSON" data)]
(cond
Expand Down Expand Up @@ -222,6 +265,15 @@
(let [out-stream (ByteArrayOutputStream. 4096)]
(.toByteArray ^ByteArrayOutputStream (write-json* out-stream jsn))))

(s/fdef write-json-str
:args (s/cat :jsn ::xs/any-json)
:ret string?)

(defn write-json-str
"Write `jsn` to a string; the string is always UTF-8 encoded."
[jsn]
(bytes->str (write-json jsn)))

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Bytes
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
Expand Down
6 changes: 3 additions & 3 deletions src/main/lrsql/util/auth.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@
[clojure.tools.logging :as log]
[buddy.core.codecs :refer [bytes->hex]]
[buddy.core.nonce :refer [random-bytes]]
[lrsql.spec.auth :as as])
[lrsql.spec.auth :as as]
[lrsql.util :as u])
(:import [java.util Base64 Base64$Decoder]))

;; NOTE: Additional scopes may be implemented in the future.
Expand Down Expand Up @@ -49,8 +50,7 @@
(when auth-header
(try (let [^String auth-part (second (re-matches #"Basic\s+(.*)"
auth-header))
^String decoded (String. (.decode decoder
auth-part))
^String decoded (u/bytes->str (.decode decoder auth-part))
[?api-key ?srt-key] (cstr/split decoded #":")]
{:api-key (if ?api-key ?api-key "")
:secret-key (if ?srt-key ?srt-key "")})
Expand Down
2 changes: 1 addition & 1 deletion src/main/lrsql/util/statement.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,4 +126,4 @@
(form-encode
(cond-> query-params
true (assoc :from next-cursor)
?agent (assoc :agent (String. (u/write-json ?agent))))))))
?agent (assoc :agent (u/write-json-str ?agent)))))))
66 changes: 32 additions & 34 deletions src/test/lrsql/admin/route_test.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,18 +62,18 @@
;; Seed information
{:keys [api-key-default
api-secret-default]} (get-in sys' [:lrs :config])
seed-body (String. (u/write-json
{"username" api-key-default
"password" api-secret-default}))
seed-body (u/write-json-str
{"username" api-key-default
"password" api-secret-default})
seed-jwt (-> (login-account content-type seed-body)
:body
u/parse-json
(get "json-web-token"))
seed-auth {"Authorization" (str "Bearer " seed-jwt)}
;; New data information
headers (merge content-type seed-auth)
req-body (String. (u/write-json {"username" "myname"
"password" "swordfish"}))]
req-body (u/write-json-str {"username" "myname"
"password" "swordfish"})]
(testing "seed jwt retrieved"
;; Sanity check that the test credentials are in place
(is (some? seed-jwt)))
Expand Down Expand Up @@ -121,14 +121,12 @@
(let [bad-body "{\"username\": \"foo\", \"password\": bar\"}"]
(is-err-code (login-account content-type bad-body) 400)))
;; Other errors
(let [bad-body (String. (u/write-json
{"username" "foo"
"password" "swordfish"}))]
(let [bad-body (u/write-json-str {"username" "foo"
"password" "swordfish"})]
;; Bad User 401
(is-err-code (login-account content-type bad-body) 401))
(let [bad-body (String. (u/write-json
{"username" "myname"
"password" "badpass"}))]
(let [bad-body (u/write-json-str {"username" "myname"
"password" "badpass"})]
;; Bad Pass 401
(is-err-code (login-account content-type bad-body) 401))
;; Bad Request
Expand All @@ -147,7 +145,7 @@
(get "account-id"))
del-auth {"Authorization" (str "Bearer " del-jwt)}
del-head (merge content-type del-auth)
del-body (String. (u/write-json {"account-id" del-id}))]
del-body (u/write-json-str {"account-id" del-id})]
(let [delete-res (delete-account headers del-body)]
(is (= 200 (:status delete-res)))
(is (= del-id (-> delete-res
Expand All @@ -167,14 +165,14 @@
(is-err-code (curl/post
"http://0.0.0.0:8080/admin/creds"
{:headers del-head
:body (String. (u/write-json {"scopes" ss}))})
:body (u/write-json-str {"scopes" ss})})
401)
(is-err-code (curl/put
"http://0.0.0.0:8080/admin/creds"
{:headers del-head
:body (String. (u/write-json {"api-key" pk
"secret-key" sk
"scopes" ss}))})
:body (u/write-json-str {"api-key" pk
"secret-key" sk
"scopes" ss})})
401)
(is-err-code (curl/get
"http://0.0.0.0:8080/admin/creds"
Expand All @@ -183,8 +181,8 @@
(is-err-code (curl/delete
"http://0.0.0.0:8080/admin/creds"
{:headers del-head
:body (String. (u/write-json {"api-key" pk
"secret-key" sk}))})
:body (u/write-json-str {"api-key" pk
"secret-key" sk})})
401)))))
(component/stop sys')))

Expand All @@ -193,9 +191,9 @@
sys' (component/start sys)
jwt (-> (curl/post "http://0.0.0.0:8080/admin/account/login"
{:headers content-type
:body (String. (u/write-json
{"username" "username"
"password" "password"}))})
:body (u/write-json-str
{"username" "username"
"password" "password"})})
:body
u/parse-json
(get "json-web-token"))
Expand All @@ -205,8 +203,8 @@
(let [{:keys [status body]}
(curl/post "http://0.0.0.0:8080/admin/creds"
{:headers hdr
:body (String. (u/write-json
{"scopes" ["all" "all/read"]}))})
:body (u/write-json-str
{"scopes" ["all" "all/read"]})})
{:strs [api-key secret-key scopes]}
(u/parse-json body)]
(is (= 200 status))
Expand Down Expand Up @@ -234,9 +232,9 @@
(curl/put
"http://0.0.0.0:8080/admin/creds"
{:headers hdr
:body (String. (u/write-json {"api-key" api-key
"secret-key" secret-key
"scopes" req-scopes}))})
:body (u/write-json-str {"api-key" api-key
"secret-key" secret-key
"scopes" req-scopes})})
{:strs [scopes]}
(u/parse-json body)]
(is (= 200 status))
Expand Down Expand Up @@ -266,9 +264,9 @@
(curl/put
"http://0.0.0.0:8080/admin/creds"
{:headers hdr
:body (String. (u/write-json {"api-key" api-key
"secret-key" secret-key
"scopes" req-scopes}))})
:body (u/write-json-str {"api-key" api-key
"secret-key" secret-key
"scopes" req-scopes})})
{:strs [scopes]}
(u/parse-json body)]
(is (= 200 status))
Expand All @@ -279,9 +277,9 @@
(curl/put
"http://0.0.0.0:8080/admin/creds"
{:headers hdr
:body (String. (u/write-json {"api-key" api-key
"secret-key" secret-key
"scopes" []}))})
:body (u/write-json-str {"api-key" api-key
"secret-key" secret-key
"scopes" []})})
{:strs [scopes]}
(u/parse-json body)]
(is (= 200 status))
Expand All @@ -291,8 +289,8 @@
(curl/delete
"http://0.0.0.0:8080/admin/creds"
{:headers hdr
:body (String. (u/write-json {"api-key" api-key
"secret-key" secret-key}))})]
:body (u/write-json-str {"api-key" api-key
"secret-key" secret-key})})]
(is (= 200 status))))
(testing "and reading after deletion"
(let [{:keys [status body]}
Expand Down
3 changes: 1 addition & 2 deletions src/test/lrsql/concurrency_test.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,8 +66,7 @@
(map (fn [batch]
{:headers headers
:basic-auth basic-auth
:body (String. (u/write-json
(vec batch)))})))
:body (u/write-json-str (vec batch))})))
insert-res (do-async-op! curl/post
endpoint
insert-reqs
Expand Down
Loading

0 comments on commit 9e668b0

Please sign in to comment.