diff --git a/deps.edn b/deps.edn index 97e4f393c..ed762ead3 100644 --- a/deps.edn +++ b/deps.edn @@ -5,6 +5,7 @@ org.clojure/tools.logging {:mvn/version "1.1.0"} org.clojure/core.memoize {:mvn/version "1.0.250"} clojure-interop/java.security {:mvn/version "1.0.5"} + org.clojure/core.async {:mvn/version "1.6.681"} ;; Util deps camel-snake-kebab/camel-snake-kebab {:mvn/version "0.4.2"} diff --git a/doc/env_vars.md b/doc/env_vars.md index d19446357..7f451a7d3 100644 --- a/doc/env_vars.md +++ b/doc/env_vars.md @@ -104,6 +104,8 @@ The following options are used for advanced database performance tuning and may | `LRSQL_OIDC_SCOPE_PREFIX` | `oidcScopePrefix` | An optional prefix prepended to OIDC scope. For example, setting this to `lrs:` would change the expected `all` scope to `lrs:all` | `""` | | `LRSQL_STMT_RETRY_LIMIT` | `stmtRetryLimit` | The number of times to retry a statement post transaction before failing. | `10` | | `LRSQL_STMT_RETRY_BUDGET` | `stmtRetryBudget` | The max amount of time allowed for statement POST transaction retries before failing (ms). | `1000` | +| `LRSQL_ENABLE_REACTIONS` | `enableReactions` | Whether or not to enable statement reactions. | `false` | +| `LRSQL_REACTION_BUFFER_SIZE` | `reactionBufferSize` | Number of pending reactions to allow. Additional reactions will be dropped with a warning message. | `10000` | _NOTE:_ `LRSQL_STMT_RETRY_LIMIT` and `LRSQL_STMT_RETRY_BUDGET` are used to mitigate a rare scenario where specific Actors or Activities are updated many times in large concurrent batches. In this situation the DBMS can encounter locking and these settings are used to allow retries that eventually write all the conflicting transactions, but may incur performance degradation. If you are experiencing this situation the first step would be to look at why your data needs to rewrite specific Actors or Activities rapidly with different values, which could potentially solve it at the source. If the issue cannot be avoided by data design alone, another possible solution is reducing batch sizes to decrease or eliminate locks. As a last resort, increasing these settings will at least ensure the statements get written but as mentioned may incur a slowdown in concurrent throughput. diff --git a/resources/lrsql/config/prod/default/lrs.edn b/resources/lrsql/config/prod/default/lrs.edn index 364c9d133..0f5d23a55 100644 --- a/resources/lrsql/config/prod/default/lrs.edn +++ b/resources/lrsql/config/prod/default/lrs.edn @@ -10,4 +10,6 @@ :oidc-authority-template #or [#env LRSQL_OIDC_AUTHORITY_TEMPLATE "config/oidc_authority.json.template"] :oidc-scope-prefix #or [#env LRSQL_OIDC_SCOPE_PREFIX ""] :stmt-retry-limit #or [#env LRSQL_STMT_RETRY_LIMIT 10] - :stmt-retry-budget #or [#env LRSQL_STMT_RETRY_BUDGET 1000]} + :stmt-retry-budget #or [#env LRSQL_STMT_RETRY_BUDGET 1000] + :enable-reactions #boolean #or [#env LRSQL_ENABLE_REACTIONS false] + :reaction-buffer-size #long #or [#env LRSQL_REACTION_BUFFER_SIZE 10000]} diff --git a/resources/lrsql/config/test/default/lrs.edn b/resources/lrsql/config/test/default/lrs.edn index cf8fe9558..d08633c7c 100644 --- a/resources/lrsql/config/test/default/lrs.edn +++ b/resources/lrsql/config/test/default/lrs.edn @@ -10,4 +10,6 @@ :oidc-authority-template "config/oidc_authority.json.template" :oidc-scope-prefix "" :stmt-retry-limit 20 - :stmt-retry-budget 10000} + :stmt-retry-budget 10000 + :enable-reactions true + :reaction-buffer-size 10000} diff --git a/src/db/postgres/lrsql/postgres/main.clj b/src/db/postgres/lrsql/postgres/main.clj index 5fa122b7d..1c1ce5a11 100644 --- a/src/db/postgres/lrsql/postgres/main.clj +++ b/src/db/postgres/lrsql/postgres/main.clj @@ -1,6 +1,7 @@ (ns lrsql.postgres.main (:require [com.stuartsierra.component :as component] [lrsql.system :as system] + [lrsql.system.util :as su] [lrsql.postgres.record :as pr]) (:gen-class)) @@ -10,8 +11,11 @@ "Run a Postgres-backed LRSQL instance based on the `:test-postgres` config profile. For use with `clojure -X:db-postgres`." [_] ; Need to pass in a map for -X - (component/start (system/system postgres-backend :test-postgres))) + (-> (system/system postgres-backend :test-postgres) + component/start + su/add-shutdown-hook!)) (defn -main [& _args] (-> (system/system postgres-backend :prod-postgres) - component/start)) + component/start + su/add-shutdown-hook!)) diff --git a/src/db/sqlite/lrsql/sqlite/main.clj b/src/db/sqlite/lrsql/sqlite/main.clj index eda5a257c..dc8d88da0 100644 --- a/src/db/sqlite/lrsql/sqlite/main.clj +++ b/src/db/sqlite/lrsql/sqlite/main.clj @@ -1,6 +1,7 @@ (ns lrsql.sqlite.main (:require [com.stuartsierra.component :as component] [lrsql.system :as system] + [lrsql.system.util :as su] [lrsql.sqlite.record :as sr]) (:gen-class)) @@ -14,7 +15,9 @@ override-profile]}] (let [profile (or override-profile (if ephemeral? :test-sqlite-mem :test-sqlite))] - (component/start (system/system sqlite-backend profile)))) + (-> (system/system sqlite-backend profile) + component/start + su/add-shutdown-hook!))) (defn -main "Main entrypoint for SQLite-backed LRSQL instances. Passing `--ephemeral true` @@ -25,4 +28,5 @@ ephemeral? (Boolean/parseBoolean ?per-str) profile (if ephemeral? :prod-sqlite-mem :prod-sqlite)] (-> (system/system sqlite-backend profile) - component/start))) + component/start + su/add-shutdown-hook!))) diff --git a/src/main/lrsql/init/reaction.clj b/src/main/lrsql/init/reaction.clj new file mode 100644 index 000000000..0955c76d4 --- /dev/null +++ b/src/main/lrsql/init/reaction.clj @@ -0,0 +1,90 @@ +(ns lrsql.init.reaction + "Reaction initialization functions." + (:require [clojure.core.async :as a] + [clojure.spec.alpha :as s] + [clojure.tools.logging :as log] + [lrsql.reaction.protocol :as rp] + [lrsql.util :as u] + [lrsql.spec.config :as config-spec] + [lrsql.spec.common :as common-spec] + [lrsql.spec.reaction :as rs] + [xapi-schema.spec :as xs] + [clojure.string :as cs])) + +(s/fdef reaction-channel + :args (s/cat :config ::config-spec/lrs) + :ret (s/nilable ::common-spec/channel)) + +(defn reaction-channel + "Based on config, return a channel to receive reactions or nil if reactions + are disabled" + [{enable-reactions :enable-reactions + reaction-buffer-size :reaction-buffer-size}] + (when enable-reactions + (a/chan reaction-buffer-size))) + +(s/fdef offer-trigger! + :args (s/cat :?reaction-channel (s/nilable ::common-spec/channel) + :trigger-id ::xs/uuid) + :ret nil?) + +(defn offer-trigger! + "Given a (possibly nil) reaction channel and a string statement ID, submit the + ID to the channel as a UUID if it exists, or do nothing if it is nil. + Log if the channel exists but the ID cannot be submitted." + [?reaction-channel trigger-id] + (when ?reaction-channel + (when-not (a/offer! ?reaction-channel (u/str->uuid trigger-id)) + (log/warnf "Reaction channel full, dropping statement ID: %s" + trigger-id)))) + +(s/fdef reaction-executor + :args (s/cat :?reaction-channel (s/nilable ::common-spec/channel) + :reactor rs/reactor?) + :ret (s/nilable ::common-spec/channel)) + +(defn reaction-executor + "Given a (possibly nil) reaction channel and a reactor implementation, process + reactions in a thread pool. If the channel is nil, returns nil." + [?reaction-channel reactor] + (when ?reaction-channel + (log/info "Starting reaction processor...") + (let [reaction-executor + (a/go-loop [] + (log/debug "Listening for reaction trigger...") + (if-let [trigger-id (a/LearningRecordStore {}) [:connection :backend]) + :reactor (component/using + (reactor/map->Reactor {}) + [:backend :lrs]) :webserver (component/using (webserver/map->Webserver {}) - [:lrs])) + [:lrs :reactor])) assoc-config (fn [m config-m] (assoc m :config config-m))] - ;; This code can be confusing. What is happening is that the above creates - ;; a system map with empty maps and then based on the key of the system, + ;; This code can be confusing. What is happening is that the above creates + ;; a system map with empty maps and then based on the key of the system, ;; populates the corresponding config (by key) from the overall aero config (-> (merge-with assoc-config initial-sys config) (component/system-using {})))) diff --git a/src/main/lrsql/system/lrs.clj b/src/main/lrsql/system/lrs.clj index 556157135..08d0af5e0 100644 --- a/src/main/lrsql/system/lrs.clj +++ b/src/main/lrsql/system/lrs.clj @@ -7,6 +7,7 @@ [lrsql.admin.protocol :as adp] [lrsql.init :as init] [lrsql.init.oidc :as oidc-init] + [lrsql.init.reaction :as react-init] [lrsql.backend.protocol :as bp] [lrsql.input.actor :as agent-input] [lrsql.input.activity :as activity-input] @@ -28,7 +29,6 @@ [lrsql.ops.query.document :as doc-q] [lrsql.ops.query.reaction :as react-q] [lrsql.ops.query.statement :as stmt-q] - [lrsql.reaction.protocol :as rp] [lrsql.spec.config :as cs] [lrsql.util.auth :as auth-util] [lrsql.util.oidc :as oidc-util] @@ -49,7 +49,8 @@ backend config authority-fn - oidc-authority-fn] + oidc-authority-fn + reaction-channel] cmp/Lifecycle (start [lrs] @@ -78,11 +79,16 @@ (assoc lrs :connection connection :authority-fn auth-fn - :oidc-authority-fn oidc-auth-fn)))) + :oidc-authority-fn oidc-auth-fn + :reaction-channel (react-init/reaction-channel config))))) (stop [lrs] (log/info "Stopping LRS...") - (assoc lrs :connection nil :authority-fn nil)) + (assoc lrs + :connection nil + :authority-fn nil + :oidc-authority-fn nil + :reaction-channel nil)) lrsp/AboutResource (-get-about @@ -136,8 +142,11 @@ stmt-result) ;; Non-error result - continue (if-some [stmt-id (:statement-id stmt-result)] - (recur (rest stmt-ins) - (update stmt-res :statement-ids conj stmt-id)) + (do + ;; Submit statement for reaction if enabled + (react-init/offer-trigger! reaction-channel stmt-id) + (recur (rest stmt-ins) + (update stmt-res :statement-ids conj stmt-id))) (recur (rest stmt-ins) stmt-res)))) ;; No more statement inputs - return @@ -376,37 +385,4 @@ (let [conn (lrs-conn this) input (react-input/delete-reaction-input reaction-id)] (jdbc/with-transaction [tx conn] - (react-cmd/delete-reaction! backend tx input)))) - rp/StatementReactor - (-react-to-statement [this statement-id] - (let [conn (lrs-conn this) - statement-results - (jdbc/with-transaction [tx conn] - (reduce - (fn [acc {:keys [reaction-id - error] - :as result}] - (if error - (let [input (react-input/error-reaction-input - reaction-id error)] - (react-cmd/error-reaction! backend tx input) - acc) - (conj acc (select-keys result [:statement :authority])))) - [] - (:result - (react-q/query-statement-reactions - backend tx {:trigger-id statement-id}))))] - ;; Submit statements one at a time with varying authority - {:statement-ids - (reduce - (fn [acc {:keys [statement authority]}] - (into acc - (:statement-ids - (lrsp/-store-statements - this - {:agent authority - :scopes #{:scope/statements.write}} - [statement] - [])))) - [] - statement-results)}))) + (react-cmd/delete-reaction! backend tx input))))) diff --git a/src/main/lrsql/system/reactor.clj b/src/main/lrsql/system/reactor.clj new file mode 100644 index 000000000..2476ed524 --- /dev/null +++ b/src/main/lrsql/system/reactor.clj @@ -0,0 +1,63 @@ +(ns lrsql.system.reactor + (:require [com.stuartsierra.component :as component] + [next.jdbc :as jdbc] + [com.yetanalytics.lrs.protocol :as lrsp] + [lrsql.reaction.protocol :as rp] + [lrsql.init.reaction :as react-init] + [lrsql.input.reaction :as react-input] + [lrsql.ops.command.reaction :as react-cmd] + [lrsql.ops.query.reaction :as react-q])) + +(defrecord Reactor [backend + lrs + reaction-executor] + component/Lifecycle + (start [this] + (assoc this + :reaction-executor + (react-init/reaction-executor + (:reaction-channel lrs) + this))) + (stop [this] + (react-init/shutdown-reactions! + (:reaction-channel lrs) + reaction-executor) + (assoc this + :backend nil + :lrs nil + :reaction-executor nil)) + rp/StatementReactor + (-react-to-statement [_ statement-id] + (let [conn (-> lrs + :connection + :conn-pool) + statement-results + (jdbc/with-transaction [tx conn] + (reduce + (fn [acc {:keys [reaction-id + error] + :as result}] + (if error + (let [input (react-input/error-reaction-input + reaction-id error)] + (react-cmd/error-reaction! backend tx input) + acc) + (conj acc (select-keys result [:statement :authority])))) + [] + (:result + (react-q/query-statement-reactions + backend tx {:trigger-id statement-id}))))] + ;; Submit statements one at a time with varying authority + {:statement-ids + (reduce + (fn [acc {:keys [statement authority]}] + (into acc + (:statement-ids + (lrsp/-store-statements + lrs + {:agent authority + :scopes #{:scope/statements.write}} + [statement] + [])))) + [] + statement-results)}))) diff --git a/src/main/lrsql/system/util.clj b/src/main/lrsql/system/util.clj index c65f45fcb..2cfb5d019 100644 --- a/src/main/lrsql/system/util.clj +++ b/src/main/lrsql/system/util.clj @@ -1,5 +1,6 @@ (ns lrsql.system.util - (:require [clojure.spec.alpha :as s] + (:require [com.stuartsierra.component :as component] + [clojure.spec.alpha :as s] [clojure.walk :as w] [clojure.tools.logging :as log] [next.jdbc.connection :as jdbc-conn] @@ -81,3 +82,16 @@ :port db-port} db-properties (merge (parse-db-props db-properties)))))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Graceful Shutdown +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defn add-shutdown-hook! + "Given a running system, add a shutdown hook to gracefully stop it." + [system] + (.addShutdownHook (Runtime/getRuntime) + (Thread. ^Runnable + (fn [] + (component/stop system)))) + system) diff --git a/src/main/lrsql/system/webserver.clj b/src/main/lrsql/system/webserver.clj index 9b7b78b8b..b6ed78fac 100644 --- a/src/main/lrsql/system/webserver.clj +++ b/src/main/lrsql/system/webserver.clj @@ -99,6 +99,7 @@ (defrecord Webserver [service server lrs + reactor config] component/Lifecycle (start @@ -144,6 +145,7 @@ (assoc this :service nil :server nil - :lrs nil)) + :lrs nil + :reactor nil)) (do (log/info "Webserver already stopped; do nothing.") this)))) diff --git a/src/test/lrsql/lrs_test.clj b/src/test/lrsql/lrs_test.clj index f51ba49f5..82f543e55 100644 --- a/src/test/lrsql/lrs_test.clj +++ b/src/test/lrsql/lrs_test.clj @@ -5,8 +5,10 @@ [com.yetanalytics.datasim.input :as sim-input] [com.yetanalytics.datasim.sim :as sim] [com.yetanalytics.lrs.protocol :as lrsp] - [lrsql.test-support :as support] - [lrsql.util :as u])) + [lrsql.admin.protocol :as adp] + [lrsql.test-support :as support] + [lrsql.test-constants :as tc] + [lrsql.util :as u])) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Init Test Config @@ -1026,6 +1028,85 @@ (get-in [:statement "id"])))))) (component/stop sys'))) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Statement Reaction Tests +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defn- remove-id + [statement] + (dissoc statement "id")) + +(deftest reaction-test + (let [sys (support/test-system) + sys' (component/start sys) + {:keys [lrs]} sys'] + (try + (testing "Processes simple reaction" + (let [;; Add a good reaction + {reaction-id + :result} (adp/-create-reaction + lrs tc/simple-reaction-ruleset true) + ;; Add a bad reaction + bad-ruleset + (assoc + tc/simple-reaction-ruleset + :template + ;; Template with invalid path + {"actor" {"mbox" {"$templatePath" ["x" "actor" "mbox"]}} + "verb" {"id" "https://example.com/verbs/completed"} + "object" {"id" "https://example.com/activities/a-and-b" + "objectType" "Activity"}}) + {bad-reaction-id + :result} (adp/-create-reaction + lrs bad-ruleset true)] + ;; Add statements + (doseq [s [tc/reaction-stmt-a + tc/reaction-stmt-b]] + (Thread/sleep 100) + (lrsp/-store-statements lrs tc/auth-ident [s] [])) + ;; Wait a little bit for the reactor + (Thread/sleep 300) + (testing "New statement added" + (is (= {:statement-result + {:statements + [{"actor" {"mbox" "mailto:bob@example.com"}, + "verb" {"id" "https://example.com/verbs/completed"}, + "object" + {"id" "https://example.com/activities/a-and-b", + "objectType" "Activity"}} + (remove-id tc/reaction-stmt-b) + (remove-id tc/reaction-stmt-a)] + :more ""} + :attachments []} + (-> (lrsp/-get-statements + lrs + tc/auth-ident + {} + []) + ;; Remove LRS fields + (update-in + [:statement-result :statements] + #(mapv (comp + remove-id + remove-props) + %)))))) + (testing "Bad ruleset error is retrievable" + (is (= [{:id reaction-id + :ruleset tc/simple-reaction-ruleset + :active true + :error nil} + {:id bad-reaction-id + :ruleset bad-ruleset + :active false + :error + {:type "ReactionTemplateError", + :message "No value found at [\"x\" \"actor\" \"mbox\"]"}}] + (mapv + #(dissoc % :created :modified) + (adp/-get-all-reactions lrs))))))) + (finally + (component/stop sys'))))) + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Document Tests ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/src/test/lrsql/ops/query/reaction_test.clj b/src/test/lrsql/ops/query/reaction_test.clj index b3222aad6..aadb57549 100644 --- a/src/test/lrsql/ops/query/reaction_test.clj +++ b/src/test/lrsql/ops/query/reaction_test.clj @@ -54,7 +54,9 @@ (deftest query-statement-reactions-valid-test (testing "valid reaction" - (let [sys (support/test-system) + (let [sys (support/test-system + :conf-overrides + {[:lrs :enable-reactions] false}) sys' (component/start sys) lrs (-> sys' :lrs) bk (:backend lrs) @@ -86,7 +88,9 @@ (deftest query-statement-reactions-custom-authority-test (testing "Valid reaction with custom authority" - (let [sys (support/test-system) + (let [sys (support/test-system + :conf-overrides + {[:lrs :enable-reactions] false}) sys' (component/start sys) lrs (-> sys' :lrs) bk (:backend lrs) @@ -124,7 +128,9 @@ (deftest query-statement-reactions-template-error-test (testing "Invalid template" - (let [sys (support/test-system) + (let [sys (support/test-system + :conf-overrides + {[:lrs :enable-reactions] false}) sys' (component/start sys) lrs (-> sys' :lrs) bk (:backend lrs) @@ -164,7 +170,9 @@ ;; This will keep the instrumentation error from clobbering (support/unstrument-lrsql) (testing "Invalid statement output" - (let [sys (support/test-system) + (let [sys (support/test-system + :conf-overrides + {[:lrs :enable-reactions] false}) sys' (component/start sys) lrs (-> sys' :lrs) bk (:backend lrs) diff --git a/src/test/lrsql/ops/util/reaction_test.clj b/src/test/lrsql/ops/util/reaction_test.clj index c39676f6c..bcfbac5e1 100644 --- a/src/test/lrsql/ops/util/reaction_test.clj +++ b/src/test/lrsql/ops/util/reaction_test.clj @@ -32,7 +32,9 @@ (use-fixtures :each support/fresh-db-fixture) (deftest query-reaction-test - (let [sys (support/test-system) + (let [sys (support/test-system + :conf-overrides + {[:lrs :enable-reactions] false}) sys' (component/start sys) lrs (-> sys' :lrs) bk (:backend lrs) @@ -115,7 +117,9 @@ (finally (component/stop sys'))))) (deftest query-active-reactions-test - (let [sys (support/test-system) + (let [sys (support/test-system + :conf-overrides + {[:lrs :enable-reactions] false}) sys' (component/start sys) lrs (-> sys' :lrs) bk (:backend lrs) @@ -139,7 +143,9 @@ (finally (component/stop sys'))))) (deftest query-reaction-history-test - (let [sys (support/test-system) + (let [sys (support/test-system + :conf-overrides + {[:lrs :enable-reactions] false}) sys' (component/start sys) lrs (-> sys' :lrs) bk (:backend lrs) diff --git a/src/test/lrsql/reaction/protocol_test.clj b/src/test/lrsql/reaction/protocol_test.clj index 8f6c4c0a2..ed51dd1f0 100644 --- a/src/test/lrsql/reaction/protocol_test.clj +++ b/src/test/lrsql/reaction/protocol_test.clj @@ -30,10 +30,12 @@ (use-fixtures :each support/fresh-db-fixture) (deftest react-to-statement-test - (let [sys (support/test-system) - sys' (component/start sys) - lrs (-> sys' :lrs) - ds (-> sys' :lrs :connection :conn-pool)] + (let [sys (support/test-system + :conf-overrides + {[:lrs :enable-reactions] false}) + sys' (component/start sys) + {:keys [lrs reactor]} sys' + ds (-> sys' :lrs :connection :conn-pool)] (try (testing "Processes simple reaction" ;; Add a reaction @@ -49,7 +51,7 @@ (lrsp/-store-statements lrs tc/auth-ident [s] [])) ;; React to last statement {[reaction-s-id] :statement-ids} - (rp/-react-to-statement lrs trigger-id)] + (rp/-react-to-statement reactor trigger-id)] (testing "New statement added" (is (= {:statement-result {:statements @@ -85,9 +87,11 @@ (component/stop sys'))))) (deftest react-to-statement-error-test - (let [sys (support/test-system) - sys' (component/start sys) - lrs (-> sys' :lrs)] + (let [sys (support/test-system + :conf-overrides + {[:lrs :enable-reactions] false}) + sys' (component/start sys) + {:keys [lrs reactor]} sys'] (try (testing "Stores reaction errors" ;; Add a reaction with a bad template @@ -115,7 +119,7 @@ (lrsp/-store-statements lrs tc/auth-ident [s] []))] (testing "No statement id results" (is (= {:statement-ids []} - (rp/-react-to-statement lrs trigger-id)))) + (rp/-react-to-statement reactor trigger-id)))) (testing "No statement added" (is (= {:statement-result {:statements