From 62654db355ce906879e74f43d752165d658582ed Mon Sep 17 00:00:00 2001 From: Andrea Maria Piana Date: Thu, 6 Feb 2020 16:17:30 +0100 Subject: [PATCH] offload chat messages This commit does a few things: 1) Messages are offloaded from any chat once we go back from the home. This allows us to ignore any message that is coming in from a chat we are not currently focused. 2) After 5 seconds of not-scrolling activity, any received message that is not currently visible will be offloaded to the database. 3) Similarly received messages that are not visible will be offloaded to the database directly --- externs.js | 1 + src/status_im/chat/models/loading.cljs | 99 ++++++++++++++++--- src/status_im/chat/models/message.cljs | 48 ++++++--- src/status_im/data_store/messages.cljs | 8 +- src/status_im/events.cljs | 5 +- src/status_im/ui/screens/chat/state.cljs | 3 + src/status_im/ui/screens/chat/views.cljs | 15 ++- src/status_im/ui/screens/events.cljs | 3 + src/status_im/ui/screens/routing/core.cljs | 2 +- .../status_im/test/chat/models/message.cljs | 2 +- 10 files changed, 148 insertions(+), 38 deletions(-) create mode 100644 src/status_im/ui/screens/chat/state.cljs diff --git a/externs.js b/externs.js index 1cef884148cc..08fc9b21280f 100644 --- a/externs.js +++ b/externs.js @@ -558,6 +558,7 @@ var TopLevel = { "version" : function () {}, "vibrate" : function () {}, "View" : function () {}, + "viewableItems": function() {}, "FlatList" : function () {}, "warn" : function () {}, "WebView" : function () {}, diff --git a/src/status_im/chat/models/loading.cljs b/src/status_im/chat/models/loading.cljs index 2752eeb743f6..56537cb98806 100644 --- a/src/status_im/chat/models/loading.cljs +++ b/src/status_im/chat/models/loading.cljs @@ -10,10 +10,15 @@ [status-im.utils.config :as config] [status-im.utils.datetime :as time] [status-im.utils.fx :as fx] - [status-im.utils.priority-map :refer [empty-message-map]] [status-im.chat.models.message-list :as message-list] [taoensso.timbre :as log])) +(defn cursor->clock-value [cursor] + (js/parseInt (.substring cursor 51 64))) + +(defn clock-value->cursor [clock-value] + (str "000000000000000000000000000000000000000000000000000" clock-value "0x0000000000000000000000000000000000000000000000000000000000000000")) + (fx/defn update-chats-in-app-db {:events [:chats-list/load-success]} [{:keys [db] :as cofx} new-chats] @@ -22,7 +27,7 @@ (assoc acc chat-id (assoc chat :messages-initialized? false - :messages empty-message-map))) + :messages {}))) {} new-chats) chats (merge old-chats chats)] @@ -31,29 +36,81 @@ :chats/loading? false)} (filters/load-filters)))) +(fx/defn offload-all-messages + [{:keys [db] :as cofx}] + (when-let [current-chat-id (:current-chat-id db)] + {:db (update-in db [:chats current-chat-id] + assoc + :all-loaded? false + :cursor nil + :messages-initialized? false + :messages {} + :message-list nil)})) + +(fx/defn handle-chat-visibility-changed + {:events [:chat.ui/message-visibility-changed]} + [{:keys [db] :as cofx} event] + (let [viewable-items (.-viewableItems event) + last-element (aget viewable-items (dec (.-length viewable-items)))] + (when last-element + (let [last-element-clock-value (:clock-value (.-item last-element)) + chat-id (:chat-id (.-item last-element))] + (when (and last-element-clock-value + (get-in db [:chats chat-id :messages-initialized?])) + (let [new-messages (reduce-kv (fn [acc message-id {:keys [clock-value] :as v}] + (if (<= last-element-clock-value clock-value) + (assoc acc message-id v) + acc)) + {} + (get-in db [:chats chat-id :messages]))] + (fx/merge cofx + {:db (-> db + + (update-in [:chats chat-id] + assoc + :messages new-messages + :all-loaded? false + :message-list (message-list/add-many nil (vals new-messages)) + :cursor (clock-value->cursor last-element-clock-value)))}))))))) + (fx/defn initialize-chats "Initialize persisted chats on startup" [cofx] (data-store.chats/fetch-chats-rpc cofx {:on-success #(re-frame/dispatch [:chats-list/load-success %])})) +(fx/defn handle-failed-loading-messages + {:events [::failed-loading-messages]} + [{:keys [db]} current-chat-id _ err] + (log/error "failed loading messages" current-chat-id err) + (when current-chat-id + {:db (assoc-in db [:chats current-chat-id :loading-messages?] false)})) (fx/defn messages-loaded "Loads more messages for current chat" {:events [::messages-loaded]} [{{:keys [current-chat-id] :as db} :db :as cofx} chat-id + session-id {:keys [cursor messages]}] (when-not (or (nil? current-chat-id) - (not= chat-id current-chat-id)) + (not= chat-id current-chat-id) + (and (get-in db [:chats current-chat-id :messages-initialized?]) + (not= session-id + (get-in db [:chats current-chat-id :messages-initialized?])))) (let [already-loaded-messages (get-in db [:chats current-chat-id :messages]) loaded-unviewed-messages-ids (get-in db [:chats current-chat-id :loaded-unviewed-messages-ids] #{}) ;; We remove those messages that are already loaded, as we might get some duplicates {:keys [all-messages new-messages - unviewed-message-ids]} (reduce (fn [{:keys [all-messages] :as acc} - {:keys [seen message-id] :as message}] + last-clock-value + unviewed-message-ids]} (reduce (fn [{:keys [last-clock-value all-messages] :as acc} + {:keys [clock-value seen message-id] :as message}] (cond-> acc + (or (nil? last-clock-value) + (> last-clock-value clock-value)) + (assoc :last-clock-value clock-value) + (not seen) (update :unviewed-message-ids conj message-id) @@ -68,8 +125,9 @@ messages)] (fx/merge cofx {:db (-> db + (assoc-in [:chats current-chat-id :cursor-clock-value] (when (seq cursor) (cursor->clock-value cursor))) (assoc-in [:chats current-chat-id :loaded-unviewed-messages-ids] unviewed-message-ids) - (assoc-in [:chats current-chat-id :messages-initialized?] true) + (assoc-in [:chats current-chat-id :loading-messages?] false) (assoc-in [:chats current-chat-id :messages] all-messages) (update-in [:chats current-chat-id :message-list] message-list/add-many new-messages) (assoc-in [:chats current-chat-id :cursor] cursor) @@ -78,11 +136,26 @@ (chat-model/mark-messages-seen current-chat-id))))) (fx/defn load-more-messages - [{:keys [db]}] + [{:keys [db] :as cofx}] + (when-let [current-chat-id (:current-chat-id db)] + (when-let [session-id (get-in db [:chats current-chat-id :messages-initialized?])] + (when-not (or (get-in db [:chats current-chat-id :all-loaded?]) + (get-in db [:chats current-chat-id :loading-messages?])) + (let [cursor (get-in db [:chats current-chat-id :cursor]) + load-messages-fx (data-store.messages/messages-by-chat-id-rpc current-chat-id + cursor + constants/default-number-of-messages + #(re-frame/dispatch [::messages-loaded current-chat-id session-id %]) + #(re-frame/dispatch [::failed-loading-messages current-chat-id session-id %]))] + (fx/merge cofx + load-messages-fx + (mailserver/load-gaps-fx current-chat-id))))))) + +(fx/defn load-messages + [{:keys [db now] :as cofx}] (when-let [current-chat-id (:current-chat-id db)] - (when-not (get-in db [:chats current-chat-id :all-loaded?]) - (let [cursor (get-in db [:chats current-chat-id :cursor])] - (data-store.messages/messages-by-chat-id-rpc current-chat-id - cursor - constants/default-number-of-messages - #(re-frame/dispatch [::messages-loaded current-chat-id %])))))) + (when-not (get-in db [:chats current-chat-id :messages-initialized?]) + (fx/merge cofx + {:db (assoc-in db [:chats current-chat-id :messages-initialized?] now)} + (load-more-messages))))) + diff --git a/src/status_im/chat/models/message.cljs b/src/status_im/chat/models/message.cljs index e3e43169e1fb..c1fa3c60fe8e 100644 --- a/src/status_im/chat/models/message.cljs +++ b/src/status_im/chat/models/message.cljs @@ -13,6 +13,7 @@ [status-im.ethereum.core :as ethereum] [status-im.mailserver.core :as mailserver] [status-im.native-module.core :as status] + [status-im.ui.screens.chat.state :as view.state] [status-im.transport.message.protocol :as protocol] [status-im.transport.utils :as transport.utils] [status-im.ui.components.react :as react] @@ -82,12 +83,6 @@ message-to-be-removed (when replace (get-in db [:chats chat-id :messages replace])) prepared-message (prepare-message message chat-id current-chat?)] - (when (and platform/desktop? - (not= from current-public-key) - (get-in db [:multiaccount :desktop-notifications?]) - (< (time/seconds-ago (time/to-date timestamp)) constants/one-earth-day)) - (let [{:keys [title body prioritary?]} (build-desktop-notification cofx message)] - (.displayNotification react/desktop-notification title body prioritary?))) (fx/merge cofx (when message-to-be-removed (hide-message chat-id message-to-be-removed)) @@ -103,29 +98,52 @@ (and (not current-chat?) (not= from current-public-key)) (update-in [:chats chat-id :loaded-unviewed-messages-ids] - (fnil conj #{}) message-id))}) - (when (and platform/desktop? - (not (system-message? prepared-message))) - (chat-model/update-dock-badge-label))))) + (fnil conj #{}) message-id))})))) (fx/defn add-received-message [{:keys [db] :as cofx} - {:keys [from message-id chat-id content] :as message}] + {:keys [from + message-id + chat-id + clock-value + content] :as message}] (let [{:keys [current-chat-id view-id]} db + cursor-clock-value (get-in db [:chats current-chat-id :cursor-clock-value]) current-chat? (and (or (= :chat view-id) (= :chat-modal view-id)) (= current-chat-id chat-id))] - (fx/merge cofx - (add-message {:message message - :current-chat? current-chat?})))) + (when (and current-chat? + (or (not cursor-clock-value) + (<= cursor-clock-value clock-value))) + ;; Not in the current view, offload to db and update cursor if necessary + (if (or (not @view.state/viewable-item) + (not= current-chat-id + (:chat-id @view.state/viewable-item)) + (<= (:clock-value @view.state/viewable-item) + clock-value)) + (add-message cofx {:message message + :current-chat? current-chat?}) + (when (and (< clock-value + cursor-clock-value) + (= current-chat-id + (:chat-id (.-item view.state/viewable-item)))) + {:db (assoc-in db [:chats chat-id :cursor] (chat-loading/clock-value->cursor clock-value))}))))) (defn- add-to-chat? [{:keys [db]} {:keys [chat-id clock-value message-id from]}] - (let [{:keys [deleted-at-clock-value messages]} + (let [{:keys [cursor-clock-value deleted-at-clock-value messages]} (get-in db [:chats chat-id])] (not (or (get messages message-id) (>= deleted-at-clock-value clock-value))))) +(fx/defn offload-message-from [{:keys [db] :as cofx} chat-id message-id] + (let [old-messages (get-in db [:chats chat-id :messages])] + (when-let [last-clock-value (get-in old-messages [message-id :clock-value])] + (let [new-messages (select-keys old-messages (for [[k v] old-messages :when (<= last-clock-value (:clock-value v))] k))] + (fx/merge cofx + {:db (assoc-in db [:chats chat-id :messages] new-messages)} + (rebuild-message-list chat-id)))))) + (defn extract-chat-id [cofx {:keys [chat-id from message-type]}] "Validate and return a valid chat-id" (cond diff --git a/src/status_im/data_store/messages.cljs b/src/status_im/data_store/messages.cljs index 758352ae6fe6..cb6102f700a2 100644 --- a/src/status_im/data_store/messages.cljs +++ b/src/status_im/data_store/messages.cljs @@ -57,12 +57,16 @@ :on-success #(re-frame/dispatch [:messages/system-messages-saved (map <-rpc %)]) :on-failure #(log/error "failed to save messages" %)})) -(defn messages-by-chat-id-rpc [chat-id cursor limit on-success] +(defn messages-by-chat-id-rpc [chat-id + cursor + limit + on-success + on-failure] {::json-rpc/call [{:method "shhext_chatMessages" :params [chat-id cursor limit] :on-success (fn [result] (on-success (update result :messages #(map <-rpc %)))) - :on-failure #(log/error "failed to get messages" %)}]}) + :on-failure on-failure}]}) (defn mark-seen-rpc [chat-id ids] {::json-rpc/call [{:method "shhext_markMessagesSeen" diff --git a/src/status_im/events.cljs b/src/status_im/events.cljs index 1597432c3334..5e815080e02c 100644 --- a/src/status_im/events.cljs +++ b/src/status_im/events.cljs @@ -494,10 +494,7 @@ (handlers/register-handler-fx :chat.ui/load-more-messages (fn [cofx _] - (let [chat-id (get-in cofx [:db :current-chat-id])] - (fx/merge cofx - (chat.loading/load-more-messages) - (mailserver/load-gaps-fx chat-id))))) + (chat.loading/load-more-messages cofx))) (handlers/register-handler-fx :chat.ui/start-chat diff --git a/src/status_im/ui/screens/chat/state.cljs b/src/status_im/ui/screens/chat/state.cljs new file mode 100644 index 000000000000..827cf99fc72d --- /dev/null +++ b/src/status_im/ui/screens/chat/state.cljs @@ -0,0 +1,3 @@ +(ns status-im.ui.screens.chat.state) + +(defonce viewable-item (atom nil)) diff --git a/src/status_im/ui/screens/chat/views.cljs b/src/status_im/ui/screens/chat/views.cljs index c3a8034c6496..5a1b747d17a4 100644 --- a/src/status_im/ui/screens/chat/views.cljs +++ b/src/status_im/ui/screens/chat/views.cljs @@ -26,6 +26,8 @@ [status-im.ui.screens.profile.tribute-to-talk.views :as tribute-to-talk.views] + [status-im.ui.screens.chat.state :as state] + [status-im.utils.debounce :as debounce] [status-im.utils.platform :as platform] [status-im.ui.screens.chat.extensions.views :as extensions]) (:require-macros [status-im.utils.views :refer [defview letsubs]])) @@ -307,6 +309,16 @@ (defonce messages-list-ref (atom nil)) +(defn on-viewable-items-changed [e] + (reset! state/viewable-item + (let [element (->> (.-viewableItems e) + reverse + (filter (fn [e] + (= :message (:type (.-item e))))) + first)] + (when element (.-item element)))) + (debounce/debounce-and-dispatch [:chat.ui/message-visibility-changed e] 5000)) + (defview messages-view [{:keys [group-chat chat-id pending-invite-inviter-name contact] :as chat} modal?] @@ -314,8 +326,6 @@ current-public-key [:multiaccount/public-key]] {:component-did-update (fn [args] - (when-not (:messages-initialized? (second (.-argv (.-props args)))) - (re-frame/dispatch [:chat.ui/load-more-messages])) (re-frame/dispatch [:chat.ui/set-chat-ui-props {:messages-focused? true :input-focused? false}]))} @@ -334,6 +344,7 @@ :idx idx :list-ref messages-list-ref}]) :inverted true + :onViewableItemsChanged on-viewable-items-changed :onEndReached #(re-frame/dispatch [:chat.ui/load-more-messages]) :onScrollToIndexFailed #() :keyboardShouldPersistTaps :handled} diff --git a/src/status_im/ui/screens/events.cljs b/src/status_im/ui/screens/events.cljs index f62b28977f13..0ea04be441bd 100644 --- a/src/status_im/ui/screens/events.cljs +++ b/src/status_im/ui/screens/events.cljs @@ -9,6 +9,7 @@ status-im.ui.screens.wallet.navigation [re-frame.core :as re-frame] [status-im.chat.models :as chat] + [status-im.chat.models.loading :as chat.loading] [status-im.hardwallet.core :as hardwallet] [status-im.mailserver.core :as mailserver] [status-im.multiaccounts.recover.core :as recovery] @@ -200,6 +201,8 @@ (fx/merge cofx {:db (assoc db :view-id view-id)} #(case view-id + :chat (chat.loading/load-messages cofx) + :home (chat.loading/offload-all-messages cofx) :keycard-settings (hardwallet/settings-screen-did-load %) :reset-card (hardwallet/reset-card-screen-did-load %) :enter-pin-login (hardwallet/enter-pin-screen-did-load %) diff --git a/src/status_im/ui/screens/routing/core.cljs b/src/status_im/ui/screens/routing/core.cljs index a7571967576a..189b938a703d 100644 --- a/src/status_im/ui/screens/routing/core.cljs +++ b/src/status_im/ui/screens/routing/core.cljs @@ -244,4 +244,4 @@ :onTransitionStart (fn [])})}]]) {:initialRouteName (if (= view-id :intro) :intro-stack - :login-stack)}))) \ No newline at end of file + :login-stack)}))) diff --git a/test/cljs/status_im/test/chat/models/message.cljs b/test/cljs/status_im/test/chat/models/message.cljs index 808ca790e4a4..dc0ae7388e4c 100644 --- a/test/cljs/status_im/test/chat/models/message.cljs +++ b/test/cljs/status_im/test/chat/models/message.cljs @@ -158,7 +158,7 @@ (testing "our own message" (is (get-in (message/receive-one cofx own-message) [:db :chats "matching" :messages "1"]))) (testing "a message with non matching chat-id" - (is (get-in (message/receive-one cofx bad-chat-id-message) [:db :chats "not-matching" :messages "1"])))))) + (is (not (get-in (message/receive-one cofx bad-chat-id-message) [:db :chats "not-matching" :messages "1"]))))))) (deftest delete-message (with-redefs [time/day-relative (constantly "day-relative")