offload chat messages

This commit does a few things:

1) Messages are offloaded from any chat once we go back from the home.
This allows us to ignore any message that is coming in from a chat we
are not currently focused.
2) After 5 seconds of not-scrolling activity, any received message that
is not currently visible will be offloaded to the database.
3) Similarly received messages that are not visible will be offloaded to
the database directly

externs.js

@@ -558,6 +558,7 @@ var TopLevel = {
   "version" : function () {},
   "vibrate" : function () {},
   "View" : function () {},
+  "viewableItems": function() {},
   "FlatList" : function () {},
   "warn" : function () {},
   "WebView" : function () {},

src/status_im/chat/models/loading.cljs

@@ -10,10 +10,15 @@
             [status-im.utils.config :as config]
             [status-im.utils.datetime :as time]
             [status-im.utils.fx :as fx]
-            [status-im.utils.priority-map :refer [empty-message-map]]
             [status-im.chat.models.message-list :as message-list]
             [taoensso.timbre :as log]))
 
+(defn cursor->clock-value [cursor]
+  (js/parseInt (.substring cursor 51 64)))
+
+(defn clock-value->cursor [clock-value]
+  (str "000000000000000000000000000000000000000000000000000" clock-value "0x0000000000000000000000000000000000000000000000000000000000000000"))
+
 (fx/defn update-chats-in-app-db
   {:events [:chats-list/load-success]}
   [{:keys [db] :as cofx} new-chats]
@@ -22,7 +27,7 @@
                         (assoc acc chat-id
                                (assoc chat
                                       :messages-initialized? false
-                                      :messages empty-message-map)))
+                                      :messages {})))
                       {}
                       new-chats)
         chats (merge old-chats chats)]
@@ -31,29 +36,81 @@
                           :chats/loading? false)}
               (filters/load-filters))))
 
+(fx/defn offload-all-messages
+  [{:keys [db] :as cofx}]
+  (when-let [current-chat-id (:current-chat-id db)]
+    {:db (update-in db [:chats current-chat-id]
+                    assoc
+                    :all-loaded? false
+                    :cursor nil
+                    :messages-initialized? false
+                    :messages {}
+                    :message-list nil)}))
+
+(fx/defn handle-chat-visibility-changed
+  {:events [:chat.ui/message-visibility-changed]}
+  [{:keys [db] :as cofx} event]
+  (let [viewable-items (.-viewableItems event)
+        last-element (aget viewable-items (dec (.-length viewable-items)))]
+    (when last-element
+      (let [last-element-clock-value (:clock-value (.-item last-element))
+            chat-id (:chat-id (.-item last-element))]
+        (when (and last-element-clock-value
+                   (get-in db [:chats chat-id :messages-initialized?]))
+          (let [new-messages (reduce-kv (fn [acc message-id {:keys [clock-value] :as v}]
+                                          (if (<= last-element-clock-value clock-value)
+                                            (assoc acc message-id v)
+                                            acc))
+                                        {}
+                                        (get-in db [:chats chat-id :messages]))]
+            (fx/merge cofx
+                      {:db (-> db
+
+                               (update-in [:chats chat-id]
+                                          assoc
+                                          :messages new-messages
+                                          :all-loaded? false
+                                          :message-list (message-list/add-many nil (vals new-messages))
+                                          :cursor (clock-value->cursor last-element-clock-value)))})))))))
+
 (fx/defn initialize-chats
   "Initialize persisted chats on startup"
   [cofx]
   (data-store.chats/fetch-chats-rpc cofx {:on-success
                                           #(re-frame/dispatch
                                             [:chats-list/load-success %])}))
+(fx/defn handle-failed-loading-messages
+  {:events [::failed-loading-messages]}
+  [{:keys [db]} current-chat-id _ err]
+  (log/error "failed loading messages" current-chat-id err)
+  (when current-chat-id
+    {:db (assoc-in db [:chats current-chat-id :loading-messages?] false)}))
 
 (fx/defn messages-loaded
   "Loads more messages for current chat"
   {:events [::messages-loaded]}
   [{{:keys [current-chat-id] :as db} :db :as cofx}
    chat-id
+   session-id
    {:keys [cursor messages]}]
   (when-not (or (nil? current-chat-id)
-                (not= chat-id current-chat-id))
+                (not= chat-id current-chat-id)
+                (and (get-in db [:chats current-chat-id :messages-initialized?])
+                     (not= session-id
+                           (get-in db [:chats current-chat-id :messages-initialized?]))))
     (let [already-loaded-messages    (get-in db [:chats current-chat-id :messages])
           loaded-unviewed-messages-ids (get-in db [:chats current-chat-id :loaded-unviewed-messages-ids] #{})
           ;; We remove those messages that are already loaded, as we might get some duplicates
           {:keys [all-messages
                   new-messages
-                  unviewed-message-ids]} (reduce (fn [{:keys [all-messages] :as acc}
-                                                      {:keys [seen message-id] :as message}]
+                  last-clock-value
+                  unviewed-message-ids]} (reduce (fn [{:keys [last-clock-value all-messages] :as acc}
+                                                      {:keys [clock-value seen message-id] :as message}]
                                                    (cond-> acc
+                                                     (or (nil? last-clock-value)
+                                                         (> last-clock-value clock-value))
+                                                     (assoc :last-clock-value clock-value)
+
                                                      (not seen)
                                                      (update :unviewed-message-ids conj message-id)
 
@@ -68,8 +125,9 @@
                                                  messages)]
       (fx/merge cofx
                 {:db (-> db
+                         (assoc-in [:chats current-chat-id :cursor-clock-value] (when (seq cursor) (cursor->clock-value cursor)))
                          (assoc-in [:chats current-chat-id :loaded-unviewed-messages-ids] unviewed-message-ids)
-                         (assoc-in [:chats current-chat-id :messages-initialized?] true)
+                         (assoc-in [:chats current-chat-id :loading-messages?] false)
                          (assoc-in [:chats current-chat-id :messages] all-messages)
                          (update-in [:chats current-chat-id :message-list] message-list/add-many new-messages)
                          (assoc-in [:chats current-chat-id :cursor] cursor)
@@ -78,11 +136,26 @@
                 (chat-model/mark-messages-seen current-chat-id)))))
 
 (fx/defn load-more-messages
-  [{:keys [db]}]
+  [{:keys [db] :as cofx}]
+  (when-let [current-chat-id (:current-chat-id db)]
+    (when-let [session-id (get-in db [:chats current-chat-id :messages-initialized?])]
+      (when-not (or (get-in db [:chats current-chat-id :all-loaded?])
+                    (get-in db [:chats current-chat-id :loading-messages?]))
+        (let [cursor (get-in db [:chats current-chat-id :cursor])
+              load-messages-fx (data-store.messages/messages-by-chat-id-rpc current-chat-id
+                                                                            cursor
+                                                                            constants/default-number-of-messages
+                                                                            #(re-frame/dispatch [::messages-loaded current-chat-id session-id %])
+                                                                            #(re-frame/dispatch [::failed-loading-messages current-chat-id session-id %]))]
+          (fx/merge cofx
+                    load-messages-fx
+                    (mailserver/load-gaps-fx current-chat-id)))))))
+
+(fx/defn load-messages
+  [{:keys [db now] :as cofx}]
   (when-let [current-chat-id (:current-chat-id db)]
-    (when-not (get-in db [:chats current-chat-id :all-loaded?])
-      (let [cursor (get-in db [:chats current-chat-id :cursor])]
-        (data-store.messages/messages-by-chat-id-rpc current-chat-id
-                                                     cursor
-                                                     constants/default-number-of-messages
-                                                     #(re-frame/dispatch [::messages-loaded current-chat-id %]))))))
+    (when-not (get-in db [:chats current-chat-id :messages-initialized?])
+      (fx/merge cofx
+                {:db (assoc-in db [:chats current-chat-id :messages-initialized?] now)}
+                (load-more-messages)))))
+

src/status_im/chat/models/message.cljs

@@ -13,6 +13,7 @@
             [status-im.ethereum.core :as ethereum]
             [status-im.mailserver.core :as mailserver]
             [status-im.native-module.core :as status]
+            [status-im.ui.screens.chat.state :as view.state]
             [status-im.transport.message.protocol :as protocol]
             [status-im.transport.utils :as transport.utils]
             [status-im.ui.components.react :as react]
@@ -82,12 +83,6 @@
         message-to-be-removed (when replace
                                 (get-in db [:chats chat-id :messages replace]))
         prepared-message (prepare-message message chat-id current-chat?)]
-    (when (and platform/desktop?
-               (not= from current-public-key)
-               (get-in db [:multiaccount :desktop-notifications?])
-               (< (time/seconds-ago (time/to-date timestamp)) constants/one-earth-day))
-      (let [{:keys [title body prioritary?]} (build-desktop-notification cofx message)]
-        (.displayNotification react/desktop-notification title body prioritary?)))
     (fx/merge cofx
               (when message-to-be-removed
                 (hide-message chat-id message-to-be-removed))
@@ -103,29 +98,52 @@
                                   (and (not current-chat?)
                                        (not= from current-public-key))
                                   (update-in [:chats chat-id :loaded-unviewed-messages-ids]
-                                             (fnil conj #{}) message-id))})
-              (when (and platform/desktop?
-                         (not (system-message? prepared-message)))
-                (chat-model/update-dock-badge-label)))))
+                                             (fnil conj #{}) message-id))}))))
 
 (fx/defn add-received-message
   [{:keys [db] :as cofx}
-   {:keys [from message-id chat-id content] :as message}]
+   {:keys [from
+           message-id
+           chat-id
+           clock-value
+           content] :as message}]
   (let [{:keys [current-chat-id view-id]} db
+        cursor-clock-value             (get-in db [:chats current-chat-id :cursor-clock-value])
         current-chat?                  (and (or (= :chat view-id)
                                                 (= :chat-modal view-id))
                                             (= current-chat-id chat-id))]
-    (fx/merge cofx
-              (add-message {:message      message
-                            :current-chat? current-chat?}))))
+    (when (and current-chat?
+               (or (not cursor-clock-value)
+                   (<= cursor-clock-value clock-value)))
+      ;; Not in the current view, offload to db and update cursor if necessary
+      (if (or (not @view.state/viewable-item)
+              (not= current-chat-id
+                    (:chat-id @view.state/viewable-item))
+              (<= (:clock-value @view.state/viewable-item)
+                  clock-value))
+        (add-message cofx {:message      message
+                           :current-chat? current-chat?})
+        (when (and (< clock-value
+                      cursor-clock-value)
+                   (= current-chat-id
+                      (:chat-id (.-item view.state/viewable-item))))
+          {:db (assoc-in db [:chats chat-id :cursor] (chat-loading/clock-value->cursor clock-value))})))))
 
 (defn- add-to-chat?
   [{:keys [db]} {:keys [chat-id clock-value message-id from]}]
-  (let [{:keys [deleted-at-clock-value messages]}
+  (let [{:keys [cursor-clock-value deleted-at-clock-value messages]}
         (get-in db [:chats chat-id])]
     (not (or (get messages message-id)
              (>= deleted-at-clock-value clock-value)))))
 
+(fx/defn offload-message-from [{:keys [db] :as cofx} chat-id message-id]
+  (let [old-messages (get-in db [:chats chat-id :messages])]
+    (when-let [last-clock-value (get-in old-messages [message-id :clock-value])]
+      (let [new-messages (select-keys old-messages (for [[k v] old-messages :when (<= last-clock-value (:clock-value v))] k))]
+        (fx/merge cofx
+                  {:db (assoc-in db [:chats chat-id :messages] new-messages)}
+                  (rebuild-message-list chat-id))))))
+
 (defn extract-chat-id [cofx {:keys [chat-id from message-type]}]
   "Validate and return a valid chat-id"
   (cond

src/status_im/data_store/messages.cljs

@@ -57,12 +57,16 @@
                   :on-success #(re-frame/dispatch [:messages/system-messages-saved (map <-rpc %)])
                   :on-failure #(log/error "failed to save messages" %)}))
 
-(defn messages-by-chat-id-rpc [chat-id cursor limit on-success]
+(defn messages-by-chat-id-rpc [chat-id
+                               cursor
+                               limit
+                               on-success
+                               on-failure]
   {::json-rpc/call [{:method "shhext_chatMessages"
                      :params [chat-id cursor limit]
                      :on-success (fn [result]
                                    (on-success (update result :messages #(map <-rpc %))))
-                     :on-failure #(log/error "failed to get messages" %)}]})
+                     :on-failure on-failure}]})
 
 (defn mark-seen-rpc [chat-id ids]
   {::json-rpc/call [{:method "shhext_markMessagesSeen"

src/status_im/events.cljs

@@ -494,10 +494,7 @@
 (handlers/register-handler-fx
  :chat.ui/load-more-messages
  (fn [cofx _]
-   (let [chat-id (get-in cofx [:db :current-chat-id])]
-     (fx/merge cofx
-               (chat.loading/load-more-messages)
-               (mailserver/load-gaps-fx chat-id)))))
+   (chat.loading/load-more-messages cofx)))
 
 (handlers/register-handler-fx
  :chat.ui/start-chat

src/status_im/ui/screens/chat/state.cljs

@@ -0,0 +1,3 @@
+(ns status-im.ui.screens.chat.state)
+
+(defonce viewable-item (atom nil))

src/status_im/ui/screens/chat/views.cljs

@@ -26,6 +26,8 @@
             [status-im.ui.screens.profile.tribute-to-talk.views
              :as
              tribute-to-talk.views]
+            [status-im.ui.screens.chat.state :as state]
+            [status-im.utils.debounce :as debounce]
             [status-im.utils.platform :as platform]
             [status-im.ui.screens.chat.extensions.views :as extensions])
   (:require-macros [status-im.utils.views :refer [defview letsubs]]))
@@ -307,15 +309,23 @@
 
 (defonce messages-list-ref (atom nil))
 
+(defn on-viewable-items-changed [e]
+  (reset! state/viewable-item
+          (let [element (->> (.-viewableItems e)
+                             reverse
+                             (filter (fn [e]
+                                       (= :message (:type (.-item e)))))
+                             first)]
+            (when element (.-item element))))
+  (debounce/debounce-and-dispatch [:chat.ui/message-visibility-changed e] 5000))
+
 (defview messages-view
   [{:keys [group-chat chat-id pending-invite-inviter-name contact] :as chat}
    modal?]
   (letsubs [messages           [:chats/current-chat-messages-stream]
             current-public-key [:multiaccount/public-key]]
     {:component-did-update
      (fn [args]
-       (when-not (:messages-initialized? (second (.-argv (.-props args))))
-         (re-frame/dispatch [:chat.ui/load-more-messages]))
        (re-frame/dispatch [:chat.ui/set-chat-ui-props
                            {:messages-focused? true
                             :input-focused?    false}]))}
@@ -334,6 +344,7 @@
                                           :idx                idx
                                           :list-ref           messages-list-ref}])
            :inverted                  true
+           :onViewableItemsChanged    on-viewable-items-changed
            :onEndReached              #(re-frame/dispatch [:chat.ui/load-more-messages])
            :keyboardShouldPersistTaps :handled}
           group-header {:header [group-chat-footer chat-id]}]

src/status_im/ui/screens/events.cljs

@@ -9,6 +9,7 @@
             status-im.ui.screens.wallet.navigation
             [re-frame.core :as re-frame]
             [status-im.chat.models :as chat]
+            [status-im.chat.models.loading :as chat.loading]
             [status-im.hardwallet.core :as hardwallet]
             [status-im.mailserver.core :as mailserver]
             [status-im.multiaccounts.recover.core :as recovery]
@@ -200,6 +201,8 @@
    (fx/merge cofx
              {:db (assoc db :view-id view-id)}
              #(case view-id
+                :chat (chat.loading/load-messages cofx)
+                :home (chat.loading/offload-all-messages cofx)
                 :keycard-settings (hardwallet/settings-screen-did-load %)
                 :reset-card (hardwallet/reset-card-screen-did-load %)
                 :enter-pin-login (hardwallet/enter-pin-screen-did-load %)

src/status_im/ui/screens/routing/core.cljs

@@ -244,4 +244,4 @@
                :onTransitionStart (fn [])})}]])
     {:initialRouteName (if (= view-id :intro)
                          :intro-stack
-                         :login-stack)})))
+                         :login-stack)})))

test/cljs/status_im/test/chat/models/message.cljs

@@ -158,7 +158,7 @@
       (testing "our own message"
         (is (get-in (message/receive-one cofx own-message) [:db :chats "matching" :messages "1"])))
       (testing "a message with non matching chat-id"
-        (is (get-in (message/receive-one cofx bad-chat-id-message) [:db :chats "not-matching" :messages "1"]))))))
+        (is (not (get-in (message/receive-one cofx bad-chat-id-message) [:db :chats "not-matching" :messages "1"])))))))
 
 (deftest delete-message
   (with-redefs [time/day-relative (constantly "day-relative")