Adds topic negotiation and partitioned topic

All the code has been implemented in statusgo: status-im/status-go#1466 Basically all the whisper filter management is done at that level. Technical description On startup we load all chats and send a list of them to status go: For a public chat: {:chatId "status"}, we create a single filter, based on the name of the chat. For each contact added by us, each user in a group chat and each one to one chat open, we send: {:chatId "0x", :oneToOne true}. This will create a chats, to listen to their contact code. Any previously negotiated topic is also returned. Once loaded, we create our filters, and upsert the mailserver topics, both of which are solely based on the filters loaded. In order to remove a chat, we delete/stopwatching first the the filter in status-react and then ask status-go to remove the filter. For a public chat we always remove, for a one-to-one we remove only if the user is not in our contacts, or in a group chat or we have a chat open. Negotiated topics are never removed, as otherwise the other user won't be able to contact us anymore. On stopping whisper we don't have to ask status-go to remove filters as they are removed automatically. Some more logic can be pushed in status-go, but that will be in subsequent PRs.
status-im · Jun 13, 2019 · 541fec7 · 541fec7
1 parent 7097de6
commit 541fec7
Show file tree

Hide file tree

Showing 51 changed files with 1,098 additions and 940 deletions.
diff --git a/STATUS_GO_SHA256 b/STATUS_GO_SHA256
@@ -1,3 +1,3 @@
 ## DO NOT EDIT THIS FILE BY HAND. USE `scripts/update-status-go.sh <tag>` instead
 
-0049i6znvl45hc651bqyzwgmzlv0fp40maggfjsrv13q5avd0g6d
+04k3ifs4j523q358r0df7xrpgzykmd68bnixg3ryiv92dh0ns7k9
diff --git a/STATUS_GO_VERSION b/STATUS_GO_VERSION
@@ -1,3 +1,3 @@
 ## DO NOT EDIT THIS FILE BY HAND. USE `scripts/update-status-go.sh <tag>` instead
 
-v0.25.0-beta.1
+v0.26.0-alpha.46
diff --git a/externs.js b/externs.js
@@ -260,6 +260,7 @@ var TopLevel = {
     "NavigationEvents" : function () {},
     "newKeyPair" : function () {},
     "newMessageFilter" : function () {},
+    "newRawMessageFilter": function() {},
     "newSymKey" : function () {},
     "nfcIsEnabled" : function () {},
     "nfcIsSupported" : function () {},

diff --git a/mobile_files/package.json.orig b/mobile_files/package.json.orig
@@ -70,7 +70,7 @@
     "string_decoder": "0.10.31",
     "text-encoding": "^0.6.4",
     "url": "0.10.3",
-    "web3": "git+https://github.com/status-im/web3.js.git#0.20.2-status",
+    "web3": "git+https://github.com/status-im/web3.js.git#0.20.3-status.alpha.3",
     "web3-utils": "1.0.0-beta.36"
   }
 }
diff --git a/mobile_files/yarn.lock b/mobile_files/yarn.lock
@@ -7081,9 +7081,9 @@ web3-utils@1.0.0-beta.36:
     underscore "1.8.3"
     utf8 "2.1.1"
 
-"web3@git+https://github.com/status-im/web3.js.git#0.20.2-status":
+"web3@git+https://github.com/status-im/web3.js.git#0.20.3-status.alpha.3":
   version "0.20.1"
-  resolved "git+https://github.com/status-im/web3.js.git#958dbabff2c77615e23f5de678a6fae0b0d70147"
+  resolved "git+https://github.com/status-im/web3.js.git#29677754c8b87c55c828df2694e35b0b760dca5f"
   dependencies:
     bignumber.js "git+https://github.com/status-im/bignumber.js.git#v4.0.2-status"
     crypto-js "^3.1.4"

diff --git a/src/status_im/accounts/core.cljs b/src/status_im/accounts/core.cljs
@@ -94,13 +94,14 @@
                                   {:desktop-notifications? desktop-notifications?}
                                   {}))
 
-(fx/defn toggle-pfs [{:keys [db] :as cofx} enabled?]
+(fx/defn toggle-device-to-device [{:keys [db] :as cofx} enabled?]
   (let [settings (get-in db [:account/account :settings])
-        warning  {:utils/show-popup {:title (i18n/label :t/pfs-warning-title)
-                                     :content (i18n/label :t/pfs-warning-content)}}]
+        warning  {:utils/show-popup {:title (i18n/label :t/device-to-device-warning-title)
+                                     :content (i18n/label :t/device-to-device-warning-content)}}]
 
     (fx/merge cofx
               (when enabled? warning)
+              ;; Set to pfs? for backward compatibility
               (accounts.update/update-settings (assoc settings :pfs? enabled?)
                                                {}))))
 

diff --git a/src/status_im/chat/db.cljs b/src/status_im/chat/db.cljs
@@ -5,8 +5,7 @@
             [status-im.chat.commands.input :as commands.input]
             [status-im.contact.db :as contact.db]
             [status-im.group-chats.db :as group-chats.db]
-            [status-im.mailserver.core :as mailserver]
-            [status-im.transport.partitioned-topic :as topic]
+            [status-im.mailserver.constants :as mailserver.constants]
             [status-im.utils.gfycat.core :as gfycat]))
 
 (defn group-chat-name
@@ -53,14 +52,6 @@
              {}
              chats))
 
-(defn topic-by-current-chat
-  [{:keys [current-chat-id chats] :as db}]
-  (let [{:keys [public?]} (get chats current-chat-id)
-        public-key (get-in db [:account/account :public-key])]
-    (if public?
-      (get-in db [:transport/chats current-chat-id :topic])
-      (topic/public-key->discovery-topic-hash public-key))))
-
 (defn sort-message-groups
   "Sorts message groups according to timestamp of first message in group"
   [message-groups messages]
@@ -164,7 +155,7 @@
              (not (nil? highest-request-to))
              (not (nil? lowest-request-from))
              (< (- highest-request-to lowest-request-from)
-                mailserver/max-gaps-range))
+                mailserver.constants/max-gaps-range))
           (update acc :messages conj {:type       :gap
                                       :value      (str :first-gap)
                                       :first-gap? true})

diff --git a/src/status_im/chat/models.cljs b/src/status_im/chat/models.cljs
@@ -1,14 +1,13 @@
 (ns status-im.chat.models
   (:require [re-frame.core :as re-frame]
             [status-im.accounts.model :as accounts.model]
-            [status-im.contact-code.core :as contact-code]
+            [status-im.transport.filters.core :as transport.filters]
             [status-im.contact.core :as contact.core]
             [status-im.data-store.chats :as chats-store]
             [status-im.data-store.messages :as messages-store]
             [status-im.i18n :as i18n]
             [status-im.mailserver.core :as mailserver]
-            [status-im.transport.chat.core :as transport.chat]
-            [status-im.transport.message.public-chat :as public-chat]
+            [status-im.transport.message.protocol :as transport.protocol]
             [status-im.tribute-to-talk.core :as tribute-to-talk]
             [status-im.ui.components.colors :as colors]
             [status-im.ui.components.desktop.events :as desktop.events]
@@ -169,22 +168,14 @@
                                   (log/error "can't remove a chat:" error))}]}
               (navigation/navigate-to-cofx :home {}))
     (fx/merge cofx
-              ;; TODO: There's a race condition here, as the removal of the filter (async)
-              ;; is done at the same time as the removal of the chat, so a message
-              ;; might come between and restore the chat. Multiple way to handle this
-              ;; (remove chat only after the filter has been removed, probably the safest,
-              ;; flag the chat to ignore new messages, change receive method for public/group chats)
-              ;; For now to keep the code simplier and avoid significant changes, best to leave as it is.
-              #(when (public-chat? % chat-id)
-                 (transport.chat/unsubscribe-from-chat % chat-id))
-              #(when (group-chat? % chat-id)
-                 (mailserver/remove-chat-from-mailserver-topic % chat-id))
               (mailserver/remove-gaps chat-id)
               (mailserver/remove-range chat-id)
               (deactivate-chat chat-id)
               (clear-history chat-id)
-              #(when (one-to-one-chat? % chat-id)
-                 (contact-code/stop-listening % chat-id))
+              (transport.filters/stop-listening chat-id)
+              ;; TODO: this is not accurate, if there's a pending contact
+              ;; request it will not be sent anymore
+              (transport.protocol/remove-chat chat-id)
               (navigation/navigate-to-cofx :home {}))))
 
 (defn- unread-messages-number [chats]
@@ -249,7 +240,10 @@
   (fx/merge cofx
             {:db (-> (assoc db :current-chat-id chat-id)
                      (set-chat-ui-props {:validation-messages nil}))}
-            (contact-code/listen-to-chat chat-id)
+            ;; Group chat don't need this to load as all the loading of topics
+            ;; happens on membership changes
+            (when-not (group-chat? cofx chat-id)
+              (transport.filters/load-chat chat-id))
             (when platform/desktop?
               (mark-messages-seen chat-id))
             (tribute-to-talk/check-tribute chat-id)))
@@ -293,6 +287,7 @@
       (fx/merge cofx
                 (upsert-chat {:chat-id   chat-id
                               :is-active true})
+                (transport.filters/load-chat chat-id)
                 (navigate-to-chat chat-id opts)))))
 
 (fx/defn start-public-chat
@@ -303,9 +298,9 @@
       (navigate-to-chat cofx topic opts))
     (fx/merge cofx
               (add-public-chat topic)
+              (transport.filters/load-chat topic)
               #(when-not dont-navigate?
                  (navigate-to-chat % topic opts))
-              (public-chat/join-public-chat topic)
               #(when platform/desktop?
                  (desktop.events/change-tab % :home)))))
 

diff --git a/src/status_im/chat/models/loading.cljs b/src/status_im/chat/models/loading.cljs
@@ -111,6 +111,7 @@
     get-unviewed-message-ids :get-unviewed-message-ids :as cofx}]
   ;; TODO: re-implement functionality for status-go protocol
   (when-not (or config/use-status-go-protocol?
+                (nil? current-chat-id)
                 (get-in db [:chats current-chat-id :all-loaded?]))
     (let [previous-pagination-info   (get-in db [:chats current-chat-id :pagination-info])
           {:keys [messages

diff --git a/src/status_im/contact/core.cljs b/src/status_im/contact/core.cljs
@@ -1,14 +1,13 @@
 (ns status-im.contact.core
   (:require [status-im.accounts.model :as accounts.model]
-            [status-im.contact-code.core :as contact-code]
+            [status-im.transport.filters.core :as transport.filters]
             [status-im.contact.db :as contact.db]
             [status-im.contact.device-info :as device-info]
             [status-im.ethereum.core :as ethereum]
             [status-im.data-store.contacts :as contacts-store]
             [status-im.mailserver.core :as mailserver]
             [status-im.transport.message.contact :as message.contact]
             [status-im.transport.message.protocol :as protocol]
-            [status-im.transport.partitioned-topic :as transport.topic]
             [status-im.tribute-to-talk.db :as tribute-to-talk]
             [status-im.tribute-to-talk.whitelist :as whitelist]
             [status-im.ui.screens.navigation :as navigation]
@@ -50,8 +49,7 @@
             {:db            (-> db
                                 (update-in [:contacts/contacts public-key] merge contact))
              :data-store/tx [(contacts-store/save-contact-tx contact)]}
-            #(when (contact.db/added? contact)
-               (contact-code/listen-to-chat % public-key))))
+            (transport.filters/load-contact contact)))
 
 (fx/defn send-contact-request
   [{:keys [db] :as cofx} {:keys [public-key] :as contact}]
@@ -70,10 +68,6 @@
       (fx/merge cofx
                 {:db (assoc-in db [:contacts/new-identity] "")}
                 (upsert-contact contact)
-                (mailserver/upsert-mailserver-topic
-                 {:chat-ids [public-key]
-                  :topic    (transport.topic/discovery-topic-hash)
-                  :fetch?   false})
                 (whitelist/add-to-whitelist public-key)
                 (send-contact-request contact)
                 (mailserver/process-next-messages-request)))))
@@ -87,24 +81,6 @@
                 {:db (assoc-in db [:contacts/new-identity] "")}
                 (upsert-contact contact)))))
 
-(fx/defn add-contacts-filter [{:keys [db]} public-key action]
-  (when (not= (get-in db [:account/account :public-key]) public-key)
-    (let [current-public-key (get-in db [:account/account :public-key])]
-      {:db
-       (cond-> db
-         config/partitioned-topic-enabled?
-         (assoc :filters/after-adding-discovery-filter
-                {:action     action
-                 :public-key public-key}))
-
-       :shh/add-discovery-filters
-       {:web3           (:web3 db)
-        :private-key-id current-public-key
-        :topics         [{:topic    (transport.topic/partitioned-topic-hash public-key)
-                          :chat-id  public-key
-                          :minPow   1
-                          :callback (constantly nil)}]}})))
-
 (defn handle-contact-update
   [public-key
    timestamp

diff --git a/src/status_im/contact_recovery/core.cljs b/src/status_im/contact_recovery/core.cljs
@@ -56,7 +56,6 @@
      [{:web3    web3
        :src     current-public-key
        :dst     public-key
-       :topics  (get-in cofx [:db :mailserver/topics])
        :payload ""}]}))
 
 (re-frame/reg-fx

diff --git a/src/status_im/data_store/chats.cljs b/src/status_im/data_store/chats.cljs
@@ -4,6 +4,7 @@
             [status-im.data-store.messages :as messages]
             [status-im.data-store.realm.core :as core]
             [status-im.ethereum.core :as ethereum]
+            [taoensso.timbre :as log]
             [status-im.utils.clocks :as utils.clocks]
             [status-im.utils.core :as utils]))
 
@@ -72,6 +73,7 @@
   "Returns tx function for saving chat"
   [chat]
   (fn [realm]
+    (log/debug "saving chat" chat)
     (core/create
      realm
      :chat

diff --git a/src/status_im/data_store/mailservers.cljs b/src/status_im/data_store/mailservers.cljs
@@ -1,6 +1,7 @@
 (ns status-im.data-store.mailservers
   (:require [cljs.tools.reader.edn :as edn]
             [re-frame.core :as re-frame]
+            [taoensso.timbre :as log]
             [status-im.data-store.realm.core :as core]))
 
 (re-frame/reg-cofx
@@ -31,7 +32,6 @@
 
 (defn deserialize-mailserver-topic [serialized-mailserver-topic]
   (-> serialized-mailserver-topic
-      (dissoc :topic)
       (update :chat-ids edn/read-string)))
 
 (re-frame/reg-cofx
@@ -50,6 +50,7 @@
   "Returns tx function for saving mailserver topic"
   [{:keys [topic mailserver-topic]}]
   (fn [realm]
+    (log/debug "saving mailserver-topic:" topic mailserver-topic)
     (core/create realm
                  :mailserver-topic
                  (-> mailserver-topic
@@ -61,6 +62,7 @@
   "Returns tx function for deleting mailserver topic"
   [topic]
   (fn [realm]
+    (log/debug "deleting mailserver-topic:" topic)
     (let [mailserver-topic (.objectForPrimaryKey realm
                                                  "mailserver-topic"
                                                  topic)]
@@ -69,6 +71,7 @@
 (defn save-chat-requests-range
   [chat-requests-range]
   (fn [realm]
+    (log/debug "saving ranges" chat-requests-range)
     (core/create realm :chat-requests-range chat-requests-range true)))
 
 (re-frame/reg-cofx
@@ -98,11 +101,13 @@
 (defn save-mailserver-requests-gap
   [gap]
   (fn [realm]
+    (log/debug "saving gap" gap)
     (core/create realm :mailserver-requests-gap gap true)))
 
 (defn delete-mailserver-requests-gaps
   [ids]
   (fn [realm]
+    (log/debug "deleting gaps" ids)
     (doseq [id ids]
       (core/delete
        realm
@@ -111,11 +116,13 @@
 (defn delete-all-gaps-by-chat
   [chat-id]
   (fn [realm]
+    (log/debug "deleting all gaps for chat" chat-id)
     (core/delete realm
                  (core/get-by-field realm :mailserver-requests-gap :chat-id chat-id))))
 
 (defn delete-range
   [chat-id]
   (fn [realm]
+    (log/debug "deleting range" chat-id)
     (core/delete realm
                  (core/get-by-field realm :chat-requests-range :chat-id chat-id))))
diff --git a/src/status_im/data_store/realm/schemas/account/core.cljs b/src/status_im/data_store/realm/schemas/account/core.cljs
@@ -560,6 +560,21 @@
           contact-recovery/v1
           mailserver-requests-gap/v1])
 
+(def v47 [chat/v15
+          chat-requests-range/v1
+          transport/v10
+          contact/v8
+          message/v11
+          mailserver/v11
+          mailserver-topic/v2
+          membership-update/v1
+          installation/v3
+          browser/v8
+          dapp-permissions/v9
+          contact-device-info/v1
+          contact-recovery/v1
+          mailserver-requests-gap/v1])
+
 ;; put schemas ordered by version
 (def schemas [{:schema        v1
                :schemaVersion 1
@@ -698,4 +713,7 @@
                :migration     (constantly nil)}
               {:schema        v46
                :schemaVersion 46
-               :migration     migrations/v46}])
+               :migration     migrations/v46}
+              {:schema        v47
+               :schemaVersion 47
+               :migration     (constantly nil)}])
diff --git a/src/status_im/data_store/realm/schemas/account/transport.cljs b/src/status_im/data_store/realm/schemas/account/transport.cljs
@@ -72,3 +72,5 @@
 (def v9 (update v8 :properties
                 dissoc
                 :ack :seen :pending-ack :pending-send))
+
+(def v10 (update v9 :properties dissoc :one-to-one :topic :sym-key-id :sym-key))
diff --git a/src/status_im/data_store/transport.cljs b/src/status_im/data_store/transport.cljs
@@ -1,5 +1,6 @@
 (ns status-im.data-store.transport
   (:require [re-frame.core :as re-frame]
+            [taoensso.timbre :as log]
             [status-im.data-store.realm.core :as core]))
 
 (defn deserialize-chat [serialized-chat]
@@ -21,6 +22,7 @@
   "Returns tx function for saving transport"
   [{:keys [chat-id chat]}]
   (fn [realm]
+    (log/debug "saving transport, chat-id:" chat-id "chat" chat)
     (core/create realm
                  :transport
                  (assoc chat :chat-id chat-id)
@@ -30,5 +32,6 @@
   "Returns tx function for deleting transport"
   [chat-id]
   (fn [realm]
+    (log/debug "deleting transport, chat-id:" chat-id)
     (let [transport (.objectForPrimaryKey realm "transport" chat-id)]
       (core/delete realm transport))))