Adds topic negotiation and partitioned topic

All the code has been implemented in statusgo: status-im/status-go#1466 Basically all the whisper filter management is done at that level. Technical description On startup we load all chats and send a list of them to status go: For a public chat: {:chatId "status"}, we create a single filter, based on the name of the chat. For each contact added by us, each user in a group chat and each one to one chat open, we send: {:chatId "0x", :oneToOne true}. This will create a chats, to listen to their contact code. Any previously negotiated topic is also returned. Once loaded, we create our filters, and upsert the mailserver topics, both of which are solely based on the filters loaded. In order to remove a chat, we delete/stopwatching first the the filter in status-react and then ask status-go to remove the filter. For a public chat we always remove, for a one-to-one we remove only if the user is not in our contacts, or in a group chat or we have a chat open. Negotiated topics are never removed, as otherwise the other user won't be able to contact us anymore. On stopping whisper we don't have to ask status-go to remove filters as they are removed automatically. Some more logic can be pushed in status-go, but that will be in subsequent PRs.
status-im · Jun 24, 2019 · 57a4a60 · 57a4a60
1 parent e454791
commit 57a4a60
Show file tree

Hide file tree

Showing 59 changed files with 1,201 additions and 1,066 deletions.
diff --git a/STATUS_GO_SHA256 b/STATUS_GO_SHA256
@@ -1,3 +1,3 @@
 ## DO NOT EDIT THIS FILE BY HAND. USE `scripts/update-status-go.sh <tag>` instead
 
-0049i6znvl45hc651bqyzwgmzlv0fp40maggfjsrv13q5avd0g6d
+1bhwwmypb64kf180kp634jwfppd3h6yda3qyh2icj3lz7y10pgvp
diff --git a/STATUS_GO_VERSION b/STATUS_GO_VERSION
@@ -1,3 +1,3 @@
 ## DO NOT EDIT THIS FILE BY HAND. USE `scripts/update-status-go.sh <tag>` instead
 
-v0.25.0-beta.1
+v0.27.0-beta.1
diff --git a/desktop_files/package.json.orig b/desktop_files/package.json.orig
@@ -98,7 +98,7 @@
     "string_decoder": "0.10.31",
     "text-encoding": "^0.6.4",
     "url": "0.10.3",
-    "web3": "git+https://github.com/status-im/web3.js.git#0.20.2-status",
+    "web3": "git+https://github.com/status-im/web3.js.git#0.20.3-status.alpha.3",
     "web3-utils": "1.0.0-beta.36",
     "hi-base32": "0.5.0"
   },

diff --git a/desktop_files/yarn.lock b/desktop_files/yarn.lock
@@ -7139,7 +7139,7 @@ react-native-invertible-scroll-view@1.1.0:
     react-native-scrollable-mixin "^1.0.1"
 
 "react-native-keychain@git+https://github.com/status-im/react-native-keychain.git#v.3.0.0-3-status":
-  version "3.0.0-rc.4"
+  version "3.0.0-rc.3"
   resolved "git+https://github.com/status-im/react-native-keychain.git#5895bafa11e734325eaaffd56dda8ca50bfc5275"
 
 "react-native-languages@git+https://github.com/status-im/react-native-languages.git#v0.1.1-status":
@@ -7297,8 +7297,8 @@ react-native-udp@2.2.1:
     xcode "^1.0.0"
     xmldoc "^0.4.0"
     yargs "^9.0.0"
-    yeoman-environment "^1.2.7"
-    yeoman-generator "^0.20.3"
+    yeoman-environment "^2.3.4"
+    yeoman-generator "^3.2.0"
 
 react-navigation-deprecated-tab-navigator@1.3.0:
   version "1.3.0"
@@ -9089,9 +9089,9 @@ web3-utils@1.0.0-beta.36:
     underscore "1.8.3"
     utf8 "2.1.1"
 
-"web3@git+https://github.com/status-im/web3.js.git#0.20.2-status":
+"web3@git+https://github.com/status-im/web3.js.git#0.20.3-status.alpha.3":
   version "0.20.1"
-  resolved "git+https://github.com/status-im/web3.js.git#958dbabff2c77615e23f5de678a6fae0b0d70147"
+  resolved "git+https://github.com/status-im/web3.js.git#29677754c8b87c55c828df2694e35b0b760dca5f"
   dependencies:
     bignumber.js "git+https://github.com/status-im/bignumber.js.git#v4.0.2-status"
     crypto-js "^3.1.4"

diff --git a/externs.js b/externs.js
@@ -260,6 +260,7 @@ var TopLevel = {
     "NavigationEvents" : function () {},
     "newKeyPair" : function () {},
     "newMessageFilter" : function () {},
+    "newRawMessageFilter": function() {},
     "newSymKey" : function () {},
     "nfcIsEnabled" : function () {},
     "nfcIsSupported" : function () {},

diff --git a/mobile_files/package.json.orig b/mobile_files/package.json.orig
@@ -71,7 +71,7 @@
     "string_decoder": "0.10.31",
     "text-encoding": "^0.6.4",
     "url": "0.10.3",
-    "web3": "git+https://github.com/status-im/web3.js.git#0.20.2-status",
+    "web3": "git+https://github.com/status-im/web3.js.git#0.20.3-status.alpha.3",
     "web3-utils": "1.0.0-beta.36"
   }
 }
diff --git a/mobile_files/yarn.lock b/mobile_files/yarn.lock
@@ -7058,9 +7058,9 @@ web3-utils@1.0.0-beta.36:
     underscore "1.8.3"
     utf8 "2.1.1"
 
-"web3@git+https://github.com/status-im/web3.js.git#0.20.2-status":
+"web3@git+https://github.com/status-im/web3.js.git#0.20.3-status.alpha.3":
   version "0.20.1"
-  resolved "git+https://github.com/status-im/web3.js.git#958dbabff2c77615e23f5de678a6fae0b0d70147"
+  resolved "git+https://github.com/status-im/web3.js.git#29677754c8b87c55c828df2694e35b0b760dca5f"
   dependencies:
     bignumber.js "git+https://github.com/status-im/bignumber.js.git#v4.0.2-status"
     crypto-js "^3.1.4"

diff --git a/src/status_im/accounts/core.cljs b/src/status_im/accounts/core.cljs
@@ -94,13 +94,14 @@
                                   {:desktop-notifications? desktop-notifications?}
                                   {}))
 
-(fx/defn toggle-pfs [{:keys [db] :as cofx} enabled?]
+(fx/defn toggle-device-to-device [{:keys [db] :as cofx} enabled?]
   (let [settings (get-in db [:account/account :settings])
-        warning  {:utils/show-popup {:title (i18n/label :t/pfs-warning-title)
-                                     :content (i18n/label :t/pfs-warning-content)}}]
+        warning  {:utils/show-popup {:title (i18n/label :t/device-to-device-warning-title)
+                                     :content (i18n/label :t/device-to-device-warning-content)}}]
 
     (fx/merge cofx
               (when enabled? warning)
+              ;; Set to pfs? for backward compatibility
               (accounts.update/update-settings (assoc settings :pfs? enabled?)
                                                {}))))
 

diff --git a/src/status_im/chat/db.cljs b/src/status_im/chat/db.cljs
@@ -5,8 +5,7 @@
             [status-im.chat.commands.input :as commands.input]
             [status-im.contact.db :as contact.db]
             [status-im.group-chats.db :as group-chats.db]
-            [status-im.mailserver.core :as mailserver]
-            [status-im.transport.partitioned-topic :as topic]
+            [status-im.mailserver.constants :as mailserver.constants]
             [status-im.utils.gfycat.core :as gfycat]))
 
 (defn group-chat-name
@@ -53,14 +52,6 @@
              {}
              chats))
 
-(defn topic-by-current-chat
-  [{:keys [current-chat-id chats] :as db}]
-  (let [{:keys [public?]} (get chats current-chat-id)
-        public-key (get-in db [:account/account :public-key])]
-    (if public?
-      (get-in db [:transport/chats current-chat-id :topic])
-      (topic/public-key->discovery-topic-hash public-key))))
-
 (defn sort-message-groups
   "Sorts message groups according to timestamp of first message in group"
   [message-groups messages]
@@ -164,7 +155,7 @@
              (not (nil? highest-request-to))
              (not (nil? lowest-request-from))
              (< (- highest-request-to lowest-request-from)
-                mailserver/max-gaps-range))
+                mailserver.constants/max-gaps-range))
           (update acc :messages conj {:type       :gap
                                       :value      (str :first-gap)
                                       :first-gap? true})

diff --git a/src/status_im/chat/models.cljs b/src/status_im/chat/models.cljs
@@ -1,14 +1,13 @@
 (ns status-im.chat.models
   (:require [re-frame.core :as re-frame]
             [status-im.accounts.model :as accounts.model]
-            [status-im.contact-code.core :as contact-code]
+            [status-im.transport.filters.core :as transport.filters]
             [status-im.contact.core :as contact.core]
             [status-im.data-store.chats :as chats-store]
             [status-im.data-store.messages :as messages-store]
             [status-im.i18n :as i18n]
             [status-im.mailserver.core :as mailserver]
-            [status-im.transport.chat.core :as transport.chat]
-            [status-im.transport.message.public-chat :as public-chat]
+            [status-im.transport.message.protocol :as transport.protocol]
             [status-im.tribute-to-talk.core :as tribute-to-talk]
             [status-im.ui.components.colors :as colors]
             [status-im.ui.components.desktop.events :as desktop.events]
@@ -169,22 +168,14 @@
                                   (log/error "can't remove a chat:" error))}]}
               (navigation/navigate-to-cofx :home {}))
     (fx/merge cofx
-              ;; TODO: There's a race condition here, as the removal of the filter (async)
-              ;; is done at the same time as the removal of the chat, so a message
-              ;; might come between and restore the chat. Multiple way to handle this
-              ;; (remove chat only after the filter has been removed, probably the safest,
-              ;; flag the chat to ignore new messages, change receive method for public/group chats)
-              ;; For now to keep the code simplier and avoid significant changes, best to leave as it is.
-              #(when (public-chat? % chat-id)
-                 (transport.chat/unsubscribe-from-chat % chat-id))
-              #(when (group-chat? % chat-id)
-                 (mailserver/remove-chat-from-mailserver-topic % chat-id))
               (mailserver/remove-gaps chat-id)
               (mailserver/remove-range chat-id)
               (deactivate-chat chat-id)
               (clear-history chat-id)
-              #(when (one-to-one-chat? % chat-id)
-                 (contact-code/stop-listening % chat-id))
+              (transport.filters/stop-listening chat-id)
+              ;; TODO: this is not accurate, if there's a pending contact
+              ;; request it will not be sent anymore
+              (transport.protocol/remove-chat chat-id)
               (navigation/navigate-to-cofx :home {}))))
 
 (defn- unread-messages-number [chats]
@@ -249,7 +240,10 @@
   (fx/merge cofx
             {:db (-> (assoc db :current-chat-id chat-id)
                      (set-chat-ui-props {:validation-messages nil}))}
-            (contact-code/listen-to-chat chat-id)
+            ;; Group chat don't need this to load as all the loading of topics
+            ;; happens on membership changes
+            (when-not (group-chat? cofx chat-id)
+              (transport.filters/load-chat chat-id))
             (when platform/desktop?
               (mark-messages-seen chat-id))
             (tribute-to-talk/check-tribute chat-id)))
@@ -286,6 +280,7 @@
       (fx/merge cofx
                 (upsert-chat {:chat-id   chat-id
                               :is-active true})
+                (transport.filters/load-chat chat-id)
                 (navigate-to-chat chat-id opts)))))
 
 (fx/defn start-public-chat
@@ -296,9 +291,9 @@
       (navigate-to-chat cofx topic opts))
     (fx/merge cofx
               (add-public-chat topic)
+              (transport.filters/load-chat topic)
               #(when-not dont-navigate?
                  (navigate-to-chat % topic opts))
-              (public-chat/join-public-chat topic)
               #(when platform/desktop?
                  (desktop.events/change-tab % :home)))))
 

diff --git a/src/status_im/chat/models/loading.cljs b/src/status_im/chat/models/loading.cljs
@@ -111,6 +111,7 @@
     get-unviewed-message-ids :get-unviewed-message-ids :as cofx}]
   ;; TODO: re-implement functionality for status-go protocol
   (when-not (or config/use-status-go-protocol?
+                (nil? current-chat-id)
                 (get-in db [:chats current-chat-id :all-loaded?]))
     (let [previous-pagination-info   (get-in db [:chats current-chat-id :pagination-info])
           {:keys [messages

diff --git a/src/status_im/chat/models/message.cljs b/src/status_im/chat/models/message.cljs
@@ -74,10 +74,15 @@
                              {}))}))
 
 (defn add-outgoing-status
-  [{:keys [from] :as message} current-public-key]
+  [{:keys [from outgoing-status] :as message} current-public-key]
   (if (and (= from current-public-key)
            (not (system-message? message)))
-    (assoc message :outgoing true)
+    (assoc message
+           :outgoing true
+           ;; We don't override outgoing-status if there, which means
+           ;; that our device has sent the message, while if empty is coming
+           ;; from a different device
+           :outgoing-status (or outgoing-status :sent))
     message))
 
 (defn build-desktop-notification

diff --git a/src/status_im/contact/core.cljs b/src/status_im/contact/core.cljs
@@ -1,14 +1,13 @@
 (ns status-im.contact.core
   (:require [status-im.accounts.model :as accounts.model]
-            [status-im.contact-code.core :as contact-code]
+            [status-im.transport.filters.core :as transport.filters]
             [status-im.contact.db :as contact.db]
             [status-im.contact.device-info :as device-info]
             [status-im.ethereum.core :as ethereum]
             [status-im.data-store.contacts :as contacts-store]
             [status-im.mailserver.core :as mailserver]
             [status-im.transport.message.contact :as message.contact]
             [status-im.transport.message.protocol :as protocol]
-            [status-im.transport.partitioned-topic :as transport.topic]
             [status-im.tribute-to-talk.db :as tribute-to-talk]
             [status-im.tribute-to-talk.whitelist :as whitelist]
             [status-im.ui.screens.navigation :as navigation]
@@ -50,8 +49,7 @@
             {:db            (-> db
                                 (update-in [:contacts/contacts public-key] merge contact))
              :data-store/tx [(contacts-store/save-contact-tx contact)]}
-            #(when (contact.db/added? contact)
-               (contact-code/listen-to-chat % public-key))))
+            (transport.filters/load-contact contact)))
 
 (fx/defn send-contact-request
   [{:keys [db] :as cofx} {:keys [public-key] :as contact}]
@@ -70,10 +68,6 @@
       (fx/merge cofx
                 {:db (assoc-in db [:contacts/new-identity] "")}
                 (upsert-contact contact)
-                (mailserver/upsert-mailserver-topic
-                 {:chat-ids [public-key]
-                  :topic    (transport.topic/discovery-topic-hash)
-                  :fetch?   false})
                 (whitelist/add-to-whitelist public-key)
                 (send-contact-request contact)
                 (mailserver/process-next-messages-request)))))
@@ -87,24 +81,6 @@
                 {:db (assoc-in db [:contacts/new-identity] "")}
                 (upsert-contact contact)))))
 
-(fx/defn add-contacts-filter [{:keys [db]} public-key action]
-  (when (not= (get-in db [:account/account :public-key]) public-key)
-    (let [current-public-key (get-in db [:account/account :public-key])]
-      {:db
-       (cond-> db
-         config/partitioned-topic-enabled?
-         (assoc :filters/after-adding-discovery-filter
-                {:action     action
-                 :public-key public-key}))
-
-       :shh/add-discovery-filters
-       {:web3           (:web3 db)
-        :private-key-id current-public-key
-        :topics         [{:topic    (transport.topic/partitioned-topic-hash public-key)
-                          :chat-id  public-key
-                          :minPow   1
-                          :callback (constantly nil)}]}})))
-
 (defn handle-contact-update
   [public-key
    timestamp

diff --git a/src/status_im/contact_code/core.cljs b/src/status_im/contact_code/core.cljs