Skip to content

Commit

Permalink
[#18608] Immprove collectibles fetching performance (#18921)
Browse files Browse the repository at this point in the history 
* Add memoized versions to convert keys

* Add placeholder for SVG collectibles due to errors and warnings

* Add events to request collectibles per account

* Update subs to list all accounts collectibles evenly

* Update collectibles tab to pull new data when end is reached

* Use memoized version of key transformation
  • Loading branch information
@ulisesmac
ulisesmac authored Mar 7, 2024
1 parent 3145ecd commit ad546f9
Show file tree
Hide file tree
Showing 12 changed files with 195 additions and 117 deletions.
29 changes: 23 additions & 6 deletions src/quo/components/profile/collectible/view.cljs
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
(ns quo.components.profile.collectible.view
(:require
[clojure.string :as string]
[quo.components.markdown.text :as text]
[quo.components.profile.collectible.style :as style]
[react-native.core :as rn]
[react-native.svg :as svg]))
[react-native.core :as rn]))

(defn remaining-tiles
[amount]
Expand All @@ -19,11 +19,28 @@
(let [svg? (and (map? resource) (:svg? resource))
image-style (style/tile-style-by-size size)]
[rn/view {:style style}
(if svg?
(cond
svg?
[rn/view
{:style {:border-radius (:border-radius image-style)
:overflow :hidden}}
[svg/svg-uri (assoc image-style :uri (:uri resource))]]
{:style (assoc image-style
:border-radius (:border-radius image-style)
:overflow :hidden
:justify-content :center
:align-items :center
:background-color :lightblue)}
[text/text "SVG Content"]]

(or (string/blank? resource) (string/blank? (:uri resource)))
[rn/view
{:style (assoc image-style
:border-radius (:border-radius image-style)
:overflow :hidden
:justify-content :center
:align-items :center
:background-color :lightgray)}
[text/text "Missing image"]]

:else
;; NOTE: using react-native-fast-image here causes a crash on devices when used inside a
;; large flatlist. The library seems to have issues with memory consumption when used with
;; large images/GIFs.
Expand Down
4 changes: 3 additions & 1 deletion src/status_im/contexts/wallet/account/tabs/view.cljs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,9 @@
(case selected-tab
:assets [assets/view]
:collectibles [collectibles/view
{:collectibles collectible-list
{:collectibles collectible-list
:on-end-reached #(rf/dispatch
[:wallet/request-collectibles-for-current-viewing-account])
:on-collectible-press (fn [{:keys [id]}]
(rf/dispatch [:wallet/get-collectible-details id]))}]
:activity [activity/view]
Expand Down
23 changes: 13 additions & 10 deletions src/status_im/contexts/wallet/common/collectibles_tab/view.cljs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
[utils.i18n :as i18n]))

(defn- view-internal
[{:keys [theme collectibles filtered? on-collectible-press]}]
[{:keys [theme collectibles filtered? on-collectible-press on-end-reached]}]
(let [no-results-match-query? (and filtered? (empty? collectibles))]
(cond
no-results-match-query?
Expand All @@ -26,14 +26,17 @@

:else
[rn/flat-list
{:data collectibles
:style {:flex 1}
:content-container-style {:align-items :center}
:num-columns 2
:render-fn (fn [{:keys [preview-url] :as collectible}]
[quo/collectible
{:images [preview-url]
:on-press #(when on-collectible-press
(on-collectible-press collectible))}])}])))
{:data collectibles
:style {:flex 1}
:content-container-style {:align-items :center}
:window-size 11
:num-columns 2
:render-fn (fn [{:keys [preview-url] :as collectible}]
[quo/collectible
{:images [preview-url]
:on-press #(when on-collectible-press
(on-collectible-press collectible))}])
:on-end-reached on-end-reached
:on-end-reached-threshold 4}])))

(def view (quo.theme/with-theme view-internal))
8 changes: 4 additions & 4 deletions src/status_im/contexts/wallet/data_store.cljs
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
(ns status-im.contexts.wallet.data-store
(:require
[camel-snake-kebab.core :as csk]
[camel-snake-kebab.extras :as cske]
[clojure.set :as set]
[clojure.string :as string]
[status-im.constants :as constants]
[utils.money :as money]
[utils.number :as utils.number]))
[utils.number :as utils.number]
[utils.transforms :as transforms]))

(defn chain-ids-string->set
[ids-string]
Expand Down Expand Up @@ -78,7 +78,7 @@
[tokens]
(-> tokens
(update-keys name)
(update-vals #(cske/transform-keys csk/->kebab-case %))
(update-vals #(cske/transform-keys transforms/->kebab-case-keyword %))
(update-vals remove-tokens-with-empty-values)
(update-vals #(mapv rpc->balances-per-chain %))))

Expand Down Expand Up @@ -122,4 +122,4 @@
(defn parse-keypairs
[keypairs]
(let [renamed-data (rename-color-id-in-data keypairs)]
(cske/transform-keys csk/->kebab-case-keyword renamed-data)))
(cske/transform-keys transforms/->kebab-case-keyword renamed-data)))
2 changes: 1 addition & 1 deletion src/status_im/contexts/wallet/events.cljs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@
[:wallet :accounts]
(utils.collection/index-by :address (data-store/rpc->accounts wallet-accounts)))
:fx [[:dispatch [:wallet/get-wallet-token]]
[:dispatch [:wallet/request-collectibles {:start-at-index 0 :new-request? true}]]
[:dispatch [:wallet/request-collectibles-for-all-accounts {:new-request? true}]]
(when new-account?
[:dispatch [:wallet/navigate-to-new-account navigate-to-account]])]})))

Expand Down
174 changes: 112 additions & 62 deletions src/status_im/contexts/wallet/events/collectibles.cljs
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,9 @@
(ns status-im.contexts.wallet.events.collectibles
(:require [camel-snake-kebab.core :as csk]
[camel-snake-kebab.extras :as cske]
[clojure.string :as string]
(:require [camel-snake-kebab.extras :as cske]
[taoensso.timbre :as log]
[utils.ethereum.chain :as chain]
[utils.re-frame :as rf]
[utils.transforms :as types]))
[utils.transforms :as transforms]))

(def collectible-data-types
{:unique-id 0
Expand All @@ -20,33 +18,23 @@
:fetch-if-cache-old 3})

(def max-cache-age-seconds 3600)
(def collectibles-request-batch-size 1000)

(defn displayable-collectible?
[collectible]
(let [{{:keys [image-url animation-url]} :collectible-data
{collection-image-url :image-url} :collection-data} collectible]
(or (not (string/blank? animation-url))
(not (string/blank? image-url))
(not (string/blank? collection-image-url)))))

(defn- add-collectibles-to-accounts
[accounts collectibles]
(reduce (fn [acc {:keys [ownership] :as collectible}]
(->> ownership
(map :address) ; In ERC1155 tokens a collectible can be owned by multiple addresses.
(reduce (fn add-collectible-to-address [acc address]
(update-in acc [address :collectibles] conj collectible))
acc)))
accounts
collectibles))

(defn store-collectibles
[{:keys [db]} [collectibles]]
(let [displayable-collectibles (filter displayable-collectible? collectibles)]
{:db (update-in db [:wallet :accounts] add-collectibles-to-accounts displayable-collectibles)}))

(rf/reg-event-fx :wallet/store-collectibles store-collectibles)
(def collectibles-request-batch-size 25)

(defn- move-collectibles-to-accounts
[accounts new-collectibles-per-account]
(reduce-kv (fn [acc account new-collectibles]
(update-in acc [account :collectibles] #(reduce conj (or % []) new-collectibles)))
accounts
new-collectibles-per-account))

(defn flush-collectibles
[{:keys [db]}]
(let [collectibles-per-account (get-in db [:wallet :ui :collectibles :fetched])]
{:db (-> db
(update-in [:wallet :ui :collectibles] dissoc :pending-requests :fetched)
(update-in [:wallet :accounts] move-collectibles-to-accounts collectibles-per-account))}))

(rf/reg-event-fx :wallet/flush-collectibles-fetched flush-collectibles)

(defn clear-stored-collectibles
[{:keys [db]}]
Expand All @@ -62,48 +50,110 @@
(rf/reg-event-fx :wallet/store-last-collectible-details store-last-collectible-details)

(rf/reg-event-fx
:wallet/request-collectibles
(fn [{:keys [db]} [{:keys [start-at-index new-request?]}]]
(let [request-id 0
collectibles-filter nil
data-type (collectible-data-types :header)
fetch-criteria {:fetch-type (fetch-type :fetch-if-not-cached)
:max-cache-age-seconds max-cache-age-seconds}
chain-ids (chain/chain-ids db)
request-params [request-id
chain-ids
(keys (get-in db [:wallet :accounts]))
collectibles-filter
start-at-index
collectibles-request-batch-size
data-type
fetch-criteria]]
:wallet/request-new-collectibles-for-account
(fn [{:keys [db]} [{:keys [request-id account amount]}]]
(let [current-collectible-idx (get-in db [:wallet :accounts account :current-collectible-idx] 0)
collectibles-filter nil
data-type (collectible-data-types :header)
fetch-criteria {:fetch-type (fetch-type :fetch-if-not-cached)
:max-cache-age-seconds max-cache-age-seconds}
chain-ids (chain/chain-ids db)
request-params [request-id
chain-ids
[account]
collectibles-filter
current-collectible-idx
amount
data-type
fetch-criteria]]
{:fx [[:json-rpc/call
[{:method "wallet_getOwnedCollectiblesAsync"
:params request-params
:on-error (fn [error]
(log/error "failed to request collectibles"
{:event :wallet/request-collectibles
(log/error "failed to request collectibles for account"
{:event :wallet/request-new-collectibles-for-account
:error error
:params request-params}))}]]
(when new-request?
[:dispatch [:wallet/clear-stored-collectibles]])]})))
:params request-params}))}]]]})))

(defonce collectibles-request-ids (atom 0))

(defn- get-unique-collectible-request-id
[amount]
(let [initial-id (deref collectibles-request-ids)
last-id (+ initial-id amount)]
(reset! collectibles-request-ids last-id)
(range initial-id last-id)))

(rf/reg-event-fx
:wallet/request-collectibles-for-all-accounts
(fn [{:keys [db]} [{:keys [new-request?]}]]
(let [accounts (->> (get-in db [:wallet :accounts])
(filter (fn [[_ {:keys [has-more-collectibles?]}]]
(or (nil? has-more-collectibles?)
(true? has-more-collectibles?))))
(keys))
num-accounts (count accounts)
collectibles-per-account (quot collectibles-request-batch-size num-accounts)
;; We need to pass unique IDs for simultaneous requests, otherwise they'll fail
request-ids (get-unique-collectible-request-id num-accounts)
collectible-requests (map (fn [id account]
[:dispatch
[:wallet/request-new-collectibles-for-account
{:request-id id
:account account
:amount collectibles-per-account}]])
request-ids
accounts)]
{:db (cond-> db
:always (assoc-in [:wallet :ui :collectibles :pending-requests] num-accounts)
new-request? (update-in [:wallet :accounts] update-vals #(dissoc % :collectibles)))
:fx collectible-requests})))

(rf/reg-event-fx
:wallet/request-collectibles-for-current-viewing-account
(fn [{:keys [db]} _]
(let [current-viewing-account (-> db :wallet :current-viewing-account-address)
[request-id] (get-unique-collectible-request-id 1)]
{:db (assoc-in db [:wallet :ui :collectibles :pending-requests] 1)
:fx [[:dispatch
[:wallet/request-new-collectibles-for-account
{:request-id request-id
:account current-viewing-account
:amount collectibles-request-batch-size}]]]})))

(defn- update-fetched-collectibles-progress
[db owner-address collectibles offset has-more?]
(-> db
(assoc-in [:wallet :ui :collectibles :fetched owner-address] collectibles)
(assoc-in [:wallet :accounts owner-address :current-collectible-idx]
(+ offset (count collectibles)))
(assoc-in [:wallet :accounts owner-address :has-more-collectibles?] has-more?)))

(rf/reg-event-fx
:wallet/owned-collectibles-filtering-done
(fn [_ [{:keys [message]}]]
(let [{:keys [has-more offset
collectibles]} (cske/transform-keys csk/->kebab-case-keyword (types/json->clj message))
start-at-index (+ offset (count collectibles))]
{:fx [[:dispatch [:wallet/store-collectibles collectibles]]
(when has-more
[:dispatch [:wallet/request-collectibles {:start-at-index start-at-index}]])]})))
(fn [{:keys [db]} [{:keys [message]}]]
(let [{:keys [offset ownershipStatus collectibles
hasMore]} (transforms/json->clj message)
collectibles (cske/transform-keys transforms/->kebab-case-keyword collectibles)
pending-requests (dec (get-in db [:wallet :ui :collectibles :pending-requests]))
owner-address (some->> ownershipStatus
first
key
name)]
{:db (cond-> db
:always (assoc-in [:wallet :ui :collectibles :pending-requests] pending-requests)
owner-address (update-fetched-collectibles-progress owner-address
collectibles
offset
hasMore))
:fx [(when (zero? pending-requests)
[:dispatch [:wallet/flush-collectibles-fetched]])]})))

(rf/reg-event-fx
:wallet/get-collectible-details
(fn [_ [collectible-id]]
(let [request-id 0
collectible-id-converted (cske/transform-keys csk/->PascalCaseKeyword collectible-id)
collectible-id-converted (cske/transform-keys transforms/->PascalCaseKeyword collectible-id)
data-type (collectible-data-types :details)
request-params [request-id [collectible-id-converted] data-type]]
{:fx [[:json-rpc/call
Expand All @@ -118,8 +168,8 @@
(rf/reg-event-fx
:wallet/get-collectible-details-done
(fn [_ [{:keys [message]}]]
(let [response (cske/transform-keys csk/->kebab-case-keyword
(types/json->clj message))
(let [response (cske/transform-keys transforms/->kebab-case-keyword
(transforms/json->clj message))
{:keys [collectibles]} response
collectible (first collectibles)]
(if collectible
Expand Down
34 changes: 12 additions & 22 deletions src/status_im/contexts/wallet/events_test.cljs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,24 +56,8 @@
(is (match? (:db effects) expected-db))))

(deftest store-collectibles
(testing "(displayable-collectible?) helper function"
(let [expected-results [[true
{:collectible-data {:image-url "https://..." :animation-url "https://..."}}]
[true {:collectible-data {:image-url "" :animation-url "https://..."}}]
[true {:collectible-data {:image-url nil :animation-url "https://..."}}]
[true {:collectible-data {:image-url "https://..." :animation-url ""}}]
[true {:collectible-data {:image-url "https://..." :animation-url nil}}]
[false {:collectible-data {:image-url "" :animation-url nil}}]
[false {:collectible-data {:image-url nil :animation-url nil}}]
[false {:collectible-data {:image-url nil :animation-url ""}}]
[false {:collectible-data {:image-url "" :animation-url ""}}]]]
(doseq [[result collection] expected-results]
(is (match? result (collectibles/displayable-collectible? collection))))))

(testing "save-collectibles-request-details"
(let [db {:wallet {:accounts {"0x1" {}
"0x3" {}}}}
collectible-1 {:collectible-data {:image-url "https://..." :animation-url "https://..."}
(testing "flush-collectibles"
(let [collectible-1 {:collectible-data {:image-url "https://..." :animation-url "https://..."}
:ownership [{:address "0x1"
:balance "1"}]}
collectible-2 {:collectible-data {:image-url "" :animation-url "https://..."}
Expand All @@ -82,12 +66,18 @@
collectible-3 {:collectible-data {:image-url "" :animation-url nil}
:ownership [{:address "0x2"
:balance "1"}]}
collectibles [collectible-1 collectible-2 collectible-3]
expected-db {:wallet {:accounts {"0x1" {:collectibles (list collectible-2 collectible-1)}
db {:wallet {:ui {:collectibles {:pending-requests 0
:fetched {"0x1" [collectible-1
collectible-2]
"0x2" [collectible-3]}}}
:accounts {"0x1" {}
"0x3" {}}}}
expected-db {:wallet {:ui {:collectibles {}}
:accounts {"0x1" {:collectibles (list collectible-1 collectible-2)}
"0x2" {:collectibles (list collectible-3)}
"0x3" {}}}}
effects (collectibles/store-collectibles {:db db} [collectibles])
result-db (:db effects)]
result-db (:db (collectibles/flush-collectibles {:db db}))]

(is (match? result-db expected-db)))))

(deftest clear-stored-collectibles
Expand Down
Loading

0 comments on commit ad546f9

Please sign in to comment.