Prevent yielding duplicated nodes from iterator

elementary/package.json

@@ -27,6 +27,7 @@
     "@testing-library/user-event": "^12.6",
     "@types/react-router-dom": "^5.3.0",
     "armoury": "0.0.1",
+    "async-mutex": "^0.4.0",
     "eslint": "^8.16.0",
     "image-extensions": "^1.1.0",
     "is-url": "^1.2.4",

elementary/src/grid/SearchGrid.tsx

@@ -1,13 +1,11 @@
 /** @jsxImportSource @emotion/react */
 
-import React, { useEffect, useRef, useState } from 'react'
-import { useAsyncEffect } from 'use-async-effect'
+import React, { useMemo, useEffect, useRef, useState } from 'react'
 
 import styled from '@emotion/styled'
 
 import { css } from '@emotion/react'
 import { useHistory } from 'react-router-dom'
-import lodash from 'lodash'
 
 import { Spinner } from '../spinner/mod'
 import { SmallCard } from '../SmallCard'
@@ -54,6 +52,12 @@ export const GridCard = ({
   )
 }
 
+const Mutex = require('async-mutex').Mutex
+
+type SearchGridState = {
+  nodes: TNode[]
+  iter: INodeIterator
+}
 export const SearchGrid = ({
   q,
   children,
@@ -69,59 +73,12 @@ export const SearchGrid = ({
   defaultSearch?: boolean
   className?: string
   storage: StorageApi
-}>) => {
-  const [search, setUpSearch] = useState<{
-    iter: INodeIterator
-    beagle: Beagle
-  } | null>(null)
-  useAsyncEffect(async () => {
-    setUpSearch({
-      iter: await storage.node.iterate(),
-      beagle: Beagle.fromString(q || undefined),
-    })
-  }, [q])
-  if (q == null && !defaultSearch) {
-    return null
-  }
-  if (search == null) {
-    return null
-  }
-  const { iter, beagle } = search
-  return (
-    <SearchGridScroll
-      beagle={beagle}
-      iter={iter}
-      onCardClick={onCardClick}
-      portable={portable}
-      className={className}
-      storage={storage}
-    >
-      {children}
-    </SearchGridScroll>
-  )
-}
-
-const SearchGridScroll = ({
-  beagle,
-  iter,
-  children,
-  onCardClick,
-  portable,
-  className,
-  storage,
-}: React.PropsWithChildren<{
-  beagle: Beagle
-  iter: INodeIterator
-  onCardClick?: (arg0: TNode) => void
-  portable?: boolean
-  className?: string
-  storage: StorageApi
 }>) => {
   const history = useHistory()
   const ref = useRef<HTMLDivElement>(null)
-  const [nodes, setNodes] = useState<TNode[]>([])
+  const beagle = useMemo(() => Beagle.fromString(q || undefined), [q])
+  const [state, setState] = useState<SearchGridState | null>(null)
   const [fetching, setFetching] = useState<boolean>(false)
-
   const isScrolledToBottom = () => {
     let height: number = 0
     let scrollTop: number = 0
@@ -137,58 +94,75 @@ const SearchGridScroll = ({
     offsetHeight = document.documentElement.offsetHeight
     return height + scrollTop + 300 >= offsetHeight
   }
-
-  const fetchNextBatch = React.useCallback(
-    lodash.throttle(
-      async () => {
-        // Continue fetching until visual space is filled with cards to the bottom and beyond.
-        // Thus if use scrolled to the bottom this loop would start fetching again adding more cards.
-        if (fetching) {
-          // Don't run more than 1 instance of fetcher
-          return
-        }
-        setFetching(true)
-        try {
-          while (isScrolledToBottom()) {
-            const node = await iter.next()
-            if (node == null) {
-              iter.abort()
-              break
-            }
-            if (beagle.searchNode(node) != null) {
-              setNodes((prev) => prev.concat(node))
-            }
+  const mutex = useRef(new Mutex())
+  const fetchNextBatch = React.useCallback(async () => {
+    mutex.current.runExclusive(async () => {
+      // Continue fetching until visual space is filled with cards to the bottom and beyond.
+      // Thus if use scrolled to the bottom this loop would start fetching again adding more cards.
+      if (!isScrolledToBottom()) {
+        // Don't run more than 1 instance of fetcher
+        return
+      }
+      setFetching(true)
+      let iter: INodeIterator
+      let nodes: TNode[]
+      if (state == null) {
+        iter = await storage.node.iterate()
+        nodes = []
+      } else {
+        iter = state.iter
+        nodes = state.nodes
+      }
+      try {
+        // FIXME(Alexnader): With this batch size we predict N of cards to fill
+        // the entire screen. As you can see this is a dirty hack, feel free to
+        // replace it when you get there next time.
+        const batchSize =
+          (window.innerWidth * window.innerHeight * 2) / (240 * 240)
+        let counter = 0
+        while (counter < batchSize) {
+          const node = await iter.next()
+          if (node == null) {
+            iter.abort()
+            break
           }
-        } catch (err) {
-          const error = errorise(err)
-          if (!isAbortError(error)) {
-            log.exception(error)
+          if (beagle.searchNode(node) != null) {
+            ++counter
+            nodes.push(node)
           }
         }
+        setState({ iter, nodes })
         setFetching(false)
-      },
-      100,
-      { leading: true, trailing: false }
-    ),
-    [beagle, iter]
-  )
+      } catch (err) {
+        setFetching(false)
+        const error = errorise(err)
+        if (!isAbortError(error)) {
+          log.exception(error)
+        }
+      }
+    })
+  }, [beagle, state])
   useEffect(() => {
     if (!portable) {
-      window.addEventListener('scroll', fetchNextBatch, { passive: true })
+      window.addEventListener('scroll', fetchNextBatch, {
+        passive: true,
+      })
       return () => {
         window.removeEventListener('scroll', fetchNextBatch)
       }
     }
     return () => {}
   }, [fetchNextBatch])
   useEffect(() => {
-    // First fetch call to kick start the process
     fetchNextBatch()
     return () => {
       // Clean up on changed search parameters
-      setNodes([])
+      setState(null)
     }
-  }, [beagle, iter])
+  }, [beagle])
+  if (q == null && !defaultSearch) {
+    return null
+  }
   const fetchingLoader = fetching ? (
     <div
       css={css`
@@ -198,7 +172,7 @@ const SearchGridScroll = ({
       <Spinner.Wheel />
     </div>
   ) : null
-  const cards: JSX.Element[] = nodes.map((node) => {
+  const cards = state?.nodes.map((node) => {
     const onClick = () => {
       if (onCardClick) {
         onCardClick(node)

smuggler-api/src/storage_api_msg_proxy.ts

@@ -151,11 +151,10 @@ class MsgProxyNodeIterator implements INodeIterator {
   }
 
   async next(): Promise<TNode | null> {
-    const nids = this.nids
-    if (this.index >= nids.length) {
+    if (this.index >= this.nids.length) {
       return null
     }
-    const nid: Nid = nids[this.index]
+    const nid: Nid = this.nids[this.index]
     const apiName = 'node.get'
     const value = await this.forward({ apiName, args: { nid } })
     if (apiName !== value.apiName) throw mismatchError(apiName, value.apiName)

yarn.lock

@@ -5913,6 +5913,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"async-mutex@npm:^0.4.0":
+  version: 0.4.0
+  resolution: "async-mutex@npm:0.4.0"
+  dependencies:
+    tslib: ^2.4.0
+  checksum: 813a71728b35a4fbfd64dba719f04726d9133c67b577fcd951b7028c4a675a13ee34e69beb82d621f87bf81f5d4f135c4c44be0448550c7db728547244ef71fc
+  languageName: node
+  linkType: hard
+
 "async@npm:^3.2.0, async@npm:^3.2.3":
   version: 3.2.3
   resolution: "async@npm:3.2.3"
@@ -8767,6 +8776,7 @@ __metadata:
     "@types/react-dom": ^16.9.16
     "@types/react-router-dom": ^5.3.0
     armoury: 0.0.1
+    async-mutex: ^0.4.0
     eslint: ^8.16.0
     eslint-plugin-import: ^2.18.2
     image-extensions: ^1.1.0
@@ -21053,6 +21063,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"tslib@npm:^2.4.0":
+  version: 2.5.0
+  resolution: "tslib@npm:2.5.0"
+  checksum: ae3ed5f9ce29932d049908ebfdf21b3a003a85653a9a140d614da6b767a93ef94f460e52c3d787f0e4f383546981713f165037dc2274df212ea9f8a4541004e1
+  languageName: node
+  linkType: hard
+
 "tsutils@npm:^3.21.0":
   version: 3.21.0
   resolution: "tsutils@npm:3.21.0"