Manual import of fragments from smuggler in local mode

truthsayer-archaeologist-communication/src/message/types.ts

@@ -14,16 +14,37 @@ import type {
   StorageApiMsgReturnValue,
 } from 'smuggler-api'
 import { VersionStruct } from '../Version'
+import { log, errorise } from 'armoury'
+
+export type StorageType =
+  | 'datacenter' /** Stores user data in @file storage_api_datacenter.ts */
+  | 'browser_ext' /** Stores user data in @file storage_api_browser_ext.ts */
 
 export type AppSettings = {
-  storageType:
-    | 'datacenter' /** Stores user data in @file storage_api_datacenter.ts */
-    | 'browser_ext' /** Stores user data in @file storage_api_browser_ext.ts */
+  storageType: StorageType
 }
+
 export function defaultSettings(): AppSettings {
   return { storageType: 'datacenter' }
 }
 
+export async function getAppSettings(): Promise<AppSettings> {
+  try {
+    const response = await FromTruthsayer.sendMessage({
+      type: 'GET_APP_SETTINGS_REQUEST',
+    })
+    return response.settings
+  } catch (err) {
+    const defaults = defaultSettings()
+    log.warning(
+      `Failed to get user's app settings,` +
+        ` falling back to ${JSON.stringify(defaults)}; ` +
+        `error - ${errorise(err).message}`
+    )
+    return defaults
+  }
+}
+
 export namespace FromTruthsayer {
   export type GetArchaeologistStateRequest = {
     type: 'GET_ARCHAEOLOGIST_STATE_REQUEST'

truthsayer/src/external-import/DataCentreImporter.tsx

@@ -0,0 +1,167 @@
+import React from 'react'
+import styled from '@emotion/styled'
+import { useAsyncEffect } from 'use-async-effect'
+
+import * as truthsayer_archaeologist_communication from 'truthsayer-archaeologist-communication'
+import { Spinner } from 'elementary'
+import { genOriginId } from 'armoury'
+import { makeDatacenterStorageApi, StorageApi, Nid } from 'smuggler-api'
+
+import { getLogoImage } from '../util/env'
+import { MzdGlobalContext } from '../lib/global'
+
+export { getLogoImage }
+
+const Box = styled.div``
+const Title = styled.div`
+  margin-bottom: 10px;
+`
+export function NotImplementedMessage() {
+  return (
+    <Title>To store data locally please enable local mode in Settings</Title>
+  )
+}
+
+async function downloadUserDataFromMazedBackend(
+  dstStorage: StorageApi,
+  srcStorage: StorageApi,
+  setLoadingState: (value: LoadingState) => void
+): Promise<void> {
+  const oldToNewNids: Map<Nid, Nid> = new Map()
+  // Data centre implementation of iterator is buggy and sometimes returs same
+  // node twice, check for duplicates to prevent creating duplicates in local
+  // storage
+  const oldNids: Set<Nid> = new Set()
+  const iter = await srcStorage.node.iterate()
+  let progressCounter = 0
+  // Clone all nodes, saving mapping between { old-nid → new-nid }
+  while (true) {
+    const node = await iter.next()
+    if (node == null) {
+      break
+    }
+    setLoadingState({
+      type: 'loading',
+      progress: `Downloading fragments (${++progressCounter})...`,
+    })
+    if (oldNids.has(node.nid)) {
+      continue
+    }
+    oldNids.add(node.nid)
+    const url = node.extattrs?.web?.url ?? node.extattrs?.web_quote?.url
+    const origin = url != null ? genOriginId(url) : undefined
+    const r = await dstStorage.node.create({
+      text: node.text,
+      index_text: node.index_text,
+      extattrs: node.extattrs,
+      ntype: node.ntype,
+      origin: origin ? { ...origin } : undefined,
+      created_at: node.created_at.unix(),
+    })
+    oldToNewNids.set(node.nid, r.nid)
+  }
+  // Clone all edges
+  progressCounter = 0
+  for (const oldNid of oldToNewNids.keys()) {
+    const edges = await srcStorage.edge.get({ nid: oldNid })
+    for (const oldEdge of edges.from_edges.concat(edges.to_edges)) {
+      setLoadingState({
+        type: 'loading',
+        progress: `Creating associations (${++progressCounter})...`,
+      })
+      const fromNid = oldToNewNids.get(oldEdge.from_nid)
+      const toNid = oldToNewNids.get(oldEdge.to_nid)
+      if (fromNid && toNid) {
+        await dstStorage.edge.create({ from: fromNid, to: toNid })
+      }
+    }
+  }
+  setLoadingState({ type: 'done' })
+}
+
+const ControlBox = styled.div``
+const BoxButtons = styled.div``
+const Button = styled.button`
+  background-color: #ffffff;
+  border-style: solid;
+  border-width: 0;
+  border-radius: 32px;
+
+  vertical-align: middle;
+  &:hover {
+    background-color: #d0d1d2;
+  }
+`
+
+type LoadingState =
+  | { type: 'standby' }
+  | { type: 'loading'; progress: string }
+  | { type: 'done' }
+export function DownloadUserDataFromMazedBackendControl() {
+  const [loadingState, setLoadingState] = React.useState<LoadingState>({
+    type: 'standby',
+  })
+  const currentStorage = React.useContext(MzdGlobalContext).storage
+  const sync = React.useCallback(() => {
+    setLoadingState({ type: 'loading', progress: '...' })
+    const sourceStorage = makeDatacenterStorageApi()
+    downloadUserDataFromMazedBackend(
+      currentStorage,
+      sourceStorage,
+      setLoadingState
+    )
+  }, [currentStorage])
+  let buttonText
+  switch (loadingState.type) {
+    case 'loading':
+      buttonText = (
+        <>
+          Downloading <Spinner.Ring /> {loadingState.progress}
+        </>
+      )
+      break
+    case 'standby':
+      buttonText = <>Download</>
+      break
+    case 'done':
+      buttonText = <>Done 🏁 </>
+      break
+  }
+  return (
+    <ControlBox>
+      <Title>
+        Download fragments from <b>Mazed datacenter</b> to <b>local storage</b>
+      </Title>
+      <BoxButtons>
+        <Button onClick={sync} disabled={loadingState.type !== 'standby'}>
+          {buttonText}
+        </Button>
+      </BoxButtons>
+    </ControlBox>
+  )
+}
+
+export function DataCentreImporter({ className }: { className?: string }) {
+  const [storageType, setStorageType] = React.useState<
+    'loading' | truthsayer_archaeologist_communication.StorageType
+  >('loading')
+  useAsyncEffect(async () => {
+    const settings =
+      await truthsayer_archaeologist_communication.getAppSettings()
+    setStorageType(settings.storageType)
+  })
+  let element
+  switch (storageType) {
+    case 'loading':
+      element = <Spinner.Ring />
+      break
+    case 'browser_ext':
+      element = <DownloadUserDataFromMazedBackendControl />
+      break
+    case 'datacenter':
+      element = <NotImplementedMessage />
+      break
+  }
+
+  return <Box className={className}>{element}</Box>
+}

truthsayer/src/external-import/ExternalImport.tsx

@@ -14,6 +14,7 @@ import {
   BrowserHistoryImporter,
   BrowserLogo as BrowserHistoryImporterLogo,
 } from './BrowserHistoryImporter'
+import { DataCentreImporter, getLogoImage } from './DataCentreImporter'
 
 const Box = styled.div`
   padding: 18px;
@@ -65,6 +66,10 @@ export function ExternalImport({
           <LogoImg src={MicrosoftOfficeOneDriveLogoImg} />
           <MicrosoftOfficeOneDriveImporter />
         </Item>
+        <Item key={'data-centre-importer'}>
+          <LogoImg src={getLogoImage()} />
+          <DataCentreImporter />
+        </Item>
       </ItemsBox>
     </Box>
   )

truthsayer/src/lib/global.tsx

@@ -22,9 +22,8 @@ import { NotificationToast } from './Toaster'
 import { errorise, log, productanalytics } from 'armoury'
 import { useAsyncEffect } from 'use-async-effect'
 import {
-  defaultSettings,
   FromTruthsayer,
-  ToTruthsayer,
+  getAppSettings,
 } from 'truthsayer-archaeologist-communication'
 import type { AppSettings } from 'truthsayer-archaeologist-communication'
 import { Loader } from './loader'
@@ -106,18 +105,8 @@ export function MzdGlobal(props: React.PropsWithChildren<MzdGlobalProps>) {
 
   const [storage, setStorage] = React.useState<StorageApi | null>(null)
   useAsyncEffect(async () => {
-    const response = await FromTruthsayer.sendMessage({
-      type: 'GET_APP_SETTINGS_REQUEST',
-    }).catch((reason): ToTruthsayer.GetAppSettingsResponse => {
-      const defaults = defaultSettings()
-      log.warning(
-        `Failed to get user's app settings,` +
-          ` falling back to ${JSON.stringify(defaults)}; ` +
-          `error - ${errorise(reason).message}`
-      )
-      return { type: 'GET_APP_SETTINGS_RESPONSE', settings: defaults }
-    })
-    setStorage(makeStorageApi(response.settings))
+    const settings = await getAppSettings()
+    setStorage(makeStorageApi(settings))
   }, [])
 
   const [state] = React.useState<Omit<MzdGlobalState, 'account' | 'storage'>>({