Decompress wasm runtime before validating (#4182)

* decompress wasm runtime before validating

* use @oneidentity/zstd-js/decompress

* better comments for maybeDecompressRuntimeBlob()

* Add a AddNewProposalModal Runtime Upgrade test

* Remove unnecessary exports

* Forward the test `runtime-upgrade-input` id

* Fix the runtime file validation

* Fix `react-hook-form` type inference error

* Small refactors

* Pass the runtime bytes through the react context

* Fix the validation hook

* Revert "Fix `react-hook-form` type inference error"

This reverts commit 92eef39.

* Forward the runtime file without the extra context

* Fix AddNewProposalModal tests

* Fix types on node.js < 18

* Generate the file array buffer only once

* Add a loader under the file drop zone

* Move `maybeDecompressRuntimeBlob` in a web worker

* Push runtime upgrade proposal down the list

* Move the loader to the `FileDropzone` component

---------

Co-authored-by: Theophile Sandoz <theophile.sandoz@gmail.com>

packages/ui/package.json

@@ -32,6 +32,7 @@
     "@nivo/bar": "^0.79.1",
     "@nivo/core": "^0.79.0",
     "@noble/hashes": "^1.1.5",
+    "@oneidentity/zstd-js": "^1.0.3",
     "@polkadot/api": "8.9.1",
     "@polkadot/extension-dapp": "0.44.2-4",
     "@polkadot/keyring": "9.5.1",

packages/ui/src/accounts/hooks/useTransactionFee.ts

@@ -1,14 +1,16 @@
-import { DependencyList, useContext, useEffect } from 'react'
+import { DependencyList, useContext, useEffect, useState } from 'react'
 
 import { Transaction, TransactionFeesContext, Fee } from '@/common/providers/transactionFees/context'
 import { Address } from '@/common/types'
 
+export type UseTransactionFee = Fee & { isLoading: boolean }
 export function useTransactionFee(
   signer?: Address,
-  getTransaction?: () => Transaction | undefined,
+  getTransaction?: () => Transaction | undefined | Promise<Transaction | undefined>,
   deps: DependencyList = []
-): Fee {
+): UseTransactionFee {
   const { transaction, feeInfo, setSigner, setTransaction } = useContext(TransactionFeesContext)
+  const [isLoading, setIsLoading] = useState(false)
 
   useEffect(() => {
     if (signer) {
@@ -18,9 +20,13 @@ export function useTransactionFee(
 
   useEffect(() => {
     if (signer && getTransaction) {
-      setTransaction(getTransaction())
+      setIsLoading(true)
+      Promise.resolve(getTransaction()).then((tx) => {
+        setIsLoading(false)
+        setTransaction(tx)
+      })
     }
   }, [signer, !!getTransaction, setTransaction, ...deps])
 
-  return { transaction, feeInfo }
+  return { transaction, isLoading, feeInfo }
 }

packages/ui/src/common/components/FileDropzone/FileDropzone.tsx

@@ -1,9 +1,10 @@
-import React from 'react'
+import React, { useCallback, useState } from 'react'
 import { DropzoneOptions, useDropzone } from 'react-dropzone'
 import styled, { css } from 'styled-components'
 
 import { getFilesFromRawEvent } from '@/common/components/FileDropzone/helpers'
 import { Label } from '@/common/components/forms'
+import { Loading } from '@/common/components/Loading'
 import { RowGapBlock } from '@/common/components/page/PageContent'
 import { TextMedium, TextSmall } from '@/common/components/typography'
 import { BorderRad, Colors, Transitions } from '@/common/constants'
@@ -15,6 +16,7 @@ interface DragResponseProps {
 }
 
 interface FileDropzoneProps extends Omit<DropzoneOptions, 'getFilesFromEvent'> {
+  id?: string
   title: string
   subtitle: string
   isRequired?: boolean
@@ -24,16 +26,27 @@ interface FileDropzoneProps extends Omit<DropzoneOptions, 'getFilesFromEvent'> {
 const MEGABYTE = 1024 * 1024
 
 export const FileDropzone = ({
+  id,
   title,
   subtitle,
-  getFilesFromEvent,
+  getFilesFromEvent: _getFilesFromEvent,
   isRequired,
   ...dropzoneOptions
 }: FileDropzoneProps) => {
+  const [isProcessingFile, setIsProcessingFile] = useState(false)
+  const getFilesFromEvent = useCallback<Required<DropzoneOptions>['getFilesFromEvent']>(
+    async (event) => {
+      setIsProcessingFile(true)
+      const res = await _getFilesFromEvent(getFilesFromRawEvent(event))
+      setIsProcessingFile(false)
+      return res
+    },
+    [_getFilesFromEvent]
+  )
   const { isDragActive, isDragAccept, isDragReject, getRootProps, getInputProps, acceptedFiles, fileRejections } =
     useDropzone({
       ...dropzoneOptions,
-      getFilesFromEvent: (event) => getFilesFromEvent(getFilesFromRawEvent(event)),
+      getFilesFromEvent,
     })
 
   return (
@@ -50,7 +63,7 @@ export const FileDropzone = ({
             isDragAccept={isDragAccept}
             isDragReject={isDragReject}
           >
-            <input {...getInputProps()} />
+            <input {...getInputProps({ id })} />
             <DropZoneText>
               Drop your file here or <DropZoneTextUnderline>browse</DropZoneTextUnderline>
             </DropZoneText>
@@ -60,23 +73,29 @@ export const FileDropzone = ({
           <TextSmall lighter>Maximum upload file size is {dropzoneOptions.maxSize / MEGABYTE} MB</TextSmall>
         )}
       </RowGapBlock>
-      <RowGapBlock gap={8}>
-        {acceptedFiles.map((file) => (
-          <ReceivedFile key={file.name} valid={true}>
-            <AcceptedFileText>
-              <b>{file.name}</b> ({file.size} B) was loaded successfully!
-            </AcceptedFileText>
-          </ReceivedFile>
-        ))}
-        {fileRejections.map(({ file, errors }) => (
-          <ReceivedFile key={file.name} valid={false}>
-            <AcceptedFileText>
-              <b>{file.name}</b> ({file.size} B) was not loaded because of:{' '}
-              {errors.map((error) => `"${error.message}"`).join(', ')}.
-            </AcceptedFileText>
-          </ReceivedFile>
-        ))}
-      </RowGapBlock>
+      {isProcessingFile ? (
+        <LoaderBox>
+          <Loading text="Processing your file..." withoutMargin />
+        </LoaderBox>
+      ) : (
+        <RowGapBlock gap={8}>
+          {acceptedFiles.map((file) => (
+            <ReceivedFile key={file.name} valid={true}>
+              <AcceptedFileText>
+                <b>{file.name}</b> ({file.size} B) was loaded successfully!
+              </AcceptedFileText>
+            </ReceivedFile>
+          ))}
+          {fileRejections.map(({ file, errors }) => (
+            <ReceivedFile key={file.name} valid={false}>
+              <AcceptedFileText>
+                <b>{file.name}</b> ({file.size} B) was not loaded because of:{' '}
+                {errors.map((error) => `"${error.message}"`).join(', ')}.
+              </AcceptedFileText>
+            </ReceivedFile>
+          ))}
+        </RowGapBlock>
+      )}
     </RowGapBlock>
   )
 }
@@ -151,6 +170,13 @@ export const DropZone = styled.div<DragResponseProps>`
     `}
 `
 
+const LoaderBox = styled(RowGapBlock)`
+  > div {
+    width: 100%;
+    margin: 10px 0;
+  }
+`
+
 const AcceptedFileText = styled.span`
   font-size: 14px;
   line-height: 20px;

packages/ui/src/common/utils/crypto/index.ts

@@ -1,5 +1,6 @@
 import { generateCommitmentFromPayloadFile } from '@joystream/js/content'
 import { blake3 } from '@noble/hashes/blake3'
+import { ZstdInit } from '@oneidentity/zstd-js/decompress'
 import { encode as encodeHash, toB58String } from 'multihashes'
 
 // FROM Atlas 5e5f2fed Klaudiusz Dembler (2022-01-11 11:09): Giza: update content extrinsics, enable uploads (#1882)
@@ -15,3 +16,22 @@ export const merkleRootFromBinary = (file: Blob): Promise<string> => {
   const read = async (start: number, end: number) => new Uint8Array(await file.slice(start, end + 1).arrayBuffer())
   return generateCommitmentFromPayloadFile(read)
 }
+
+// https://github.com/paritytech/substrate/blob/master/primitives/maybe-compressed-blob/src/lib.rs
+// Convert blob to Buffer type to make it easier work with (compare|subarray),
+// although it is adding a copy step.
+// Looks for 8-byte magic prefix in blob to determine if it is compressed with zstd algorithm.
+// If compressed, strips the prefix and decompresses remaining bytes returning
+// decompressed value, otherwise returns original blob.
+export const maybeDecompressRuntimeBlob = async (blob: ArrayBuffer): Promise<Buffer | Uint8Array> => {
+  const ZSTD_PREFIX = Buffer.from([82, 188, 83, 118, 70, 219, 142, 5])
+  let wasm: Buffer | Uint8Array = Buffer.from(blob)
+  const prefix = wasm.subarray(0, 8)
+  const isCompressed = Buffer.compare(prefix, ZSTD_PREFIX) === 0
+  if (isCompressed) {
+    const { ZstdStream } = await ZstdInit()
+    // strip the prefix and decompress the rest
+    wasm = ZstdStream.decompress(wasm.subarray(8))
+  }
+  return wasm
+}

packages/ui/src/common/utils/crypto/worker/client.ts

@@ -10,8 +10,12 @@ export const hashFile = computeInWorker('HASH_FILE')
 
 export const merkleRootFromBinary = computeInWorker('MERKLE_ROOT')
 
-function computeInWorker(type: WorkerRequestType) {
-  return async (file: Blob): Promise<string> => {
+export const maybeDecompressRuntimeBlob = computeInWorker('DECOMPRESS_RUNTIME')
+
+function computeInWorker(type: 'HASH_FILE' | 'MERKLE_ROOT'): (file: Blob) => Promise<string>
+function computeInWorker(type: 'DECOMPRESS_RUNTIME'): (file: ArrayBuffer) => Promise<Buffer | Uint8Array>
+function computeInWorker(type: WorkerRequestType): (file: any) => Promise<any> {
+  return async (file) => {
     if (!worker) {
       worker = new Worker(new URL('./worker', import.meta.url), { type: 'module' })
       messages = fromEvent<MessageEvent<WorkerResponse>>(worker, 'message')

packages/ui/src/common/utils/crypto/worker/utils.ts

@@ -1,24 +1,26 @@
-import { merkleRootFromBinary, hashFile } from '..'
+import { merkleRootFromBinary, hashFile, maybeDecompressRuntimeBlob } from '..'
 
-export type WorkerRequestType = 'HASH_FILE' | 'MERKLE_ROOT'
+export type WorkerRequestType = 'HASH_FILE' | 'MERKLE_ROOT' | 'DECOMPRESS_RUNTIME'
 export type WorkerRequest = {
   type: WorkerRequestType
   id: string
-  file: Blob
+  file: Blob | ArrayBuffer
 }
 
 export type WorkerResponse = {
   type: WorkerRequestType
   id: string
-  value: string
+  value: any
   error?: boolean
 }
 
-export const compute = async (type: WorkerRequestType, file: Blob): Promise<string> => {
+export async function compute(type: WorkerRequestType, file: Blob | ArrayBuffer): Promise<any> {
   switch (type) {
     case 'HASH_FILE':
-      return await hashFile(file)
+      return await hashFile(file as Blob)
     case 'MERKLE_ROOT':
-      return await merkleRootFromBinary(file)
+      return await merkleRootFromBinary(file as Blob)
+    case 'DECOMPRESS_RUNTIME':
+      return await maybeDecompressRuntimeBlob(file as ArrayBuffer)
   }
 }

packages/ui/src/common/utils/file.ts

@@ -0,0 +1,3 @@
+import { memoize } from 'lodash'
+
+export const asArrayBuffer = memoize((file?: File) => file?.arrayBuffer() ?? new ArrayBuffer(0))

packages/ui/src/common/utils/validation.tsx

@@ -214,7 +214,7 @@ export const useYupValidationResolver = <T extends FieldValues>(
       } catch (errors: any) {
         return {
           values: {},
-          errors: convertYupErrorVectorToFieldErrors<T>(errors?.inner ?? []),
+          errors: convertYupErrorVectorToFieldErrors<T>(errors?.inner ?? [errors ?? Error('Unknown')]),
         }
       }
     },