Merge pull request #4407 from Lezek123/argus-colossus-tests

Argus/Colossus and CLI create/update video tests

cli/src/ExitCodes.ts

@@ -14,6 +14,7 @@ enum ExitCodes {
   StorageNodeError = 53,
   ActionCurrentlyUnavailable = 54,
   QueryNodeError = 55,
+  UnexpectedRuntimeState = 56,
 
   // NOTE: never exceed exit code 255 or it will be modulated by `256` and create problems
 }

cli/src/base/UploadCommandBase.ts

@@ -339,9 +339,12 @@ export default abstract class UploadCommandBase extends ContentDirectoryCommandB
         }
       })
     )
+    multiBar.stop()
     errors.forEach(([objectId, message]) => this.warn(`Upload of object ${objectId} failed: ${message}`))
     this.handleRejectedUploads(bagId, assets, results, inputFilePath, outputFilePostfix)
-    multiBar.stop()
+    if (errors.length) {
+      this.exit(ExitCodes.StorageNodeError)
+    }
   }
 
   async prepareAssetsForExtrinsic(

cli/src/commands/content/createChannel.ts

@@ -3,6 +3,7 @@ import { createType } from '@joystream/types'
 import { ChannelId } from '@joystream/types/primitives'
 import { flags } from '@oclif/command'
 import chalk from 'chalk'
+import ExitCodes from '../../ExitCodes'
 import ContentDirectoryCommandBase from '../../base/ContentDirectoryCommandBase'
 import UploadCommandBase from '../../base/UploadCommandBase'
 import { getInputJson } from '../../helpers/InputOutput'
@@ -83,6 +84,12 @@ export default class CreateChannelCommand extends UploadCommandBase {
     this.log(chalk.green(`Channel with id ${chalk.cyanBright(channelId.toString())} successfully created!`))
     this.output(channelId.toString())
 
+    if (dataObjects.size !== (assets?.objectCreationList.length || 0)) {
+      this.error('Unexpected number of channel assets in ChannelCreated event!', {
+        exit: ExitCodes.UnexpectedRuntimeState,
+      })
+    }
+
     if (dataObjects.size) {
       await this.uploadAssets(
         `dynamic:channel:${channelId.toString()}`,

cli/src/commands/content/createVideo.ts

@@ -4,7 +4,6 @@ import { asValidatedMetadata, metadataToBytes } from '../../helpers/serializatio
 import { VideoInputParameters, VideoFileMetadata } from '../../Types'
 import { createType } from '@joystream/types'
 import { flags } from '@oclif/command'
-import { VideoId } from '@joystream/types/primitives'
 import { ContentMetadata, IVideoMetadata, VideoMetadata } from '@joystream/metadata-protobuf'
 import { VideoInputSchema } from '../../schemas/ContentDirectory'
 import chalk from 'chalk'
@@ -126,15 +125,19 @@ export default class CreateVideoCommand extends UploadCommandBase {
       videoCreationParameters,
     ])
 
-    const videoId: VideoId = this.getEvent(result, 'content', 'VideoCreated').data[2]
+    const [, , videoId, , dataObjects] = this.getEvent(result, 'content', 'VideoCreated').data
     this.log(chalk.green(`Video with id ${chalk.cyanBright(videoId.toString())} successfully created!`))
 
-    const dataObjectsUploadedEvent = this.findEvent(result, 'storage', 'DataObjectsUploaded')
-    if (dataObjectsUploadedEvent) {
-      const [objectIds] = dataObjectsUploadedEvent.data
+    if (dataObjects.size !== (assets?.objectCreationList.length || 0)) {
+      this.error('Unexpected number of video assets in VideoCreated event!', {
+        exit: ExitCodes.UnexpectedRuntimeState,
+      })
+    }
+
+    if (dataObjects.size) {
       await this.uploadAssets(
         `dynamic:channel:${channelId.toString()}`,
-        [...objectIds.values()].map((id, index) => ({
+        [...dataObjects].map((id, index) => ({
           dataObjectId: id,
           path: [...resolvedVideoAssets, ...resolvedSubtitleAssets][index].path,
         })),

cli/src/commands/content/updateChannel.ts

@@ -148,9 +148,17 @@ export default class UpdateChannelCommand extends UploadCommandBase {
       channelId,
       channelUpdateParameters,
     ])
-    const channelUpdatedEvent = this.findEvent(result, 'content', 'ChannelUpdated')
-    if (channelUpdatedEvent) {
-      const objectIds = channelUpdatedEvent.data[3]
+
+    const channelUpdatedEvent = this.getEvent(result, 'content', 'ChannelUpdated')
+    const objectIds = channelUpdatedEvent.data[3]
+
+    if (objectIds.size !== (assetsToUpload?.objectCreationList.length || 0)) {
+      this.error('Unexpected number of channel assets in ChannelUpdated event!', {
+        exit: ExitCodes.UnexpectedRuntimeState,
+      })
+    }
+
+    if (objectIds.size) {
       await this.uploadAssets(
         `dynamic:channel:${channelId.toString()}`,
         [...objectIds].map((id, index) => ({ dataObjectId: id, path: resolvedAssets[index].path })),

cli/src/commands/content/updateVideo.ts

@@ -1,5 +1,5 @@
 import { getInputJson } from '../../helpers/InputOutput'
-import { VideoInputParameters } from '../../Types'
+import { VideoFileMetadata, VideoInputParameters } from '../../Types'
 import { asValidatedMetadata, metadataToBytes } from '../../helpers/serialization'
 import UploadCommandBase from '../../base/UploadCommandBase'
 import { flags } from '@oclif/command'
@@ -9,7 +9,7 @@ import {
   PalletContentIterableEnumsChannelActionPermission as ChannelActionPermission,
 } from '@polkadot/types/lookup'
 import { VideoInputSchema } from '../../schemas/ContentDirectory'
-import { ContentMetadata, VideoMetadata } from '@joystream/metadata-protobuf'
+import { ContentMetadata, IVideoMetadata, VideoMetadata } from '@joystream/metadata-protobuf'
 import { DataObjectInfoFragment } from '../../graphql/generated/queries'
 import BN from 'bn.js'
 import { formatBalance } from '@polkadot/util'
@@ -67,6 +67,20 @@ export default class UpdateVideoCommand extends UploadCommandBase {
     return assetsToRemove.map((a) => Number(a.id))
   }
 
+  setVideoMetadataDefaults(metadata: IVideoMetadata, videoFileMetadata: VideoFileMetadata): IVideoMetadata {
+    return {
+      duration: videoFileMetadata.duration,
+      mediaPixelWidth: videoFileMetadata.width,
+      mediaPixelHeight: videoFileMetadata.height,
+      mediaType: {
+        codecName: videoFileMetadata.codecName,
+        container: videoFileMetadata.container,
+        mimeMediaType: videoFileMetadata.mimeType,
+      },
+      ...metadata,
+    }
+  }
+
   async run(): Promise<void> {
     const {
       flags: { input, context },
@@ -80,7 +94,7 @@ export default class UpdateVideoCommand extends UploadCommandBase {
     const keypair = await this.getDecodedPair(address)
 
     const videoInput = await getInputJson<VideoInputParameters>(input, VideoInputSchema)
-    const meta = asValidatedMetadata(VideoMetadata, videoInput)
+    let meta = asValidatedMetadata(VideoMetadata, videoInput)
     const { enableComments } = videoInput
 
     // Video assets
@@ -110,6 +124,13 @@ export default class UpdateVideoCommand extends UploadCommandBase {
       )
     ).filter((r) => r)
 
+    // Try to get updated video file metadata
+    if (videoAssetIndices.videoPath !== undefined) {
+      const videoFileMetadata = await this.getVideoFileMetadata(resolvedVideoAssets[videoAssetIndices.videoPath].path)
+      this.log('Video media file parameters established:', videoFileMetadata)
+      meta = this.setVideoMetadataDefaults(meta, videoFileMetadata)
+    }
+
     // Prepare and send the extrinsic
     const serializedMeta = metadataToBytes(VideoMetadata, meta)
     const expectedDataObjectStateBloatBond = await this.getApi().dataObjectStateBloatBond()
@@ -157,12 +178,20 @@ export default class UpdateVideoCommand extends UploadCommandBase {
       videoId,
       videoUpdateParameters,
     ])
-    const dataObjectsUploadedEvent = this.findEvent(result, 'storage', 'DataObjectsUpdated')
-    if (dataObjectsUploadedEvent) {
-      const [, objectIds] = dataObjectsUploadedEvent.data
+
+    const VideoUpdatedEvent = this.getEvent(result, 'content', 'VideoUpdated')
+    const objectIds = VideoUpdatedEvent.data[3]
+
+    if (objectIds.size !== (assetsToUpload?.objectCreationList.length || 0)) {
+      this.error('Unexpected number of video assets in VideoUpdated event!', {
+        exit: ExitCodes.UnexpectedRuntimeState,
+      })
+    }
+
+    if (objectIds.size) {
       await this.uploadAssets(
         `dynamic:channel:${video.inChannel.toString()}`,
-        [...objectIds.values()].map((id, index) => ({
+        [...objectIds].map((id, index) => ({
           dataObjectId: id,
           path: [...resolvedVideoAssets, ...resolvedSubtitleAssets][index].path,
         })),

distributor-node/.eslintignore

@@ -1,2 +1,3 @@
 src/types/generated
 src/services/networking/query-node/generated
+client

distributor-node/.gitignore

@@ -8,3 +8,4 @@
 node_modules
 /local
 /out
+client/lib

distributor-node/.prettierignore

@@ -2,3 +2,4 @@
 lib
 local
 /**/*.md
+client

distributor-node/CHANGELOG.md

@@ -1,13 +1,14 @@
 ### 0.2.0
 
-- **Carthage release:** Support for mime-type provided as part of `SubtitleMetadata` (video subtitles). It is now treated as fallback `mime-type` as long as it's a valid `text/*` type and the `file-type` package fails to detect any type from the file signature (magic number).
-- **Security:** Sensitive information (like private keys) is now hidden in the node's logs.
-- Requesting an asset which has not been accepted by any storage provider yet (`isAccepted == false`) now results in `404: Data object has not been uploaded yet` (previously `500: Failed to download object {id} from any availablable storage provider`)
-- Fixed links in CLI docs
+- **Carthage release:** Support for mime-type provided as part of `SubtitleMetadata` (video subtitles). It is now treated as fallback `mime-type` as long as it's a valid `text/*` type and the `file-type` package fails to detect any type from the file signature (magic number),
+- **Security:** Sensitive information (like private keys) is now hidden in the node's logs,
+- Requesting an asset which has not been accepted by any storage provider yet (`isAccepted == false`) now results in `404: Data object has not been uploaded yet` (previously `500: Failed to download object {id} from any availablable storage provider`),
+- Fixed links in CLI docs,
+- Autogenerated API client library (`@joystream/distributor-node-client`) is now part of the distributor node codebase.
 
 ### 0.1.2
-- Fix cache `cache-control: max-age` header for objects served from the filesystem ([`send`](https://www.npmjs.com/package/send) library requires `max-age` to be provided as miliseconds) 
+- Fix cache `cache-control: max-age` header for objects served from the filesystem ([`send`](https://www.npmjs.com/package/send) library requires `max-age` to be provided as miliseconds).
 
 ### 0.1.1
-- Replace `read-chunk` dependency w/ custom `readFileChunk` implementation in order to fix the issue w/ data object `mime-type` detection (https://github.com/Joystream/joystream/pull/3723)
-- Add `version` property to `/status` endpoint result
+- Replace `read-chunk` dependency w/ custom `readFileChunk` implementation in order to fix the issue w/ data object `mime-type` detection (https://github.com/Joystream/joystream/pull/3723),
+- Add `version` property to `/status` endpoint result.

distributor-node/src/services/networking/storage-node/generated/.openapi-generator-ignore → distributor-node/client/.openapi-generator-ignore

@@ -21,7 +21,6 @@
 #docs/*.md
 # Then explicitly reverse the ignore rule for a single file:
 #!docs/README.md
-
 git_push.sh
 .npmignore
-.gitignore
+.gitignore

distributor-node/client/.openapi-generator/VERSION

@@ -0,0 +1 @@
+6.0.1