Create gradient video with ffmpeg [C-1446] [C-1413] (#2250)

* create gradient video with ffmpeg

* fixes

* fix build

* actual fix

* try again

* try again

Co-authored-by: Nikki Kang <kangaroo233@gmail.com>

packages/mobile/android/build.gradle

@@ -8,6 +8,7 @@ buildscript {
         targetSdkVersion = 30
         castFrameworkVersion = "21.0.0"
         ndkVersion = "21.4.7075529"
+        ffmpegKitPackage = "https-lts"
     }
     repositories {
         google()

packages/mobile/ios/AudiusReactNative.xcodeproj/project.pbxproj

@@ -376,10 +376,26 @@
 			);
 			inputPaths = (
 				"${PODS_ROOT}/Target Support Files/Pods-AudiusReactNative/Pods-AudiusReactNative-frameworks.sh",
+				"${PODS_ROOT}/ffmpeg-kit-ios-https/ffmpegkit.framework",
+				"${PODS_ROOT}/ffmpeg-kit-ios-https/libavcodec.framework",
+				"${PODS_ROOT}/ffmpeg-kit-ios-https/libavdevice.framework",
+				"${PODS_ROOT}/ffmpeg-kit-ios-https/libavfilter.framework",
+				"${PODS_ROOT}/ffmpeg-kit-ios-https/libavformat.framework",
+				"${PODS_ROOT}/ffmpeg-kit-ios-https/libavutil.framework",
+				"${PODS_ROOT}/ffmpeg-kit-ios-https/libswresample.framework",
+				"${PODS_ROOT}/ffmpeg-kit-ios-https/libswscale.framework",
 				"${PODS_XCFRAMEWORKS_BUILD_DIR}/FingerprintPro/FingerprintPro.framework/FingerprintPro",
 			);
 			name = "[CP] Embed Pods Frameworks";
 			outputPaths = (
+				"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/ffmpegkit.framework",
+				"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/libavcodec.framework",
+				"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/libavdevice.framework",
+				"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/libavfilter.framework",
+				"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/libavformat.framework",
+				"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/libavutil.framework",
+				"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/libswresample.framework",
+				"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/libswscale.framework",
 				"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/FingerprintPro.framework",
 			);
 			runOnlyForDeploymentPostprocessing = 0;

packages/mobile/ios/Podfile

@@ -47,6 +47,7 @@ target 'AudiusReactNative' do
   pod 'glog', :podspec => '../node_modules/react-native/third-party-podspecs/glog.podspec'
   pod 'RCT-Folly', :podspec => '../node_modules/react-native/third-party-podspecs/RCT-Folly.podspec'
   pod 'boost', :podspec => '../node_modules/react-native/third-party-podspecs/boost.podspec'
+  pod 'ffmpeg-kit-react-native', :subspecs => ['https-lts'], :podspec => '../node_modules/ffmpeg-kit-react-native/ffmpeg-kit-react-native.podspec'
 
   config = use_native_modules!
   use_react_native!(

packages/mobile/ios/Podfile.lock

@@ -15,6 +15,10 @@ PODS:
     - React-Core (= 0.66.3)
     - React-jsi (= 0.66.3)
     - ReactCommon/turbomodule/core (= 0.66.3)
+  - ffmpeg-kit-ios-https (5.1.LTS)
+  - ffmpeg-kit-react-native/https-lts (5.1.0):
+    - ffmpeg-kit-ios-https (= 5.1.LTS)
+    - React-Core
   - FingerprintPro (2.1.3)
   - Firebase/Analytics (8.12.1):
     - Firebase/Core
@@ -506,6 +510,7 @@ DEPENDENCIES:
   - DoubleConversion (from `../node_modules/react-native/third-party-podspecs/DoubleConversion.podspec`)
   - FBLazyVector (from `../node_modules/react-native/Libraries/FBLazyVector`)
   - FBReactNativeSpec (from `../node_modules/react-native/React/FBReactNativeSpec`)
+  - ffmpeg-kit-react-native/https-lts (from `../node_modules/ffmpeg-kit-react-native/ffmpeg-kit-react-native.podspec`)
   - glog (from `../node_modules/react-native/third-party-podspecs/glog.podspec`)
   - lottie-ios (from `../node_modules/lottie-ios`)
   - lottie-react-native (from `../node_modules/lottie-react-native`)
@@ -580,6 +585,7 @@ DEPENDENCIES:
 SPEC REPOS:
   trunk:
     - Amplitude
+    - ffmpeg-kit-ios-https
     - FingerprintPro
     - Firebase
     - FirebaseAnalytics
@@ -611,6 +617,8 @@ EXTERNAL SOURCES:
     :path: "../node_modules/react-native/Libraries/FBLazyVector"
   FBReactNativeSpec:
     :path: "../node_modules/react-native/React/FBReactNativeSpec"
+  ffmpeg-kit-react-native:
+    :podspec: "../node_modules/ffmpeg-kit-react-native/ffmpeg-kit-react-native.podspec"
   glog:
     :podspec: "../node_modules/react-native/third-party-podspecs/glog.podspec"
   lottie-ios:
@@ -756,6 +764,8 @@ SPEC CHECKSUMS:
   DoubleConversion: 831926d9b8bf8166fd87886c4abab286c2422662
   FBLazyVector: de148e8310b8b878db304ceea2fec13f2c02e3a0
   FBReactNativeSpec: 6192956c9e346013d5f1809ba049af720b11c6a4
+  ffmpeg-kit-ios-https: 9e50ffa7eaa6272a0021829e054ef241f2ecffb2
+  ffmpeg-kit-react-native: a8ba22b56048e8449372ba307f6bfc4f263c8445
   FingerprintPro: 4c5fafa1f42648a6d0fb8eda63fa4a9e0eed8229
   Firebase: 580d09e8edafc3073ebf09c03fd42e4d80d35fe9
   FirebaseAnalytics: bd10d7706ba8d6e01ea2816f8fe9e9b881cb0918
@@ -842,6 +852,6 @@ SPEC CHECKSUMS:
   SRSRadialGradient: 71378a83e52b616c40367c37ee7cb6865f9f7c6e
   Yoga: 32a18c0e845e185f4a2a66ec76e1fd1f958f22fa
 
-PODFILE CHECKSUM: 5b3673b9ec4477000fe1c9c3090d7b84553eeb1f
+PODFILE CHECKSUM: c9b95587012acb2b0b533d3c6d6b8cd587bc5b55
 
 COCOAPODS: 1.11.3

packages/mobile/package.json

@@ -70,6 +70,7 @@
     "bn.js": "5.2.0",
     "color": "3.2.1",
     "expo-crypto": "9.2.0",
+    "ffmpeg-kit-react-native": "5.1.0",
     "formik": "2.2.9",
     "fxa-common-password-list": "0.0.4",
     "hermes-engine": "0.9.0",

packages/mobile/src/components/share-drawer/ShareDrawer.tsx

@@ -1,4 +1,6 @@
-import { useCallback, useContext } from 'react'
+import React, { useCallback, useContext } from 'react'
+
+import path from 'path'
 
 import {
   FeatureFlags,
@@ -7,11 +9,14 @@ import {
   tracksSocialActions,
   usersSocialActions,
   shareModalUISelectors,
-  shareSoundToTiktokModalActions
+  shareSoundToTiktokModalActions,
+  uuid
 } from '@audius/common'
 import Clipboard from '@react-native-clipboard/clipboard'
+import { FFmpegKit, ReturnCode } from 'ffmpeg-kit-react-native'
 import { Linking, View } from 'react-native'
 import Config from 'react-native-config'
+import RNFS from 'react-native-fs'
 import Share from 'react-native-share'
 import { useDispatch, useSelector } from 'react-redux'
 
@@ -38,14 +43,6 @@ const { shareTrack } = tracksSocialActions
 const { shareCollection } = collectionsSocialActions
 const { getAccountUser } = accountSelectors
 
-const shareOptions = {
-  backgroundVideo: '', // TODO(nkang): Base64 video goes here
-  stickerImage: '', // TODO(nkang): Base64 sticker image goes here
-  attributionURL: Config.AUDIUS_URL,
-  social: Share.Social.INSTAGRAM_STORIES,
-  appId: Config.INSTAGRAM_APP_ID
-}
-
 const useStyles = makeStyles(({ palette }) => ({
   shareToTwitterAction: {
     color: palette.staticTwitterBlue
@@ -59,6 +56,9 @@ const useStyles = makeStyles(({ palette }) => ({
   copyLinkAction: {
     color: palette.secondary
   },
+  shareToInstagramStoryAction: {
+    color: palette.primary
+  },
   title: {
     display: 'flex',
     flexDirection: 'row',
@@ -90,7 +90,7 @@ export const ShareDrawer = () => {
   const { isEnabled: isShareToInstagramStoryEnabled } = useFeatureFlag(
     FeatureFlags.SHARE_TO_STORY
   )
-  const { secondary, neutral, staticTwitterBlue } = useThemeColors()
+  const { primary, secondary, neutral, staticTwitterBlue } = useThemeColors()
   const themeVariant = useThemeVariant()
   const isLightMode = themeVariant === Theme.DEFAULT
   const dispatch = useDispatch()
@@ -123,9 +123,35 @@ export const ShareDrawer = () => {
   // nkang: WIP, will probably be moved:
   const handleShareToInstagramStory = useCallback(async () => {
     if (content?.type === 'track') {
+      const storyVideoPath = path.join(
+        RNFS.TemporaryDirectoryPath,
+        `storyVideo-${uuid()}.mp4`
+      )
+      const session = await FFmpegKit.execute(
+        `-f lavfi -i gradients=n=2:type=linear:s=270x480:duration=10:speed=0.05:c0=#AA1F3B:c1=#671525:x0=0:x1=0:y0=0:y1=280,format=rgb0 -t 10 ${storyVideoPath}`
+      )
+      // TODO(nkang): Add loading state
+      const returnCode = await session.getReturnCode()
+
+      if (ReturnCode.isSuccess(returnCode)) {
+      } else {
+        const output = await session.getOutput()
+        // TODO(nkang): Make this a toast?
+        console.error('Error sharing story: ', output)
+        return
+      }
+
+      const shareOptions = {
+        backgroundVideo: storyVideoPath,
+        // stickerImage: image, TODO(nkang): Base64 sticker image goes here
+        attributionURL: Config.AUDIUS_URL,
+        social: Share.Social.INSTAGRAM_STORIES,
+        appId: Config.INSTAGRAM_APP_ID
+      }
       try {
         await Share.shareSingle(shareOptions)
       } catch (error) {
+        // TODO (nkang): Make this a toast?
         console.error('Error sharing story: ', error)
       }
     }
@@ -208,35 +234,43 @@ export const ShareDrawer = () => {
 
     const shareToInstagramStoriesAction = {
       text: messages.instagramStory,
-      icon: <IconInstagram fill={secondary} height={26} width={26} />,
-      // TODO(nkang) Replace with style from pending design
-      style: styles.copyLinkAction,
+      icon: <IconInstagram fill={primary} height={26} width={26} />,
+      style: styles.shareToInstagramStoryAction,
       callback: handleShareToInstagramStory
     }
 
-    const result = [shareToTwitterAction, copyLinkAction, shareSheetAction]
+    const result: {
+      text: string
+      icon: React.ReactElement
+      style: Record<string, string>
+      callback: (() => void) | (() => Promise<void>)
+    }[] = [shareToTwitterAction]
 
     if (shouldIncludeTikTokAction) {
-      result.splice(1, 0, shareToTikTokAction)
+      result.push(shareToTikTokAction)
     }
     if (shouldIncludeInstagramStoryAction) {
-      result.splice(2, 0, shareToInstagramStoriesAction)
+      result.push(shareToInstagramStoriesAction)
     }
 
+    result.push(copyLinkAction, shareSheetAction)
+
     return result
   }, [
     staticTwitterBlue,
     styles.shareToTwitterAction,
     styles.shareToTikTokAction,
     styles.shareToTikTokActionDark,
     styles.copyLinkAction,
+    styles.shareToInstagramStoryAction,
     handleShareToTwitter,
     isLightMode,
     handleShareToTikTok,
     shareType,
     secondary,
     handleCopyLink,
     handleOpenShareSheet,
+    primary,
     handleShareToInstagramStory,
     shouldIncludeTikTokAction,
     shouldIncludeInstagramStoryAction