[C-2823] Improve autogenerated image logic (#3729)

packages/common/src/context/appContext.ts

@@ -3,6 +3,7 @@ import { createContext, useContext } from 'react'
 import { StorageNodeSelectorService } from '@audius/sdk'
 
 import { AnalyticsEvent, AllTrackingEvents } from 'models/Analytics'
+import { AudiusBackend } from 'services/audius-backend'
 
 type AppContextType = {
   analytics: {
@@ -15,6 +16,12 @@ type AppContextType = {
     }
   }
   storageNodeSelector?: StorageNodeSelectorService
+  imageUtils: {
+    generatePlaylistArtwork: (
+      imageUrls: string[]
+    ) => Promise<{ url: string; file: File }>
+  }
+  audiusBackend: AudiusBackend
 }
 
 export const AppContext = createContext<AppContextType | null>(null)

packages/common/src/hooks/index.ts

@@ -18,3 +18,4 @@ export * from './useThrottledCallback'
 export * from './useDebouncedCallback'
 export * from './useSavedCollections'
 export * from './chats'
+export * from './useGeneratePlaylistArtwork'

packages/common/src/hooks/useGeneratePlaylistArtwork.ts

@@ -0,0 +1,53 @@
+import { useCallback } from 'react'
+
+import { useSelector } from 'react-redux'
+
+import { ID } from 'models/Identifiers'
+import { SquareSizes } from 'models/ImageSizes'
+import { useAppContext } from 'src/context'
+import { getCollection } from 'store/cache/collections/selectors'
+import { getTrack } from 'store/cache/tracks/selectors'
+import { CommonState } from 'store/index'
+import { removeNullable } from 'utils/typeUtils'
+
+export const useGeneratePlaylistArtwork = (collectionId: ID) => {
+  const collection = useSelector(
+    (state: CommonState) => getCollection(state, { id: collectionId })!
+  )
+
+  const collectionTracks = useSelector((state: CommonState) => {
+    const trackIds = collection.playlist_contents.track_ids.map(
+      ({ track }) => track
+    )
+    const tracks = trackIds
+      .map((trackId) => getTrack(state, { id: trackId }))
+      .filter(removeNullable)
+      .slice(0, 4)
+
+    if (tracks.length < 4) {
+      return tracks.slice(0, 1)
+    }
+
+    return tracks
+  })
+
+  const { imageUtils, audiusBackend } = useAppContext()
+
+  return useCallback(async () => {
+    if (collectionTracks.length === 0) {
+      return { url: '', file: undefined }
+    }
+
+    const trackArtworkUrls = await Promise.all(
+      collectionTracks.map(async (track) => {
+        const { cover_art_sizes, cover_art } = track
+        return await audiusBackend.getImageUrl(
+          cover_art_sizes || cover_art,
+          SquareSizes.SIZE_1000_BY_1000
+        )
+      })
+    )
+
+    return await imageUtils.generatePlaylistArtwork(trackArtworkUrls)
+  }, [collectionTracks, audiusBackend, imageUtils])
+}

packages/common/src/models/Collection.ts

@@ -35,6 +35,7 @@ export type CollectionMetadata = {
   has_current_user_saved: boolean
   is_album: boolean
   is_delete: boolean
+  is_image_autogenerated?: boolean
   is_private: boolean
   playlist_contents: {
     track_ids: PlaylistTrackId[]
@@ -70,7 +71,7 @@ export type ComputedCollectionProperties = {
   _cover_art_sizes: CoverArtSizes
   _moved?: UID
   _temp?: boolean
-  artwork?: { file?: File; url?: string }
+  artwork?: { file?: File; url?: string; source?: string; error?: string }
 }
 
 export type Collection = CollectionMetadata & ComputedCollectionProperties

packages/common/src/services/audius-backend/AudiusBackend.ts

@@ -458,7 +458,6 @@ export const audiusBackend = ({
     if (imagePreloader) {
       try {
         const preloaded = await imagePreloader(imageUrl)
-        console.log('we doing this actually?')
         if (preloaded) {
           return imageUrl
         }

packages/common/src/store/cache/collections/types.ts

@@ -19,7 +19,7 @@ export type Image = {
   size?: number
   fileType?: string
   url: string
-  file?: string
+  file?: string | File
 }
 
 export type EditPlaylistValues = Collection & {

packages/common/src/store/storeContext.ts

@@ -73,7 +73,7 @@ export type CommonStoreContext = {
   openSeaClient: OpenSeaClient
   audiusSdk: () => Promise<AudiusSdk>
   imageUtils: {
-    createPlaylistArtwork: (
+    generatePlaylistArtwork: (
       urls: string[]
     ) => Promise<{ file: File; url: string }>
   }

packages/mobile/src/app/AppContextProvider.tsx

@@ -4,7 +4,9 @@ import { AppContext } from '@audius/common'
 import { useAsync } from 'react-use'
 
 import * as analytics from 'app/services/analytics'
+import { audiusBackendInstance } from 'app/services/audius-backend-instance'
 import { getStorageNodeSelector } from 'app/services/sdk/storageNodeSelector'
+import { generatePlaylistArtwork } from 'app/utils/generatePlaylistArtwork'
 
 type AppContextProviderProps = {
   children: ReactNode
@@ -18,7 +20,9 @@ export const AppContextProvider = (props: AppContextProviderProps) => {
   const value = useMemo(
     () => ({
       analytics,
-      storageNodeSelector
+      storageNodeSelector,
+      imageUtils: { generatePlaylistArtwork },
+      audiusBackend: audiusBackendInstance
     }),
     [storageNodeSelector]
   )

packages/mobile/src/components/fields/PickArtworkField.tsx

@@ -1,6 +1,7 @@
-import { useCallback, useMemo, useState } from 'react'
+import { useCallback, useMemo } from 'react'
 
-import { useField } from 'formik'
+import { useGeneratePlaylistArtwork } from '@audius/common'
+import { useField, useFormikContext } from 'formik'
 import { capitalize } from 'lodash'
 import { View } from 'react-native'
 import type { Asset } from 'react-native-image-picker'
@@ -56,7 +57,10 @@ const useStyles = makeStyles(({ palette, spacing }) => ({
 
 const messages = {
   addArtwork: 'Add Artwork',
-  changeArtwork: 'Change Artwork'
+  changeArtwork: 'Change Artwork',
+  removeArtwork: 'Remove Artwork',
+  removingArtwork: 'Removing Artwork',
+  updatingArtwork: 'Updating Artwork'
 }
 
 type PickArtworkFieldProps = {
@@ -67,18 +71,21 @@ export const PickArtworkField = (props: PickArtworkFieldProps) => {
   const { name } = props
   const styles = useStyles()
   const { neutralLight8 } = useThemeColors()
-  // const name = 'artwork'
-  const [{ value }, { error, touched }, { setValue }] = useField(name)
+  const [{ value }, { error, touched }, { setValue: setArtwork }] =
+    useField(name)
   const [{ value: existingTrackArtwork }] = useField('trackArtwork')
   const trackArtworkUrl = value?.url ?? existingTrackArtwork
+  const [{ value: collectionId }] = useField('playlist_id')
+  const [{ value: isImageAutogenerated }, , { setValue: setIsAutogenerated }] =
+    useField('is_image_autogenerated')
 
   const { secondary } = useThemeColors()
 
-  const [isLoading, setIsLoading] = useState(false)
+  const { status, setStatus } = useFormikContext()
 
-  const handlePress = useCallback(() => {
+  const handleChangeArtwork = useCallback(() => {
     const handleImageSelected = (_image: Image, rawResponse: Asset) => {
-      setValue({
+      setArtwork({
         url: rawResponse.uri,
         file: {
           uri: rawResponse.uri,
@@ -87,23 +94,35 @@ export const PickArtworkField = (props: PickArtworkFieldProps) => {
         },
         source: 'original'
       })
-      setIsLoading(true)
+      setIsAutogenerated(false)
+      setStatus({ imageLoading: true })
     }
     launchSelectImageActionSheet(handleImageSelected, secondary)
-  }, [secondary, setValue])
+  }, [secondary, setArtwork, setIsAutogenerated, setStatus])
 
   const source = useMemo(() => ({ uri: trackArtworkUrl }), [trackArtworkUrl])
 
+  const generatePlaylistArtwork = useGeneratePlaylistArtwork(collectionId)
+
+  const handleRemoveArtwork = useCallback(async () => {
+    setStatus({ imageLoading: true, imageGenerating: true })
+    const { file, url } = await generatePlaylistArtwork()
+    setArtwork({ url, file })
+    setIsAutogenerated(true)
+  }, [setStatus, generatePlaylistArtwork, setArtwork, setIsAutogenerated])
+
   return (
     <View style={styles.root}>
       <DynamicImage
         source={source}
-        onLoad={() => setIsLoading(false)}
+        onLoad={() =>
+          setStatus({ imageLoading: false, imageGenerating: false })
+        }
         style={styles.image}
         noSkeleton
       >
         <View style={styles.iconPicture}>
-          {isLoading ? (
+          {status.imageLoading || status.imageGenerating ? (
             <LoadingSpinner style={styles.loading} />
           ) : trackArtworkUrl ? null : (
             <IconImage height={128} width={128} fill={neutralLight8} />
@@ -114,11 +133,23 @@ export const PickArtworkField = (props: PickArtworkFieldProps) => {
             variant='secondaryAlt'
             size='large'
             title={
-              trackArtworkUrl ? messages.changeArtwork : messages.addArtwork
+              status.imageGenerating && isImageAutogenerated
+                ? messages.updatingArtwork
+                : trackArtworkUrl && status.imageGenerating
+                ? messages.removingArtwork
+                : trackArtworkUrl && !isImageAutogenerated
+                ? messages.removeArtwork
+                : trackArtworkUrl
+                ? messages.changeArtwork
+                : messages.addArtwork
             }
             icon={IconPencil}
             iconPosition='left'
-            onPress={handlePress}
+            onPress={
+              trackArtworkUrl && !isImageAutogenerated
+                ? handleRemoveArtwork
+                : handleChangeArtwork
+            }
           />
         </View>
       </DynamicImage>

packages/mobile/src/screens/edit-playlist-screen/EditPlaylistScreen.tsx

@@ -50,7 +50,11 @@ export const EditPlaylistScreen = () => {
   }
 
   return (
-    <Formik initialValues={initialValues} onSubmit={handleSubmit}>
+    <Formik
+      initialValues={initialValues}
+      initialStatus={{ imageLoading: false, imageGenerating: false }}
+      onSubmit={handleSubmit}
+    >
       {(formikProps) => (
         <EditPlaylistNavigator
           {...formikProps}

packages/mobile/src/store/storeContext.ts

@@ -19,7 +19,7 @@ import {
 import { audiusSdk } from 'app/services/sdk/audius-sdk'
 import { trackDownload } from 'app/services/track-download'
 import { walletClient } from 'app/services/wallet-client'
-import { createPlaylistArtwork } from 'app/utils/createPlaylistArtwork'
+import { generatePlaylistArtwork } from 'app/utils/generatePlaylistArtwork'
 import { reportToSentry } from 'app/utils/reportToSentry'
 import share from 'app/utils/share'
 
@@ -54,6 +54,6 @@ export const storeContext: CommonStoreContext = {
   openSeaClient: new OpenSeaClient(Config.OPENSEA_API_URL),
   audiusSdk,
   imageUtils: {
-    createPlaylistArtwork
+    generatePlaylistArtwork
   }
 }

packages/mobile/src/utils/createPlaylistArtwork.ts

@@ -16,19 +16,26 @@ export async function createPlaylistArtwork(imageUrls: string[]) {
   const images = await Promise.all(
     imageUrls.map((imageUrl) => Jimp.read(imageUrl))
   )
-  const newImage = new Jimp(canvasWidth, canvasHeight)
 
-  for (let i = 0; i < 4; i++) {
-    const image = images[i]
-    if (image) {
-      image.resize(imageWidth, imageHeight)
+  let newImage: Jimp
 
-      // Calculate the x and y position based on the quadrant
-      const x = i % 2 === 0 ? 0 : imageWidth
-      const y = i < 2 ? 0 : imageHeight
+  if (images.length === 1) {
+    newImage = images[0]
+  } else {
+    newImage = new Jimp(canvasWidth, canvasHeight)
 
-      // Composite the image onto the canvas
-      newImage.composite(image, x, y)
+    for (let i = 0; i < 4; i++) {
+      const image = images[i]
+      if (image) {
+        image.resize(imageWidth, imageHeight)
+
+        // Calculate the x and y position based on the quadrant
+        const x = i % 2 === 0 ? 0 : imageWidth
+        const y = i < 2 ? 0 : imageHeight
+
+        // Composite the image onto the canvas
+        newImage.composite(image, x, y)
+      }
     }
   }
 

packages/mobile/src/utils/generatePlaylistArtwork.ts

@@ -0,0 +1,45 @@
+import Jimp from 'jimp'
+import RNFetchBlob from 'rn-fetch-blob'
+
+const {
+  fs: { dirs, writeFile }
+} = RNFetchBlob
+
+const canvasWidth = 1000 // Adjust the width and height as per your requirements
+const canvasHeight = 1000
+const imageWidth = canvasWidth / 2
+const imageHeight = canvasHeight / 2
+
+const mimeType = Jimp.MIME_JPEG
+
+export async function generatePlaylistArtwork(imageUrls: string[]) {
+  const images = await Promise.all(
+    imageUrls.map((imageUrl) => Jimp.read(imageUrl))
+  )
+  const newImage = new Jimp(canvasWidth, canvasHeight)
+
+  for (let i = 0; i < 4; i++) {
+    const image = images[i]
+    if (image) {
+      image.resize(imageWidth, imageHeight)
+
+      // Calculate the x and y position based on the quadrant
+      const x = i % 2 === 0 ? 0 : imageWidth
+      const y = i < 2 ? 0 : imageHeight
+
+      // Composite the image onto the canvas
+      newImage.composite(image, x, y)
+    }
+  }
+
+  const fileName = 'playlist-artwork'
+  const url = `${dirs.DocumentDir}/${fileName}.jpg`
+  const imageContents = await newImage.getBase64Async(mimeType)
+  const [, base64Contents] = imageContents.split(',')
+  await writeFile(url, base64Contents, 'base64')
+
+  return {
+    url,
+    file: { uri: url, name: fileName, type: mimeType } as unknown as File
+  }
+}