Skip to content
This repository has been archived by the owner on Aug 21, 2024. It is now read-only.

Commit

Permalink
Simplify Static Resources (#9061)
Browse files Browse the repository at this point in the history 
* simplify static resource parsing of scenes and uploads

* add download button in file browser input

* fix
  • Loading branch information
@HexaField
HexaField committed Oct 18, 2023
1 parent 718eaa0 commit 0f72f9c
Show file tree
Hide file tree
Showing 11 changed files with 70 additions and 386 deletions.
1 change: 0 additions & 1 deletion .env.local.default
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,6 @@ LOCAL_STORAGE_PROVIDER_PORT=8642
GOOGLE_ANALYTICS_TRACKING_ID=
HUB_ENDPOINT=https://etherealengine.io
INSTANCESERVER_UNREACHABLE_TIMEOUT_SECONDS=10
CLONE_STATIC_RESOURCES=false


MATCHMAKER_EMULATION_MODE=true
Expand Down
6 changes: 3 additions & 3 deletions packages/editor/src/components/inputs/ArrayInputGroup.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@ All portions of the code written by the Ethereal Engine team are Copyright © 20
Ethereal Engine. All Rights Reserved.
*/

import Icon from '@etherealengine/ui/src/primitives/mui/Icon'
import AddIcon from '@mui/icons-material/Add'
import DeleteIcon from '@mui/icons-material/Delete'
import IconButton from '@mui/material/IconButton'
import React from 'react'
import styles from './ArrayInputGroup.module.scss'
Expand Down Expand Up @@ -105,7 +105,7 @@ const ArrayInputGroup = ({
padding: 0
}}
>
<AddIcon sx={{ color: 'primary.contrastText' }} />
<AddIcon sx={{ color: 'var(--textColor)' }} />
</IconButton>
</InputGroup>
{values &&
Expand All @@ -125,7 +125,7 @@ const ArrayInputGroup = ({
}}
onClick={() => deleteInput(index + 1)}
>
<DeleteIcon sx={{ color: 'primary.contrastText' }} />
<Icon type="Delete" style={{ color: 'var(--textColor)' }} />
</IconButton>
</InputGroup>
))}
Expand Down
44 changes: 41 additions & 3 deletions packages/editor/src/components/inputs/FileBrowserInput.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,12 @@ All portions of the code written by the Ethereal Engine team are Copyright © 20
Ethereal Engine. All Rights Reserved.
*/

import IconButton from '@mui/material/IconButton'
import React from 'react'
import { useDrop } from 'react-dnd'

import config from '@etherealengine/common/src/config'
import Icon from '@etherealengine/ui/src/primitives/mui/Icon'
import { ItemTypes } from '../../constants/AssetTypes'
import useUpload from '../assets/useUpload'
import { ControlledStringInput, StringInputProps } from './StringInput'
Expand Down Expand Up @@ -54,6 +57,32 @@ export function FileBrowserInput({
}
const onUpload = useUpload(uploadOptions)

// todo fix for invalid URLs
const assetIsExternal = value && !value?.includes(config.client.fileServer)
const uploadExternalAsset = () => {
onUpload([
{
isFile: true,
name: value?.split('/').pop(),
file: async (onSuccess, onFail) => {
try {
const asset = await fetch(value!)
const blob = await asset.blob()
const file = new File([blob], value!.split('/').pop()!)
onSuccess(file)
} catch (error) {
if (onFail) onFail(error)
else throw error
}
}
} as Partial<FileSystemFileEntry>
] as any).then((assets) => {
if (assets) {
onChange?.(assets[0])
}
})
}

const [{ canDrop, isOver }, dropRef] = useDrop({
accept: [...acceptDropItems, ItemTypes.File],
async drop(item: any, monitor) {
Expand All @@ -73,9 +102,7 @@ export function FileBrowserInput({

onUpload(entries).then((assets) => {
if (assets) {
for (let index = 0; index < assets.length; index++) {
onChange?.(assets[index])
}
onChange?.(assets[0])
}
})
}
Expand All @@ -96,6 +123,17 @@ export function FileBrowserInput({
canDrop={isOver && canDrop}
{...rest}
/>
{assetIsExternal && (
<IconButton
disableRipple
style={{
padding: 0
}}
onClick={uploadExternalAsset}
>
<Icon type="Download" style={{ color: 'var(--textColor)' }} />
</IconButton>
)}
</>
)
}
Expand Down
2 changes: 0 additions & 2 deletions packages/server-core/src/appconfig.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,8 +145,6 @@ const server = {
corsServerPort: process.env.CORS_SERVER_PORT!,
storageProvider: process.env.STORAGE_PROVIDER!,
storageProviderExternalEndpoint: process.env.STORAGE_PROVIDER_EXTERNAL_ENDPOINT!,
cloneProjectStaticResources:
typeof process.env.CLONE_STATIC_RESOURCES === 'undefined' ? true : process.env.CLONE_STATIC_RESOURCES === 'true',
gaTrackingId: process.env.GOOGLE_ANALYTICS_TRACKING_ID!,
hub: {
endpoint: process.env.HUB_ENDPOINT!
Expand Down
209 changes: 1 addition & 208 deletions packages/server-core/src/media/static-resource/static-resource-helper.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,19 +25,10 @@ Ethereal Engine. All Rights Reserved.

import appRootPath from 'app-root-path'
import assert from 'assert'
import fs from 'fs'
import path from 'path'

import { destroyEngine } from '@etherealengine/engine/src/ecs/classes/Engine'

import { StaticResourceType, staticResourcePath } from '@etherealengine/engine/src/schemas/media/static-resource.schema'
import { Paginated } from '@feathersjs/feathers'
import { Application } from '../../../declarations'
import { mockFetch, restoreFetch } from '../../../tests/util/mockFetch'
import { createFeathersKoaApp } from '../../createApp'
import { getCachedURL } from '../storageprovider/getCachedURL'
import { getStorageProvider } from '../storageprovider/storageprovider'
import { addAssetFromProject, downloadResourceAndMetadata } from './static-resource-helper'
import { downloadResourceAndMetadata } from './static-resource-helper'

describe('static-resource-helper', () => {
before(() => {
Expand Down Expand Up @@ -91,201 +82,3 @@ describe('static-resource-helper', () => {
})
})
})

const testProject = 'test-project'

describe('audio-upload', () => {
let app: Application

before(async () => {
app = createFeathersKoaApp()
await app.setup()
const url = 'https://test.com/projects/default-project/assets/test.mp3'
const url2 = getCachedURL('/projects/default-project/assets/test.mp3', getStorageProvider().cacheDomain)
mockFetch({
[url]: {
contentType: 'audio/mpeg',
response: fs.readFileSync(
path.join(appRootPath.path, '/packages/projects/default-project/assets/SampleAudio.mp3')
)
},
[url2]: {
contentType: 'audio/mpeg',
response: fs.readFileSync(
path.join(appRootPath.path, '/packages/projects/default-project/assets/SampleAudio.mp3')
)
}
})
})

afterEach(async () => {
const storageProvider = getStorageProvider()
if (await storageProvider.doesExist('test.mp3', 'static-resources/test-project/'))
await storageProvider.deleteResources(['static-resources/test-project/test.mp3'])

const existingResource = (await app.service(staticResourcePath).find({
query: {
mimeType: 'audio/mpeg'
},
paginate: false
})) as StaticResourceType[]
await Promise.all(existingResource.map((resource) => app.service(staticResourcePath).remove(resource.id)))
})

after(() => {
restoreFetch()
return destroyEngine()
})

describe('addAssetFromProject', () => {
it('should download audio asset as a new static resource from external url when forced to download', async () => {
const storageProvider = getStorageProvider()
const url = 'https://test.com/projects/default-project/assets/test.mp3'

const staticResource = await addAssetFromProject(app, url, testProject, true)

assert(staticResource.id)
assert.equal(staticResource.url, getCachedURL(staticResource.key!, storageProvider.cacheDomain))
assert.equal(staticResource.key, 'static-resources/test-project/test.mp3')
assert.equal(staticResource.mimeType, 'audio/mpeg')
assert.equal(staticResource.project, testProject)

assert(await storageProvider.doesExist('test.mp3', 'static-resources/test-project/'))

const file = await storageProvider.getObject(staticResource.key!)
assert.equal(file.ContentType, 'audio/mpeg')
})

it('should link audio asset as a new static resource from external url when not forced to download', async () => {
const storageProvider = getStorageProvider()
const url = 'https://test.com/projects/default-project/assets/test.mp3'

const staticResource = await addAssetFromProject(app, url, testProject, false)

assert(staticResource.id)
assert.equal(staticResource.url, 'https://test.com/projects/default-project/assets/test.mp3')
assert.equal(staticResource.key, 'https://test.com/projects/default-project/assets/test.mp3')
assert.equal(staticResource.mimeType, 'audio/mpeg')
assert.equal(staticResource.project, testProject)

const fileExists = await storageProvider.doesExist('test.mp3', 'static-resources/test-project/')
assert(!fileExists)
})

it('should download audio asset as a new static resource from another project', async () => {
const storageProvider = getStorageProvider()
const url = getCachedURL('/projects/default-project/assets/test.mp3', storageProvider.cacheDomain)

const staticResource = await addAssetFromProject(app, url, testProject, true)

assert.equal(staticResource.key, 'static-resources/test-project/test.mp3')
assert.equal(staticResource.url, getCachedURL(staticResource.key!, storageProvider.cacheDomain))
assert.equal(staticResource.mimeType, 'audio/mpeg')
assert.equal(staticResource.project, testProject)

assert(await storageProvider.doesExist('test.mp3', 'static-resources/test-project/'))

const file = await storageProvider.getObject(staticResource.key!)
assert.equal(file.ContentType, 'audio/mpeg')
})

it('should link audio asset as a new static resource from another project', async () => {
const storageProvider = getStorageProvider()
const url = getCachedURL('/projects/default-project/assets/test.mp3', storageProvider.cacheDomain)

const staticResource = await addAssetFromProject(app, url, testProject, false)

assert.equal(staticResource.key, url)
assert.equal(staticResource.url, url)
assert.equal(staticResource.mimeType, 'audio/mpeg')
assert.equal(staticResource.project, testProject)

// should not exist under static resources
const fileExists = await storageProvider.doesExist('test.mp3', 'static-resources/test-project/')
assert(!fileExists)
})

it('should link audio asset as a new static resource from url if from the same project', async () => {
const storageProvider = getStorageProvider()
const url = getCachedURL('/projects/default-project/assets/test.mp3', storageProvider.cacheDomain)

const staticResource = await addAssetFromProject(app, url, 'default-project', false)

assert.equal(staticResource.key, 'projects/default-project/assets/test.mp3')
assert.equal(staticResource.url, url)
assert.equal(staticResource.mimeType, 'audio/mpeg')
assert.equal(staticResource.project, 'default-project')

// should not exist under static resources
const fileExists = await storageProvider.doesExist('test.mp3', 'static-resources/test-project/')
assert(!fileExists)
})

it('should return existing static resource with the same hash and project', async () => {
const storageProvider = getStorageProvider()
const url = getCachedURL('/projects/default-project/assets/test.mp3', storageProvider.cacheDomain)

const response = await addAssetFromProject(app, url, 'default-project', false)
const response2 = await addAssetFromProject(app, url, 'default-project', false)

const staticResources = (await app.service(staticResourcePath).find({
query: {
url
}
})) as Paginated<StaticResourceType>

assert.equal(staticResources.data.length, 1)
assert.equal(response.id, response2.id)
assert.equal(response.url, response2.url)
assert.equal(response.key, response2.key)
})

it('should return new static resource with the same hash exists in another project when forcing download', async () => {
const storageProvider = getStorageProvider()
const url = getCachedURL('/projects/default-project/assets/test.mp3', storageProvider.cacheDomain)

const response = await addAssetFromProject(app, url, 'default-project', true)
const response2 = await addAssetFromProject(app, url, 'test-project', true)

const staticResources = (await app.service(staticResourcePath).find({
query: {
url: response.url
}
})) as Paginated<StaticResourceType>
assert.equal(staticResources.data.length, 1)

const staticResources2 = (await app.service(staticResourcePath).find({
query: {
url: response2.url
}
})) as Paginated<StaticResourceType>
assert.equal(staticResources2.data.length, 1)

assert.notEqual(response.id, response2.id)
assert.notEqual(response.url, response2.url)
assert.notEqual(response.key, response2.key)
assert.equal(response.hash, response2.hash)
})

it('should different static resource with the same url and hash if it exists in another project when not forcing download', async () => {
const storageProvider = getStorageProvider()
const url = getCachedURL('/projects/default-project/assets/test.mp3', storageProvider.cacheDomain)

const response = await addAssetFromProject(app, url, 'default-project', false)
const response2 = await addAssetFromProject(app, url, 'test-project', false)

const staticResources = (await app.service(staticResourcePath).find({
query: {
url: response.url
}
})) as Paginated<StaticResourceType>
assert.equal(staticResources.data.length, 2)

assert(response.id !== response2.id)
assert.equal(response.url, response2.url)
// key is different as it is a different project
assert.notEqual(response.key, response2.key)
assert.equal(response.hash, response2.hash)
})
})
})
Loading

0 comments on commit 0f72f9c

Please sign in to comment.