diff --git a/arduino-ide-extension/src/browser/create/create-api.ts b/arduino-ide-extension/src/browser/create/create-api.ts index 0e81ecb90..55a5d6387 100644 --- a/arduino-ide-extension/src/browser/create/create-api.ts +++ b/arduino-ide-extension/src/browser/create/create-api.ts @@ -93,7 +93,11 @@ export class CreateApi { async readDirectory( posixPath: string, - options: { recursive?: boolean; match?: string } = {} + options: { + recursive?: boolean; + match?: string; + skipSketchCache?: boolean; + } = {} ): Promise { const url = new URL( `${this.domain()}/files/d/$HOME/sketches_v2${posixPath}` @@ -106,21 +110,29 @@ export class CreateApi { } const headers = await this.headers(); - return this.run(url, { - method: 'GET', - headers, - }) - .then(async (result) => { - // add arduino_secrets.h to the results, when reading a sketch main folder - if (posixPath.length && posixPath !== posix.sep) { - const sketch = this.sketchCache.getSketch(posixPath); + const cachedSketch = this.sketchCache.getSketch(posixPath); + const sketchPromise = options.skipSketchCache + ? (cachedSketch && this.sketch(cachedSketch.id)) || Promise.resolve(null) + : Promise.resolve(this.sketchCache.getSketch(posixPath)); + + return Promise.all([ + sketchPromise, + this.run(url, { + method: 'GET', + headers, + }), + ]) + .then(async ([sketch, result]) => { + if (posixPath.length && posixPath !== posix.sep) { if (sketch && sketch.secrets && sketch.secrets.length > 0) { result.push(this.getSketchSecretStat(sketch)); } } - return result; + return result.filter( + (res) => !Create.do_not_sync_files.includes(res.name) + ); }) .catch((reason) => { if (reason?.status === 404) return [] as Create.Resource[]; diff --git a/arduino-ide-extension/src/browser/create/create-fs-provider.ts b/arduino-ide-extension/src/browser/create/create-fs-provider.ts index 8199f693d..091ebc979 100644 --- a/arduino-ide-extension/src/browser/create/create-fs-provider.ts +++ b/arduino-ide-extension/src/browser/create/create-fs-provider.ts @@ -30,8 +30,6 @@ import { SketchesService } from '../../common/protocol'; import { ArduinoPreferences } from '../arduino-preferences'; import { Create } from './typings'; -export const REMOTE_ONLY_FILES = ['sketch.json']; - @injectable() export class CreateFsProvider implements @@ -109,14 +107,10 @@ export class CreateFsProvider const resources = await this.getCreateApi.readDirectory( uri.path.toString() ); - return resources - .filter((res) => !REMOTE_ONLY_FILES.includes(res.name)) - .map(({ name, type }) => [name, this.toFileType(type)]); + return resources.map(({ name, type }) => [name, this.toFileType(type)]); } async delete(uri: URI, opts: FileDeleteOptions): Promise { - return; - if (!opts.recursive) { throw new Error( 'Arduino Create file-system provider does not support non-recursive deletion.' diff --git a/arduino-ide-extension/src/browser/create/typings.ts b/arduino-ide-extension/src/browser/create/typings.ts index e951ac794..b5fb0e2d3 100644 --- a/arduino-ide-extension/src/browser/create/typings.ts +++ b/arduino-ide-extension/src/browser/create/typings.ts @@ -21,7 +21,7 @@ export namespace Create { export type ResourceType = 'sketch' | 'folder' | 'file'; export const arduino_secrets_file = 'arduino_secrets.h'; - export const do_not_sync_files = ['.theia']; + export const do_not_sync_files = ['.theia', 'sketch.json']; export interface Resource { readonly name: string; /** diff --git a/arduino-ide-extension/src/browser/widgets/cloud-sketchbook/cloud-sketch-cache.ts b/arduino-ide-extension/src/browser/widgets/cloud-sketchbook/cloud-sketch-cache.ts index 09a9779f6..6c877e15a 100644 --- a/arduino-ide-extension/src/browser/widgets/cloud-sketchbook/cloud-sketch-cache.ts +++ b/arduino-ide-extension/src/browser/widgets/cloud-sketchbook/cloud-sketch-cache.ts @@ -22,6 +22,14 @@ export class SketchCache { return this.fileStats[path] || null; } + purgeByPath(path: string): void { + for (const itemPath in this.fileStats) { + if (itemPath.indexOf(path) === 0) { + delete this.fileStats[itemPath]; + } + } + } + addSketch(sketch: Create.Sketch): void { const { path } = sketch; const posixPath = toPosixPath(path); diff --git a/arduino-ide-extension/src/browser/widgets/cloud-sketchbook/cloud-sketchbook-tree.ts b/arduino-ide-extension/src/browser/widgets/cloud-sketchbook/cloud-sketchbook-tree.ts index 6ecac6ddd..a315a8f32 100644 --- a/arduino-ide-extension/src/browser/widgets/cloud-sketchbook/cloud-sketchbook-tree.ts +++ b/arduino-ide-extension/src/browser/widgets/cloud-sketchbook/cloud-sketchbook-tree.ts @@ -17,7 +17,6 @@ import { PreferenceScope, } from '@theia/core/lib/browser/preferences/preference-service'; import { MessageService } from '@theia/core/lib/common/message-service'; -import { REMOTE_ONLY_FILES } from './../../create/create-fs-provider'; import { CreateApi } from '../../create/create-api'; import { CreateUri } from '../../create/create-uri'; import { CloudSketchbookTreeModel } from './cloud-sketchbook-tree-model'; @@ -33,10 +32,17 @@ import { ArduinoPreferences } from '../../arduino-preferences'; import { SketchesServiceClientImpl } from '../../../common/protocol/sketches-service-client-impl'; import { FileStat } from '@theia/filesystem/lib/common/files'; import { WorkspaceNode } from '@theia/navigator/lib/browser/navigator-tree'; +import { posix, splitSketchPath } from '../../create/create-paths'; +import { Create } from '../../create/typings'; const MESSAGE_TIMEOUT = 5 * 1000; const deepmerge = require('deepmerge').default; +type FilesToWrite = { source: URI; dest: URI }; +type FilesToSync = { + filesToWrite: FilesToWrite[]; + filesToDelete: URI[]; +}; @injectable() export class CloudSketchbookTree extends SketchbookTree { @inject(FileService) @@ -94,7 +100,7 @@ export class CloudSketchbookTree extends SketchbookTree { async pull(arg: any): Promise { const { - model, + // model, node, }: { model: CloudSketchbookTreeModel; @@ -127,47 +133,12 @@ export class CloudSketchbookTree extends SketchbookTree { const commandsCopy = node.commands; node.commands = []; - // check if the sketch dir already exist - if (CloudSketchbookTree.CloudSketchTreeNode.isSynced(node)) { - const filesToPull = ( - await this.createApi.readDirectory(node.remoteUri.path.toString()) - ).filter((file: any) => !REMOTE_ONLY_FILES.includes(file.name)); - - await Promise.all( - filesToPull.map((file: any) => { - const uri = CreateUri.toUri(file); - this.fileService.copy(uri, LocalCacheUri.root.resolve(uri.path), { - overwrite: true, - }); - }) - ); + const localUri = await this.fileService.toUnderlyingResource( + LocalCacheUri.root.resolve(node.remoteUri.path) + ); + await this.sync(node.remoteUri, localUri); - // open the pulled files in the current workspace - const currentSketch = await this.sketchServiceClient.currentSketch(); - - if ( - !CreateUri.is(node.uri) && - currentSketch && - currentSketch.uri === node.uri.toString() - ) { - filesToPull.forEach(async (file) => { - const localUri = LocalCacheUri.root.resolve( - CreateUri.toUri(file).path - ); - const underlying = await this.fileService.toUnderlyingResource( - localUri - ); - - model.open(underlying); - }); - } - } else { - await this.fileService.copy( - node.remoteUri, - LocalCacheUri.root.resolve(node.uri.path), - { overwrite: true } - ); - } + this.sketchCache.purgeByPath(node.remoteUri.path.toString()); node.commands = commandsCopy; this.messageService.info(`Done pulling ‘${node.fileStat.name}’.`, { @@ -214,17 +185,107 @@ export class CloudSketchbookTree extends SketchbookTree { } const commandsCopy = node.commands; node.commands = []; - // delete every first level file, then push everything - const result = await this.fileService.copy(node.uri, node.remoteUri, { - overwrite: true, - }); + + const localUri = await this.fileService.toUnderlyingResource( + LocalCacheUri.root.resolve(node.remoteUri.path) + ); + await this.sync(localUri, node.remoteUri); + + this.sketchCache.purgeByPath(node.remoteUri.path.toString()); + node.commands = commandsCopy; - this.messageService.info(`Done pushing ‘${result.name}’.`, { + this.messageService.info(`Done pushing ‘${node.fileStat.name}’.`, { timeout: MESSAGE_TIMEOUT, }); }); } + async recursiveURIs(uri: URI): Promise { + // remote resources can be fetched one-shot via api + if (CreateUri.is(uri)) { + const resources = await this.createApi.readDirectory( + uri.path.toString(), + { recursive: true, skipSketchCache: true } + ); + return resources.map((resource) => + CreateUri.toUri(splitSketchPath(resource.path)[1]) + ); + } + + const fileStat = await this.fileService.resolve(uri, { + resolveMetadata: false, + }); + + if (!fileStat.children || !fileStat.isDirectory) { + return [fileStat.resource]; + } + + let childrenUris: URI[] = []; + + for await (const child of fileStat.children) { + childrenUris = [ + ...childrenUris, + ...(await this.recursiveURIs(child.resource)), + ]; + } + + return [fileStat.resource, ...childrenUris]; + } + + private URIsToMap(uris: URI[], basepath: string): Record { + return uris.reduce((prev: Record, curr) => { + const path = curr.toString().split(basepath); + + if (path.length !== 2 || path[1].length === 0) { + return prev; + } + + // do not map "do_not_sync" files/directoris and their descendants + const segments = path[1].split(posix.sep) || []; + if ( + segments.some((segment) => Create.do_not_sync_files.includes(segment)) + ) { + return prev; + } + + // skip when the filename is a hidden file (starts with `.`) + if (segments[segments.length - 1].indexOf('.') === 0) { + return prev; + } + + return { ...prev, [path[1]]: curr }; + }, {}); + } + + async getUrisMap(uri: URI) { + const basepath = uri.toString(); + const exists = await this.fileService.exists(uri); + const uris = + (exists && this.URIsToMap(await this.recursiveURIs(uri), basepath)) || {}; + return uris; + } + + async treeDiff(source: URI, dest: URI): Promise { + const [sourceURIs, destURIs] = await Promise.all([ + this.getUrisMap(source), + this.getUrisMap(dest), + ]); + + const destBase = dest.toString(); + const filesToWrite: FilesToWrite[] = []; + + Object.keys(sourceURIs).forEach((path) => { + const destUri = destURIs[path] || new URI(destBase + path); + + filesToWrite.push({ source: sourceURIs[path], dest: destUri }); + delete destURIs[path]; + }); + + const filesToDelete = Object.values(destURIs); + + return { filesToWrite, filesToDelete }; + } + async refresh( node?: CompositeTreeNode ): Promise { @@ -266,6 +327,25 @@ export class CloudSketchbookTree extends SketchbookTree { } } + async sync(source: URI, dest: URI) { + const { filesToWrite, filesToDelete } = await this.treeDiff(source, dest); + await Promise.all( + filesToWrite.map(async ({ source, dest }) => { + if ((await this.fileService.resolve(source)).isFile) { + const content = await this.fileService.read(source); + return this.fileService.write(dest, content.value); + } + return this.fileService.createFolder(dest); + }) + ); + + await Promise.all( + filesToDelete.map((file) => + this.fileService.delete(file, { recursive: true }) + ) + ); + } + async resolveChildren(parent: CompositeTreeNode): Promise { return (await super.resolveChildren(parent)).sort((a, b) => { if ( @@ -295,7 +375,7 @@ export class CloudSketchbookTree extends SketchbookTree { /** * Retrieve fileStats for the given node, merging the local and remote childrens - * Local children take prevedence over remote ones + * Local children take precedence over remote ones * @param node * @returns */ @@ -376,6 +456,7 @@ export class CloudSketchbookTree extends SketchbookTree { const node = this.getNode(id); if (fileStat.isDirectory) { if (DirNode.is(node)) { + node.uri = uri; node.fileStat = fileStat; return node; } @@ -391,6 +472,7 @@ export class CloudSketchbookTree extends SketchbookTree { } if (FileNode.is(node)) { node.fileStat = fileStat; + node.uri = uri; return node; } return {