ipfs.add + globSource + many files will freeze

[bluelovers]

let ret = await addDirectoryToIPFS(ipfs, `target path`, {
			options: {
				pin:  false,
				//progress: createProgressBar,
			},
			globSourceOptions: {
				hidden: true,
			},
		});

		console.dir(ret.files.length);
		console.dir(ret.root.cid.toString());

use v1 will can't see any thing and freeze
use v2 will see some logs, but after many runs will freeze

both can't finish job

test file
https://ipfs.io/ipfs/QmQtUGvQnYv2ZDw5GdRsaMXqjsYGp3h74BwoFHRRSFiWRZ?filename=.cache.7z

v1

export async function addDirectoryToIPFS(ipfs: IIPFSFileApi, targetDirPath: string, {
	options,
	globSourceOptions,
}: {
	options?: IIPFSFileApiAddOptions,
	globSourceOptions?: IGlobSourceOptions,
} = {})
{
	let files: IIPFSFileApiAddReturnEntry[] = [];

	for await (const file of ipfs.add(globSource(targetDirPath, {
		recursive: true,
		...globSourceOptions,
	}), options))
	{
		console.log(file)
		files.push(file);
	}

	const root = files[files.length - 1]

	return {
		targetDirPath,
		root,
		files,
	}
}

v2

export async function addDirectoryToIPFS(ipfs: IIPFSFileApi, targetDirPath: string, {
	options,
	globSourceOptions,
}: {
	options?: IIPFSFileApiAddOptions,
	globSourceOptions?: IGlobSourceOptions,
} = {})
{

	const stream = globSource(targetDirPath, {
		recursive: true,
		...globSourceOptions,
	})

	let files: IIPFSFileApiAddReturnEntry[] = [];
	let root: IIPFSFileApiAddReturnEntry;

	let i = 0;

	// @ts-ignore
	for await (const file of ipfs.add(stream, options))
	{
		if ((i++ % 100) === 0)
		{
			console.dir(file.path)
			console.log(file.cid.toString());
			//console.dir(root = file)
		}

		root = file;
	}

	return {
		targetDirPath,
		root,
		files: {
			length: i,
		},
	}
}

[achingbrain]

Are you running this over http (e.g. to a daemon process) or direct to an in-process node?

[bluelovers]

over http and also try use direct by cli ( jsipfs add ) too

but all freeze

[achingbrain]

Is the daemon running in the background when you use the CLI?

[bluelovers]

yes

[bluelovers]

video
https://ipfs.io/ipfs/QmeaxfayZPFnhV8L6VpkfDYv2fTtN6BRsAYdXKHuGEZQZi?filename=bandicam%202020-03-25%2002-56-27-468.avi

first delay is by search files,

but............

after upload to 2x % is freezed, not more print any message

[achingbrain]

The next release will included reworked multipart requests and it should resolve this issue (hopefully later this week) - previously in order to use FormData requests we had to process every file that's being uploaded in advance as that's what the FormData API requires, but now we create a streaming multipart request.

This will work for node, but for the browser it will be fixed when they support streaming uploads - for more see #2838

[bluelovers]

when i use v3 for upload, it still will freeze after upload many file

v3

export async function addDirectoryToIPFS(ipfs: IIPFSFilesApi & IIPFSFilesApi, targetDirPath: string, {
	options,
	globSourceOptions,
	ignoreExists,
}: {
	options?: IIPFSFileApiAddOptions,
	globSourceOptions?: IGlobSourceOptions,
	ignoreExists?: boolean,
} = {})
{

	const stream = globSource(targetDirPath, {
		recursive: true,
		...globSourceOptions,
	})

	let i = 0;
	let cid;

	ignoreExists = !!ignoreExists;

	for await (const entry of stream)
	{
		if (entry.content)
		{
			if (ignoreExists === true && await ipfsFilesExists(ipfs, entry.path))
			{
				console.gray.debug(entry.path)
				continue;
			}

			console.debug(entry.path)

			let buf = await getStream.buffer(entry.content)

			await ipfs.files.write(entry.path, buf, {
				create: true,
				parents: true,
				mode: entry.mode,
				mtime: entry.mtime,
			})

			i++;

			if ((i % 100) === 0)
			{
				const cid = await ipfs.files.flush()

				console.debug(cid.toString())
			}
		}
		else
		{
			console.debug(entry.path)
		}

	}

	cid = await ipfs.files.flush()

	return {
		targetDirPath,
		root: {
			cid,
		},
		files: {
			length: i,
		},
	}
}

[bluelovers]

at v4 is same, it still freeze after update many files

so i think maybe is something happen in ipfs daemon

export async function addDirectoryToIPFS(ipfs: IIPFSFilesApi & IIPFSFilesApi, targetDirPath: string, {
	options,
	globSourceOptions,
	ignoreExists,
}: {
	options?: IIPFSFileApiAddOptions,
	globSourceOptions?: IGlobSourceOptions,
	ignoreExists?: boolean,
} = {})
{
	let i = 0;
	let cid;

	ignoreExists = !!ignoreExists;

	const rootPath = '/' + path.basename(targetDirPath) + '/';

	for await (let filename of FastGlob.stream([
		'**/*',
		'**/*.txt',
	], {
		cwd: targetDirPath,
		onlyFiles: true,
		//deep: Infinity,
	}))
	{
		filename = filename.toString();

		//console.dir(filename)

		let entry = {
			path: rootPath + filename,
			// @ts-ignore
			//content: createReadStream(path.join(targetDirPath, filename)),
			mode: undefined,
			mtime: undefined,
		}

		if (ignoreExists === true && await ipfsFilesExists(ipfs, entry.path))
		{
			console.gray.debug(entry.path)
			continue;
		}

		console.debug(entry.path)

		let buf = await readFile(path.join(targetDirPath, filename))

		await ipfs.files.write(entry.path, buf, {
			create: true,
			parents: true,
			mode: entry.mode,
			mtime: entry.mtime,
		})

		i++;

		if ((i % 100) === 0)
		{
			const cid = await ipfs.files.flush()

			console.success(cid.toString())
		}

	}

	cid = await ipfs.files.flush()

	return {
		targetDirPath,
		root: {
			cid,
		},
		files: {
			length: i,
		},
	}
}

[bluelovers]

when it freeze

webui can't connect to daemon

and daemon is still live in console