Skip to content

Commit

Permalink
Link dependencies before installation (#7145)
Browse files Browse the repository at this point in the history
## Proposed Changes

- core aspects, legacy, harmony, and local deps from the capsule are
linked before installation.
- the information about all the links is passed to the package manager,
so that the package manager won't remove them.
  • Loading branch information
zkochan authored Mar 20, 2023
1 parent 24c0c1e commit b2434f5
Show file tree
Hide file tree
Showing 4 changed files with 145 additions and 49 deletions.
92 changes: 80 additions & 12 deletions scopes/component/isolator/isolator.main.runtime.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ import {
DependencyResolverAspect,
DependencyResolverMain,
LinkingOptions,
LinkDetail,
LinkResults,
WorkspacePolicy,
InstallOptions,
DependencyList,
Expand Down Expand Up @@ -330,17 +332,33 @@ export class IsolatorMain {
await Promise.all(
capsuleList.map(async (capsule) => {
const newCapsuleList = CapsuleList.fromArray([capsule]);
await this.installInCapsules(capsule.path, newCapsuleList, installOptions, cachePackagesOnCapsulesRoot);
await this.linkInCapsules(capsulesDir, newCapsuleList, capsulesWithPackagesData, linkingOptions);
const linkedDependencies = await this.linkInCapsules(
capsulesDir,
newCapsuleList,
capsulesWithPackagesData,
linkingOptions
);
await this.installInCapsules(capsule.path, newCapsuleList, installOptions, {
cachePackagesOnCapsulesRoot,
linkedDependencies,
});
})
);
} else {
// When nesting is used, the first component (which is the entry component) is installed in the root
// and all other components (which are the dependencies of the entry component) are installed in
// a subdirectory.
const rootDir = installOptions?.useNesting ? capsuleList[0].path : capsulesDir;
await this.installInCapsules(rootDir, capsuleList, installOptions, cachePackagesOnCapsulesRoot);
await this.linkInCapsules(capsulesDir, capsuleList, capsulesWithPackagesData, linkingOptions);
const linkedDependencies = await this.linkInCapsules(
capsulesDir,
capsuleList,
capsulesWithPackagesData,
linkingOptions
);
await this.installInCapsules(rootDir, capsuleList, installOptions, {
cachePackagesOnCapsulesRoot,
linkedDependencies,
});
}
longProcessLogger.end();
this.logger.consoleSuccess();
Expand All @@ -365,11 +383,14 @@ export class IsolatorMain {
capsulesDir: string,
capsuleList: CapsuleList,
isolateInstallOptions: IsolateComponentsInstallOptions,
cachePackagesOnCapsulesRoot?: boolean
opts: {
cachePackagesOnCapsulesRoot?: boolean;
linkedDependencies?: Record<string, Record<string, string>>;
}
) {
const installer = this.dependencyResolver.getInstaller({
rootDir: capsulesDir,
cacheRootDirectory: cachePackagesOnCapsulesRoot ? capsulesDir : undefined,
cacheRootDirectory: opts.cachePackagesOnCapsulesRoot ? capsulesDir : undefined,
});
// When using isolator we don't want to use the policy defined in the workspace directly,
// we only want to instal deps from components and the peer from the workspace
Expand All @@ -379,6 +400,7 @@ export class IsolatorMain {
installTeambitBit: !!isolateInstallOptions.installTeambitBit,
packageManagerConfigRootDir: isolateInstallOptions.packageManagerConfigRootDir,
resolveVersionsFromDependenciesOnly: true,
linkedDependencies: opts.linkedDependencies,
};

const packageManagerInstallOptions: PackageManagerInstallOptions = {
Expand All @@ -405,20 +427,22 @@ export class IsolatorMain {
capsuleList: CapsuleList,
capsulesWithPackagesData: CapsulePackageJsonData[],
linkingOptions: LinkingOptions
) {
): Promise<Record<string, Record<string, string>>> {
const linker = this.dependencyResolver.getLinker({
rootDir: capsulesDir,
linkingOptions,
});
const peerOnlyPolicy = this.getWorkspacePeersOnlyPolicy();
const capsulesWithModifiedPackageJson = this.getCapsulesWithModifiedPackageJson(capsulesWithPackagesData);
await linker.link(capsulesDir, peerOnlyPolicy, this.toComponentMap(capsuleList), {
const linkResults = await linker.link(capsulesDir, peerOnlyPolicy, this.toComponentMap(capsuleList), {
...linkingOptions,
linkNestedDepsInNM: !this.dependencyResolver.hasRootComponents() && linkingOptions.linkNestedDepsInNM,
});
let rootLinks: LinkDetail[] | undefined;
let nestedLinks: Record<string, Record<string, string>> | undefined;
if (!this.dependencyResolver.hasRootComponents()) {
await symlinkOnCapsuleRoot(capsuleList, this.logger, capsulesDir);
await symlinkDependenciesToCapsules(capsulesWithModifiedPackageJson, capsuleList, this.logger);
rootLinks = await symlinkOnCapsuleRoot(capsuleList, this.logger, capsulesDir);
const capsulesWithModifiedPackageJson = this.getCapsulesWithModifiedPackageJson(capsulesWithPackagesData);
nestedLinks = await symlinkDependenciesToCapsules(capsulesWithModifiedPackageJson, capsuleList, this.logger);
} else {
const coreAspectIds = this.aspectLoader.getCoreAspectIds();
const coreAspectCapsules = CapsuleList.fromArray(
Expand All @@ -427,8 +451,52 @@ export class IsolatorMain {
return coreAspectIds.includes(compIdWithoutVersion);
})
);
await symlinkOnCapsuleRoot(coreAspectCapsules, this.logger, capsulesDir);
rootLinks = await symlinkOnCapsuleRoot(coreAspectCapsules, this.logger, capsulesDir);
}
return {
...nestedLinks,
[capsulesDir]: this.toLocalLinks(linkResults, rootLinks),
};
}

private toLocalLinks(linkResults: LinkResults, rootLinks: LinkDetail[] | undefined): Record<string, string> {
const localLinks: Array<[string, string]> = [];
if (linkResults.teambitBitLink) {
localLinks.push(this.linkDetailToLocalDepEntry(linkResults.teambitBitLink.linkDetail));
}
if (linkResults.coreAspectsLinks) {
linkResults.coreAspectsLinks.forEach((link) => {
localLinks.push(this.linkDetailToLocalDepEntry(link.linkDetail));
});
}
if (linkResults.harmonyLink) {
localLinks.push(this.linkDetailToLocalDepEntry(linkResults.harmonyLink));
}
if (linkResults.teambitLegacyLink) {
localLinks.push(this.linkDetailToLocalDepEntry(linkResults.teambitLegacyLink));
}
if (linkResults.resolvedFromEnvLinks) {
linkResults.resolvedFromEnvLinks.forEach((link) => {
link.linksDetail.forEach((linkDetail) => {
localLinks.push(this.linkDetailToLocalDepEntry(linkDetail));
});
});
}
if (linkResults.linkToDirResults) {
linkResults.linkToDirResults.forEach((link) => {
localLinks.push(this.linkDetailToLocalDepEntry(link.linksDetail));
});
}
if (rootLinks) {
rootLinks.forEach((link) => {
localLinks.push(this.linkDetailToLocalDepEntry(link));
});
}
return Object.fromEntries(localLinks.map(([key, value]) => [key, `link:${value}`]));
}

private linkDetailToLocalDepEntry(linkDetail: LinkDetail): [string, string] {
return [linkDetail.packageName, linkDetail.from];
}

private getCapsulesWithModifiedPackageJson(capsulesWithPackagesData: CapsulePackageJsonData[]) {
Expand Down
57 changes: 34 additions & 23 deletions scopes/component/isolator/symlink-dependencies-to-capsules.ts
Original file line number Diff line number Diff line change
@@ -1,58 +1,69 @@
import { ComponentID } from '@teambit/component';
import { LinkDetail } from '@teambit/dependency-resolver';
import { Logger } from '@teambit/logger';
import { BitId } from '@teambit/legacy-bit-id';
import ConsumerComponent from '@teambit/legacy/dist/consumer/component';
import Symlink from '@teambit/legacy/dist/links/symlink';
import componentIdToPackageName from '@teambit/legacy/dist/utils/bit/component-id-to-package-name';
import path from 'path';

import { Capsule } from './capsule';
import CapsuleList from './capsule-list';

export async function symlinkDependenciesToCapsules(capsules: Capsule[], capsuleList: CapsuleList, logger: Logger) {
export async function symlinkDependenciesToCapsules(
capsules: Capsule[],
capsuleList: CapsuleList,
logger: Logger
): Promise<Record<string, Record<string, string>>> {
logger.debug(`symlinkDependenciesToCapsules, ${capsules.length} capsules`);
await Promise.all(
capsules.map((capsule) => {
return symlinkComponent(capsule.component.state._consumer, capsuleList, logger);
})
return Object.fromEntries(
await Promise.all(
capsules.map((capsule) => {
return symlinkComponent(capsule.component.state._consumer, capsuleList, logger);
})
)
);
}

export async function symlinkOnCapsuleRoot(capsuleList: CapsuleList, logger: Logger, capsuleRoot: string) {
export async function symlinkOnCapsuleRoot(
capsuleList: CapsuleList,
logger: Logger,
capsuleRoot: string
): Promise<LinkDetail[]> {
const modulesPath = path.join(capsuleRoot, 'node_modules');
const symlinks = capsuleList.map((capsule) => {
return capsuleList.map((capsule) => {
const packageName = componentIdToPackageName(capsule.component.state._consumer);
const dest = path.join(modulesPath, packageName);
const src = path.relative(path.resolve(dest, '..'), capsule.path);

return new Symlink(src, dest, capsule.component.id._legacy);
return {
from: capsule.path,
to: dest,
packageName,
};
});

await Promise.all(symlinks.map((symlink) => symlink.write()));
}

async function symlinkComponent(component: ConsumerComponent, capsuleList: CapsuleList, logger: Logger) {
async function symlinkComponent(
component: ConsumerComponent,
capsuleList: CapsuleList,
logger: Logger
): Promise<[string, Record<string, string>]> {
const componentCapsule = capsuleList.getCapsuleIgnoreScopeAndVersion(new ComponentID(component.id));
if (!componentCapsule) throw new Error(`unable to find the capsule for ${component.id.toString()}`);
const allDeps = component.getAllDependenciesIds();
const symlinks = allDeps.map((depId: BitId) => {
const linkResults = allDeps.reduce((acc, depId: BitId) => {
// TODO: this is dangerous - we might have 2 capsules for the same component with different version, then we might link to the wrong place
const devCapsule = capsuleList.getCapsuleIgnoreScopeAndVersion(new ComponentID(depId));
if (!devCapsule) {
// happens when a dependency is not in the workspace. (it gets installed via the package manager)
logger.debug(
`symlinkComponentToCapsule: unable to find the capsule for ${depId.toStringWithoutVersion()}. skipping`
);
return null;
return acc;
}
const packageName = componentIdToPackageName(devCapsule.component.state._consumer);
const devCapsulePath = devCapsule.path;
// @todo: this is a hack, the capsule should be the one responsible to symlink, this works only for FS capsules.
const dest = path.join(componentCapsule.path, 'node_modules', packageName);
// use relative symlink in capsules to make it really isolated from the machine fs
const src = path.relative(path.resolve(dest, '..'), devCapsulePath);
return new Symlink(src, dest, component.id);
});
acc[packageName] = `link:${devCapsulePath}`;
return acc;
}, {});

await Promise.all(symlinks.map((symlink) => symlink && symlink.write()));
return [componentCapsule.path, linkResults];
}
22 changes: 22 additions & 0 deletions scopes/dependencies/dependency-resolver/dependency-installer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ export type InstallOptions = {
installTeambitBit: boolean;
packageManagerConfigRootDir?: string;
resolveVersionsFromDependenciesOnly?: boolean;
linkedDependencies?: Record<string, Record<string, string>>;
};

export type GetComponentManifestsOptions = {
Expand Down Expand Up @@ -138,6 +139,27 @@ export class DependencyInstaller {
if (!finalRootDir) {
throw new RootDirNotDefined();
}
if (options.linkedDependencies) {
const directDeps = new Set<string>();
Object.values(manifests).forEach((manifest) => {
for (const depName of Object.keys({ ...manifest.dependencies, ...manifest.devDependencies })) {
directDeps.add(depName);
}
});
if (options.linkedDependencies[finalRootDir]) {
for (const manifest of Object.values(manifests)) {
if (manifest.name && directDeps.has(manifest.name)) {
delete options.linkedDependencies[finalRootDir][manifest.name];
}
}
}
Object.entries(options.linkedDependencies).forEach(([dir, linkedDeps]) => {
if (!manifests[dir]) {
manifests[dir] = {};
}
manifests[dir].dependencies = Object.assign({}, manifests[dir].dependencies, linkedDeps);
});
}
// Make sure to take other default if passed options with only one option
const calculatedPmOpts = {
...DEFAULT_PM_INSTALL_OPTIONS,
Expand Down
23 changes: 9 additions & 14 deletions scopes/dependencies/dependency-resolver/dependency-linker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ const DEFAULT_LINKING_OPTIONS: LinkingOptions = {
linkNestedDepsInNM: true,
};

export type LinkDetail = { from: string; to: string };
export type LinkDetail = { packageName: string; from: string; to: string };

export type CoreAspectLinkResult = {
aspectId: string;
Expand Down Expand Up @@ -209,6 +209,7 @@ export class DependencyLinker {
return {
componentId: component.id.toString(),
linksDetail: {
packageName: componentPackageName,
from: path.join(rootDir, 'node_modules', componentPackageName),
to: path.join(targetDir, 'node_modules', componentPackageName),
},
Expand Down Expand Up @@ -297,6 +298,7 @@ export class DependencyLinker {
// const linkSrc = folderEntry.origPath || folderEntry.path;
const origPath = folderEntry.origPath ? `(${folderEntry.origPath})` : '';
const linkDetail: LinkDetail = {
packageName: folderEntry.moduleName,
from: `${linkSrc} ${origPath}`,
to: linkTarget,
};
Expand Down Expand Up @@ -374,6 +376,7 @@ export class DependencyLinker {
}
const linkSrc = resolveModuleDirFromFile(resolvedModule, depEntry.dependencyId);
const linkDetail: LinkDetail = {
packageName: depEntry.dependencyId,
from: linkSrc,
to: linkTarget,
};
Expand Down Expand Up @@ -427,15 +430,7 @@ export class DependencyLinker {
const src = this.aspectLoader.mainAspect.path;
await fs.ensureDir(path.dirname(target));
createSymlinkOrCopy(src, target);
return { from: src, to: target };
}

async linkCoreAspects(dir: string): Promise<Array<CoreAspectLinkResult | undefined>> {
const coreAspectsIds = this.aspectLoader.getCoreAspectIds();
const coreAspectsIdsWithoutMain = coreAspectsIds.filter((id) => id !== this.aspectLoader.mainAspect.id);
return coreAspectsIdsWithoutMain.map((id) => {
return this.linkCoreAspect(dir, id, getCoreAspectName(id), getCoreAspectPackageName(id));
});
return { packageName: this.aspectLoader.mainAspect.packageName, from: src, to: target };
}

private async linkNonExistingCoreAspects(
Expand Down Expand Up @@ -498,7 +493,7 @@ export class DependencyLinker {
this.logger.debug(`linkCoreAspect: aspectDir ${aspectDir} does not exist, linking it to ${target}`);
aspectDir = getAspectDir(id);
createSymlinkOrCopy(aspectDir, target);
return { aspectId: id, linkDetail: { from: aspectDir, to: target } };
return { aspectId: id, linkDetail: { packageName, from: aspectDir, to: target } };
}

try {
Expand All @@ -513,7 +508,7 @@ export class DependencyLinker {
}
this.logger.debug(`linkCoreAspect: linking aspectPath ${aspectPath} to ${target}`);
createSymlinkOrCopy(aspectPath, target);
return { aspectId: id, linkDetail: { from: aspectPath, to: target } };
return { aspectId: id, linkDetail: { packageName, from: aspectPath, to: target } };
} catch (err: any) {
throw new CoreAspectLinkError(id, err);
}
Expand Down Expand Up @@ -577,7 +572,7 @@ export class DependencyLinker {
if (!isDistDirExist) {
const newDir = getDistDirForDevEnv(packageName);
createSymlinkOrCopy(newDir, target);
return { from: newDir, to: target };
return { packageName, from: newDir, to: target };
}

try {
Expand All @@ -590,7 +585,7 @@ export class DependencyLinker {
return undefined;
}
createSymlinkOrCopy(resolvedPath, target);
return { from: resolvedPath, to: target };
return { packageName, from: resolvedPath, to: target };
} catch (err: any) {
throw new NonAspectCorePackageLinkError(err, packageName);
}
Expand Down

0 comments on commit b2434f5

Please sign in to comment.