Skip to content

Commit

Permalink
Ability to store a built package in offline mirror (#5314)
Browse files Browse the repository at this point in the history 
* WIP: ability to store a built package in offline mirror

* added tests

* more tests

* flow fixes and tests

* lockfile updates

* fixed more tests

* feedback

* fixed test

* feedback
  • Loading branch information
@bestander
bestander authored Feb 22, 2018
1 parent 0fb0fc0 commit eab88b8
Show file tree
Hide file tree
Showing 23 changed files with 483 additions and 191 deletions.
166 changes: 0 additions & 166 deletions __tests__/commands/install/integration.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ import {run as cache} from '../../../src/cli/commands/cache.js';
import {run as check} from '../../../src/cli/commands/check.js';
import * as constants from '../../../src/constants.js';
import * as reporters from '../../../src/reporters/index.js';
import {parse} from '../../../src/lockfile';
import {Install, run as install} from '../../../src/cli/commands/install.js';
import Lockfile from '../../../src/lockfile';
import * as fs from '../../../src/util/fs.js';
Expand All @@ -16,7 +15,6 @@ import {getPackageVersion, explodeLockfile, runInstall, runLink, createLockfile,
jasmine.DEFAULT_TIMEOUT_INTERVAL = 150000;

let request = require('request');
const semver = require('semver');
const path = require('path');
const stream = require('stream');

Expand Down Expand Up @@ -145,22 +143,6 @@ test('reading a lockfile should not optimize it', async () => {
});
});

test('creates the file in the mirror when fetching a git repository', async () => {
await runInstall({}, 'install-git', async (config, reporter): Promise<void> => {
const lockfile = await Lockfile.fromDirectory(config.cwd);

expect(await fs.glob('example-yarn-package.git-*', {cwd: `${config.cwd}/offline-mirror`})).toHaveLength(1);

await fs.unlink(path.join(config.cwd, 'offline-mirror'));
await fs.unlink(path.join(config.cwd, 'node_modules'));

const firstReinstall = new Install({}, config, reporter, lockfile);
await firstReinstall.init();

expect(await fs.glob('example-yarn-package.git-*', {cwd: `${config.cwd}/offline-mirror`})).toHaveLength(1);
});
});

test.concurrent('creates a symlink to a directory when using the link: protocol', async () => {
await runInstall({}, 'install-link', async (config): Promise<void> => {
const expectPath = path.join(config.cwd, 'node_modules', 'test-absolute');
Expand Down Expand Up @@ -570,24 +552,6 @@ test.concurrent('install renamed packages', (): Promise<void> => {
});
});

test.concurrent('install from offline mirror', (): Promise<void> => {
return runInstall({}, 'install-from-offline-mirror', async (config): Promise<void> => {
const allFiles = await fs.walk(config.cwd);

expect(
allFiles.findIndex((file): boolean => {
return file.relative === path.join('node_modules', 'fake-dependency', 'package.json');
}),
).toBeGreaterThanOrEqual(0);

expect(
allFiles.findIndex((file): boolean => {
return file.relative === path.join('node_modules', '@fakescope', 'fake-dependency', 'package.json');
}),
).toBeGreaterThanOrEqual(0);
});
});

test.concurrent('install from git cache', (): Promise<void> => {
return runInstall({}, 'install-from-git-cache', async (config): Promise<void> => {
expect(await getPackageVersion(config, 'dep-a')).toEqual('0.0.1');
Expand Down Expand Up @@ -712,26 +676,6 @@ test.concurrent('install should be idempotent', (): Promise<void> => {
});
});

test.concurrent('install should add missing deps to yarn and mirror (PR import scenario)', (): Promise<void> => {
return runInstall({}, 'install-import-pr', async config => {
expect(await getPackageVersion(config, 'mime-types')).toEqual('2.0.0');
expect(semver.satisfies(await getPackageVersion(config, 'mime-db'), '~1.0.1')).toEqual(true);
expect(await getPackageVersion(config, 'fake-yarn-dependency')).toEqual('1.0.1');

const mirror = await fs.walk(path.join(config.cwd, 'mirror-for-offline'));
expect(mirror).toHaveLength(3);
expect(mirror[0].relative).toEqual('fake-yarn-dependency-1.0.1.tgz');
expect(mirror[1].relative.indexOf('mime-db-1.0.')).toEqual(0);
expect(mirror[2].relative).toEqual('mime-types-2.0.0.tgz');

const lockFileContent = await fs.readFile(path.join(config.cwd, 'yarn.lock'));
const lockFileLines = explodeLockfile(lockFileContent);
expect(lockFileLines).toHaveLength(11);
expect(lockFileLines[3].indexOf('mime-db@')).toEqual(0);
expect(lockFileLines[6].indexOf('mime-types@2.0.0')).toEqual(0);
});
});

test.concurrent('install should update checksums in yarn.lock (--update-checksums)', (): Promise<void> => {
const packageRealHash = '5faad9c2c07f60dd76770f71cf025b62a63cfd4e';
const packageCacheName = `npm-abab-1.0.4-${packageRealHash}`;
Expand All @@ -747,47 +691,6 @@ test.concurrent('install should update checksums in yarn.lock (--update-checksum
});
});

test.concurrent('install should update a dependency to yarn and mirror (PR import scenario 2)', (): Promise<void> => {
// mime-types@2.0.0 is gets updated to mime-types@2.1.11 via
// a change in package.json,
// files in mirror, yarn.lock, package.json and node_modules should reflect that

return runInstall({}, 'install-import-pr-2', async (config, reporter): Promise<void> => {
expect(semver.satisfies(await getPackageVersion(config, 'mime-db'), '~1.0.1')).toEqual(true);

expect(await getPackageVersion(config, 'mime-types')).toEqual('2.0.0');

await fs.copy(path.join(config.cwd, 'package.json.after'), path.join(config.cwd, 'package.json'), reporter);

const reinstall = new Install({}, config, reporter, await Lockfile.fromDirectory(config.cwd));
await reinstall.init();

expect(semver.satisfies(await getPackageVersion(config, 'mime-db'), '~1.23.0')).toEqual(true);

expect(await getPackageVersion(config, 'mime-types')).toEqual('2.1.11');

const lockFileWritten = await fs.readFile(path.join(config.cwd, 'yarn.lock'));
const lockFileLines = explodeLockfile(lockFileWritten);

expect(lockFileLines[0]).toEqual('mime-db@~1.23.0:');
expect(lockFileLines[2]).toMatch(/resolved "https:\/\/registry\.yarnpkg\.com\/mime-db\/-\/mime-db-/);
expect(lockFileLines[3]).toEqual('mime-types@2.1.11:');
expect(lockFileLines[5]).toMatch(
/resolved "https:\/\/registry\.yarnpkg\.com\/mime-types\/-\/mime-types-2\.1\.11\.tgz#[a-f0-9]+"/,
);

const mirror = await fs.walk(path.join(config.cwd, 'mirror-for-offline'));
expect(mirror).toHaveLength(4);

const newFilesInMirror = mirror.filter((elem): boolean => {
return elem.relative !== 'mime-db-1.0.3.tgz' && elem.relative !== 'mime-types-2.0.0.tgz';
});

expect(newFilesInMirror).toHaveLength(2);
});
});

if (process.platform !== 'win32') {
// TODO: This seems like a real issue, not just a config issue
test.concurrent('install cache symlinks properly', (): Promise<void> => {
Expand All @@ -805,51 +708,6 @@ if (process.platform !== 'win32') {
});
}

test.concurrent('offline mirror can be enabled from parent dir', (): Promise<void> => {
const fixture = {
source: 'offline-mirror-configuration',
cwd: 'enabled-from-parent',
};
return runInstall({}, fixture, async (config, reporter) => {
const rawLockfile = await fs.readFile(path.join(config.cwd, 'yarn.lock'));
const {object: lockfile} = parse(rawLockfile);
expect(lockfile['mime-types@2.1.14'].resolved).toEqual(
'https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.14.tgz#f7ef7d97583fcaf3b7d282b6f8b5679dab1e94ee',
);
expect(await fs.exists(path.join(config.cwd, '../offline-mirror/mime-types-2.1.14.tgz'))).toBe(true);
});
});

test.concurrent('offline mirror can be enabled from parent dir, with merging of own .yarnrc', (): Promise<void> => {
const fixture = {
source: 'offline-mirror-configuration',
cwd: 'enabled-from-parent-merge',
};
return runInstall({}, fixture, async (config, reporter) => {
const rawLockfile = await fs.readFile(path.join(config.cwd, 'yarn.lock'));
const {object: lockfile} = parse(rawLockfile);
expect(lockfile['mime-types@2.1.14'].resolved).toEqual(
'https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.14.tgz#f7ef7d97583fcaf3b7d282b6f8b5679dab1e94ee',
);
expect(await fs.exists(path.join(config.cwd, '../offline-mirror/mime-types-2.1.14.tgz'))).toBe(true);
});
});

test.concurrent('offline mirror can be disabled locally', (): Promise<void> => {
const fixture = {
source: 'offline-mirror-configuration',
cwd: 'disabled-locally',
};
return runInstall({}, fixture, async (config, reporter) => {
const rawLockfile = await fs.readFile(path.join(config.cwd, 'yarn.lock'));
const {object: lockfile} = parse(rawLockfile);
expect(lockfile['mime-types@2.1.14'].resolved).toEqual(
'https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.14.tgz#f7ef7d97583fcaf3b7d282b6f8b5679dab1e94ee',
);
expect(await fs.exists(path.join(config.cwd, '../offline-mirror/mime-types-2.1.14.tgz'))).toBe(false);
});
});

// sync test because we need to get all the requests to confirm their validity
test('install a scoped module from authed private registry', (): Promise<void> => {
return runInstall({}, 'install-from-authed-private-registry', async config => {
Expand Down Expand Up @@ -1022,30 +880,6 @@ test.concurrent('should install if symlink source does not exist', async (): Pro
await runInstall({}, 'relative-symlinks-work', () => {});
});

test.concurrent('prunes the offline mirror tarballs after pruning is enabled', (): Promise<void> => {
return runInstall({}, 'prune-offline-mirror', async (config): Promise<void> => {
const mirrorPath = 'mirror-for-offline';
// Scenario:
// dep-a 1.0.0 was originally installed, and it depends on dep-b 1.0.0, so
// both of these were added to the offline mirror. Then dep-a was upgraded
// to 1.1.0 which doesn't depend on dep-b. After this, pruning was enabled,
// so the next install should remove dep-a-1.0.0.tgz and dep-b-1.0.0.tgz.
expect(await fs.exists(path.join(config.cwd, `${mirrorPath}/dep-a-1.0.0.tgz`))).toEqual(false);
expect(await fs.exists(path.join(config.cwd, `${mirrorPath}/dep-b-1.0.0.tgz`))).toEqual(false);
expect(await fs.exists(path.join(config.cwd, `${mirrorPath}/dummy.txt`))).toEqual(true);
});
});

test.concurrent('scoped packages remain in offline mirror after pruning is enabled', (): Promise<void> => {
return runInstall({}, 'prune-offline-mirror-scoped', async (config): Promise<void> => {
const mirrorPath = 'mirror-for-offline';
// scoped package exists
expect(await fs.exists(path.join(config.cwd, `${mirrorPath}/@fakescope-fake-dependency-1.0.1.tgz`))).toEqual(true);
// unscoped package exists
expect(await fs.exists(path.join(config.cwd, `${mirrorPath}/fake-dependency-1.0.1.tgz`))).toEqual(true);
});
});

test.concurrent('bailout should work with --production flag too', (): Promise<void> => {
return runInstall({production: true}, 'bailout-prod', async (config, reporter): Promise<void> => {
// remove file
Expand Down
Loading

0 comments on commit eab88b8

Please sign in to comment.