Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Bundle Node 14 as a fallback for operating systems that cannot run Node 18 #4151

Merged
merged 1 commit into from
May 30, 2023
Merged

Bundle Node 14 as a fallback for operating systems that cannot run Node 18 #4151

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
- [Table Visualization] Move format table, consolidate types and add unit tests ([#3397](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3397))
- [Multiple Datasource] Support Amazon OpenSearch Serverless ([#3957](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3957))
- Add support for Node.js >=14.20.1 <19 ([#4071](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/4071))
- Bundle Node.js 14 as a fallback for operating systems that cannot run Node.js 18 ([#4151](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/4151))

### 🐛 Bug Fixes

Expand Down
10 changes: 8 additions & 2 deletions scripts/use_node
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,11 +57,17 @@ elif [ ! -z "$NODE_HOME" ]; then
NODE_ERROR_MSG="in NODE_HOME"
NODE_ERROR_SHOW=true
else
# Set these variables outside, as catchalls, to show meaningful errors if needed
NODE="$OSD_HOME/node/bin/node"
NODE_ERROR_MSG="bundled with OpenSearch Dashboards"
# A bin folder at the root is only present in release builds that have a bundled Node.js binary
if [ -x "OSD_HOME/bin" ]; then
if [ -d "${OSD_HOME}/bin" ]; then
NODE_ERROR_SHOW=true
# Not all operating systems can run the latest Node.js and the fallback is for them
"${NODE}" -v 2> /dev/null
if [ $? -ne 0 ] && [ -d "${OSD_HOME}/node/fallback" ]; then
NODE="$OSD_HOME/node/fallback/bin/node"
fi
fi
fi

Expand All @@ -75,7 +81,7 @@ else
fi

if [ ! -x "$NODE" ]; then
# Irrespective of NODE_ERROR_SHOW, show the error
# Irrespective of NODE_ERROR_SHOW, if NODE is not found or executable, show the error
echo "Could not find a Node.js runtime binary $NODE_ERROR_MSG or on the system" >&2
exit 1
fi
Expand Down
16 changes: 15 additions & 1 deletion src/dev/build/tasks/create_archives_sources_task.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
*/

import { scanCopy, Task } from '../lib';
import { getNodeDownloadInfo } from './nodejs';
import { getNodeDownloadInfo, getNodeVersionDownloadInfo, NODE14_FALLBACK_VERSION } from './nodejs';

export const CreateArchivesSources: Task = {
description: 'Creating platform-specific archive source directories',
Expand All @@ -54,6 +54,20 @@ export const CreateArchivesSources: Task = {
destination: build.resolvePathForPlatform(platform, 'node'),
});

// ToDo [NODE14]: Remove this Node.js 14 fallback download
// Copy the Node.js 14 binaries into node/fallback to be used by `use_node`
await scanCopy({
source: (
await getNodeVersionDownloadInfo(
NODE14_FALLBACK_VERSION,
platform.getNodeArch(),
platform.isWindows(),
config.resolveFromRepo()
)
).extractDir,
destination: build.resolvePathForPlatform(platform, 'node', 'fallback'),
});

log.debug('Node.js copied into', platform.getNodeArch(), 'specific build directory');
})
);
Expand Down
50 changes: 49 additions & 1 deletion src/dev/build/tasks/nodejs/download_node_builds_task.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,9 @@ jest.mock('../../lib/get_build_number');

expect.addSnapshotSerializer(createAnyInstanceSerializer(ToolingLog));

const { getNodeDownloadInfo } = jest.requireMock('./node_download_info');
const { getNodeDownloadInfo, getNodeVersionDownloadInfo } = jest.requireMock(
'./node_download_info'
);
const { getNodeShasums } = jest.requireMock('./node_shasums');
const { download } = jest.requireMock('../../lib/download');

Expand Down Expand Up @@ -76,6 +78,16 @@ async function setup({ failOnUrl }: { failOnUrl?: string } = {}) {
};
});

getNodeVersionDownloadInfo.mockImplementation((version, architecture, isWindows, repoRoot) => {
return {
url: `https://mirrors.nodejs.org/dist/v${version}/node-v${version}-${architecture}.tar.gz`,
downloadName: `node-v${version}-${architecture}.tar.gz`,
downloadPath: `/mocked/path/.node_binaries/${version}/node-v${version}-${architecture}.tar.gz`,
extractDir: `/mocked/path/.node_binaries/${version}/${architecture}`,
version,
};
});

getNodeShasums.mockReturnValue({
'linux:downloadName': 'linux:sha256',
'darwin:downloadName': 'darwin:sha256',
Expand Down Expand Up @@ -134,6 +146,42 @@ it('downloads node builds for each platform', async () => {
"url": "win32:url",
},
],
Array [
Object {
"destination": "/mocked/path/.node_binaries/14.21.3/node-v14.21.3-linux-x64.tar.gz",
"log": <ToolingLog>,
"retries": 3,
"sha256": undefined,
"url": "https://mirrors.nodejs.org/dist/v14.21.3/node-v14.21.3-linux-x64.tar.gz",
},
],
Array [
Object {
"destination": "/mocked/path/.node_binaries/14.21.3/node-v14.21.3-linux-arm64.tar.gz",
"log": <ToolingLog>,
"retries": 3,
"sha256": undefined,
"url": "https://mirrors.nodejs.org/dist/v14.21.3/node-v14.21.3-linux-arm64.tar.gz",
},
],
Array [
Object {
"destination": "/mocked/path/.node_binaries/14.21.3/node-v14.21.3-darwin-x64.tar.gz",
"log": <ToolingLog>,
"retries": 3,
"sha256": undefined,
"url": "https://mirrors.nodejs.org/dist/v14.21.3/node-v14.21.3-darwin-x64.tar.gz",
},
],
Array [
Object {
"destination": "/mocked/path/.node_binaries/14.21.3/node-v14.21.3-win32-x64.tar.gz",
"log": <ToolingLog>,
"retries": 3,
"sha256": undefined,
"url": "https://mirrors.nodejs.org/dist/v14.21.3/node-v14.21.3-win32-x64.tar.gz",
},
],
]
`);
expect(testWriter.messages).toMatchInlineSnapshot(`Array []`);
Expand Down
35 changes: 30 additions & 5 deletions src/dev/build/tasks/nodejs/download_node_builds_task.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,16 +30,25 @@

import { download, GlobalTask } from '../../lib';
import { getNodeShasums } from './node_shasums';
import { getNodeDownloadInfo, getRequiredVersion } from './node_download_info';
import {
getNodeDownloadInfo,
getNodeVersionDownloadInfo,
getRequiredVersion,
NODE14_FALLBACK_VERSION,
} from './node_download_info';

export const DownloadNodeBuilds: GlobalTask = {
global: true,
description: 'Downloading node.js builds for all platforms',
async run(config, log) {
const requiredNodeVersion = getRequiredVersion(config);
const shasums = await getNodeShasums(log, requiredNodeVersion);
await Promise.all(
config.getTargetPlatforms().map(async (platform) => {

// ToDo [NODE14]: Remove this Node.js 14 fallback download
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wonder if there would be a slight improvement to not even attempt to download node 14 if the the current node version fails as well (for platform availability reasons).

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

At this stage we won't know since it is a redistributable package that we build here.

const node14ShaSums = await getNodeShasums(log, NODE14_FALLBACK_VERSION);

await Promise.all([
...config.getTargetPlatforms().map(async (platform) => {
const { url, downloadPath, downloadName } = await getNodeDownloadInfo(config, platform);
await download({
log,
Expand All @@ -48,7 +57,23 @@ export const DownloadNodeBuilds: GlobalTask = {
destination: downloadPath,
retries: 3,
});
})
);
}),
// ToDo [NODE14]: Remove this Node.js 14 fallback download
...config.getTargetPlatforms().map(async (platform) => {
const { url, downloadPath, downloadName } = await getNodeVersionDownloadInfo(
NODE14_FALLBACK_VERSION,
platform.getNodeArch(),
platform.isWindows(),
config.resolveFromRepo()
);
await download({
log,
url,
sha256: node14ShaSums[downloadName],
destination: downloadPath,
retries: 3,
});
}),
]);
},
};
28 changes: 28 additions & 0 deletions src/dev/build/tasks/nodejs/extract_node_builds_task.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,27 @@ it('runs expected fs operations', async () => {
"strip": 1,
},
],
Array [
<absolute path>/.node_binaries/14.21.3/node-v14.21.3-linux-x64.tar.gz,
<absolute path>/.node_binaries/14.21.3/linux-x64,
Object {
"strip": 1,
},
],
Array [
<absolute path>/.node_binaries/14.21.3/node-v14.21.3-linux-arm64.tar.gz,
<absolute path>/.node_binaries/14.21.3/linux-arm64,
Object {
"strip": 1,
},
],
Array [
<absolute path>/.node_binaries/14.21.3/node-v14.21.3-darwin-x64.tar.gz,
<absolute path>/.node_binaries/14.21.3/darwin-x64,
Object {
"strip": 1,
},
],
],
"unzip": Array [
Array [
Expand All @@ -132,6 +153,13 @@ it('runs expected fs operations', async () => {
"strip": 1,
},
],
Array [
<absolute path>/.node_binaries/14.21.3/node-v14.21.3-win-x64.zip,
<absolute path>/.node_binaries/14.21.3/win32-x64,
Object {
"strip": 1,
},
],
],
}
`);
Expand Down
28 changes: 23 additions & 5 deletions src/dev/build/tasks/nodejs/extract_node_builds_task.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,21 +29,39 @@
*/

import { untar, unzip, GlobalTask } from '../../lib';
import { getNodeDownloadInfo } from './node_download_info';
import {
getNodeDownloadInfo,
getNodeVersionDownloadInfo,
NODE14_FALLBACK_VERSION,
} from './node_download_info';

export const ExtractNodeBuilds: GlobalTask = {
global: true,
description: 'Extracting node.js builds for all platforms',
async run(config) {
await Promise.all(
config.getTargetPlatforms().map(async (platform) => {
await Promise.all([
...config.getTargetPlatforms().map(async (platform) => {
const { downloadPath, extractDir } = await getNodeDownloadInfo(config, platform);
if (platform.isWindows()) {
await unzip(downloadPath, extractDir, { strip: 1 });
} else {
await untar(downloadPath, extractDir, { strip: 1 });
}
})
);
}),
// ToDo [NODE14]: Remove this Node.js 14 fallback download
...config.getTargetPlatforms().map(async (platform) => {
const { downloadPath, extractDir } = await getNodeVersionDownloadInfo(
NODE14_FALLBACK_VERSION,
platform.getNodeArch(),
platform.isWindows(),
config.resolveFromRepo()
);
if (platform.isWindows()) {
await unzip(downloadPath, extractDir, { strip: 1 });
} else {
await untar(downloadPath, extractDir, { strip: 1 });
}
}),
]);
},
};
27 changes: 26 additions & 1 deletion src/dev/build/tasks/nodejs/node_download_info.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,14 +28,16 @@
* under the License.
*/

import { basename } from 'path';
import { basename, resolve } from 'path';
import fetch from 'node-fetch';
import semver from 'semver';

import { Config, Platform } from '../../lib';

const NODE_RANGE_CACHE: { [key: string]: string } = {};

export const NODE14_FALLBACK_VERSION = '14.21.3';
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

would it be over-engineering a .node_version_14 ?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I agree that we would rather have one source of truth. Let's do a quick-follow on this.


export async function getNodeDownloadInfo(config: Config, platform: Platform) {
const version = getRequiredVersion(config);
const arch = platform.getNodeArch();
Expand All @@ -57,6 +59,29 @@ export async function getNodeDownloadInfo(config: Config, platform: Platform) {
};
}

export async function getNodeVersionDownloadInfo(
version: string,
architecture: string,
isWindows: boolean,
repoRoot: string
) {
const downloadName = isWindows
? `node-v${version}-win-x64.zip`
: `node-v${version}-${architecture}.tar.gz`;

const url = `https://mirrors.nodejs.org/dist/v${version}/${downloadName}`;
const downloadPath = resolve(repoRoot, '.node_binaries', version, basename(downloadName));
const extractDir = resolve(repoRoot, '.node_binaries', version, architecture);

return {
url,
downloadName,
downloadPath,
extractDir,
version,
};
}

export async function getLatestNodeVersion(config: Config) {
const range = config.getNodeRange();
// Check cache and return if known
Expand Down