#714: Use the build cache to speed up the LS

arduino-ide-extension/package.json

@@ -155,7 +155,7 @@
   ],
   "arduino": {
     "cli": {
-      "version": "0.24.0"
+      "version": "0.25.0-rc1"
     },
     "fwuploader": {
       "version": "2.2.0"
@@ -164,7 +164,7 @@
       "version": "14.0.0"
     },
     "languageServer": {
-      "version": "0.6.0"
+      "version": "0.7.1"
     }
   }
 }

arduino-ide-extension/scripts/download-cli.js

@@ -1,141 +1,87 @@
 // @ts-check
 
 (async () => {
+  const path = require('path');
+  const shell = require('shelljs');
+  const semver = require('semver');
+  const moment = require('moment');
+  const downloader = require('./downloader');
+  const { goBuildFromGit } = require('./utils');
+
+  const version = (() => {
+    const pkg = require(path.join(__dirname, '..', 'package.json'));
+    if (!pkg) {
+      return undefined;
+    }
 
-    const fs = require('fs');
-    const path = require('path');
-    const temp = require('temp');
-    const shell = require('shelljs');
-    const semver = require('semver');
-    const moment = require('moment');
-    const downloader = require('./downloader');
-
-    const version = (() => {
-        const pkg = require(path.join(__dirname, '..', 'package.json'));
-        if (!pkg) {
-            return undefined;
-        }
+    const { arduino } = pkg;
+    if (!arduino) {
+      return undefined;
+    }
 
-        const { arduino } = pkg;
-        if (!arduino) {
-            return undefined;
-        }
+    const { cli } = arduino;
+    if (!cli) {
+      return undefined;
+    }
 
-        const { cli } = arduino;
-        if (!cli) {
-            return undefined;
+    const { version } = cli;
+    return version;
+  })();
+
+  if (!version) {
+    shell.echo(`Could not retrieve CLI version info from the 'package.json'.`);
+    shell.exit(1);
+  }
+
+  const { platform, arch } = process;
+  const buildFolder = path.join(__dirname, '..', 'build');
+  const cliName = `arduino-cli${platform === 'win32' ? '.exe' : ''}`;
+  const destinationPath = path.join(buildFolder, cliName);
+
+  if (typeof version === 'string') {
+    const suffix = (() => {
+      switch (platform) {
+        case 'darwin':
+          return 'macOS_64bit.tar.gz';
+        case 'win32':
+          return 'Windows_64bit.zip';
+        case 'linux': {
+          switch (arch) {
+            case 'arm':
+              return 'Linux_ARMv7.tar.gz';
+            case 'arm64':
+              return 'Linux_ARM64.tar.gz';
+            case 'x64':
+              return 'Linux_64bit.tar.gz';
+            default:
+              return undefined;
+          }
         }
-
-        const { version } = cli;
-        return version;
+        default:
+          return undefined;
+      }
     })();
-
-    if (!version) {
-        shell.echo(`Could not retrieve CLI version info from the 'package.json'.`);
-        shell.exit(1);
+    if (!suffix) {
+      shell.echo(`The CLI is not available for ${platform} ${arch}.`);
+      shell.exit(1);
     }
-
-    const { platform, arch } = process;
-    const buildFolder = path.join(__dirname, '..', 'build');
-    const cliName = `arduino-cli${platform === 'win32' ? '.exe' : ''}`;
-    const destinationPath = path.join(buildFolder, cliName);
-
-    if (typeof version === 'string') {
-        const suffix = (() => {
-            switch (platform) {
-                case 'darwin': return 'macOS_64bit.tar.gz';
-                case 'win32': return 'Windows_64bit.zip';
-                case 'linux': {
-                    switch (arch) {
-                        case 'arm': return 'Linux_ARMv7.tar.gz';
-                        case 'arm64': return 'Linux_ARM64.tar.gz';
-                        case 'x64': return 'Linux_64bit.tar.gz';
-                        default: return undefined;
-                    }
-                }
-                default: return undefined;
-            }
-        })();
-        if (!suffix) {
-            shell.echo(`The CLI is not available for ${platform} ${arch}.`);
-            shell.exit(1);
-        }
-        if (semver.valid(version)) {
-            const url = `https://downloads.arduino.cc/arduino-cli/arduino-cli_${version}_${suffix}`;
-            shell.echo(`📦  Identified released version of the CLI. Downloading version ${version} from '${url}'`);
-            await downloader.downloadUnzipFile(url, destinationPath, 'arduino-cli');
-        } else if (moment(version, 'YYYYMMDD', true).isValid()) {
-            const url = `https://downloads.arduino.cc/arduino-cli/nightly/arduino-cli_nightly-${version}_${suffix}`;
-            shell.echo(`🌙  Identified nightly version of the CLI. Downloading version ${version} from '${url}'`);
-            await downloader.downloadUnzipFile(url, destinationPath, 'arduino-cli');
-        } else {
-            shell.echo(`🔥  Could not interpret 'version': ${version}`);
-            shell.exit(1);
-        }
+    if (semver.valid(version)) {
+      const url = `https://downloads.arduino.cc/arduino-cli/arduino-cli_${version}_${suffix}`;
+      shell.echo(
+        `📦  Identified released version of the CLI. Downloading version ${version} from '${url}'`
+      );
+      await downloader.downloadUnzipFile(url, destinationPath, 'arduino-cli');
+    } else if (moment(version, 'YYYYMMDD', true).isValid()) {
+      const url = `https://downloads.arduino.cc/arduino-cli/nightly/arduino-cli_nightly-${version}_${suffix}`;
+      shell.echo(
+        `🌙  Identified nightly version of the CLI. Downloading version ${version} from '${url}'`
+      );
+      await downloader.downloadUnzipFile(url, destinationPath, 'arduino-cli');
     } else {
-
-        // We assume an object with `owner`, `repo`, commitish?` properties.
-        const { owner, repo, commitish } = version;
-        if (!owner) {
-            shell.echo(`Could not retrieve 'owner' from ${JSON.stringify(version)}`);
-            shell.exit(1);
-        }
-        if (!repo) {
-            shell.echo(`Could not retrieve 'repo' from ${JSON.stringify(version)}`);
-            shell.exit(1);
-        }
-        const url = `https://github.com/${owner}/${repo}.git`;
-        shell.echo(`Building CLI from ${url}. Commitish: ${commitish ? commitish : 'HEAD'}`);
-
-        if (fs.existsSync(destinationPath)) {
-            shell.echo(`Skipping the CLI build because it already exists: ${destinationPath}`);
-            return;
-        }
-
-        if (shell.mkdir('-p', buildFolder).code !== 0) {
-            shell.echo('Could not create build folder.');
-            shell.exit(1);
-        }
-
-        const tempRepoPath = temp.mkdirSync();
-        shell.echo(`>>> Cloning CLI source to ${tempRepoPath}...`);
-        if (shell.exec(`git clone ${url} ${tempRepoPath}`).code !== 0) {
-            shell.exit(1);
-        }
-        shell.echo('<<< Cloned CLI repo.')
-
-        if (commitish) {
-            shell.echo(`>>> Checking out ${commitish}...`);
-            if (shell.exec(`git -C ${tempRepoPath} checkout ${commitish}`).code !== 0) {
-                shell.exit(1);
-            }
-            shell.echo(`<<< Checked out ${commitish}.`);
-        }
-
-        shell.echo(`>>> Building the CLI...`);
-        if (shell.exec('go build', { cwd: tempRepoPath }).code !== 0) {
-            shell.exit(1);
-        }
-        shell.echo('<<< CLI build done.')
-
-        if (!fs.existsSync(path.join(tempRepoPath, cliName))) {
-            shell.echo(`Could not find the CLI at ${path.join(tempRepoPath, cliName)}.`);
-            shell.exit(1);
-        }
-
-        const builtCliPath = path.join(tempRepoPath, cliName);
-        shell.echo(`>>> Copying CLI from ${builtCliPath} to ${destinationPath}...`);
-        if (shell.cp(builtCliPath, destinationPath).code !== 0) {
-            shell.exit(1);
-        }
-        shell.echo(`<<< Copied the CLI.`);
-
-        shell.echo('<<< Verifying CLI...');
-        if (!fs.existsSync(destinationPath)) {
-            shell.exit(1);
-        }
-        shell.echo('>>> Verified CLI.');
-
+      shell.echo(`🔥  Could not interpret 'version': ${version}`);
+      shell.exit(1);
     }
-
+  } else {
+    goBuildFromGit(version, destinationPath, 'CLI');
+  }
 })();

arduino-ide-extension/scripts/download-ls.js

@@ -7,22 +7,23 @@
   const path = require('path');
   const shell = require('shelljs');
   const downloader = require('./downloader');
+  const { goBuildFromGit } = require('./utils');
 
-  const [DEFAULT_ALS_VERSION, DEFAULT_CLANGD_VERSION] = (() => {
+  const [DEFAULT_LS_VERSION, DEFAULT_CLANGD_VERSION] = (() => {
     const pkg = require(path.join(__dirname, '..', 'package.json'));
-    if (!pkg) return undefined;
+    if (!pkg) return [undefined, undefined];
 
     const { arduino } = pkg;
-    if (!arduino) return undefined;
+    if (!arduino) return [undefined, undefined];
 
     const { languageServer, clangd } = arduino;
-    if (!languageServer) return undefined;
-    if (!clangd) return undefined;
+    if (!languageServer) return [undefined, undefined];
+    if (!clangd) return [undefined, undefined];
 
     return [languageServer.version, clangd.version];
   })();
 
-  if (!DEFAULT_ALS_VERSION) {
+  if (!DEFAULT_LS_VERSION) {
     shell.echo(
       `Could not retrieve Arduino Language Server version info from the 'package.json'.`
     );
@@ -39,8 +40,8 @@
   const yargs = require('yargs')
     .option('ls-version', {
       alias: 'lv',
-      default: DEFAULT_ALS_VERSION,
-      describe: `The version of the 'arduino-language-server' to download. Defaults to ${DEFAULT_ALS_VERSION}.`,
+      default: DEFAULT_LS_VERSION,
+      describe: `The version of the 'arduino-language-server' to download. Defaults to ${DEFAULT_LS_VERSION}.`,
     })
     .option('clangd-version', {
       alias: 'cv',
@@ -56,7 +57,7 @@
     .version(false)
     .parse();
 
-  const alsVersion = yargs['ls-version'];
+  const lsVersion = yargs['ls-version'];
   const clangdVersion = yargs['clangd-version'];
   const force = yargs['force-download'];
   const { platform, arch } = process;
@@ -87,6 +88,8 @@
       lsSuffix = 'Windows_64bit.zip';
       clangdSuffix = 'Windows_64bit';
       break;
+    default:
+      throw new Error(`Unsupported platform/arch: ${platformArch}.`);
   }
   if (!lsSuffix || !clangdSuffix) {
     shell.echo(
@@ -95,12 +98,16 @@
     shell.exit(1);
   }
 
-  const alsUrl = `https://downloads.arduino.cc/arduino-language-server/${
-    alsVersion === 'nightly'
-      ? 'nightly/arduino-language-server'
-      : 'arduino-language-server_' + alsVersion
-  }_${lsSuffix}`;
-  downloader.downloadUnzipAll(alsUrl, build, lsExecutablePath, force);
+  if (typeof lsVersion === 'string') {
+    const lsUrl = `https://downloads.arduino.cc/arduino-language-server/${
+      lsVersion === 'nightly'
+        ? 'nightly/arduino-language-server'
+        : 'arduino-language-server_' + lsVersion
+    }_${lsSuffix}`;
+    downloader.downloadUnzipAll(lsUrl, build, lsExecutablePath, force);
+  } else {
+    goBuildFromGit(lsVersion, lsExecutablePath, 'language-server');
+  }
 
   const clangdUrl = `https://downloads.arduino.cc/tools/clangd_${clangdVersion}_${clangdSuffix}.tar.bz2`;
   downloader.downloadUnzipAll(clangdUrl, build, clangdExecutablePath, force, {

arduino-ide-extension/scripts/downloader.js

@@ -86,6 +86,7 @@ exports.downloadUnzipFile = async (
  * @param targetDir {string}  Directory into which to decompress the archive
  * @param targetFile {string} Path to the main file expected after decompressing
  * @param force {boolean}     Whether to download even if the target file exists
+ * @param decompressOptions {import('decompress').DecompressOptions}
  */
 exports.downloadUnzipAll = async (
   url,