Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat(cli): Optimise serving static assets #4182

Merged
merged 5 commits into from
Sep 29, 2022
Merged

feat(cli): Optimise serving static assets #4182

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
db6d0a5
simplify static server setup
netroy Sep 23, 2022
e0506db
use consistent ignoredEndpoints across all routes
netroy Sep 23, 2022
a32581a
remove unnecessary lodash usage
netroy Sep 23, 2022
710a1ea
stop keeping generated assets in memory
netroy Sep 23, 2022
d180cc8
pre-generate all static assets
netroy Sep 28, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 36 additions & 0 deletions package-lock.json
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions packages/cli/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@
"@types/parseurl": "^1.3.1",
"@types/passport-jwt": "^3.0.6",
"@types/psl": "^1.1.0",
"@types/send": "^0.17.1",
"@types/shelljs": "^0.8.11",
"@types/superagent": "4.1.13",
"@types/supertest": "^2.0.11",
Expand Down
125 changes: 41 additions & 84 deletions packages/cli/src/Server.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,14 +30,15 @@
/* eslint-disable no-await-in-loop */

import { exec as callbackExec } from 'child_process';
import { promises, readFileSync } from 'fs';
import { existsSync, readFileSync } from 'fs';
import { access as fsAccess, readFile, writeFile, mkdir } from 'fs/promises';
import os from 'os';
import { dirname as pathDirname, join as pathJoin, resolve as pathResolve } from 'path';
import { createHmac } from 'crypto';
import { promisify } from 'util';
import cookieParser from 'cookie-parser';
import express from 'express';
import _ from 'lodash';
import send from 'send';
import { FindManyOptions, getConnectionManager, In } from 'typeorm';
// eslint-disable-next-line import/no-extraneous-dependencies
import axios, { AxiosRequestConfig } from 'axios';
Expand Down Expand Up @@ -397,15 +398,12 @@ class App {
this.endpointWebhook,
this.endpointWebhookTest,
this.endpointPresetCredentials,
];
if (!config.getEnv('publicApi.disabled')) {
ignoredEndpoints.push(this.publicApiEndpoint);
}
// eslint-disable-next-line prefer-spread
ignoredEndpoints.push.apply(ignoredEndpoints, excludeEndpoints.split(':'));
config.getEnv('publicApi.disabled') ? this.publicApiEndpoint : '',
...excludeEndpoints.split(':'),
].filter((u) => !!u);

// eslint-disable-next-line no-useless-escape
const authIgnoreRegex = new RegExp(`^\/(${_(ignoredEndpoints).compact().join('|')})\/?.*$`);
const authIgnoreRegex = new RegExp(`^\/(${ignoredEndpoints.join('|')})\/?.*$`);

// Check for basic auth credentials if activated
const basicAuthActive = config.getEnv('security.basicAuth.active');
Expand Down Expand Up @@ -664,18 +662,7 @@ class App {
history({
rewrites: [
{
from: new RegExp(
`^/(${[
'healthz',
'metrics',
'css',
'js',
this.restEndpoint,
this.endpointWebhook,
this.endpointWebhookTest,
...(excludeEndpoints.length ? excludeEndpoints.split(':') : []),
].join('|')})/?.*$`,
),
from: new RegExp(`^/(${[this.restEndpoint, ...ignoredEndpoints].join('|')})/?.*$`),
to: (context) => {
return context.parsedUrl.pathname!.toString();
},
Expand Down Expand Up @@ -967,7 +954,7 @@ class App {
const headersPath = pathJoin(packagesPath, 'nodes-base', 'dist', 'nodes', 'headers');

try {
await promises.access(`${headersPath}.js`);
await fsAccess(`${headersPath}.js`);
} catch (_) {
return; // no headers available
}
Expand Down Expand Up @@ -1192,12 +1179,12 @@ class App {
timezone,
);

const signatureMethod = _.get(oauthCredentials, 'signatureMethod') as string;
const signatureMethod = oauthCredentials.signatureMethod as string;

const oAuthOptions: clientOAuth1.Options = {
consumer: {
key: _.get(oauthCredentials, 'consumerKey') as string,
secret: _.get(oauthCredentials, 'consumerSecret') as string,
key: oauthCredentials.consumerKey as string,
secret: oauthCredentials.consumerSecret as string,
},
signature_method: signatureMethod,
// eslint-disable-next-line @typescript-eslint/naming-convention
Expand All @@ -1220,7 +1207,7 @@ class App {

const options: RequestOptions = {
method: 'POST',
url: _.get(oauthCredentials, 'requestTokenUrl') as string,
url: oauthCredentials.requestTokenUrl as string,
data: oauthRequestData,
};

Expand All @@ -1237,9 +1224,7 @@ class App {

const responseJson = Object.fromEntries(paramsParser.entries());

const returnUri = `${_.get(oauthCredentials, 'authUrl')}?oauth_token=${
responseJson.oauth_token
}`;
const returnUri = `${oauthCredentials.authUrl}?oauth_token=${responseJson.oauth_token}`;

// Encrypt the data
const credentials = new Credentials(
Expand Down Expand Up @@ -1332,7 +1317,7 @@ class App {

const options: AxiosRequestConfig = {
method: 'POST',
url: _.get(oauthCredentials, 'accessTokenUrl') as string,
url: oauthCredentials.accessTokenUrl as string,
params: {
oauth_token,
oauth_verifier,
Expand Down Expand Up @@ -1753,35 +1738,11 @@ class App {

if (!config.getEnv('endpoints.disableUi')) {
// Read the index file and replace the path placeholder
const editorUiPath = require.resolve('n8n-editor-ui');
const filePath = pathJoin(pathDirname(editorUiPath), 'dist', 'index.html');
const n8nPath = config.getEnv('path');
const basePathRegEx = /\/{{BASE_PATH}}\//g;

let readIndexFile = readFileSync(filePath, 'utf8');
readIndexFile = readIndexFile.replace(basePathRegEx, n8nPath);
readIndexFile = readIndexFile.replace(/\/favicon.ico/g, `${n8nPath}favicon.ico`);

const cssPath = pathJoin(pathDirname(editorUiPath), 'dist', '**/*.css');
const cssFiles: Record<string, string> = {};
glob.sync(cssPath).forEach((filePath) => {
let readFile = readFileSync(filePath, 'utf8');
readFile = readFile.replace(basePathRegEx, n8nPath);
cssFiles[filePath.replace(pathJoin(pathDirname(editorUiPath), 'dist'), '')] = readFile;
});

const jsPath = pathJoin(pathDirname(editorUiPath), 'dist', '**/*.js');
const jsFiles: Record<string, string> = {};
glob.sync(jsPath).forEach((filePath) => {
let readFile = readFileSync(filePath, 'utf8');
readFile = readFile.replace(basePathRegEx, n8nPath);
jsFiles[filePath.replace(pathJoin(pathDirname(editorUiPath), 'dist'), '')] = readFile;
});

const hooksUrls = config.getEnv('externalFrontendHooksUrls');

let scriptsString = '';

if (hooksUrls) {
scriptsString = hooksUrls.split(';').reduce((acc, curr) => {
return `${acc}<script src="${curr}"></script>`;
Expand All @@ -1802,42 +1763,38 @@ class App {
scriptsString += phLoadingScript;
}

const editorUiDistDir = pathJoin(pathDirname(require.resolve('n8n-editor-ui')), 'dist');
const generatedStaticDir = pathJoin(__dirname, '../public');

const firstLinkedScriptSegment = '<link href="/js/';
readIndexFile = readIndexFile.replace(
firstLinkedScriptSegment,
scriptsString + firstLinkedScriptSegment,
);
const compileFile = async (fileName: string) => {
const filePath = pathJoin(editorUiDistDir, fileName);
if (/(index.html)|.*\.(js|css)/.test(filePath) && existsSync(filePath)) {
const srcFile = await readFile(filePath, 'utf8');
let payload = srcFile.replace(basePathRegEx, n8nPath);
if (filePath === 'index.html') {
payload = payload
.replace(/\/favicon.ico/g, `${n8nPath}favicon.ico`)
.replace(firstLinkedScriptSegment, scriptsString + firstLinkedScriptSegment);
}
const destFile = pathJoin(generatedStaticDir, fileName);
await mkdir(pathDirname(destFile), { recursive: true });
await writeFile(destFile, payload, 'utf-8');
}
};

// Serve the altered index.html file separately
this.app.get(`/index.html`, async (req: express.Request, res: express.Response) => {
res.send(readIndexFile);
});
await compileFile('index.html');
const files = await glob('**/*.{css,js}', { cwd: editorUiDistDir });
await Promise.all(files.map(compileFile));

this.app.get('/assets/*.css', async (req: express.Request, res: express.Response) => {
res.type('text/css').send(cssFiles[req.url]);
});
this.app.use('/', express.static(generatedStaticDir), express.static(editorUiDistDir));

this.app.get('/assets/*.js', async (req: express.Request, res: express.Response) => {
res.type('text/javascript').send(jsFiles[req.url]);
const startTime = new Date().toUTCString();
this.app.use('/index.html', (req, res, next) => {
res.setHeader('Last-Modified', startTime);
next();
});

// Serve the website
this.app.use(
'/',
express.static(pathJoin(pathDirname(editorUiPath), 'dist'), {
index: 'index.html',
setHeaders: (res, path) => {
if (res.req && res.req.url === '/index.html') {
// Set last modified date manually to n8n start time so
// that it hopefully refreshes the page when a new version
// got used
res.setHeader('Last-Modified', startTime);
}
},
}),
);
}
const startTime = new Date().toUTCString();
}
}

Expand Down