From c6bb560d3205a3ef2686e6df06b7d0ccbecaed05 Mon Sep 17 00:00:00 2001 From: Florent Beauchamp Date: Tue, 28 Dec 2021 11:39:42 +0100 Subject: [PATCH] feat(xo-proxy): implement reverse proxy + tests --- @xen-orchestra/proxy/package.json | 1 + @xen-orchestra/proxy/src/app/mixins/api.mjs | 20 ++-- .../proxy/src/app/mixins/reverseProxy.mjs | 109 ++++++++++++++++++ @xen-orchestra/proxy/tests/cert.pem | 19 +++ @xen-orchestra/proxy/tests/key.pem | 27 +++++ @xen-orchestra/proxy/tests/localServer.mjs | 66 +++++++++++ .../proxy/tests/reverseProxy.unit.spec.mjs | 84 ++++++++++++++ CHANGELOG.unreleased.md | 1 + yarn.lock | 5 + 9 files changed, 323 insertions(+), 9 deletions(-) create mode 100644 @xen-orchestra/proxy/src/app/mixins/reverseProxy.mjs create mode 100644 @xen-orchestra/proxy/tests/cert.pem create mode 100644 @xen-orchestra/proxy/tests/key.pem create mode 100644 @xen-orchestra/proxy/tests/localServer.mjs create mode 100644 @xen-orchestra/proxy/tests/reverseProxy.unit.spec.mjs diff --git a/@xen-orchestra/proxy/package.json b/@xen-orchestra/proxy/package.json index 913de3c9e03..cc748e835ca 100644 --- a/@xen-orchestra/proxy/package.json +++ b/@xen-orchestra/proxy/package.json @@ -46,6 +46,7 @@ "getopts": "^2.2.3", "golike-defer": "^0.5.1", "http-server-plus": "^0.11.0", + "http2-proxy": "^5.0.53", "json-rpc-protocol": "^0.13.1", "jsonrpc-websocket-client": "^0.7.2", "koa": "^2.5.1", diff --git a/@xen-orchestra/proxy/src/app/mixins/api.mjs b/@xen-orchestra/proxy/src/app/mixins/api.mjs index a79b631935e..6d9b4753cb6 100644 --- a/@xen-orchestra/proxy/src/app/mixins/api.mjs +++ b/@xen-orchestra/proxy/src/app/mixins/api.mjs @@ -45,8 +45,8 @@ export default class Api { constructor(app, { appVersion, httpServer }) { this._ajv = new Ajv({ allErrors: true }) this._methods = { __proto__: null } - this.PREFIX_URL = '/api/v1' - const router = new Router({ prefix: this.PREFIX_URL }).post('/', async ctx => { + const PREFIX = '/api/v1' + const router = new Router({ prefix: PREFIX }).post('/', async ctx => { // Before Node 13.0 there was an inactivity timeout of 2 mins, which may // not be enough for the API. ctx.req.setTimeout(0) @@ -102,6 +102,7 @@ export default class Api { // breaks, send some data every 10s to keep it opened. const stopTimer = clearInterval.bind( undefined, + // @to check : can this add space inside binary data ? setInterval(() => stream.push(' '), keepAliveInterval) ) stream.on('end', stopTimer).on('error', stopTimer) @@ -113,18 +114,19 @@ export default class Api { const koa = new Koa() .on('error', warn) - // only answers to query to the root url of this mixin - .use( async(ctx, next)=>{ - if(ctx.req.url.startsWith(this.PREFIX_URL)){ - await next() - } - }) .use(helmet()) .use(compress()) .use(router.routes()) .use(router.allowedMethods()) - httpServer.on('request', koa.callback()) + const callback = koa.callback() + httpServer.on('request', (req, res)=>{ + // only answers to query to the root url of this mixin + // do it before giving the request to Koa to ensure it's not modified + if(req.url.startsWith(PREFIX)){ + callback(req, res) + } + } ) this.addMethods({ system: { diff --git a/@xen-orchestra/proxy/src/app/mixins/reverseProxy.mjs b/@xen-orchestra/proxy/src/app/mixins/reverseProxy.mjs new file mode 100644 index 00000000000..5e7a3c0f4a4 --- /dev/null +++ b/@xen-orchestra/proxy/src/app/mixins/reverseProxy.mjs @@ -0,0 +1,109 @@ +import { createLogger } from '@xen-orchestra/log' +import { urlToHttpOptions } from 'url' //node 14.18.0, 15.7.0 is it ok ? +import proxy from 'http2-proxy' + +const RE_ABSOLUTE_URL = new RegExp('^(?:[a-z]+:)?//', 'i') + +const { debug, warn } = createLogger('xo:proxy:reverse-proxy') + +function removeSlash(str){ + return str.replace(/^\/|\/$/g, '') +} + +function mergeUrl(relative, base){ + const res = new URL(base) + const relativeUrl = new URL(relative, base) + res.pathname = relativeUrl.pathname + relativeUrl.searchParams.forEach((value, name)=>{ + // we do not allow to modify params already specified by config + if(!res.searchParams.has(name)){ + res.searchParams.append(name, value) + } + }) + res.hash = relativeUrl.hash.length > 0 ? relativeUrl.hash: res.hash + return res +} +export default class ReverseProxy { + constructor(app, { httpServer }) { + this.httpServer = httpServer + this.app = app + this.configs = {} + for(const [proxyId, {path, target, ...options}] of Object.entries(app.config.get('reverseProxies'))){ + this.configs[proxyId] = { + path: '/proxy/v1/'+removeSlash(path), + target: new URL(target), + options + } + this.setupOneProxy(proxyId) + } + } + + localToBackendUrl(proxyId, localPath){ + const { path:basePath, target } = this.configs[proxyId] + let localPathWithoutBase = removeSlash(localPath).substring(basePath.length) + localPathWithoutBase = './'+removeSlash(localPathWithoutBase) + const url = mergeUrl(localPathWithoutBase, target) + return url + } + + backendToLocalPath(proxyId, backendUrl){ + const { path:basePath, target } = this.configs[proxyId] + + // keep redirect url relative to local server + const localPath = `${basePath}/${backendUrl.pathname.substring(target.pathname.length)}${backendUrl.search}${backendUrl.hash}` + return localPath + } + + proxy(proxyId, req,res){ + const {path, target, options} = this.configs[proxyId] + const url = new URL(target) + if(!req.url.startsWith(path+'/')){ + return + } + const targetUrl = this.localToBackendUrl(proxyId, req.originalUrl || req.url) + proxy.web(req, res, { + ...urlToHttpOptions(targetUrl), + ...options, + onRes: (err, req, proxyRes)=>{ + // rewrite redirect to pass through this proxy + if(proxyRes.statusCode === 301 || proxyRes.statusCode === 302){ + // handle relative/ absolute location + const redirectTargetLocation = new URL(proxyRes.headers.location, url) + + // this proxy should only allow communication between known hosts. Don't open it too much + if(redirectTargetLocation.hostname !== url.hostname || redirectTargetLocation.protocol !== url.protocol){ + throw new Error(`Can't redirect from ${url.hostname} to ${redirectTargetLocation.hostname} `) + } + res.writeHead(proxyRes.statusCode, { + ...proxyRes.headers, + 'location': this.backendToLocalPath(proxyId, redirectTargetLocation)}); + res.end() + return + } + // pass through the anwer of the remote server + res.writeHead(proxyRes.statusCode, { + ...proxyRes.headers}); + // pass through content + proxyRes.pipe(res) + } + }) + } + + upgrade(proxyId, req, socket, head){ + const {path, options} = this.configs[proxyId] + if(!req.url.startsWith(path+'/')){ + return + } + const targetUrl = this.localToBackendUrl(proxyId, req.originalUrl || req.url) + proxy.ws(req, socket, head, { + ...urlToHttpOptions(targetUrl), + ...options + }) + } + + setupOneProxy(proxyId){ + this.httpServer.on('request', (req, res) => this.proxy(proxyId, req, res)) + this.httpServer.on('upgrade', (req, socket, head) =>this.upgrade(proxyId, req, socket, head)) + } + +} diff --git a/@xen-orchestra/proxy/tests/cert.pem b/@xen-orchestra/proxy/tests/cert.pem new file mode 100644 index 00000000000..67a7ae8db08 --- /dev/null +++ b/@xen-orchestra/proxy/tests/cert.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDETCCAfkCFHXO1U7YJHI61bPNhYDvyBNJYH4LMA0GCSqGSIb3DQEBCwUAMEUx +CzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl +cm5ldCBXaWRnaXRzIFB0eSBMdGQwHhcNMjIwMTEwMTI0MTU4WhcNNDkwNTI3MTI0 +MTU4WjBFMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UE +CgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOC +AQ8AMIIBCgKCAQEA1jMLdHuZu2R1fETyB2iRect1alwv76clp/7A8tx4zNaVA9qB +BcHbI83mkozuyrXpsEUblTvvcWkheBPAvWD4gj0eWSDSiuf0edcIS6aky+Lr/n0T +W/vL5kVNrgPTlsO8OyQcXjDeuUOR1xDWIa8G71Ynd6wtATB7oXe7kaV/Z6b2fENr +4wlW0YEDnMHik59c9jXDshhQYDlErwZsSyHuLwkC7xuYO26SUW9fPcHJA3uOfxeG +BrCxMuSMOJtdmslRWhLCjbk0PT12OYCCRlvuTvPHa8N57GEQbi4xAu+XATgO1DUm +Dq/oCSj0TcWUXXOykN/PAC2cjIyqkU2e7orGaQIDAQABMA0GCSqGSIb3DQEBCwUA +A4IBAQCTshhF3V5WVhnpFGHd+tPfeHmUVrUnbC+xW7fSeWpamNmTjHb7XB6uDR0O +DGswhEitbbSOsCiwz4/zpfE3/3+X07O8NPbdHTVHCei6D0uyegEeWQ2HoocfZs3X +8CORe8TItuvQAevV17D0WkGRoJGVAOiKo+izpjI55QXQ+FjkJ0bfl1iksnUJk0+I +ZNmRRNjNyOxo7NAzomSBHfJ5rDE+E440F2uvXIE9OIwHRiq6FGvQmvGijPeeP5J0 +LzcSK98jfINFSsA/Wn5vWE+gfH9ySD2G3r2cDTS904T77PNiYH+cNSP6ujtmNzvK +Bgoa3jXZPRBi82TUOb2jj5DB33bg +-----END CERTIFICATE----- diff --git a/@xen-orchestra/proxy/tests/key.pem b/@xen-orchestra/proxy/tests/key.pem new file mode 100644 index 00000000000..a0d23620df9 --- /dev/null +++ b/@xen-orchestra/proxy/tests/key.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA1jMLdHuZu2R1fETyB2iRect1alwv76clp/7A8tx4zNaVA9qB +BcHbI83mkozuyrXpsEUblTvvcWkheBPAvWD4gj0eWSDSiuf0edcIS6aky+Lr/n0T +W/vL5kVNrgPTlsO8OyQcXjDeuUOR1xDWIa8G71Ynd6wtATB7oXe7kaV/Z6b2fENr +4wlW0YEDnMHik59c9jXDshhQYDlErwZsSyHuLwkC7xuYO26SUW9fPcHJA3uOfxeG +BrCxMuSMOJtdmslRWhLCjbk0PT12OYCCRlvuTvPHa8N57GEQbi4xAu+XATgO1DUm +Dq/oCSj0TcWUXXOykN/PAC2cjIyqkU2e7orGaQIDAQABAoIBAQC65uVq6WLWGa1O +FtbdUggGL1svyGrngYChGvB/uZMKoX57U1DbljDCCCrV23WNmbfkYBjWWervmZ1j +qlC2roOJGQ1/Fd3A6O7w1YnegPUxFrt3XunijE55iiVi3uHknryDGlpKcfgVzfjW +oVFHKPMzKYjcqnbGn+hwlwoq5y7JYFTOa57/dZbyommbodRyy9Dpn0OES0grQqwR +VD1amrQ7XJhukcxQgYPuDc/jM3CuowoBsv9f+Q2zsPgr6CpHxxLLIs+kt8NQJT9v +neg/pm8ojcwOa9qoILdtu6ue7ee3VE9cFnB1vutxS1+MPeI5wgTJjaYrgPCMxXBM +2LdJJEmBAoGBAPA6LpuU1vv5R3x66hzenSk4LS1fj24K0WuBdTwFvzQmCr70oKdo +Yywxt+ZkBw5aEtzQlB8GewolHobDJrzxMorU+qEXX3jP2BIPDVQl2orfjr03Yyus +s5mYS/Qa6Zf1yObrjulTNm8oTn1WaG3TIvi8c5DyG2OK28N/9oMI1XGRAoGBAORD +YKyII/S66gZsJSf45qmrhq1hHuVt1xae5LUPP6lVD+MCCAmuoJnReV8fc9h7Dvgd +YPMINkWUTePFr3o4p1mh2ZC7ldczgDn6X4TldY2J3Zg47xJa5hL0L6JL4NiCGRIE +FV5rLJxkGh/DDBfmC9hQQ6Yg6cHvyewso5xVnBtZAoGAI+OdWPMIl0ZrrqYyWbPM +aP8SiMfRBtCo7tW9bQUyxpi0XEjxw3Dt+AlJfysMftFoJgMnTedK9H4NLHb1T579 +PQ6KjwyN39+1WSVUiXDKUJsLmSswLrMzdcvx9PscUO6QYCdrB2K+LCcqasFBAr9b +ZyvIXCw/eUSihneUnYjxUnECgYAoPgCzKiU8ph9QFozOaUExNH4/3tl1lVHQOR8V +FKUik06DtP35xwGlXJrLPF5OEhPnhjZrYk0/IxBAUb/ICmjmknQq4gdes0Ot9QgW +A+Yfl+irR45ObBwXx1kGgd4YDYeh93pU9QweXj+Ezfw50mLQNgZXKYJMoJu2uX/2 +tdkZsQKBgQCTfDcW8qBntI6V+3Gh+sIThz+fjdv5+qT54heO4EHadc98ykEZX0M1 +sCWJiAQWM/zWXcsTndQDgDsvo23jpoulVPDitSEISp5gSe9FEN2njsVVID9h1OIM +f30s5kwcJoiV9kUCya/BFtuS7kbuQfAyPU0v3I+lUey6VCW6A83OTg== +-----END RSA PRIVATE KEY----- diff --git a/@xen-orchestra/proxy/tests/localServer.mjs b/@xen-orchestra/proxy/tests/localServer.mjs new file mode 100644 index 00000000000..84887ec83c5 --- /dev/null +++ b/@xen-orchestra/proxy/tests/localServer.mjs @@ -0,0 +1,66 @@ +import { createServer as creatServerHttps } from 'https'; +import { createServer as creatServerHttp } from 'http'; + +import { parse } from 'url'; +import { WebSocketServer } from 'ws'; +import fs from 'fs' + +const httpsServer = creatServerHttps({ + key: fs.readFileSync('key.pem'), + cert: fs.readFileSync('cert.pem') +}); +const httpServer = creatServerHttp(); + +const wss = new WebSocketServer({ noServer: true, perMessageDeflate: false }); + +function upgrade(request, socket, head) { + const { pathname } = parse(request.url); +console.log('will upgrade', pathname) + // web socket server only on /foo url + if (pathname === '/foo') { + wss.handleUpgrade(request, socket, head, function done(ws) { + wss.emit('connection', ws, request); + ws.on('message', function message(data) { + console.log('received: %s', data); + ws.send(data); + + }); + }); + } else { + socket.destroy(); + } +} + +function httpHandler(req, res){ + console.log(req.url) + switch(req.url){ + case '/index.html': + res.end('hi') + return + case '/redirect': + res.writeHead(301, { + 'Location': 'index.html' + }) + res.end() + return + case '/chainRedirect': + res.writeHead(301, { + 'Location': '/redirect' + }) + res.end() + return + default: + res.writeHEad(404) + res.end() + } +} + +httpsServer.on('upgrade', upgrade); +httpServer.on('upgrade', upgrade); + +httpsServer.on('request', httpHandler) +httpServer.on('request', httpHandler) + + +httpsServer.listen(8080); +httpServer.listen(8081); diff --git a/@xen-orchestra/proxy/tests/reverseProxy.unit.spec.mjs b/@xen-orchestra/proxy/tests/reverseProxy.unit.spec.mjs new file mode 100644 index 00000000000..c2e1ea888e8 --- /dev/null +++ b/@xen-orchestra/proxy/tests/reverseProxy.unit.spec.mjs @@ -0,0 +1,84 @@ + import ReverseProxy from '../dist/app/mixins/reverseProxy.mjs' +import { strictEqual } from 'assert' + + +function makeApp(reverseProxies){ + return { + config:{ + get: ()=>reverseProxies + } + } +} + +const app = makeApp( + { + https: { + path: '/https', + target: 'https://localhost:8080/remotePath/?baseParm=1#one=2&another=3' + } + } +) + +const expectedLocalToRemote = { + https: [ + { + local: '/proxy/v1/https/', + remote: 'https://localhost:8080/remotePath/?baseParm=1#one=2&another=3' + }, + { + local: '/proxy/v1/https/sub', + remote: 'https://localhost:8080/remotePath/sub?baseParm=1#one=2&another=3' + }, + { + local: '/proxy/v1/https/sub/index.html', + remote: 'https://localhost:8080/remotePath/sub/index.html?baseParm=1#one=2&another=3' + }, + { + local: '/proxy/v1/https/sub?param=1', + remote: 'https://localhost:8080/remotePath/sub?baseParm=1¶m=1#one=2&another=3' + }, + { + local: '/proxy/v1/https/sub?baseParm=willbeoverwritten¶m=willstay', + remote: 'https://localhost:8080/remotePath/sub?baseParm=1¶m=willstay#one=2&another=3' + }, + { + local: '/proxy/v1/https/sub?param=1#another=willoverwrite', + remote: 'https://localhost:8080/remotePath/sub?baseParm=1¶m=1#another=willoverwrite' + } + ] +} +const proxy = new ReverseProxy(app, {httpServer: { on : ()=>{}}}) +for(const proxyId in expectedLocalToRemote){ + for( const {local, remote} of expectedLocalToRemote[proxyId]){ + strictEqual(proxy.localToBackendUrl('https', local).href, remote, "error converting to backend") + } +} + +const expectedRemoteToLocal = { + https: [ + { + local: '/proxy/v1/https/', + remote: 'https://localhost:8080/remotePath/' + }, + { + local: '/proxy/v1/https/sub/index.html', + remote: 'https://localhost:8080/remotePath/sub/index.html' + }, + { + local: '/proxy/v1/https/?baseParm=1#one=2&another=3', + remote: 'https://localhost:8080/remotePath/?baseParm=1#one=2&another=3' + }, + { + local: '/proxy/v1/https/sub?baseParm=1#one=2&another=3', + remote: 'https://localhost:8080/remotePath/sub?baseParm=1#one=2&another=3' + } + ] +} + +for(const proxyId in expectedRemoteToLocal){ + for( const {local, remote} of expectedRemoteToLocal[proxyId]){ + const url = new URL(remote, 'https://localhost:8080/remotePath/?baseParm=1#one=2&another=3') + strictEqual(proxy.backendToLocalPath('https', url), local, "error converting to local") + } +} + diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md index 5a3dea8cfe2..c8d0649200d 100644 --- a/CHANGELOG.unreleased.md +++ b/CHANGELOG.unreleased.md @@ -6,6 +6,7 @@ ### Enhancements > Users must be able to say: “Nice enhancement, I'm eager to test it” +- [xo-proxy] : A visible xo-proxy can be used to transfer queries to another proxy/ host hidden from xo [PR#6071](https://github.com/vatesfr/xen-orchestra/pull/6072) ### Bug fixes diff --git a/yarn.lock b/yarn.lock index d065a09f400..4973b8780a2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8321,6 +8321,11 @@ http-signature@~1.2.0: jsprim "^1.2.2" sshpk "^1.7.0" +http2-proxy@^5.0.53: + version "5.0.53" + resolved "https://registry.yarnpkg.com/http2-proxy/-/http2-proxy-5.0.53.tgz#fc6cb07d2bb977a388ebeec4449557f2011e5a1f" + integrity sha512-k9OUKrPWau/YeViJGv5peEFgSGPE2n8CDyk/G3f+JfaaJzbFMPAK5PJTd99QYSUvgUwVBGNbZJCY/BEb+kUZNQ== + https-browserify@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/https-browserify/-/https-browserify-1.0.0.tgz#ec06c10e0a34c0f2faf199f7fd7fc78fffd03c73"