From b5e7d0a8af87ae6a949af493a5796d63df09d600 Mon Sep 17 00:00:00 2001 From: Alex Rodionov Date: Thu, 30 May 2024 12:14:52 -0700 Subject: [PATCH] Integrate remote cache server for Bazel --- config.js | 5 +++++ index.js | 19 +++++++++++++--- package.json | 2 +- post.js | 20 +++++++++++++++++ remote-cache-server.js | 50 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 92 insertions(+), 4 deletions(-) create mode 100644 remote-cache-server.js diff --git a/config.js b/config.js index 9f8c900..0e368d1 100644 --- a/config.js +++ b/config.js @@ -151,4 +151,9 @@ module.exports = { name: 'repository', paths: [bazelRepository] }, + remoteCacheServer: { + enabled: true, + url: 'http://localhost:8080/cache', + logPath: `${os.tmpdir()}/remote-cache-server.log`, + } } diff --git a/index.js b/index.js index 61f0d9d..c362dab 100644 --- a/index.js +++ b/index.js @@ -6,6 +6,8 @@ const github = require('@actions/github') const glob = require('@actions/glob') const tc = require('@actions/tool-cache') const config = require('./config') +const { fork } = require('child_process') +const path = require('path') async function run() { try { @@ -28,6 +30,7 @@ async function setupBazel() { await restoreCache(config.diskCache) await restoreCache(config.repositoryCache) await restoreExternalCaches(config.externalCache) + await startRemoteCacheServer() } async function setupBazelisk() { @@ -92,8 +95,8 @@ async function downloadBazelisk() { core.debug(`Downloading from ${url}`) const downloadPath = await tc.downloadTool(url, undefined, `token ${token}`) - core.debug('Adding to the cache...'); - fs.chmodSync(downloadPath, '755'); + core.debug('Adding to the cache...') + fs.chmodSync(downloadPath, '755') const cachePath = await tc.cacheFile(downloadPath, 'bazel', 'bazelisk', version) core.debug(`Successfully cached bazelisk to ${cachePath}`) @@ -107,6 +110,7 @@ async function setupBazelrc() { `startup --output_base=${config.paths.bazelOutputBase}\n` ) fs.appendFileSync(bazelrcPath, config.bazelrc.join("\n")) + fs.appendFileSync(bazelrcPath, `build --remote_cache=${config.remoteCacheServer.url}\n`) } } @@ -174,4 +178,13 @@ async function restoreCache(cacheConfig) { }()) } -run() +async function startRemoteCacheServer() { + const log = fs.openSync(config.remoteCacheServer.logPath, 'a') + const serverProcess = fork(path.join(__dirname, 'dist', 'remote-cache-server', 'index.js'), [], { + detached: true, + stdio: ['ignore', log, log, 'ipc'] + }) + serverProcess.unref() + + core.saveState('remote-cache-server-pid', serverProcess.pid.toString()) +} diff --git a/package.json b/package.json index dcffaee..7410367 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "node": "20.x" }, "scripts": { - "build": "ncc build index.js -s -o dist/main && ncc build post.js -s -o dist/post", + "build": "ncc build index.js -s -o dist/main && ncc build post.js -s -o dist/post && ncc build remote-cache-server.js -s -o dist/remote-cache-server", "test": "echo \"Error: no test specified\" && exit 1" }, "author": "Alex Rodionov ", diff --git a/post.js b/post.js index dc5f602..5967970 100644 --- a/post.js +++ b/post.js @@ -5,11 +5,31 @@ const core = require('@actions/core') const glob = require('@actions/glob') const config = require('./config') const { getFolderSize } = require('./util') +const { execSync } = require('child_process') async function run() { + await stopRemoteCacheServer() await saveCaches() } +async function stopRemoteCacheServer() { + const pid = core.getState('remote-cache-server-pid') + if (pid) { + try { + process.kill(pid, 'SIGTERM') + core.debug(`Stopped remote cache server with PID: ${pid}`) + } catch (error) { + core.error(`Failed to stop remote cache server with PID: ${pid}. Error: ${error}`) + } + } + + const logPath = config.remoteCacheServer.logPath + if (fs.existsSync(logPath)) { + const logContent = fs.readFileSync(logPath, 'utf8') + core.debug(`Remote cache server log:\n${logContent}`) + } +} + async function saveCaches() { await saveCache(config.bazeliskCache) await saveCache(config.diskCache) diff --git a/remote-cache-server.js b/remote-cache-server.js new file mode 100644 index 0000000..50170ee --- /dev/null +++ b/remote-cache-server.js @@ -0,0 +1,50 @@ +const http = require('http') +const cache = require('@actions/cache') +const fs = require('fs') + +// https://bazel.build/remote/caching#http-caching +const server = http.createServer(async (req, res) => { + const { method, url } = req + const [, , cacheType, sha] = url.split('/') + const filePath = `/tmp/cache-${cacheType}-${sha}` + + if (method === 'GET') { + try { + const cacheId = await cache.restoreCache([filePath], sha) + if (!cacheId) { + console.log(`Cache miss for ${sha}`) + res.writeHead(404) + return res.end('Cache miss') + } + const data = fs.readFileSync(filePath) + res.writeHead(200, { 'Content-Type': 'application/octet-stream' }) + res.end(data) + } catch (error) { + console.error(`Error retrieving cache for ${sha}: ${error}`) + res.writeHead(500) + res.end('Internal Server Error') + } + } else if (method === 'PUT') { + const data = [] + req.on('data', chunk => data.push(chunk)) + req.on('end', async () => { + try { + fs.writeFileSync(filePath, Buffer.concat(data)) + await cache.saveCache([filePath], sha) + console.log(`Cache saved for ${sha}`) + res.writeHead(201) + res.end('Cache saved') + } catch (error) { + console.error(`Error saving cache for ${sha}: ${error}`) + res.writeHead(500) + res.end('Internal Server Error') + } + }) + } else { + res.writeHead(405) + res.end('Method Not Allowed') + } +}) + +const PORT = process.env.PORT || 8080 +server.listen(PORT, () => console.log(`Server listening on port ${PORT}`))