fix: interact with aws only using the CLI

libp2p · Sep 5, 2024 · 31176a4 · 31176a4
1 parent fda3183
commit 31176a4
Show file tree

Hide file tree

Showing 2 changed files with 26 additions and 30 deletions.
diff --git a/hole-punch-interop/helpers/cache.ts b/hole-punch-interop/helpers/cache.ts
@@ -3,6 +3,7 @@ const scriptDir = __dirname;
 
 import * as crypto from 'crypto';
 import * as fs from 'fs';
+import * as os from 'os';
 import * as path from 'path';
 import * as child_process from 'child_process';
 import ignore, { Ignore } from 'ignore'
@@ -79,10 +80,11 @@ async function loadCacheOrBuild(dir: string, ig: Ignore) {
             if (!AWS_BUCKET) {
                 throw new Error("AWS_BUCKET not set")
             }
-            const res = await fetch(`https://s3.amazonaws.com/${AWS_BUCKET}/imageCache/${cacheKey}-${arch}.tar.gz`, {method: "HEAD"})
-            if (res.ok) {
+            try {
+                child_process.execSync(`aws s3 ls s3://${AWS_BUCKET}/imageCache/${cacheKey}-${arch}.tar.gz`)
                 console.log("Cache already exists")
-            } else {
+            } catch (e) {
+                console.log("Cache doesn't exist", e)
                 // Read image id from image.json
                 const imageID = JSON.parse(fs.readFileSync(path.join(dir, 'image.json')).toString()).imageID;
                 console.log(`Pushing cache for ${dir}: ${imageID}`)
@@ -102,18 +104,14 @@ async function loadCacheOrBuild(dir: string, ig: Ignore) {
             if (!AWS_BUCKET) {
                 throw new Error("AWS_BUCKET not set")
             }
-            // Check if the cache exists
-            const res = await fetch(`https://s3.amazonaws.com/${AWS_BUCKET}/imageCache/${cacheKey}-${arch}.tar.gz`, {method: "HEAD"})
-            if (res.ok) {
-                const dockerLoadedMsg = child_process.execSync(`curl https://s3.amazonaws.com/${AWS_BUCKET}/imageCache/${cacheKey}-${arch}.tar.gz  | docker image load`).toString();
-                const loadedImageId = dockerLoadedMsg.match(/Loaded image( ID)?: (.*)/)[2];
-                if (loadedImageId) {
-                    console.log(`Cache hit for ${loadedImageId}`);
-                    fs.writeFileSync(path.join(dir, 'image.json'), JSON.stringify({imageID: loadedImageId}) + "\n");
-                    cacheHit = true
-                }
-            } else {
-                console.log("Cache not found")
+            const cachePath = fs.mkdtempSync(path.join(os.tmpdir(), 'cache'))
+            const archivePath = path.join(cachePath, 'archive.tar.gz')
+            const dockerLoadedMsg = child_process.execSync(`aws s3 cp https://s3.amazonaws.com/${AWS_BUCKET}/imageCache/${cacheKey}-${arch}.tar.gz ${archivePath} && docker image load -i ${archivePath}`).toString();
+            const loadedImageId = dockerLoadedMsg.match(/Loaded image( ID)?: (.*)/)[2];
+            if (loadedImageId) {
+                console.log(`Cache hit for ${loadedImageId}`);
+                fs.writeFileSync(path.join(dir, 'image.json'), JSON.stringify({imageID: loadedImageId}) + "\n");
+                cacheHit = true
             }
         } catch (e) {
             console.log("Cache not found:", e)

diff --git a/transport-interop/helpers/cache.ts b/transport-interop/helpers/cache.ts
@@ -3,6 +3,7 @@ const scriptDir = __dirname;
 
 import * as crypto from 'crypto';
 import * as fs from 'fs';
+import * as os from 'os';
 import * as path from 'path';
 import * as child_process from 'child_process';
 import ignore, { Ignore } from 'ignore'
@@ -68,10 +69,11 @@ switch (modeStr) {
                     if (!AWS_BUCKET) {
                         throw new Error("AWS_BUCKET not set")
                     }
-                    const res = await fetch(`https://s3.amazonaws.com/${AWS_BUCKET}/imageCache/${cacheKey}-${arch}.tar.gz`, { method: "HEAD" })
-                    if (res.ok) {
+                    try {
+                        child_process.execSync(`aws s3 ls s3://${AWS_BUCKET}/imageCache/${cacheKey}-${arch}.tar.gz`)
                         console.log("Cache already exists")
-                    } else {
+                    } catch (e) {
+                        console.log("Cache doesn't exist", e)
                         // Read image id from image.json
                         const imageID = JSON.parse(fs.readFileSync(path.join(implFolder, 'image.json')).toString()).imageID;
                         console.log(`Pushing cache for ${impl}: ${imageID}`)
@@ -91,18 +93,14 @@ switch (modeStr) {
                     if (!AWS_BUCKET) {
                         throw new Error("AWS_BUCKET not set")
                     }
-                    // Check if the cache exists
-                    const res = await fetch(`https://s3.amazonaws.com/${AWS_BUCKET}/imageCache/${cacheKey}-${arch}.tar.gz`, { method: "HEAD" })
-                    if (res.ok) {
-                        const dockerLoadedMsg = child_process.execSync(`curl https://s3.amazonaws.com/${AWS_BUCKET}/imageCache/${cacheKey}-${arch}.tar.gz  | docker image load`).toString();
-                        const loadedImageId = dockerLoadedMsg.match(/Loaded image( ID)?: (.*)/)[2];
-                        if (loadedImageId) {
-                            console.log(`Cache hit for ${loadedImageId}`);
-                            fs.writeFileSync(path.join(implFolder, 'image.json'), JSON.stringify({ imageID: loadedImageId }) + "\n");
-                            cacheHit = true
-                        }
-                    } else {
-                        console.log("Cache not found")
+                    const cachePath = fs.mkdtempSync(path.join(os.tmpdir(), 'cache'))
+                    const archivePath = path.join(cachePath, 'archive.tar.gz')
+                    const dockerLoadedMsg = child_process.execSync(`aws s3 cp https://s3.amazonaws.com/${AWS_BUCKET}/imageCache/${cacheKey}-${arch}.tar.gz ${archivePath} && docker image load -i ${archivePath}`).toString();
+                            const loadedImageId = dockerLoadedMsg.match(/Loaded image( ID)?: (.*)/)[2];
+                    if (loadedImageId) {
+                        console.log(`Cache hit for ${loadedImageId}`);
+                        fs.writeFileSync(path.join(implFolder, 'image.json'), JSON.stringify({ imageID: loadedImageId }) + "\n");
+                        cacheHit = true
                     }
                 } catch (e) {
                     console.log("Cache not found:", e)