oci: loadArchive to import an index from a tar archive image bundle

Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com>

.gitattributes

@@ -1,2 +1,3 @@
 /.yarn/releases/** binary
 /.yarn/plugins/** binary
+/__tests__/fixtures/oci-archive/** binary

__tests__/fixtures/hello.Dockerfile

@@ -14,7 +14,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM busybox
+FROM busybox AS build
 ARG NAME=foo
 ARG TARGETPLATFORM
-RUN echo "Hello $NAME from $TARGETPLATFORM"
+RUN echo "Hello $NAME from $TARGETPLATFORM" > foo
+
+FROM scratch
+COPY --from=build /foo /

__tests__/oci/oci.test.ts

@@ -0,0 +1,53 @@
+/**
+ * Copyright 2024 actions-toolkit authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {afterEach, beforeEach, describe, expect, jest, test} from '@jest/globals';
+import path from 'path';
+import * as rimraf from 'rimraf';
+
+import {OCI} from '../../src/oci/oci';
+
+const fixturesDir = path.join(__dirname, '..', 'fixtures');
+
+// prettier-ignore
+const tmpDir = path.join(process.env.TEMP || '/tmp', 'docker-jest');
+
+beforeEach(() => {
+  jest.clearAllMocks();
+});
+
+afterEach(function () {
+  rimraf.sync(tmpDir);
+});
+
+describe('loadArchive', () => {
+  // prettier-ignore
+  test.each([
+    ['docker~test-docker-action~COIO50.dockerbuild'],
+    ['docker~test-docker-action~SNHBPN+3.dockerbuild'],
+    ['docker~test-docker-action~TYO4JJ+15.dockerbuild'],
+    ['hello-oci-gzip.tar'],
+    ['hello-oci-multiplatform-gzip.tar'],
+    ['hello-oci-uncompressed.tar'],
+    ['hello-oci-zstd.tar']
+  ])('extracting %p', async (filename) => {
+    const res = await OCI.loadArchive({
+      file: path.join(fixturesDir, 'oci-archive', filename)
+    });
+    expect(res).toBeDefined();
+    // console.log(JSON.stringify(res, null, 2));
+  });
+});

package.json

@@ -58,15 +58,19 @@
     "@octokit/plugin-rest-endpoint-methods": "^10.4.0",
     "async-retry": "^1.3.3",
     "csv-parse": "^5.5.6",
+    "gunzip-maybe": "^1.4.2",
     "handlebars": "^4.7.8",
     "jwt-decode": "^4.0.0",
     "semver": "^7.6.2",
+    "tar-stream": "^3.1.7",
     "tmp": "^0.2.3"
   },
   "devDependencies": {
     "@types/csv-parse": "^1.2.2",
+    "@types/gunzip-maybe": "^1.4.2",
     "@types/node": "^20.12.10",
     "@types/semver": "^7.5.8",
+    "@types/tar-stream": "^3.1.3",
     "@types/tmp": "^0.2.6",
     "@typescript-eslint/eslint-plugin": "^7.8.0",
     "@typescript-eslint/parser": "^7.8.0",

src/oci/oci.ts

@@ -0,0 +1,170 @@
+/**
+ * Copyright 2024 actions-toolkit authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import fs from 'fs';
+import gunzip from 'gunzip-maybe';
+import * as path from 'path';
+import {PassThrough, Readable} from 'stream';
+import * as tar from 'tar-stream';
+
+import {Archive, LoadArchiveOpts} from '../types/oci/oci';
+import {Index} from '../types/oci';
+import {Manifest} from '../types/oci/manifest';
+import {Image} from '../types/oci/config';
+import {IMAGE_BLOBS_DIR_V1, IMAGE_INDEX_FILE_V1, IMAGE_LAYOUT_FILE_V1, ImageLayout} from '../types/oci/layout';
+import {MEDIATYPE_IMAGE_INDEX_V1} from '../types/oci/mediatype';
+
+export class OCI {
+  public static loadArchive(opts: LoadArchiveOpts): Promise<Archive | undefined> {
+    return new Promise<Archive>((resolve, reject) => {
+      const tarex: tar.Extract = tar.extract();
+
+      let rootIndex: Index;
+      let rootLayout: ImageLayout;
+      const indexes: Record<string, Index> = {};
+      const manifests: Record<string, Manifest> = {};
+      const images: Record<string, Image> = {};
+      const blobs: Record<string, unknown> = {};
+
+      tarex.on('entry', async (header, stream, next) => {
+        if (header.type === 'file') {
+          const filename = path.normalize(header.name);
+          if (filename === IMAGE_INDEX_FILE_V1) {
+            rootIndex = await OCI.streamToJson<Index>(stream);
+          } else if (filename === IMAGE_LAYOUT_FILE_V1) {
+            rootLayout = await OCI.streamToJson<ImageLayout>(stream);
+          } else if (filename.startsWith(path.join(IMAGE_BLOBS_DIR_V1, path.sep))) {
+            const manifestStream = new PassThrough();
+            const blobStream = new PassThrough();
+            const teeStream = new PassThrough();
+            stream.pipe(teeStream);
+            teeStream.pipe(manifestStream);
+            teeStream.pipe(blobStream);
+            const promises = [OCI.streamToJson(manifestStream).catch(() => undefined), OCI.extractBlob(blobStream).catch(() => undefined)];
+            const [manifest, blob] = await Promise.all(promises);
+            const digest = `${filename.split(path.sep)[1]}:${filename.split(path.sep)[filename.split(path.sep).length - 1]}`;
+            if (OCI.isManifest(manifest)) {
+              manifests[digest] = manifest;
+            } else if (OCI.isIndex(manifest)) {
+              indexes[digest] = manifest;
+            } else if (OCI.isImage(manifest)) {
+              images[digest] = manifest;
+            } else {
+              blobs[digest] = blob ?? manifest;
+            }
+          } else {
+            reject(new Error(`Invalid OCI archive: unexpected file ${filename}`));
+          }
+        }
+        stream.resume();
+        next();
+      });
+
+      tarex.on('finish', () => {
+        if (!rootIndex || !rootLayout) {
+          reject(new Error('Invalid OCI archive: missing index or layout'));
+        }
+        resolve({
+          rootIndex: rootIndex,
+          rootLayout: rootLayout,
+          indexes: indexes,
+          manifests: manifests,
+          images: images,
+          blobs: blobs
+        } as Archive);
+      });
+
+      tarex.on('error', error => {
+        reject(error);
+      });
+
+      fs.createReadStream(opts.file).pipe(gunzip()).pipe(tarex);
+    });
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  private static isIndex(index: any): index is Index {
+    return index?.mediaType === MEDIATYPE_IMAGE_INDEX_V1;
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  private static isManifest(manifest: any): manifest is Manifest {
+    return manifest?.layers && Array.isArray(manifest.layers);
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  private static isImage(image: any): image is Image {
+    return image?.rootfs;
+  }
+
+  private static async extractBlob(blobStream: Readable): Promise<unknown> {
+    let dt: unknown;
+    return new Promise<unknown>((resolve, reject) => {
+      const tarex: tar.Extract = tar.extract();
+      tarex.on('entry', async (header, stream, next) => {
+        if (header.type === 'file') {
+          const streamCopy = new PassThrough();
+          stream.pipe(streamCopy);
+          dt = await OCI.streamToString(streamCopy);
+        }
+        stream.resume();
+        next();
+      });
+      tarex.on('finish', () => {
+        resolve(dt);
+      });
+      tarex.on('error', async error => {
+        reject(error);
+      });
+      blobStream.pipe(gunzip()).pipe(tarex);
+    });
+  }
+
+  private static async streamToJson<T>(stream: Readable): Promise<T> {
+    return new Promise<T>((resolve, reject) => {
+      const chunks: string[] = [];
+      let bytes = 0;
+      stream.on('end', () => {
+        try {
+          resolve(JSON.parse(chunks.join('')));
+        } catch (error) {
+          reject(error);
+        }
+      });
+      stream.on('error', error => reject(error));
+      stream.on('data', chunk => {
+        bytes += chunk.length;
+        if (bytes <= 2 * 1024 * 1024) {
+          chunks.push(chunk.toString('utf8'));
+        } else {
+          reject(new Error('The data stream exceeds the size limit for JSON parsing.'));
+        }
+      });
+    });
+  }
+
+  private static async streamToString(stream: Readable): Promise<string> {
+    const chunks: Buffer[] = [];
+    return new Promise((resolve, reject) => {
+      stream.on('end', () => {
+        resolve(Buffer.concat(chunks).toString('utf8'));
+      });
+      stream.on('error', error => reject(error));
+      stream.on('data', chunk => {
+        chunks.push(chunk);
+      });
+    });
+  }
+}

src/types/oci/config.ts

@@ -0,0 +1,52 @@
+/**
+ * Copyright 2024 actions-toolkit authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {Digest} from './digest';
+import {Platform} from './descriptor';
+
+export interface ImageConfig {
+  User?: string;
+  ExposedPorts?: Record<string, unknown>;
+  Env?: string[];
+  Entrypoint?: string[];
+  Cmd?: string[];
+  Volumes?: Record<string, unknown>;
+  WorkingDir?: string;
+  Labels?: Record<string, string>;
+  StopSignal?: string;
+  ArgsEscaped?: boolean;
+}
+
+export interface RootFS {
+  type: string;
+  diff_ids: Digest[];
+}
+
+export interface History {
+  created?: string; // assuming RFC 3339 formatted string
+  created_by?: string;
+  author?: string;
+  comment?: string;
+  empty_layer?: boolean;
+}
+
+export interface Image extends Platform {
+  created?: string; // assuming RFC 3339 formatted string
+  author?: string;
+  config?: ImageConfig;
+  rootfs: RootFS;
+  history?: History[];
+}

src/types/oci/descriptor.ts

@@ -0,0 +1,45 @@
+/**
+ * Copyright 2024 actions-toolkit authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {Digest} from './digest';
+
+import {MEDIATYPE_EMPTY_JSON_V1} from './mediatype';
+
+export interface Descriptor {
+  mediaType: string;
+  digest: Digest;
+  size: number;
+  urls?: string[];
+  annotations?: Record<string, string>;
+  data?: string;
+  platform?: Platform;
+  artifactType?: string;
+}
+
+export interface Platform {
+  architecture: string;
+  os: string;
+  'os.version'?: string;
+  'os.features'?: string[];
+  variant?: string;
+}
+
+export const DescriptorEmptyJSON: Descriptor = {
+  mediaType: MEDIATYPE_EMPTY_JSON_V1,
+  digest: 'sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a',
+  size: 2,
+  data: '{}'
+};

src/types/oci/digest.ts

@@ -0,0 +1,17 @@
+/**
+ * Copyright 2024 actions-toolkit authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export type Digest = string;