feat(datasource): Add datasource for buildpack registry (#32721)

Co-authored-by: Nicolas Bender <nicolas.bender@sap.com>
Co-authored-by: HonkingGoose <34918129+HonkingGoose@users.noreply.github.com>
Co-authored-by: Michael Kriese <michael.kriese@visualon.de>
Co-authored-by: Pavel Busko <pavel.busko@sap.com>
Co-authored-by: Johannes Dillmann <j.dillmann@sap.com>
Co-authored-by: Michael Kriese <michael.kriese@gmx.de>

lib/modules/datasource/api.ts

@@ -6,6 +6,7 @@ import { AzurePipelinesTasksDatasource } from './azure-pipelines-tasks';
 import { BazelDatasource } from './bazel';
 import { BitbucketTagsDatasource } from './bitbucket-tags';
 import { BitriseDatasource } from './bitrise';
+import { BuildpacksRegistryDatasource } from './buildpacks-registry';
 import { CdnjsDatasource } from './cdnjs';
 import { ClojureDatasource } from './clojure';
 import { ConanDatasource } from './conan';
@@ -78,6 +79,7 @@ api.set(AzurePipelinesTasksDatasource.id, new AzurePipelinesTasksDatasource());
 api.set(BazelDatasource.id, new BazelDatasource());
 api.set(BitbucketTagsDatasource.id, new BitbucketTagsDatasource());
 api.set(BitriseDatasource.id, new BitriseDatasource());
+api.set(BuildpacksRegistryDatasource.id, new BuildpacksRegistryDatasource());
 api.set(CdnjsDatasource.id, new CdnjsDatasource());
 api.set(ClojureDatasource.id, new ClojureDatasource());
 api.set(ConanDatasource.id, new ConanDatasource());

lib/modules/datasource/buildpacks-registry/index.spec.ts

@@ -0,0 +1,66 @@
+import { getPkgReleases } from '..';
+import * as httpMock from '../../../../test/http-mock';
+import { BuildpacksRegistryDatasource } from '.';
+
+const baseUrl = 'https://registry.buildpacks.io/api/v1/buildpacks/';
+
+describe('modules/datasource/buildpacks-registry/index', () => {
+  describe('getReleases', () => {
+    it('processes real data', async () => {
+      httpMock
+        .scope(baseUrl)
+        .get('/heroku/python')
+        .reply(200, {
+          latest: {
+            version: '0.17.1',
+            namespace: 'heroku',
+            name: 'python',
+            description: "Heroku's buildpack for Python applications.",
+            homepage: 'https://github.com/heroku/buildpacks-python',
+            licenses: ['BSD-3-Clause'],
+            stacks: ['*'],
+            id: '75946bf8-3f6a-4af0-a757-614bebfdfcd6',
+          },
+          versions: [
+            {
+              version: '0.17.1',
+              _link:
+                'https://registry.buildpacks.io//api/v1/buildpacks/heroku/python/0.17.1',
+            },
+            {
+              version: '0.17.0',
+              _link:
+                'https://registry.buildpacks.io//api/v1/buildpacks/heroku/python/0.17.0',
+            },
+          ],
+        });
+      const res = await getPkgReleases({
+        datasource: BuildpacksRegistryDatasource.id,
+        packageName: 'heroku/python',
+      });
+      expect(res).toEqual({
+        registryUrl: 'https://registry.buildpacks.io',
+        releases: [{ version: '0.17.0' }, { version: '0.17.1' }],
+        sourceUrl: 'https://github.com/heroku/buildpacks-python',
+      });
+    });
+
+    it('returns null on empty result', async () => {
+      httpMock.scope(baseUrl).get('/heroku/empty').reply(200, {});
+      const res = await getPkgReleases({
+        datasource: BuildpacksRegistryDatasource.id,
+        packageName: 'heroku/empty',
+      });
+      expect(res).toBeNull();
+    });
+
+    it('handles not found', async () => {
+      httpMock.scope(baseUrl).get('/heroku/notexisting').reply(404);
+      const res = await getPkgReleases({
+        datasource: BuildpacksRegistryDatasource.id,
+        packageName: 'heroku/notexisting',
+      });
+      expect(res).toBeNull();
+    });
+  });
+});

lib/modules/datasource/buildpacks-registry/index.ts

@@ -0,0 +1,72 @@
+import urlJoin from 'url-join';
+import { ZodError } from 'zod';
+import { logger } from '../../../logger';
+import { cache } from '../../../util/cache/package/decorator';
+import { Result } from '../../../util/result';
+import { Datasource } from '../datasource';
+import { ReleasesConfig } from '../schema';
+import type { GetReleasesConfig, Release, ReleaseResult } from '../types';
+import { BuildpacksRegistryResponseSchema } from './schema';
+
+export class BuildpacksRegistryDatasource extends Datasource {
+  static readonly id = 'buildpacks-registry';
+
+  constructor() {
+    super(BuildpacksRegistryDatasource.id);
+  }
+
+  override readonly customRegistrySupport = false;
+
+  override readonly defaultRegistryUrls = ['https://registry.buildpacks.io'];
+
+  override readonly releaseTimestampSupport = true;
+  override readonly releaseTimestampNote =
+    'The release timestamp is determined from the `published_at` field in the results.';
+  override readonly sourceUrlSupport = 'release';
+  override readonly sourceUrlNote =
+    'The source URL is determined from the `source_code_url` field of the release object in the results.';
+
+  @cache({
+    namespace: `datasource-${BuildpacksRegistryDatasource.id}`,
+    key: ({ registryUrl, packageName }: GetReleasesConfig) =>
+      `${registryUrl}:${packageName}`,
+  })
+  async getReleases(config: GetReleasesConfig): Promise<ReleaseResult | null> {
+    const result = Result.parse(config, ReleasesConfig)
+      .transform(({ packageName, registryUrl }) => {
+        const url = urlJoin(
+          registryUrl,
+          'api',
+          'v1',
+          'buildpacks',
+          packageName,
+        );
+
+        return this.http.getJsonSafe(url, BuildpacksRegistryResponseSchema);
+      })
+      .transform(({ versions, latest }): ReleaseResult => {
+        const releases: Release[] = versions;
+
+        const res: ReleaseResult = { releases };
+
+        if (latest?.homepage) {
+          res.homepage = latest.homepage;
+        }
+
+        return res;
+      });
+
+    const { val, err } = await result.unwrap();
+
+    if (err instanceof ZodError) {
+      logger.debug({ err }, 'buildpacks: validation error');
+      return null;
+    }
+
+    if (err) {
+      this.handleGenericErrors(err);
+    }
+
+    return val;
+  }
+}

lib/modules/datasource/buildpacks-registry/schema.spec.ts

@@ -0,0 +1,43 @@
+import { BuildpacksRegistryResponseSchema } from './schema';
+
+describe('modules/datasource/buildpacks-registry/schema', () => {
+  it('parses buildpack-registry schema', () => {
+    const response = {
+      latest: {
+        version: '0.17.1',
+        namespace: 'heroku',
+        name: 'python',
+        description: "Heroku's buildpack for Python applications.",
+        homepage: 'https://github.com/heroku/buildpacks-python',
+        licenses: ['BSD-3-Clause'],
+        stacks: ['*'],
+        id: '75946bf8-3f6a-4af0-a757-614bebfdfcd6',
+      },
+      versions: [
+        {
+          version: '0.2.0',
+          _link:
+            'https://registry.buildpacks.io//api/v1/buildpacks/heroku/python/0.2.0',
+        },
+        {
+          version: '0.1.0',
+          _link:
+            'https://registry.buildpacks.io//api/v1/buildpacks/heroku/python/0.1.0',
+        },
+      ],
+    };
+    expect(BuildpacksRegistryResponseSchema.parse(response)).toMatchObject({
+      latest: {
+        homepage: 'https://github.com/heroku/buildpacks-python',
+      },
+      versions: [
+        {
+          version: '0.2.0',
+        },
+        {
+          version: '0.1.0',
+        },
+      ],
+    });
+  });
+});

lib/modules/datasource/buildpacks-registry/schema.ts

@@ -0,0 +1,17 @@
+import { z } from 'zod';
+
+/**
+ *  Response from registry.buildpacks.io
+ */
+export const BuildpacksRegistryResponseSchema = z.object({
+  latest: z
+    .object({
+      homepage: z.string().optional(),
+    })
+    .optional(),
+  versions: z
+    .object({
+      version: z.string(),
+    })
+    .array(),
+});

lib/modules/manager/buildpacks/extract.spec.ts

@@ -23,6 +23,7 @@ describe('modules/manager/buildpacks/extract', () => {
 [_]
 schema-version = "0.2"
 
+# valid cases
 [io.buildpacks]
 builder = "registry.corp/builder/noble:1.1.1"
 
@@ -36,16 +37,22 @@ uri = "buildpacks/nodejs:3.3.3"
 uri = "example/foo@1.0.0"
 
 [[io.buildpacks.group]]
-uri = "example/registry-cnb"
+uri = "urn:cnb:registry:example/bar@1.2.3"
 
 [[io.buildpacks.group]]
-uri = "urn:cnb:registry:example/foo@1.0.0"
+uri = "cnbs/some-bp@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
 
 [[io.buildpacks.group]]
-uri = "some-bp@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
+uri = "cnbs/some-bp:some-tag@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
 
 [[io.buildpacks.group]]
-uri = "cnbs/some-bp:some-tag@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
+id = "example/tee"
+version = "2.3.4"
+
+#invalid cases
+
+[[io.buildpacks.group]]
+uri = "example/registry-cnb"
 
 [[io.buildpacks.group]]
 uri = "from=builder:foobar"
@@ -54,7 +61,10 @@ uri = "from=builder:foobar"
 uri = "file://local.oci"
 
 [[io.buildpacks.group]]
-uri = "foo://fake.oci"`,
+uri = "foo://fake.oci"
+
+[[io.buildpacks.group]]
+id = "not/valid"`,
         'project.toml',
         {},
       );
@@ -84,15 +94,29 @@ uri = "foo://fake.oci"`,
           depName: 'buildpacks/nodejs',
           replaceString: 'buildpacks/nodejs:3.3.3',
         },
+        {
+          autoReplaceStringTemplate:
+            '{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}',
+          datasource: 'buildpacks-registry',
+          currentValue: '1.0.0',
+          packageName: 'example/foo',
+        },
+        {
+          autoReplaceStringTemplate:
+            '{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}',
+          datasource: 'buildpacks-registry',
+          currentValue: '1.2.3',
+          packageName: 'example/bar',
+        },
         {
           autoReplaceStringTemplate:
             '{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}',
           currentDigest:
             'sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef',
           datasource: 'docker',
-          depName: 'some-bp',
+          depName: 'cnbs/some-bp',
           replaceString:
-            'some-bp@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef',
+            'cnbs/some-bp@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef',
         },
         {
           autoReplaceStringTemplate:
@@ -105,6 +129,11 @@ uri = "foo://fake.oci"`,
           replaceString:
             'cnbs/some-bp:some-tag@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef',
         },
+        {
+          datasource: 'buildpacks-registry',
+          currentValue: '2.3.4',
+          packageName: 'example/tee',
+        },
       ]);
     });
   });

lib/modules/manager/buildpacks/extract.ts

@@ -1,13 +1,20 @@
 import is from '@sindresorhus/is';
 import { logger } from '../../../logger';
 import { regEx } from '../../../util/regex';
+import { BuildpacksRegistryDatasource } from '../../datasource/buildpacks-registry';
+import { isVersion } from '../../versioning/semver';
 import { getDep } from '../dockerfile/extract';
 import type {
   ExtractConfig,
   PackageDependency,
   PackageFileContent,
 } from '../types';
-import { type ProjectDescriptor, ProjectDescriptorToml } from './schema';
+import {
+  type ProjectDescriptor,
+  ProjectDescriptorToml,
+  isBuildpackByName,
+  isBuildpackByURI,
+} from './schema';
 
 const dockerPrefix = regEx(/^docker:\/?\//);
 const dockerRef = regEx(
@@ -20,6 +27,24 @@ function isDockerRef(ref: string): boolean {
   }
   return false;
 }
+const buildpackRegistryPrefix = 'urn:cnb:registry:';
+const buildpackRegistryId = regEx(
+  /^[a-z0-9\-.]+\/[a-z0-9\-.]+(?:@(?<version>.+))?$/,
+);
+
+function isBuildpackRegistryId(ref: string): boolean {
+  const bpRegistryMatch = buildpackRegistryId.exec(ref);
+  if (!bpRegistryMatch) {
+    return false;
+  } else if (!bpRegistryMatch.groups?.version) {
+    return true;
+  }
+  return isVersion(bpRegistryMatch.groups.version);
+}
+
+function isBuildpackRegistryRef(ref: string): boolean {
+  return isBuildpackRegistryId(ref) || ref.startsWith(buildpackRegistryPrefix);
+}
 
 function parseProjectToml(
   content: string,
@@ -76,7 +101,7 @@ export function extractPackageFile(
     is.array(descriptor.io.buildpacks.group)
   ) {
     for (const group of descriptor.io.buildpacks.group) {
-      if (group.uri && isDockerRef(group.uri)) {
+      if (isBuildpackByURI(group) && isDockerRef(group.uri)) {
         const dep = getDep(
           group.uri.replace(dockerPrefix, ''),
           true,
@@ -92,6 +117,31 @@ export function extractPackageFile(
         );
 
         deps.push(dep);
+      } else if (isBuildpackByURI(group) && isBuildpackRegistryRef(group.uri)) {
+        const dependency = group.uri.replace(buildpackRegistryPrefix, '');
+
+        if (dependency.includes('@')) {
+          const version = dependency.split('@')[1];
+          const dep: PackageDependency = {
+            datasource: BuildpacksRegistryDatasource.id,
+            currentValue: version,
+            packageName: dependency.split('@')[0],
+            autoReplaceStringTemplate:
+              '{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}',
+          };
+          deps.push(dep);
+        }
+      } else if (isBuildpackByName(group)) {
+        const version = group.version;
+
+        if (version) {
+          const dep: PackageDependency = {
+            datasource: BuildpacksRegistryDatasource.id,
+            currentValue: version,
+            packageName: group.id,
+          };
+          deps.push(dep);
+        }
       }
     }
   }