feat(self-hosted): convert experimental env vars to config options (#…

…29154)

Co-authored-by: HonkingGoose <34918129+HonkingGoose@users.noreply.github.com>

docs/usage/self-hosted-configuration.md

@@ -388,6 +388,16 @@ The above configuration approach will mean the values are redacted in logs like
          "customEnvVariables": {"SECRET_TOKEN": "{{ secrets.SECRET_TOKEN }}"},
 ```
 
+## deleteConfigFile
+
+If set to `true` Renovate tries to delete the self-hosted config file after reading it.
+
+The process that runs Renovate must have the correct permissions to delete the config file.
+
+<!-- prettier-ignore -->
+!!! tip
+    You can tell Renovate where to find your config file with the `RENOVATE_CONFIG_FILE` environment variable.
+
 ## detectGlobalManagerConfig
 
 The purpose of this config option is to allow you (as a bot admin) to configure manager-specific files such as a global `.npmrc` file, instead of configuring it in Renovate config.
@@ -1082,7 +1092,7 @@ Defines how the report is exposed:
 - `<unset>` If unset, no report will be provided, though the debug logs will still have partial information of the report
 - `logging` The report will be printed as part of the log messages on `INFO` level
 - `file` The report will be written to a path provided by [`reportPath`](#reportpath)
-- `s3` The report is pushed to an S3 bucket defined by [`reportPath`](#reportpath). This option reuses [`RENOVATE_X_S3_ENDPOINT`](./self-hosted-experimental.md#renovate_x_s3_endpoint) and [`RENOVATE_X_S3_PATH_STYLE`](./self-hosted-experimental.md#renovate_x_s3_path_style)
+- `s3` The report is pushed to an S3 bucket defined by [`reportPath`](#reportpath). This option reuses [`s3Endpoint`](./self-hosted-configuration.md#s3endpoint) and [`s3PathStyle`](./self-hosted-configuration.md#s3PathStyle)
 
 ## repositories
 
@@ -1140,6 +1150,22 @@ The combinations of `requireConfig` and `onboarding` are:
 | `requireConfig=optional` | An onboarding PR will be created if no config file exists. If the onboarding PR is closed and there's no config file, the repository will be processed. | Repository is processed regardless of config file presence.   |
 | `requireConfig=ignored`  | No onboarding PR will be created and repo will be processed while ignoring any config file present.                                                     | Repository is processed, any config file is ignored.          |
 
+## s3Endpoint
+
+If set, Renovate will use this string as the `endpoint` when creating the AWS S3 client instance.
+
+## s3PathStyle
+
+If set, Renovate will enable `forcePathStyle` when creating the AWS S3 client instance.
+
+For example:
+| `s3PathStyle` | Path |
+| ------------- | ---------------------------------- |
+| `off` | `https://bucket.s3.amazonaws.com/` |
+| `on` | `https://s3.amazonaws.com/bucket/` |
+
+Read the [AWS S3 docs, Interface BucketEndpointInputConfig](https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-s3/interfaces/bucketendpointinputconfig.html) to learn more about path-style URLs.
+
 ## secrets
 
 Secrets may be configured by a bot admin in `config.js`, which will then make them available for templating within repository configs.

docs/usage/self-hosted-experimental.md

@@ -28,13 +28,6 @@ Skipping the check will speed things up, but may result in versions being return
 
 If set to any value, Renovate will always paginate requests to GitHub fully, instead of stopping after 10 pages.
 
-## `RENOVATE_X_DELETE_CONFIG_FILE`
-
-If `true` Renovate tries to delete the self-hosted config file after reading it.
-You can set the config file Renovate should read with the `RENOVATE_CONFIG_FILE` environment variable.
-
-The process that runs Renovate must have the correct permissions to delete the config file.
-
 ## `RENOVATE_X_DOCKER_HUB_DISABLE_LABEL_LOOKUP`
 
 If set to any value, Renovate will skip attempting to get release labels (e.g. gitRef, sourceUrl) from manifest annotations for `https://index.docker.io`.
@@ -113,18 +106,6 @@ If set, Renovate will rewrite GitHub Enterprise Server's pagination responses to
 
 If set, Renovate will persist repository cache locally after uploading to S3.
 
-## `RENOVATE_X_S3_ENDPOINT`
-
-If set, Renovate will use this string as the `endpoint` when instantiating the AWS S3 client.
-
-## `RENOVATE_X_S3_PATH_STYLE`
-
-If set, Renovate will enable `forcePathStyle` when instantiating the AWS S3 client.
-
-> Whether to force path style URLs for S3 objects (e.g., `https://s3.amazonaws.com//` instead of `https://.s3.amazonaws.com/`)
-
-Source: [AWS S3 documentation - Interface BucketEndpointInputConfig](https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-s3/interfaces/bucketendpointinputconfig.html)
-
 ## `RENOVATE_X_SQLITE_PACKAGE_CACHE`
 
 If set, Renovate will use SQLite as the backend for the package cache.

lib/config/global.ts

@@ -37,6 +37,8 @@ export class GlobalConfig {
     'autodiscoverRepoSort',
     'autodiscoverRepoOrder',
     'userAgent',
+    's3Endpoint',
+    's3PathStyle',
     'cachePrivatePackages',
   ];
 

lib/config/options/index.ts

@@ -2931,6 +2931,29 @@ const options: RenovateOptions[] = [
     default: 90,
     globalOnly: true,
   },
+  {
+    name: 'deleteConfigFile',
+    description:
+      'If set to `true`, Renovate tries to delete the self-hosted config file after reading it.',
+    type: 'boolean',
+    default: false,
+    globalOnly: true,
+  },
+  {
+    name: 's3Endpoint',
+    description:
+      'If set, Renovate will use this string as the `endpoint` when creating the AWS S3 client instance.',
+    type: 'string',
+    globalOnly: true,
+  },
+  {
+    name: 's3PathStyle',
+    description:
+      'If set, Renovate will enable `forcePathStyle` when creating the AWS S3 client instance.',
+    type: 'boolean',
+    default: false,
+    globalOnly: true,
+  },
   {
     name: 'cachePrivatePackages',
     description:

lib/config/types.ts

@@ -125,6 +125,7 @@ export interface GlobalOnlyConfig {
   redisUrl?: string;
   repositories?: RenovateRepository[];
   useCloudMetadataServices?: boolean;
+  deleteConfigFile?: boolean;
 }
 
 // Config options used within the repository worker, but not user configurable
@@ -165,6 +166,8 @@ export interface RepoGlobalConfig {
   autodiscoverRepoSort?: RepoSortMethod;
   autodiscoverRepoOrder?: SortMethod;
   userAgent?: string;
+  s3Endpoint?: string;
+  s3PathStyle?: boolean;
   cachePrivatePackages?: boolean;
 }
 
@@ -224,6 +227,8 @@ export interface RenovateConfig
     AssigneesAndReviewersConfig,
     ConfigMigration,
     Record<string, unknown> {
+  s3Endpoint?: string;
+  s3PathStyle?: boolean;
   reportPath?: string;
   reportType?: 'logging' | 'file' | 's3' | null;
   depName?: string;

lib/instrumentation/reporting.ts

@@ -89,7 +89,7 @@ export async function exportStats(config: RenovateConfig): Promise<void> {
         ContentType: 'application/json',
       };
 
-      const client = getS3Client();
+      const client = getS3Client(config.s3Endpoint, config.s3PathStyle);
       const command = new PutObjectCommand(s3Params);
       await client.send(command);
     }

lib/util/s3.spec.ts

@@ -2,8 +2,6 @@ import { getS3Client, parseS3Url } from './s3';
 
 describe('util/s3', () => {
   afterEach(() => {
-    delete process.env.RENOVATE_X_S3_ENDPOINT;
-    delete process.env.RENOVATE_X_S3_PATH_STYLE;
     jest.resetModules();
   });
 
@@ -28,10 +26,13 @@ describe('util/s3', () => {
     expect(client1).toBe(client2);
   });
 
-  it('is uses experimental env', async () => {
-    process.env.RENOVATE_X_S3_ENDPOINT = 'https://minio.domain.test';
-    process.env.RENOVATE_X_S3_PATH_STYLE = 'true';
+  it('uses user-configured s3 values', async () => {
     const s3 = await import('./s3');
+    const globalConfig = await import('../config/global');
+    globalConfig.GlobalConfig.set({
+      s3Endpoint: 'https://minio.domain.test',
+      s3PathStyle: true,
+    });
     const client1 = s3.getS3Client();
     const client2 = getS3Client();
     expect(client1).not.toBe(client2);
@@ -44,4 +45,19 @@ describe('util/s3', () => {
     });
     expect(client1.config.forcePathStyle).toBeTrue();
   });
+
+  it('uses s3 values from globalConfig instead of GlobalConfig class', async () => {
+    const s3 = await import('./s3');
+    const client1 = s3.getS3Client('https://minio.domain.test', true);
+    const client2 = getS3Client('https://minio.domain.test', true);
+    expect(client1).not.toBe(client2);
+    expect(await client1.config.endpoint?.()).toStrictEqual({
+      hostname: 'minio.domain.test',
+      path: '/',
+      port: undefined,
+      protocol: 'https:',
+      query: undefined,
+    });
+    expect(client1.config.forcePathStyle).toBeTrue();
+  });
 });

lib/util/s3.ts

@@ -1,12 +1,20 @@
 // Singleton S3 instance initialized on-demand.
 import { S3Client } from '@aws-sdk/client-s3';
+import is from '@sindresorhus/is';
+import { GlobalConfig } from '../config/global';
 import { parseUrl } from './url';
 
 let s3Instance: S3Client | undefined;
-export function getS3Client(): S3Client {
+export function getS3Client(
+  // Only needed if GlobalConfig is not initialized due to some error
+  s3Endpoint?: string,
+  s3PathStyle?: boolean,
+): S3Client {
   if (!s3Instance) {
-    const endpoint = process.env.RENOVATE_X_S3_ENDPOINT;
-    const forcePathStyle = process.env.RENOVATE_X_S3_PATH_STYLE;
+    const endpoint = s3Endpoint ?? GlobalConfig.get('s3Endpoint');
+    const forcePathStyle = is.undefined(s3PathStyle)
+      ? !!GlobalConfig.get('s3PathStyle')
+      : s3PathStyle;
     s3Instance = new S3Client({
       ...(endpoint && { endpoint }),
       ...(forcePathStyle && { forcePathStyle: true }),

lib/workers/global/config/parse/env.spec.ts

@@ -274,13 +274,20 @@ describe('workers/global/config/parse/env', () => {
         RENOVATE_X_AUTODISCOVER_REPO_SORT: 'alpha',
         RENOVATE_X_DOCKER_MAX_PAGES: '10',
         RENOVATE_AUTODISCOVER_REPO_ORDER: 'desc',
+        RENOVATE_X_DELETE_CONFIG_FILE: 'true',
+        RENOVATE_X_S3_ENDPOINT: 'endpoint',
+        RENOVATE_X_S3_PATH_STYLE: 'true',
       };
       const config = await env.getConfig(envParam);
+      expect(config.dockerMaxPages).toBeUndefined();
       expect(config).toMatchObject({
         mergeConfidenceEndpoint: 'some-url',
         mergeConfidenceDatasources: ['docker'],
         autodiscoverRepoSort: 'alpha',
         autodiscoverRepoOrder: 'desc',
+        deleteConfigFile: true,
+        s3Endpoint: 'endpoint',
+        s3PathStyle: true,
       });
     });
 

lib/workers/global/config/parse/env.ts

@@ -88,6 +88,9 @@ function massageEnvKeyValues(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
 const convertedExperimentalEnvVars = [
   'RENOVATE_X_AUTODISCOVER_REPO_SORT',
   'RENOVATE_X_AUTODISCOVER_REPO_ORDER',
+  'RENOVATE_X_DELETE_CONFIG_FILE',
+  'RENOVATE_X_S3_ENDPOINT',
+  'RENOVATE_X_S3_PATH_STYLE',
   'RENOVATE_X_MERGE_CONFIDENCE_API_BASE_URL',
   'RENOVATE_X_MERGE_CONFIDENCE_SUPPORTED_DATASOURCES',
 ];

lib/workers/global/config/parse/file.spec.ts

@@ -141,34 +141,16 @@ describe('workers/global/config/parse/file', () => {
       expect(logger.fatal).toHaveBeenCalledWith('Unsupported file type');
       fs.unlinkSync(configFile);
     });
-
-    it('removes the config file if RENOVATE_CONFIG_FILE & RENOVATE_X_DELETE_CONFIG_FILE are set', async () => {
-      fsRemoveSpy.mockImplementationOnce(() => {
-        // no-op
-      });
-      fsPathExistsSpy
-        .mockResolvedValueOnce(true as never)
-        .mockResolvedValueOnce(true as never);
-      const configFile = upath.resolve(tmp.path, './config.json');
-      fs.writeFileSync(configFile, `{"token": "abc"}`, { encoding: 'utf8' });
-
-      await file.getConfig({
-        RENOVATE_CONFIG_FILE: configFile,
-        RENOVATE_X_DELETE_CONFIG_FILE: 'true',
-      });
-
-      expect(processExitSpy).not.toHaveBeenCalled();
-      expect(fsRemoveSpy).toHaveBeenCalledTimes(1);
-      expect(fsRemoveSpy).toHaveBeenCalledWith(configFile);
-      fs.unlinkSync(configFile);
-    });
   });
 
   describe('deleteConfigFile()', () => {
     it.each([[undefined], [' ']])(
       'skip when RENOVATE_CONFIG_FILE is not set ("%s")',
       async (configFile) => {
-        await file.deleteNonDefaultConfig({ RENOVATE_CONFIG_FILE: configFile });
+        await file.deleteNonDefaultConfig(
+          { RENOVATE_CONFIG_FILE: configFile },
+          true,
+        );
 
         expect(fsRemoveSpy).toHaveBeenCalledTimes(0);
       },
@@ -177,23 +159,27 @@ describe('workers/global/config/parse/file', () => {
     it('skip when config file does not exist', async () => {
       fsPathExistsSpy.mockResolvedValueOnce(false as never);
 
-      await file.deleteNonDefaultConfig({
-        RENOVATE_CONFIG_FILE: 'path',
-        RENOVATE_X_DELETE_CONFIG_FILE: 'true',
-      });
+      await file.deleteNonDefaultConfig(
+        {
+          RENOVATE_CONFIG_FILE: 'path',
+        },
+        true,
+      );
 
       expect(fsRemoveSpy).toHaveBeenCalledTimes(0);
     });
 
     it.each([['false'], [' ']])(
-      'skip if RENOVATE_X_DELETE_CONFIG_FILE is not set ("%s")',
+      'skip if deleteConfigFile is not set ("%s")',
       async (deleteConfig) => {
         fsPathExistsSpy.mockResolvedValueOnce(true as never);
 
-        await file.deleteNonDefaultConfig({
-          RENOVATE_X_DELETE_CONFIG_FILE: deleteConfig,
-          RENOVATE_CONFIG_FILE: '/path/to/config.js',
-        });
+        await file.deleteNonDefaultConfig(
+          {
+            RENOVATE_CONFIG_FILE: '/path/to/config.js',
+          },
+          deleteConfig === 'true',
+        );
 
         expect(fsRemoveSpy).toHaveBeenCalledTimes(0);
       },
@@ -206,10 +192,12 @@ describe('workers/global/config/parse/file', () => {
       fsPathExistsSpy.mockResolvedValueOnce(true as never);
       const configFile = '/path/to/config.js';
 
-      await file.deleteNonDefaultConfig({
-        RENOVATE_CONFIG_FILE: configFile,
-        RENOVATE_X_DELETE_CONFIG_FILE: 'true',
-      });
+      await file.deleteNonDefaultConfig(
+        {
+          RENOVATE_CONFIG_FILE: configFile,
+        },
+        true,
+      );
 
       expect(fsRemoveSpy).toHaveBeenCalledTimes(1);
       expect(fsRemoveSpy).toHaveBeenCalledWith(configFile);
@@ -226,10 +214,12 @@ describe('workers/global/config/parse/file', () => {
       fsPathExistsSpy.mockResolvedValueOnce(true as never);
       const configFile = '/path/to/config.js';
 
-      await file.deleteNonDefaultConfig({
-        RENOVATE_CONFIG_FILE: configFile,
-        RENOVATE_X_DELETE_CONFIG_FILE: 'true',
-      });
+      await file.deleteNonDefaultConfig(
+        {
+          RENOVATE_CONFIG_FILE: configFile,
+        },
+        true,
+      );
 
       expect(fsRemoveSpy).toHaveBeenCalledTimes(1);
       expect(fsRemoveSpy).toHaveBeenCalledWith(configFile);

lib/workers/global/config/parse/file.ts

@@ -79,21 +79,20 @@ export async function getConfig(env: NodeJS.ProcessEnv): Promise<AllConfig> {
     logger.debug('No config file found on disk - skipping');
   }
 
-  await deleteNonDefaultConfig(env); // Try deletion only if RENOVATE_CONFIG_FILE is specified
-
   return migrateAndValidateConfig(config, configFile);
 }
 
 export async function deleteNonDefaultConfig(
   env: NodeJS.ProcessEnv,
+  deleteConfigFile: boolean,
 ): Promise<void> {
   const configFile = env.RENOVATE_CONFIG_FILE;
 
   if (is.undefined(configFile) || is.emptyStringOrWhitespace(configFile)) {
     return;
   }
 
-  if (env.RENOVATE_X_DELETE_CONFIG_FILE !== 'true') {
+  if (!deleteConfigFile) {
     return;
   }