diff --git a/lib/modules/manager/pip-compile/artifacts.spec.ts b/lib/modules/manager/pip-compile/artifacts.spec.ts index 861c5127bc1242..6818e35d0550d6 100644 --- a/lib/modules/manager/pip-compile/artifacts.spec.ts +++ b/lib/modules/manager/pip-compile/artifacts.spec.ts @@ -86,6 +86,7 @@ describe('modules/manager/pip-compile/artifacts', () => { it('returns null if all unchanged', async () => { fs.readLocalFile.mockResolvedValueOnce(simpleHeader); + fs.readLocalFile.mockResolvedValueOnce('dependency==1.2.3'); const execSnapshots = mockExecAll(); fs.readLocalFile.mockResolvedValueOnce('new lock'); expect( @@ -106,6 +107,7 @@ describe('modules/manager/pip-compile/artifacts', () => { it('returns null if no config.lockFiles', async () => { fs.readLocalFile.mockResolvedValueOnce(simpleHeader); + fs.readLocalFile.mockResolvedValueOnce('dependency==1.2.3'); fs.readLocalFile.mockResolvedValueOnce('new lock'); expect( await updateArtifacts({ @@ -125,6 +127,7 @@ describe('modules/manager/pip-compile/artifacts', () => { it('returns updated requirements.txt', async () => { fs.readLocalFile.mockResolvedValueOnce(simpleHeader); + fs.readLocalFile.mockResolvedValueOnce('dependency==1.2.3'); const execSnapshots = mockExecAll(); git.getRepoStatus.mockResolvedValue( partial({ @@ -162,6 +165,7 @@ describe('modules/manager/pip-compile/artifacts', () => { }), ); fs.readLocalFile.mockResolvedValueOnce(simpleHeader); + fs.readLocalFile.mockResolvedValueOnce('dependency==1.2.3'); fs.ensureCacheDir.mockResolvedValueOnce('/tmp/renovate/cache/others/pip'); expect( await updateArtifacts({ @@ -215,6 +219,7 @@ describe('modules/manager/pip-compile/artifacts', () => { }), ); fs.readLocalFile.mockResolvedValueOnce(simpleHeader); + fs.readLocalFile.mockResolvedValueOnce('dependency==1.2.3'); expect( await updateArtifacts({ packageFileName: 'requirements.in', @@ -328,6 +333,7 @@ describe('modules/manager/pip-compile/artifacts', () => { it('catches errors', async () => { const execSnapshots = mockExecAll(); fs.readLocalFile.mockResolvedValueOnce('Current requirements.txt'); + fs.readLocalFile.mockResolvedValueOnce('dependency==1.2.3'); fs.writeLocalFile.mockImplementationOnce(() => { throw new Error('not found'); }); @@ -348,6 +354,7 @@ describe('modules/manager/pip-compile/artifacts', () => { it('returns updated requirements.txt when doing lockfile maintenance', async () => { fs.readLocalFile.mockResolvedValueOnce(simpleHeader); + fs.readLocalFile.mockResolvedValueOnce('dependency==1.2.3'); const execSnapshots = mockExecAll(); git.getRepoStatus.mockResolvedValue( partial({ @@ -370,6 +377,7 @@ describe('modules/manager/pip-compile/artifacts', () => { it('uses --upgrade-package only for isLockfileUpdate', async () => { fs.readLocalFile.mockResolvedValueOnce(simpleHeader); + fs.readLocalFile.mockResolvedValueOnce('dependency==1.2.3'); const execSnapshots = mockExecAll(); git.getRepoStatus.mockResolvedValue( partial({ @@ -397,6 +405,7 @@ describe('modules/manager/pip-compile/artifacts', () => { it('uses pip-compile version from config', async () => { fs.readLocalFile.mockResolvedValueOnce(simpleHeader); + fs.readLocalFile.mockResolvedValueOnce('dependency==1.2.3'); GlobalConfig.set(dockerAdminConfig); // pip-tools datasource.getPkgReleases.mockResolvedValueOnce({ diff --git a/lib/modules/manager/pip-compile/artifacts.ts b/lib/modules/manager/pip-compile/artifacts.ts index 6ce9b28b927b50..5d8bafb46d4220 100644 --- a/lib/modules/manager/pip-compile/artifacts.ts +++ b/lib/modules/manager/pip-compile/artifacts.ts @@ -1,4 +1,5 @@ import { quote } from 'shlex'; +import upath from 'upath'; import { TEMPORARY_ERROR } from '../../../constants/error-messages'; import { logger } from '../../../logger'; import { exec } from '../../../util/exec'; @@ -8,13 +9,19 @@ import { writeLocalFile, } from '../../../util/fs'; import { getRepoStatus } from '../../../util/git'; -import * as pipRequirements from '../pip_requirements'; -import type { UpdateArtifact, UpdateArtifactsResult, Upgrade } from '../types'; +import { extractPackageFileFlags as extractRequirementsFileFlags } from '../pip_requirements/common'; +import type { + PackageFileContent, + UpdateArtifact, + UpdateArtifactsResult, + Upgrade, +} from '../types'; import { extractHeaderCommand, extractPythonVersion, getExecOptions, - getRegistryCredVarsFromPackageFile, + getRegistryCredVarsFromPackageFiles, + matchManager, } from './common'; import type { PipCompileArgs } from './types'; import { inferCommandExecDir } from './utils'; @@ -113,12 +120,25 @@ export async function updateArtifacts({ ); const cwd = inferCommandExecDir(outputFileName, compileArgs.outputFile); const upgradePackages = updatedDeps.filter((dep) => dep.isLockfileUpdate); - const packageFile = pipRequirements.extractPackageFile(newInputContent); + const packageFiles: PackageFileContent[] = []; + for (const name of compileArgs.sourceFiles) { + const manager = matchManager(name); + if (manager === 'pip_requirements') { + const path = upath.join(cwd, name); + const content = await readLocalFile(path, 'utf8'); + if (content) { + const packageFile = extractRequirementsFileFlags(content); + if (packageFile) { + packageFiles.push(packageFile); + } + } + } + } const cmd = constructPipCompileCmd(compileArgs, upgradePackages); const execOptions = await getExecOptions( config, cwd, - getRegistryCredVarsFromPackageFile(packageFile), + getRegistryCredVarsFromPackageFiles(packageFiles), pythonVersion, ); logger.trace({ cwd, cmd }, 'pip-compile command'); diff --git a/lib/modules/manager/pip-compile/common.spec.ts b/lib/modules/manager/pip-compile/common.spec.ts index 165b9cededc34a..f8ae7750b36c47 100644 --- a/lib/modules/manager/pip-compile/common.spec.ts +++ b/lib/modules/manager/pip-compile/common.spec.ts @@ -5,7 +5,7 @@ import { allowedPipOptions, extractHeaderCommand, extractPythonVersion, - getRegistryCredVarsFromPackageFile, + getRegistryCredVarsFromPackageFiles, matchManager, } from './common'; import { inferCommandExecDir } from './utils'; @@ -187,7 +187,7 @@ describe('modules/manager/pip-compile/common', () => { }); }); - describe('getCredentialVarsFromPackageFile()', () => { + describe('getRegistryCredVarsFromPackageFiles()', () => { it('handles both registryUrls and additionalRegistryUrls', () => { hostRules.find.mockReturnValueOnce({ username: 'user1', @@ -198,11 +198,13 @@ describe('modules/manager/pip-compile/common', () => { password: 'password2', }); expect( - getRegistryCredVarsFromPackageFile({ - deps: [], - registryUrls: ['https://example.com/pypi/simple'], - additionalRegistryUrls: ['https://example2.com/pypi/simple'], - }), + getRegistryCredVarsFromPackageFiles([ + { + deps: [], + registryUrls: ['https://example.com/pypi/simple'], + additionalRegistryUrls: ['https://example2.com/pypi/simple'], + }, + ]), ).toEqual({ KEYRING_SERVICE_NAME_0: 'example.com', KEYRING_SERVICE_USERNAME_0: 'user1', @@ -223,13 +225,15 @@ describe('modules/manager/pip-compile/common', () => { password: 'password2', }); expect( - getRegistryCredVarsFromPackageFile({ - deps: [], - additionalRegistryUrls: [ - 'https://example.com/pypi/simple', - 'https://example2.com/pypi/simple', - ], - }), + getRegistryCredVarsFromPackageFiles([ + { + deps: [], + additionalRegistryUrls: [ + 'https://example.com/pypi/simple', + 'https://example2.com/pypi/simple', + ], + }, + ]), ).toEqual({ KEYRING_SERVICE_NAME_0: 'example.com', KEYRING_SERVICE_USERNAME_0: 'user1', @@ -245,10 +249,12 @@ describe('modules/manager/pip-compile/common', () => { username: 'user', }); expect( - getRegistryCredVarsFromPackageFile({ - deps: [], - additionalRegistryUrls: ['https://example.com/pypi/simple'], - }), + getRegistryCredVarsFromPackageFiles([ + { + deps: [], + additionalRegistryUrls: ['https://example.com/pypi/simple'], + }, + ]), ).toEqual({ KEYRING_SERVICE_NAME_0: 'example.com', KEYRING_SERVICE_USERNAME_0: 'user', @@ -261,10 +267,12 @@ describe('modules/manager/pip-compile/common', () => { password: 'password', }); expect( - getRegistryCredVarsFromPackageFile({ - deps: [], - additionalRegistryUrls: ['https://example.com/pypi/simple'], - }), + getRegistryCredVarsFromPackageFiles([ + { + deps: [], + additionalRegistryUrls: ['https://example.com/pypi/simple'], + }, + ]), ).toEqual({ KEYRING_SERVICE_NAME_0: 'example.com', KEYRING_SERVICE_USERNAME_0: '', @@ -277,14 +285,46 @@ describe('modules/manager/pip-compile/common', () => { password: 'password', }); expect( - getRegistryCredVarsFromPackageFile({ - deps: [], - additionalRegistryUrls: ['invalid-url'], - }), + getRegistryCredVarsFromPackageFiles([ + { + deps: [], + additionalRegistryUrls: ['invalid-url'], + }, + ]), ).toEqual({}); }); }); + it('handles multiple package files', () => { + hostRules.find.mockReturnValueOnce({ + username: 'user1', + password: 'password1', + }); + hostRules.find.mockReturnValueOnce({ + username: 'user2', + password: 'password2', + }); + expect( + getRegistryCredVarsFromPackageFiles([ + { + deps: [], + registryUrls: ['https://example.com/pypi/simple'], + }, + { + deps: [], + additionalRegistryUrls: ['https://example2.com/pypi/simple'], + }, + ]), + ).toEqual({ + KEYRING_SERVICE_NAME_0: 'example.com', + KEYRING_SERVICE_USERNAME_0: 'user1', + KEYRING_SERVICE_PASSWORD_0: 'password1', + KEYRING_SERVICE_NAME_1: 'example2.com', + KEYRING_SERVICE_USERNAME_1: 'user2', + KEYRING_SERVICE_PASSWORD_1: 'password2', + }); + }); + describe('matchManager()', () => { it('matches pip_setup setup.py', () => { expect(matchManager('setup.py')).toBe('pip_setup'); diff --git a/lib/modules/manager/pip-compile/common.ts b/lib/modules/manager/pip-compile/common.ts index 5f9e7255387012..8e4e4906f81a6e 100644 --- a/lib/modules/manager/pip-compile/common.ts +++ b/lib/modules/manager/pip-compile/common.ts @@ -281,13 +281,17 @@ function cleanUrl(url: string): URL | null { } } -export function getRegistryCredVarsFromPackageFile( - packageFile: PackageFileContent | null, +export function getRegistryCredVarsFromPackageFiles( + packageFiles: PackageFileContent[], ): ExtraEnv { - const urls = [ - ...(packageFile?.registryUrls ?? []), - ...(packageFile?.additionalRegistryUrls ?? []), - ]; + const urls: string[] = []; + for (const packageFile of packageFiles) { + urls.push( + ...(packageFile.registryUrls ?? []), + ...(packageFile.additionalRegistryUrls ?? []), + ); + } + logger.debug(urls, 'Extracted registry URLs from package files'); const uniqueHosts = new Set( urls.map(cleanUrl).filter(isNotNullOrUndefined),