From 77524af19f8f21eca30329ebb6b57b3e34f84905 Mon Sep 17 00:00:00 2001 From: Aleksandr Mezin Date: Fri, 24 May 2024 09:09:19 +0300 Subject: [PATCH] feat(manager/pip-compile): extract Python version from lock files (#29145) --- .../manager/pip-compile/artifacts.spec.ts | 88 +++++++++++++++++++ lib/modules/manager/pip-compile/artifacts.ts | 6 ++ .../manager/pip-compile/common.spec.ts | 16 ++++ lib/modules/manager/pip-compile/common.ts | 36 +++++++- lib/modules/manager/pip-compile/readme.md | 2 +- 5 files changed, 146 insertions(+), 2 deletions(-) diff --git a/lib/modules/manager/pip-compile/artifacts.spec.ts b/lib/modules/manager/pip-compile/artifacts.spec.ts index af72c6b299ec81..d106fe441663f6 100644 --- a/lib/modules/manager/pip-compile/artifacts.spec.ts +++ b/lib/modules/manager/pip-compile/artifacts.spec.ts @@ -1,3 +1,4 @@ +import { codeBlock } from 'common-tags'; import { mockDeep } from 'jest-mock-extended'; import { join } from 'upath'; import { envMock, mockExecAll } from '../../../../test/exec-util'; @@ -233,6 +234,93 @@ describe('modules/manager/pip-compile/artifacts', () => { ]); }); + it('installs Python version according to the lock file', async () => { + GlobalConfig.set({ ...adminConfig, binarySource: 'install' }); + datasource.getPkgReleases.mockResolvedValueOnce({ + releases: [ + { version: '3.11.0' }, + { version: '3.11.1' }, + { version: '3.12.0' }, + ], + }); + const execSnapshots = mockExecAll(); + git.getRepoStatus.mockResolvedValue( + partial({ + modified: ['requirements.txt'], + }), + ); + fs.readLocalFile.mockResolvedValueOnce(simpleHeader); + expect( + await updateArtifacts({ + packageFileName: 'requirements.in', + updatedDeps: [], + newPackageFileContent: 'some new content', + config: { + ...config, + constraints: { pipTools: '6.13.0' }, + lockFiles: ['requirements.txt'], + }, + }), + ).not.toBeNull(); + + expect(execSnapshots).toMatchObject([ + { cmd: 'install-tool python 3.11.1' }, + { cmd: 'install-tool pip-tools 6.13.0' }, + { + cmd: 'pip-compile requirements.in', + options: { cwd: '/tmp/github/some/repo' }, + }, + ]); + }); + + it('installs latest Python version if no constraints and not in header', async () => { + GlobalConfig.set({ ...adminConfig, binarySource: 'install' }); + datasource.getPkgReleases.mockResolvedValueOnce({ + releases: [ + { version: '3.11.0' }, + { version: '3.11.1' }, + { version: '3.12.0' }, + ], + }); + const execSnapshots = mockExecAll(); + git.getRepoStatus.mockResolvedValue( + partial({ + modified: ['requirements.txt'], + }), + ); + // Before 6.2.0, pip-compile didn't include Python version in header + const noPythonVersionHeader = codeBlock` + # + # This file is autogenerated by pip-compile + # To update, run: + # + # pip-compile requirements.in + # + `; + fs.readLocalFile.mockResolvedValueOnce(noPythonVersionHeader); + expect( + await updateArtifacts({ + packageFileName: 'requirements.in', + updatedDeps: [], + newPackageFileContent: 'some new content', + config: { + ...config, + constraints: { pipTools: '6.13.0' }, + lockFiles: ['requirements.txt'], + }, + }), + ).not.toBeNull(); + + expect(execSnapshots).toMatchObject([ + { cmd: 'install-tool python 3.12.0' }, + { cmd: 'install-tool pip-tools 6.13.0' }, + { + cmd: 'pip-compile requirements.in', + options: { cwd: '/tmp/github/some/repo' }, + }, + ]); + }); + it('catches errors', async () => { const execSnapshots = mockExecAll(); fs.readLocalFile.mockResolvedValueOnce('Current requirements.txt'); diff --git a/lib/modules/manager/pip-compile/artifacts.ts b/lib/modules/manager/pip-compile/artifacts.ts index 994c4ec5aa4f8b..7f8b2ad3434071 100644 --- a/lib/modules/manager/pip-compile/artifacts.ts +++ b/lib/modules/manager/pip-compile/artifacts.ts @@ -12,6 +12,7 @@ import * as pipRequirements from '../pip_requirements'; import type { UpdateArtifact, UpdateArtifactsResult, Upgrade } from '../types'; import { extractHeaderCommand, + extractPythonVersion, getExecOptions, getRegistryCredVarsFromPackageFile, } from './common'; @@ -106,6 +107,10 @@ export async function updateArtifacts({ await deleteLocalFile(outputFileName); } const compileArgs = extractHeaderCommand(existingOutput, outputFileName); + const pythonVersion = extractPythonVersion( + existingOutput, + outputFileName, + ); const cwd = inferCommandExecDir(outputFileName, compileArgs.outputFile); const upgradePackages = updatedDeps.filter((dep) => dep.isLockfileUpdate); const packageFile = pipRequirements.extractPackageFile(newInputContent); @@ -114,6 +119,7 @@ export async function updateArtifacts({ config, cwd, getRegistryCredVarsFromPackageFile(packageFile), + pythonVersion, ); logger.trace({ cwd, cmd }, 'pip-compile command'); logger.trace({ env: execOptions.extraEnv }, 'pip-compile extra env vars'); diff --git a/lib/modules/manager/pip-compile/common.spec.ts b/lib/modules/manager/pip-compile/common.spec.ts index 8d39bc116f4a79..032706d2de6ead 100644 --- a/lib/modules/manager/pip-compile/common.spec.ts +++ b/lib/modules/manager/pip-compile/common.spec.ts @@ -4,6 +4,7 @@ import { logger } from '../../../logger'; import { allowedPipOptions, extractHeaderCommand, + extractPythonVersion, getRegistryCredVarsFromPackageFile, } from './common'; import { inferCommandExecDir } from './utils'; @@ -170,6 +171,21 @@ describe('modules/manager/pip-compile/common', () => { ); }); + describe('extractPythonVersion()', () => { + it('extracts Python version from valid header', () => { + expect( + extractPythonVersion( + getCommandInHeader('pip-compile reqs.in'), + 'reqs.txt', + ), + ).toBe('3.11'); + }); + + it('returns undefined if version cannot be extracted', () => { + expect(extractPythonVersion('', 'reqs.txt')).toBeUndefined(); + }); + }); + describe('getCredentialVarsFromPackageFile()', () => { it('handles both registryUrls and additionalRegistryUrls', () => { hostRules.find.mockReturnValueOnce({ diff --git a/lib/modules/manager/pip-compile/common.ts b/lib/modules/manager/pip-compile/common.ts index 0d3600d57f92e3..b80607fa760e3c 100644 --- a/lib/modules/manager/pip-compile/common.ts +++ b/lib/modules/manager/pip-compile/common.ts @@ -13,6 +13,7 @@ import type { PipCompileArgs } from './types'; export function getPythonVersionConstraint( config: UpdateArtifactsConfig, + extractedPythonVersion: string | undefined, ): string | undefined | null { const { constraints = {} } = config; const { python } = constraints; @@ -22,6 +23,11 @@ export function getPythonVersionConstraint( return python; } + if (extractedPythonVersion) { + logger.debug('Using python constraint extracted from the lock file'); + return `==${extractedPythonVersion}`; + } + return undefined; } export function getPipToolsVersionConstraint( @@ -41,8 +47,9 @@ export async function getExecOptions( config: UpdateArtifactsConfig, cwd: string, extraEnv: ExtraEnv, + extractedPythonVersion: string | undefined, ): Promise { - const constraint = getPythonVersionConstraint(config); + const constraint = getPythonVersionConstraint(config, extractedPythonVersion); const pipToolsConstraint = getPipToolsVersionConstraint(config); const execOptions: ExecOptions = { cwd: ensureLocalPath(cwd), @@ -197,6 +204,33 @@ export function extractHeaderCommand( return result; } +const pythonVersionRegex = regEx( + /^(#.*?\r?\n)*# This file is autogenerated by pip-compile with Python (?\d+(\.\d+)*)\s/, + 'i', +); + +export function extractPythonVersion( + content: string, + fileName: string, +): string | undefined { + const match = pythonVersionRegex.exec(content); + if (match?.groups === undefined) { + logger.warn( + `pip-compile: failed to extract Python version from header in ${fileName} ${content}`, + ); + return undefined; + } + logger.trace( + `pip-compile: found Python version header in ${fileName}: \n${match[0]}`, + ); + const { pythonVersion } = match.groups; + logger.debug( + { fileName, pythonVersion }, + `pip-compile: extracted Python version from header`, + ); + return pythonVersion; +} + function throwForDisallowedOption(arg: string): void { if (disallowedPipOptions.includes(arg)) { throw new Error(`Option ${arg} not allowed for this manager`); diff --git a/lib/modules/manager/pip-compile/readme.md b/lib/modules/manager/pip-compile/readme.md index 24324ec67403cc..cb846fb0a78d42 100644 --- a/lib/modules/manager/pip-compile/readme.md +++ b/lib/modules/manager/pip-compile/readme.md @@ -64,7 +64,7 @@ Because `pip-compile` will update source files with their associated manager you ### Configuration of Python version -By default Renovate uses the latest version of Python. +By default Renovate extracts Python version from the header. To get Renovate to use another version of Python, add a constraints` rule to the Renovate config: ```json