From 5bd88c795eb0fe5e260452489ec56aa0a5380890 Mon Sep 17 00:00:00 2001 From: Mathieu Kniewallner Date: Mon, 2 Sep 2024 00:30:45 +0200 Subject: [PATCH 1/8] feat(manager): add PEP 723 --- lib/modules/manager/api.ts | 2 + lib/modules/manager/pep723/extract.spec.ts | 99 ++++++++++++++++++++++ lib/modules/manager/pep723/extract.ts | 41 +++++++++ lib/modules/manager/pep723/index.ts | 12 +++ lib/modules/manager/pep723/readme.md | 1 + lib/modules/manager/pep723/schema.ts | 18 ++++ 6 files changed, 173 insertions(+) create mode 100644 lib/modules/manager/pep723/extract.spec.ts create mode 100644 lib/modules/manager/pep723/extract.ts create mode 100644 lib/modules/manager/pep723/index.ts create mode 100644 lib/modules/manager/pep723/readme.md create mode 100644 lib/modules/manager/pep723/schema.ts diff --git a/lib/modules/manager/api.ts b/lib/modules/manager/api.ts index 8e7271cd3aa320..2e7ccf55d2fa6e 100644 --- a/lib/modules/manager/api.ts +++ b/lib/modules/manager/api.ts @@ -71,6 +71,7 @@ import * as nvm from './nvm'; import * as ocb from './ocb'; import * as osgi from './osgi'; import * as pep621 from './pep621'; +import * as pep723 from './pep723'; import * as pipCompile from './pip-compile'; import * as pip_requirements from './pip_requirements'; import * as pip_setup from './pip_setup'; @@ -174,6 +175,7 @@ api.set('nvm', nvm); api.set('ocb', ocb); api.set('osgi', osgi); api.set('pep621', pep621); +api.set('pep723', pep723); api.set('pip-compile', pipCompile); api.set('pip_requirements', pip_requirements); api.set('pip_setup', pip_setup); diff --git a/lib/modules/manager/pep723/extract.spec.ts b/lib/modules/manager/pep723/extract.spec.ts new file mode 100644 index 00000000000000..0a0a97ab7b2bd7 --- /dev/null +++ b/lib/modules/manager/pep723/extract.spec.ts @@ -0,0 +1,99 @@ +import { codeBlock } from 'common-tags'; +import { extractPackageFile } from '.'; + +describe('modules/manager/pep723/extract', () => { + describe('extractPackageFile()', () => { + it('should extract dependencies', () => { + const res = extractPackageFile( + codeBlock` +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "requests==2.32.3", +# "rich>=13.8.0", +# ] +# /// +`, + 'foo.py', + ); + + expect(res).toMatchObject({ + deps: [ + { + currentValue: '==2.32.3', + currentVersion: '2.32.3', + datasource: 'pypi', + depName: 'requests', + depType: 'project.dependencies', + packageName: 'requests', + }, + { + currentValue: '>=13.8.0', + datasource: 'pypi', + depName: 'rich', + depType: 'project.dependencies', + packageName: 'rich', + }, + ], + }); + }); + + it('should skip invalid dependencies', () => { + const res = extractPackageFile( + codeBlock` +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "requests==2.32.3", +# "==1.2.3", +# ] +# /// +`, + 'foo.py', + ); + + expect(res).toMatchObject({ + deps: [ + { + currentValue: '==2.32.3', + currentVersion: '2.32.3', + datasource: 'pypi', + depName: 'requests', + depType: 'project.dependencies', + packageName: 'requests', + }, + ], + }); + }); + + it('should return an empty list on missing dependencies', () => { + const res = extractPackageFile( + codeBlock` +# /// script +# requires-python = ">=3.11" +# /// +`, + 'foo.py', + ); + + expect(res).toMatchObject({ deps: [] }); + }); + + it('should return null on invalid TOML', () => { + const res = extractPackageFile( + codeBlock` +# /// script +# requires-python +# dependencies = [ +# "requests==2.32.3", +# "rich>=13.8.0", +# ] +# /// +`, + 'foo.py', + ); + + expect(res).toBeNull(); + }); + }); +}); diff --git a/lib/modules/manager/pep723/extract.ts b/lib/modules/manager/pep723/extract.ts new file mode 100644 index 00000000000000..583caedf7fd167 --- /dev/null +++ b/lib/modules/manager/pep723/extract.ts @@ -0,0 +1,41 @@ +import { logger } from '../../../logger'; +import { regEx } from '../../../util/regex'; +import { Result } from '../../../util/result'; +import type { PackageFileContent } from '../types'; +import { Pep723Schema } from './schema'; + +// Adapted regex from the Python reference implementation: https://packaging.python.org/en/latest/specifications/inline-script-metadata/#reference-implementation +const regex = regEx( + /^# \/\/\/ (?[a-zA-Z0-9-]+)$\s(?(^#(| .*)$\s)+)^# \/\/\/$/, + 'm', +); + +export function extractPackageFile( + content: string, + packageFile: string, +): PackageFileContent | null { + const match = regex.exec(content); + const matchedContent = match?.groups?.content; + + if (!matchedContent) { + return null; + } + + // Adapted code from the Python reference implementation: https://packaging.python.org/en/latest/specifications/inline-script-metadata/#reference-implementation + const parsedToml = matchedContent + .split('\n') + .map((line) => line.substring(line.startsWith('# ') ? 2 : 1)) + .join('\n'); + + const { val: deps, err } = Result.parse(parsedToml, Pep723Schema).unwrap(); + + if (err) { + logger.debug( + { packageFile, err }, + `Error parsing PEP 723 inline script metadata`, + ); + return null; + } + + return { deps }; +} diff --git a/lib/modules/manager/pep723/index.ts b/lib/modules/manager/pep723/index.ts new file mode 100644 index 00000000000000..fff3416ccdf6e5 --- /dev/null +++ b/lib/modules/manager/pep723/index.ts @@ -0,0 +1,12 @@ +import type { Category } from '../../../constants'; +import { PypiDatasource } from '../../datasource/pypi'; +export { extractPackageFile } from './extract'; + +export const supportedDatasources = [PypiDatasource.id]; + +export const categories: Category[] = ['python']; + +export const defaultConfig = { + // Since any Python file can embed PEP 723 metadata, make the feature opt-in, to avoid parsing all Python files. + fileMatch: [], +}; diff --git a/lib/modules/manager/pep723/readme.md b/lib/modules/manager/pep723/readme.md new file mode 100644 index 00000000000000..bc190b4149eb4c --- /dev/null +++ b/lib/modules/manager/pep723/readme.md @@ -0,0 +1 @@ +This manager supports updating dependencies inside Python files that use [inline script metadata](https://packaging.python.org/en/latest/specifications/inline-script-metadata/), also known as PEP 723. diff --git a/lib/modules/manager/pep723/schema.ts b/lib/modules/manager/pep723/schema.ts new file mode 100644 index 00000000000000..8ad29e75976282 --- /dev/null +++ b/lib/modules/manager/pep723/schema.ts @@ -0,0 +1,18 @@ +import { z } from 'zod'; +import { Toml } from '../../../util/schema-utils'; +import { depTypes, pep508ToPackageDependency } from '../pep621/utils'; + +const Pep723Dep = z + .string() + .transform((dep) => pep508ToPackageDependency(depTypes.dependencies, dep)); + +export const Pep723Schema = Toml.pipe( + z + .object({ + dependencies: z + .array(Pep723Dep) + .transform((deps) => deps.filter((dep) => !!dep)) + .optional(), + }) + .transform(({ dependencies }) => dependencies ?? []), +); From c7b32a81996090baf18bd62bcd8ce02ee22db0fb Mon Sep 17 00:00:00 2001 From: Mathieu Kniewallner Date: Sun, 8 Sep 2024 00:00:49 +0200 Subject: [PATCH 2/8] test(manager/pep723): test case without match --- lib/modules/manager/pep723/extract.spec.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lib/modules/manager/pep723/extract.spec.ts b/lib/modules/manager/pep723/extract.spec.ts index 0a0a97ab7b2bd7..b738a292125697 100644 --- a/lib/modules/manager/pep723/extract.spec.ts +++ b/lib/modules/manager/pep723/extract.spec.ts @@ -95,5 +95,17 @@ describe('modules/manager/pep723/extract', () => { expect(res).toBeNull(); }); + + it('should return null if there is no PEP 723 metadata', () => { + const res = extractPackageFile( + codeBlock` +if True: + print("requires-python>=3.11") +`, + 'foo.py', + ); + + expect(res).toBeNull(); + }); }); }); From e4e423a53e567025adb518b98e28d01241d36694 Mon Sep 17 00:00:00 2001 From: Mathieu Kniewallner Date: Sun, 8 Sep 2024 12:18:08 +0200 Subject: [PATCH 3/8] fix(manager/pep723): use `newlineRegex` --- lib/modules/manager/pep723/extract.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/modules/manager/pep723/extract.ts b/lib/modules/manager/pep723/extract.ts index 583caedf7fd167..7a9cb88e1ce664 100644 --- a/lib/modules/manager/pep723/extract.ts +++ b/lib/modules/manager/pep723/extract.ts @@ -1,5 +1,5 @@ import { logger } from '../../../logger'; -import { regEx } from '../../../util/regex'; +import {newlineRegex, regEx} from '../../../util/regex'; import { Result } from '../../../util/result'; import type { PackageFileContent } from '../types'; import { Pep723Schema } from './schema'; @@ -23,7 +23,7 @@ export function extractPackageFile( // Adapted code from the Python reference implementation: https://packaging.python.org/en/latest/specifications/inline-script-metadata/#reference-implementation const parsedToml = matchedContent - .split('\n') + .split(newlineRegex) .map((line) => line.substring(line.startsWith('# ') ? 2 : 1)) .join('\n'); From 0982b29b75c4322bfd0a5c73e92b1d75f5d24500 Mon Sep 17 00:00:00 2001 From: Mathieu Kniewallner Date: Sun, 8 Sep 2024 13:09:32 +0200 Subject: [PATCH 4/8] refactor(manager/pep723): use `safeParse` Co-authored-by: Michael Kriese --- lib/modules/manager/pep723/extract.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/modules/manager/pep723/extract.ts b/lib/modules/manager/pep723/extract.ts index 7a9cb88e1ce664..854bd4bd1e107c 100644 --- a/lib/modules/manager/pep723/extract.ts +++ b/lib/modules/manager/pep723/extract.ts @@ -1,6 +1,5 @@ import { logger } from '../../../logger'; -import {newlineRegex, regEx} from '../../../util/regex'; -import { Result } from '../../../util/result'; +import { newlineRegex, regEx } from '../../../util/regex'; import type { PackageFileContent } from '../types'; import { Pep723Schema } from './schema'; @@ -27,11 +26,11 @@ export function extractPackageFile( .map((line) => line.substring(line.startsWith('# ') ? 2 : 1)) .join('\n'); - const { val: deps, err } = Result.parse(parsedToml, Pep723Schema).unwrap(); + const { data: deps, error } = Pep723Schema.safeParse(parsedToml); - if (err) { + if (error) { logger.debug( - { packageFile, err }, + { packageFile, error }, `Error parsing PEP 723 inline script metadata`, ); return null; From aa0e7803c6bdd304bde49d98c7dc1881391e27de Mon Sep 17 00:00:00 2001 From: Mathieu Kniewallner Date: Sun, 8 Sep 2024 13:37:51 +0200 Subject: [PATCH 5/8] feat(manager/pep723): extract Python constraint --- lib/modules/manager/pep723/extract.spec.ts | 9 +++++++-- lib/modules/manager/pep723/extract.ts | 4 ++-- lib/modules/manager/pep723/schema.ts | 13 ++++++++++++- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/lib/modules/manager/pep723/extract.spec.ts b/lib/modules/manager/pep723/extract.spec.ts index b738a292125697..ddf4ba2f64b04b 100644 --- a/lib/modules/manager/pep723/extract.spec.ts +++ b/lib/modules/manager/pep723/extract.spec.ts @@ -35,6 +35,7 @@ describe('modules/manager/pep723/extract', () => { packageName: 'rich', }, ], + extractedConstraints: { python: '>=3.11' }, }); }); @@ -42,7 +43,7 @@ describe('modules/manager/pep723/extract', () => { const res = extractPackageFile( codeBlock` # /// script -# requires-python = ">=3.11" +# requires-python = "==3.11" # dependencies = [ # "requests==2.32.3", # "==1.2.3", @@ -63,6 +64,7 @@ describe('modules/manager/pep723/extract', () => { packageName: 'requests', }, ], + extractedConstraints: { python: '==3.11' }, }); }); @@ -76,7 +78,10 @@ describe('modules/manager/pep723/extract', () => { 'foo.py', ); - expect(res).toMatchObject({ deps: [] }); + expect(res).toMatchObject({ + deps: [], + extractedConstraints: { python: '>=3.11' }, + }); }); it('should return null on invalid TOML', () => { diff --git a/lib/modules/manager/pep723/extract.ts b/lib/modules/manager/pep723/extract.ts index 854bd4bd1e107c..be004ca85caa07 100644 --- a/lib/modules/manager/pep723/extract.ts +++ b/lib/modules/manager/pep723/extract.ts @@ -26,7 +26,7 @@ export function extractPackageFile( .map((line) => line.substring(line.startsWith('# ') ? 2 : 1)) .join('\n'); - const { data: deps, error } = Pep723Schema.safeParse(parsedToml); + const { data: res, error } = Pep723Schema.safeParse(parsedToml); if (error) { logger.debug( @@ -36,5 +36,5 @@ export function extractPackageFile( return null; } - return { deps }; + return res; } diff --git a/lib/modules/manager/pep723/schema.ts b/lib/modules/manager/pep723/schema.ts index 8ad29e75976282..c29ccf37ec1079 100644 --- a/lib/modules/manager/pep723/schema.ts +++ b/lib/modules/manager/pep723/schema.ts @@ -1,6 +1,8 @@ +import is from '@sindresorhus/is'; import { z } from 'zod'; import { Toml } from '../../../util/schema-utils'; import { depTypes, pep508ToPackageDependency } from '../pep621/utils'; +import type { PackageFileContent } from '../types'; const Pep723Dep = z .string() @@ -9,10 +11,19 @@ const Pep723Dep = z export const Pep723Schema = Toml.pipe( z .object({ + 'requires-python': z.string().optional(), dependencies: z .array(Pep723Dep) .transform((deps) => deps.filter((dep) => !!dep)) .optional(), }) - .transform(({ dependencies }) => dependencies ?? []), + .transform(({ 'requires-python': requiresPython, dependencies }) => { + const res: PackageFileContent = { deps: dependencies ?? [] }; + + if (is.nonEmptyString(requiresPython)) { + res.extractedConstraints = { python: requiresPython }; + } + + return res; + }), ); From 4365e801eec418b975ca7e48fbd12c086d04828d Mon Sep 17 00:00:00 2001 From: Mathieu Kniewallner Date: Sun, 8 Sep 2024 16:31:52 +0200 Subject: [PATCH 6/8] fix(manager/pep723): return `null` on empty deps Co-authored-by: Michael Kriese --- lib/modules/manager/pep723/extract.spec.ts | 7 ++----- lib/modules/manager/pep723/extract.ts | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/lib/modules/manager/pep723/extract.spec.ts b/lib/modules/manager/pep723/extract.spec.ts index ddf4ba2f64b04b..7a9cdaf400e56d 100644 --- a/lib/modules/manager/pep723/extract.spec.ts +++ b/lib/modules/manager/pep723/extract.spec.ts @@ -68,7 +68,7 @@ describe('modules/manager/pep723/extract', () => { }); }); - it('should return an empty list on missing dependencies', () => { + it('should return null on missing dependencies', () => { const res = extractPackageFile( codeBlock` # /// script @@ -78,10 +78,7 @@ describe('modules/manager/pep723/extract', () => { 'foo.py', ); - expect(res).toMatchObject({ - deps: [], - extractedConstraints: { python: '>=3.11' }, - }); + expect(res).toBeNull(); }); it('should return null on invalid TOML', () => { diff --git a/lib/modules/manager/pep723/extract.ts b/lib/modules/manager/pep723/extract.ts index be004ca85caa07..a98ae53cbd0f72 100644 --- a/lib/modules/manager/pep723/extract.ts +++ b/lib/modules/manager/pep723/extract.ts @@ -36,5 +36,5 @@ export function extractPackageFile( return null; } - return res; + return res.deps.length ? res : null; } From af46173ed3c53cd119c4b50c176434ce10c698ba Mon Sep 17 00:00:00 2001 From: Mathieu Kniewallner Date: Sun, 8 Sep 2024 16:33:38 +0200 Subject: [PATCH 7/8] test(manager/pep723): use stricter `toEqual` --- lib/modules/manager/pep723/extract.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/modules/manager/pep723/extract.spec.ts b/lib/modules/manager/pep723/extract.spec.ts index 7a9cdaf400e56d..4b9cb115f81ae0 100644 --- a/lib/modules/manager/pep723/extract.spec.ts +++ b/lib/modules/manager/pep723/extract.spec.ts @@ -17,7 +17,7 @@ describe('modules/manager/pep723/extract', () => { 'foo.py', ); - expect(res).toMatchObject({ + expect(res).toEqual({ deps: [ { currentValue: '==2.32.3', @@ -53,7 +53,7 @@ describe('modules/manager/pep723/extract', () => { 'foo.py', ); - expect(res).toMatchObject({ + expect(res).toEqual({ deps: [ { currentValue: '==2.32.3', From a54141e7893bf402fe33e682cbe872e5540a3077 Mon Sep 17 00:00:00 2001 From: Mathieu Kniewallner Date: Tue, 10 Sep 2024 20:40:39 +0200 Subject: [PATCH 8/8] test(manager/pep723): indent code blocks --- lib/modules/manager/pep723/extract.spec.ts | 62 +++++++++++----------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/lib/modules/manager/pep723/extract.spec.ts b/lib/modules/manager/pep723/extract.spec.ts index 4b9cb115f81ae0..d6221b08d7ca4d 100644 --- a/lib/modules/manager/pep723/extract.spec.ts +++ b/lib/modules/manager/pep723/extract.spec.ts @@ -6,14 +6,14 @@ describe('modules/manager/pep723/extract', () => { it('should extract dependencies', () => { const res = extractPackageFile( codeBlock` -# /// script -# requires-python = ">=3.11" -# dependencies = [ -# "requests==2.32.3", -# "rich>=13.8.0", -# ] -# /// -`, + # /// script + # requires-python = ">=3.11" + # dependencies = [ + # "requests==2.32.3", + # "rich>=13.8.0", + # ] + # /// + `, 'foo.py', ); @@ -42,14 +42,14 @@ describe('modules/manager/pep723/extract', () => { it('should skip invalid dependencies', () => { const res = extractPackageFile( codeBlock` -# /// script -# requires-python = "==3.11" -# dependencies = [ -# "requests==2.32.3", -# "==1.2.3", -# ] -# /// -`, + # /// script + # requires-python = "==3.11" + # dependencies = [ + # "requests==2.32.3", + # "==1.2.3", + # ] + # /// + `, 'foo.py', ); @@ -71,10 +71,10 @@ describe('modules/manager/pep723/extract', () => { it('should return null on missing dependencies', () => { const res = extractPackageFile( codeBlock` -# /// script -# requires-python = ">=3.11" -# /// -`, + # /// script + # requires-python = ">=3.11" + # /// + `, 'foo.py', ); @@ -84,14 +84,14 @@ describe('modules/manager/pep723/extract', () => { it('should return null on invalid TOML', () => { const res = extractPackageFile( codeBlock` -# /// script -# requires-python -# dependencies = [ -# "requests==2.32.3", -# "rich>=13.8.0", -# ] -# /// -`, + # /// script + # requires-python + # dependencies = [ + # "requests==2.32.3", + # "rich>=13.8.0", + # ] + # /// + `, 'foo.py', ); @@ -101,9 +101,9 @@ describe('modules/manager/pep723/extract', () => { it('should return null if there is no PEP 723 metadata', () => { const res = extractPackageFile( codeBlock` -if True: - print("requires-python>=3.11") -`, + if True: + print("requires-python>=3.11") + `, 'foo.py', );