From ef4e98389c480971aa911af38dbc142e21765f21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0ar=C5=ABnas=20Nejus?= Date: Mon, 2 Dec 2024 12:49:23 +0000 Subject: [PATCH 01/18] Add test for changelog formatting --- .github/workflows/test-changelog.yaml | 51 +++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 .github/workflows/test-changelog.yaml diff --git a/.github/workflows/test-changelog.yaml b/.github/workflows/test-changelog.yaml new file mode 100644 index 0000000000..c5661aadc1 --- /dev/null +++ b/.github/workflows/test-changelog.yaml @@ -0,0 +1,51 @@ +name: Temporarily test changelog formatting +on: + pull_request: + +jobs: + build: + name: Get changelog + runs-on: ubuntu-latest + outputs: + changelog: ${{ steps.generate_changelog.outputs.changelog }} + steps: + - uses: actions/checkout@v4 + with: + ref: master + + - name: Install Python tools + uses: BrandonLWhite/pipx-install-action@v0.1.1 + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: poetry + + - name: Install dependencies + run: poetry install --with=release --extras=docs + + - name: Install pandoc + run: sudo apt update && sudo apt install pandoc -y + + - name: Obtain the changelog + id: generate_changelog + run: | + { + echo 'changelog<> "$GITHUB_OUTPUT" + test: + runs-on: ubuntu-latest + permissions: + pull-requests: write + needs: build + env: + CHANGELOG: ${{ needs.build.outputs.changelog }} + steps: + - uses: mshick/add-pr-comment@v2 + with: + message: |- + ### Commit: ${{ github.sha }} + ### Changelog: + + ${{ env.CHANGELOG }} From aff43d8d622edc67019fd1428f07c4ab846f1c82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0ar=C5=ABnas=20Nejus?= Date: Mon, 2 Dec 2024 12:54:17 +0000 Subject: [PATCH 02/18] Remove poe output --- .github/workflows/make_release.yaml | 2 +- .github/workflows/test-changelog.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/make_release.yaml b/.github/workflows/make_release.yaml index cf1b12fe24..aeb4c0a329 100644 --- a/.github/workflows/make_release.yaml +++ b/.github/workflows/make_release.yaml @@ -67,7 +67,7 @@ jobs: run: | { echo 'changelog<> "$GITHUB_OUTPUT" diff --git a/.github/workflows/test-changelog.yaml b/.github/workflows/test-changelog.yaml index c5661aadc1..35ea067435 100644 --- a/.github/workflows/test-changelog.yaml +++ b/.github/workflows/test-changelog.yaml @@ -31,7 +31,7 @@ jobs: run: | { echo 'changelog<> "$GITHUB_OUTPUT" test: From e9076ffb53b916a6d55bbb07eec70d254a55b9b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0ar=C5=ABnas=20Nejus?= Date: Tue, 3 Dec 2024 03:19:08 +0000 Subject: [PATCH 03/18] Unindent list items in the changelog I found out that GitHub Actions use pandoc version 2.9.2.1 which converts bullet points like this: echo ' * Item * Another item ' | pandoc --from=rst --to=gfm - Item - Another item Note that each item has two-space indent. Meanwhile, locally I've been testing this conversion using pandoc 3.5 which does not add any indent: echo ' * Item * Another item ' | pandoc --from=rst --to=gfm - Item - Another item This commit removes the indent and cleans up the how the replacements in rst and md text are performed. --- extra/release.py | 45 ++++++++++++++++++++++----------------------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/extra/release.py b/extra/release.py index 329fbb208f..abbb57bd47 100755 --- a/extra/release.py +++ b/extra/release.py @@ -13,6 +13,7 @@ import click import tomli from packaging.version import Version, parse +from typing_extensions import TypeAlias BASE = Path(__file__).parent.parent.absolute() PYPROJECT = BASE / "pyproject.toml" @@ -23,6 +24,19 @@ RST_LATEST_CHANGES = re.compile( rf"{version_header}\n--+\s+(.+?)\n\n+{version_header}", re.DOTALL ) +Replacement: TypeAlias = "tuple[str, str | Callable[[re.Match[str]], str]]" +RST_REPLACEMENTS: list[Replacement] = [ + (r"(?<=[\s(])`([^`]+)`(?=[^_])", r"``\1``"), # ticks with verbatim ranges. + (r":bug:`(\d+)`", r":bug: (#\1)"), # Issue numbers. + (r":user:`(\w+)`", r"@\1"), # Users. +] +MD_REPLACEMENTS: list[Replacement] = [ + (r"^ ( *- )", r"\1"), # remove list indentation + (r"^(\w.+?):$", r"### \1"), # make sections headers + (r"^- `/?plugins/(\w+)`:?", r"- Plugin **`\1`**:"), # highlight plugins + (r"^- `(\w+)-cmd`:?", r"- Command **`\1`**:"), # highlight commands + (r"### [^\n]+\n+(?=### )", ""), # remove empty sections +] def update_docs_config(text: str, new: Version) -> str: @@ -95,18 +109,10 @@ def bump_version(new: Version) -> None: def rst2md(text: str) -> str: """Use Pandoc to convert text from ReST to Markdown.""" - # Other backslashes with verbatim ranges. - rst = re.sub(r"(?<=[\s(])`([^`]+)`(?=[^_])", r"``\1``", text) - - # Bug numbers. - rst = re.sub(r":bug:`(\d+)`", r":bug: (#\1)", rst) - - # Users. - rst = re.sub(r":user:`(\w+)`", r"@\1", rst) return ( subprocess.check_output( - ["/usr/bin/pandoc", "--from=rst", "--to=gfm", "--wrap=none"], - input=rst.encode(), + ["pandoc", "--from=rst", "--to=gfm", "--wrap=none"], + input=text.encode(), ) .decode() .strip() @@ -115,25 +121,18 @@ def rst2md(text: str) -> str: def changelog_as_markdown() -> str: """Get the latest changelog entry as hacked up Markdown.""" - with CHANGELOG.open() as f: - contents = f.read() + contents = CHANGELOG.read_text() m = RST_LATEST_CHANGES.search(contents) rst = m.group(1) if m else "" - # Convert with Pandoc. - md = rst2md(rst) - - # Make sections stand out - md = re.sub(r"^(\w.+?):$", r"### \1", md, flags=re.M) + for pattern, repl in RST_REPLACEMENTS: + rst = re.sub(pattern, repl, rst, flags=re.M) - # Highlight plugin names - md = re.sub( - r"^- `/?plugins/(\w+)`:?", r"- Plugin **`\1`**:", md, flags=re.M - ) + md = rst2md(rst) - # Highlights command names. - md = re.sub(r"^- `(\w+)-cmd`:?", r"- Command **`\1`**:", md, flags=re.M) + for pattern, repl in MD_REPLACEMENTS: + md = re.sub(pattern, repl, md, flags=re.M) # sort list items alphabetically for each of the sections return MD_CHANGELOG_SECTION_LIST.sub( From 914dbcb420a65dbdc987f8f236de1e90c03179f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0ar=C5=ABnas=20Nejus?= Date: Tue, 3 Dec 2024 05:21:08 +0000 Subject: [PATCH 04/18] Cache deps --- .github/workflows/test-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-changelog.yaml b/.github/workflows/test-changelog.yaml index 35ea067435..d3c9f84caa 100644 --- a/.github/workflows/test-changelog.yaml +++ b/.github/workflows/test-changelog.yaml @@ -17,7 +17,7 @@ jobs: uses: BrandonLWhite/pipx-install-action@v0.1.1 - uses: actions/setup-python@v5 with: - python-version: ${{ env.PYTHON_VERSION }} + python-version: "3.9" cache: poetry - name: Install dependencies From 13e83cdce3b7b9a7259587f928a587d617fee158 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0ar=C5=ABnas=20Nejus?= Date: Tue, 3 Dec 2024 05:23:11 +0000 Subject: [PATCH 05/18] :facepalm: --- .github/workflows/test-changelog.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/test-changelog.yaml b/.github/workflows/test-changelog.yaml index d3c9f84caa..1ff642f91f 100644 --- a/.github/workflows/test-changelog.yaml +++ b/.github/workflows/test-changelog.yaml @@ -10,8 +10,6 @@ jobs: changelog: ${{ steps.generate_changelog.outputs.changelog }} steps: - uses: actions/checkout@v4 - with: - ref: master - name: Install Python tools uses: BrandonLWhite/pipx-install-action@v0.1.1 From 7b9625bc867b391d05fb0bc2c4c69e20ef4e6805 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0ar=C5=ABnas=20Nejus?= Date: Tue, 3 Dec 2024 23:50:07 +0000 Subject: [PATCH 06/18] Test rst to md conversion --- .github/workflows/ci.yaml | 6 +-- extra/release.py | 15 +++++--- test/test_release.py | 78 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 90 insertions(+), 9 deletions(-) create mode 100644 test/test_release.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8a0c736726..6261c26d68 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -29,12 +29,12 @@ jobs: python-version: ${{ matrix.python-version }} cache: poetry - - name: Install PyGobject dependencies on Ubuntu + - name: Install PyGobject and release script dependencies on Ubuntu if: matrix.platform == 'ubuntu-latest' run: | sudo apt update - sudo apt install ffmpeg gobject-introspection libgirepository1.0-dev - poetry install --extras=replaygain --extras=reflink + sudo apt install ffmpeg gobject-introspection libgirepository1.0-dev pandoc + poetry install --with=release --extras=replaygain --extras=reflink - name: Install Python dependencies run: poetry install --only=main,test --extras=autobpm diff --git a/extra/release.py b/extra/release.py index abbb57bd47..89c6fa72d0 100755 --- a/extra/release.py +++ b/extra/release.py @@ -119,13 +119,15 @@ def rst2md(text: str) -> str: ) -def changelog_as_markdown() -> str: - """Get the latest changelog entry as hacked up Markdown.""" - contents = CHANGELOG.read_text() +def get_changelog_contents() -> str | None: + if m := RST_LATEST_CHANGES.search(CHANGELOG.read_text()): + return m.group(1) + + return None - m = RST_LATEST_CHANGES.search(contents) - rst = m.group(1) if m else "" +def changelog_as_markdown(rst: str) -> str: + """Get the latest changelog entry as hacked up Markdown.""" for pattern, repl in RST_REPLACEMENTS: rst = re.sub(pattern, repl, rst, flags=re.M) @@ -155,7 +157,8 @@ def bump(version: Version) -> None: @cli.command() def changelog(): """Get the most recent version's changelog as Markdown.""" - print(changelog_as_markdown()) + if changelog := get_changelog_contents(): + print(changelog_as_markdown(changelog)) if __name__ == "__main__": diff --git a/test/test_release.py b/test/test_release.py new file mode 100644 index 0000000000..2e46334096 --- /dev/null +++ b/test/test_release.py @@ -0,0 +1,78 @@ +"""Tests for the release utils.""" + +import os +import shutil +import sys + +import pytest + +from extra.release import changelog_as_markdown + +pytestmark = pytest.mark.skipif( + not ( + (os.environ.get("GITHUB_ACTIONS") == "true" and sys.platform != "win32") + or bool(shutil.which("pandoc")) + ), + reason="pandoc isn't available", +) + + +@pytest.fixture +def rst_changelog(): + return """New features: + +* :doc:`/plugins/substitute`: Some substitute + multi-line change. + :bug:`5467` +* :ref:`list-cmd` Update. + +Bug fixes: + +* Some fix that refers to an issue. + :bug:`5467` +* Some fix that mentions user :user:`username`. + +Empty section: + +Other changes: + +* Changed `bitesize` label to `good first issue`. Our `contribute`_ page is now + automatically populated with these issues. :bug:`4855` + +.. _contribute: https://github.com/beetbox/beets/contribute + +2.1.0 (November 22, 2024) +------------------------- + +Bug fixes: + +* Fixed something.""" + + +@pytest.fixture +def md_changelog(): + return r"""### New features + +- Command **`list`**: Update. +- Plugin **`substitute`**: Some substitute multi-line change. :bug: (\#5467) + +### Bug fixes + +- Some fix that mentions user @username. +- Some fix that refers to an issue. :bug: (\#5467) + +### Other changes + + +# 2.1.0 (November 22, 2024) +- Changed `bitesize` label to `good first issue`. Our [contribute](https://github.com/beetbox/beets/contribute) page is now automatically populated with these issues. :bug: (\#4855) + +### Bug fixes + +- Fixed something.""" # noqa: E501 + + +def test_convert_rst_to_md(rst_changelog, md_changelog): + actual = changelog_as_markdown(rst_changelog) + + assert actual == md_changelog From d98226aa07ab767792d98b747b01eac773702920 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0ar=C5=ABnas=20Nejus?= Date: Wed, 4 Dec 2024 00:04:11 +0000 Subject: [PATCH 07/18] Fix ordering bullet point lists --- extra/release.py | 13 ++++++++----- test/test_release.py | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/extra/release.py b/extra/release.py index 89c6fa72d0..5a4baf4238 100755 --- a/extra/release.py +++ b/extra/release.py @@ -7,6 +7,7 @@ import re import subprocess from datetime import datetime, timezone +from functools import partial from pathlib import Path from typing import Callable @@ -19,7 +20,6 @@ PYPROJECT = BASE / "pyproject.toml" CHANGELOG = BASE / "docs" / "changelog.rst" -MD_CHANGELOG_SECTION_LIST = re.compile(r"- .+?(?=\n\n###|$)", re.DOTALL) version_header = r"\d+\.\d+\.\d+ \([^)]+\)" RST_LATEST_CHANGES = re.compile( rf"{version_header}\n--+\s+(.+?)\n\n+{version_header}", re.DOTALL @@ -37,6 +37,10 @@ (r"^- `(\w+)-cmd`:?", r"- Command **`\1`**:"), # highlight commands (r"### [^\n]+\n+(?=### )", ""), # remove empty sections ] +order_bullet_points = partial( + re.compile("(\n- .*?(?=\n(?! *- )|$))", flags=re.DOTALL).sub, + lambda m: "\n- ".join(sorted(m.group().split("\n- "))), +) def update_docs_config(text: str, new: Version) -> str: @@ -136,10 +140,9 @@ def changelog_as_markdown(rst: str) -> str: for pattern, repl in MD_REPLACEMENTS: md = re.sub(pattern, repl, md, flags=re.M) - # sort list items alphabetically for each of the sections - return MD_CHANGELOG_SECTION_LIST.sub( - lambda m: "\n".join(sorted(m.group().splitlines())), md - ) + # order bullet points in each of the lists alphabetically to + # improve readability + return order_bullet_points(md) @click.group() diff --git a/test/test_release.py b/test/test_release.py index 2e46334096..a206b4e76e 100644 --- a/test/test_release.py +++ b/test/test_release.py @@ -63,9 +63,9 @@ def md_changelog(): ### Other changes +- Changed `bitesize` label to `good first issue`. Our [contribute](https://github.com/beetbox/beets/contribute) page is now automatically populated with these issues. :bug: (\#4855) # 2.1.0 (November 22, 2024) -- Changed `bitesize` label to `good first issue`. Our [contribute](https://github.com/beetbox/beets/contribute) page is now automatically populated with these issues. :bug: (\#4855) ### Bug fixes From dd96928f38cd3ce6282afb92fdf4ebf42e8a928a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0ar=C5=ABnas=20Nejus?= Date: Wed, 4 Dec 2024 01:52:27 +0000 Subject: [PATCH 08/18] Test nested bullet points conversion --- test/test_release.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/test/test_release.py b/test/test_release.py index a206b4e76e..c16aa43f23 100644 --- a/test/test_release.py +++ b/test/test_release.py @@ -31,6 +31,13 @@ def rst_changelog(): * Some fix that refers to an issue. :bug:`5467` * Some fix that mentions user :user:`username`. +* Some fix with its own bullet points using incorrect indentation: + * First nested bullet point + with some text that wraps to the next line + * Second nested bullet point +* Another fix with its own bullet points using correct indentation: + * First + * Second Empty section: @@ -58,9 +65,20 @@ def md_changelog(): ### Bug fixes -- Some fix that mentions user @username. - Some fix that refers to an issue. :bug: (\#5467) +- Some fix that mentions user @username. + +- - Some fix with its own bullet points using incorrect indentation: + + - First nested bullet point with some text that wraps to the next line + - Second nested bullet point + +- Another fix with its own bullet points using correct indentation: + + - First + - Second + ### Other changes - Changed `bitesize` label to `good first issue`. Our [contribute](https://github.com/beetbox/beets/contribute) page is now automatically populated with these issues. :bug: (\#4855) From c26473e6cb28fb93f2d8c0c8a0c11776004d9585 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0ar=C5=ABnas=20Nejus?= Date: Wed, 4 Dec 2024 02:02:20 +0000 Subject: [PATCH 09/18] Fix nested bullet points conversion --- extra/release.py | 5 ++++- test/test_release.py | 19 +++++++------------ 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/extra/release.py b/extra/release.py index 5a4baf4238..239d81361e 100755 --- a/extra/release.py +++ b/extra/release.py @@ -26,12 +26,15 @@ ) Replacement: TypeAlias = "tuple[str, str | Callable[[re.Match[str]], str]]" RST_REPLACEMENTS: list[Replacement] = [ + (r"(?<=\n) {3,4}(?=\*)", " "), # fix indent of nested bullet points ... + (r"(?<=\n) {5,6}(?=[\w:`])", " "), # ... and align wrapped text indent (r"(?<=[\s(])`([^`]+)`(?=[^_])", r"``\1``"), # ticks with verbatim ranges. (r":bug:`(\d+)`", r":bug: (#\1)"), # Issue numbers. (r":user:`(\w+)`", r"@\1"), # Users. ] MD_REPLACEMENTS: list[Replacement] = [ - (r"^ ( *- )", r"\1"), # remove list indentation + (r"^ (- )", r"\1"), # remove indent from top-level bullet points + (r"^ +( - )", r"\1"), # adjust nested bullet points indent (r"^(\w.+?):$", r"### \1"), # make sections headers (r"^- `/?plugins/(\w+)`:?", r"- Plugin **`\1`**:"), # highlight plugins (r"^- `(\w+)-cmd`:?", r"- Command **`\1`**:"), # highlight commands diff --git a/test/test_release.py b/test/test_release.py index c16aa43f23..9f40fc9340 100644 --- a/test/test_release.py +++ b/test/test_release.py @@ -65,19 +65,14 @@ def md_changelog(): ### Bug fixes -- Some fix that refers to an issue. :bug: (\#5467) - -- Some fix that mentions user @username. - -- - Some fix with its own bullet points using incorrect indentation: - - - First nested bullet point with some text that wraps to the next line - - Second nested bullet point - - Another fix with its own bullet points using correct indentation: - - - First - - Second + - First + - Second +- Some fix that mentions user @username. +- Some fix that refers to an issue. :bug: (\#5467) +- Some fix with its own bullet points using incorrect indentation: + - First nested bullet point with some text that wraps to the next line + - Second nested bullet point ### Other changes From 89afb8cd890c58f9e1c956b09a3606029339bcd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0ar=C5=ABnas=20Nejus?= Date: Wed, 4 Dec 2024 02:34:10 +0000 Subject: [PATCH 10/18] Test wrapped line starting with the username role --- test/test_release.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/test/test_release.py b/test/test_release.py index 9f40fc9340..cb23408394 100644 --- a/test/test_release.py +++ b/test/test_release.py @@ -31,6 +31,8 @@ def rst_changelog(): * Some fix that refers to an issue. :bug:`5467` * Some fix that mentions user :user:`username`. +* Some fix thanks to + :user:`username`. :bug:`5467` * Some fix with its own bullet points using incorrect indentation: * First nested bullet point with some text that wraps to the next line @@ -65,11 +67,14 @@ def md_changelog(): ### Bug fixes +- Some fix thanks to +- Some fix that mentions user @username. +- Some fix that refers to an issue. :bug: (\#5467) + 1. - bug + (\#5467) - Another fix with its own bullet points using correct indentation: - First - Second -- Some fix that mentions user @username. -- Some fix that refers to an issue. :bug: (\#5467) - Some fix with its own bullet points using incorrect indentation: - First nested bullet point with some text that wraps to the next line - Second nested bullet point From 806c1702fb1f0f5a6bf226101612f632cc13cf87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0ar=C5=ABnas=20Nejus?= Date: Wed, 4 Dec 2024 02:35:50 +0000 Subject: [PATCH 11/18] Fix wrapped line starting with the username role --- extra/release.py | 3 ++- test/test_release.py | 8 +++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/extra/release.py b/extra/release.py index 239d81361e..658d8abc72 100755 --- a/extra/release.py +++ b/extra/release.py @@ -30,7 +30,7 @@ (r"(?<=\n) {5,6}(?=[\w:`])", " "), # ... and align wrapped text indent (r"(?<=[\s(])`([^`]+)`(?=[^_])", r"``\1``"), # ticks with verbatim ranges. (r":bug:`(\d+)`", r":bug: (#\1)"), # Issue numbers. - (r":user:`(\w+)`", r"@\1"), # Users. + (r":user:`(\w+)`", r"\@\1"), # Users. ] MD_REPLACEMENTS: list[Replacement] = [ (r"^ (- )", r"\1"), # remove indent from top-level bullet points @@ -127,6 +127,7 @@ def rst2md(text: str) -> str: def get_changelog_contents() -> str | None: + return CHANGELOG.read_text() if m := RST_LATEST_CHANGES.search(CHANGELOG.read_text()): return m.group(1) diff --git a/test/test_release.py b/test/test_release.py index cb23408394..38afe28ced 100644 --- a/test/test_release.py +++ b/test/test_release.py @@ -67,14 +67,12 @@ def md_changelog(): ### Bug fixes -- Some fix thanks to -- Some fix that mentions user @username. -- Some fix that refers to an issue. :bug: (\#5467) - 1. - bug - (\#5467) - Another fix with its own bullet points using correct indentation: - First - Second +- Some fix thanks to @username. :bug: (\#5467) +- Some fix that mentions user @username. +- Some fix that refers to an issue. :bug: (\#5467) - Some fix with its own bullet points using incorrect indentation: - First nested bullet point with some text that wraps to the next line - Second nested bullet point From 6d602effc37fb7710b60428340afb509f8715ae3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0ar=C5=ABnas=20Nejus?= Date: Wed, 4 Dec 2024 02:40:37 +0000 Subject: [PATCH 12/18] Add a test for literal code block --- test/test_release.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/test_release.py b/test/test_release.py index 38afe28ced..d7e1c1b17d 100644 --- a/test/test_release.py +++ b/test/test_release.py @@ -26,6 +26,10 @@ def rst_changelog(): :bug:`5467` * :ref:`list-cmd` Update. +You can do something with this command:: + + $ do-something + Bug fixes: * Some fix that refers to an issue. @@ -65,6 +69,10 @@ def md_changelog(): - Command **`list`**: Update. - Plugin **`substitute`**: Some substitute multi-line change. :bug: (\#5467) +### You can do something with this command:: + + $ do-something + ### Bug fixes - Another fix with its own bullet points using correct indentation: From 0b905e1b1714026a217c766e5649df71182d2393 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0ar=C5=ABnas=20Nejus?= Date: Wed, 4 Dec 2024 04:16:33 +0000 Subject: [PATCH 13/18] Ignore literal code blocks when making headers --- extra/release.py | 4 ++-- test/test_release.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/extra/release.py b/extra/release.py index 658d8abc72..75ba0b7b51 100755 --- a/extra/release.py +++ b/extra/release.py @@ -35,7 +35,7 @@ MD_REPLACEMENTS: list[Replacement] = [ (r"^ (- )", r"\1"), # remove indent from top-level bullet points (r"^ +( - )", r"\1"), # adjust nested bullet points indent - (r"^(\w.+?):$", r"### \1"), # make sections headers + (r"^(\w[^\n]+):(?=\n\n[^ ])", r"### \1"), # make sections headers (r"^- `/?plugins/(\w+)`:?", r"- Plugin **`\1`**:"), # highlight plugins (r"^- `(\w+)-cmd`:?", r"- Command **`\1`**:"), # highlight commands (r"### [^\n]+\n+(?=### )", ""), # remove empty sections @@ -142,7 +142,7 @@ def changelog_as_markdown(rst: str) -> str: md = rst2md(rst) for pattern, repl in MD_REPLACEMENTS: - md = re.sub(pattern, repl, md, flags=re.M) + md = re.sub(pattern, repl, md, flags=re.M | re.DOTALL) # order bullet points in each of the lists alphabetically to # improve readability diff --git a/test/test_release.py b/test/test_release.py index d7e1c1b17d..f69389d545 100644 --- a/test/test_release.py +++ b/test/test_release.py @@ -69,7 +69,7 @@ def md_changelog(): - Command **`list`**: Update. - Plugin **`substitute`**: Some substitute multi-line change. :bug: (\#5467) -### You can do something with this command:: +You can do something with this command: $ do-something From 779ba791f93d66c5de8769472f65ede31a9a603a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0ar=C5=ABnas=20Nejus?= Date: Wed, 4 Dec 2024 04:35:11 +0000 Subject: [PATCH 14/18] Cap maximum sub-section name length --- extra/release.py | 3 ++- test/test_release.py | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/extra/release.py b/extra/release.py index 75ba0b7b51..b8aa28f464 100755 --- a/extra/release.py +++ b/extra/release.py @@ -35,7 +35,8 @@ MD_REPLACEMENTS: list[Replacement] = [ (r"^ (- )", r"\1"), # remove indent from top-level bullet points (r"^ +( - )", r"\1"), # adjust nested bullet points indent - (r"^(\w[^\n]+):(?=\n\n[^ ])", r"### \1"), # make sections headers + (r"^(\w[^\n]{,80}):(?=\n\n[^ ])", r"### \1"), # format section headers + (r"^(\w[^\n]{81,}):(?=\n\n[^ ])", r"**\1**"), # and bolden too long ones (r"^- `/?plugins/(\w+)`:?", r"- Plugin **`\1`**:"), # highlight plugins (r"^- `(\w+)-cmd`:?", r"- Command **`\1`**:"), # highlight commands (r"### [^\n]+\n+(?=### )", ""), # remove empty sections diff --git a/test/test_release.py b/test/test_release.py index f69389d545..39b6b63fc1 100644 --- a/test/test_release.py +++ b/test/test_release.py @@ -45,6 +45,9 @@ def rst_changelog(): * First * Second +Section naaaaaaaaaaaaaaaaaaaaaaaammmmmmmmmmmmmmmmeeeeeeeeeeeeeee with over 80 +characters: + Empty section: Other changes: @@ -85,6 +88,8 @@ def md_changelog(): - First nested bullet point with some text that wraps to the next line - Second nested bullet point +**Section naaaaaaaaaaaaaaaaaaaaaaaammmmmmmmmmmmmmmmeeeeeeeeeeeeeee with over 80 characters** + ### Other changes - Changed `bitesize` label to `good first issue`. Our [contribute](https://github.com/beetbox/beets/contribute) page is now automatically populated with these issues. :bug: (\#4855) From e579df0a98efa93c73a6b94ef419a3e688b2c47d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0ar=C5=ABnas=20Nejus?= Date: Wed, 4 Dec 2024 04:43:58 +0000 Subject: [PATCH 15/18] Can we link users to plugin docs? --- extra/release.py | 7 ++++--- test/test_release.py | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/extra/release.py b/extra/release.py index b8aa28f464..25dc940f5c 100755 --- a/extra/release.py +++ b/extra/release.py @@ -19,6 +19,7 @@ BASE = Path(__file__).parent.parent.absolute() PYPROJECT = BASE / "pyproject.toml" CHANGELOG = BASE / "docs" / "changelog.rst" +DOCS = "https://beets.readthedocs.io/en/stable" version_header = r"\d+\.\d+\.\d+ \([^)]+\)" RST_LATEST_CHANGES = re.compile( @@ -28,7 +29,7 @@ RST_REPLACEMENTS: list[Replacement] = [ (r"(?<=\n) {3,4}(?=\*)", " "), # fix indent of nested bullet points ... (r"(?<=\n) {5,6}(?=[\w:`])", " "), # ... and align wrapped text indent - (r"(?<=[\s(])`([^`]+)`(?=[^_])", r"``\1``"), # ticks with verbatim ranges. + (r"(?<=[\s(])(`[^`]+`)(?!_)", r"`\1`"), # double quotes for inline code (r":bug:`(\d+)`", r":bug: (#\1)"), # Issue numbers. (r":user:`(\w+)`", r"\@\1"), # Users. ] @@ -37,8 +38,8 @@ (r"^ +( - )", r"\1"), # adjust nested bullet points indent (r"^(\w[^\n]{,80}):(?=\n\n[^ ])", r"### \1"), # format section headers (r"^(\w[^\n]{81,}):(?=\n\n[^ ])", r"**\1**"), # and bolden too long ones - (r"^- `/?plugins/(\w+)`:?", r"- Plugin **`\1`**:"), # highlight plugins - (r"^- `(\w+)-cmd`:?", r"- Command **`\1`**:"), # highlight commands + (r"^- `/?plugins/(\w+)`:?", rf"- Plugin [\1]({DOCS}/plugins/\1.html):"), + (r"^- `(\w+)-cmd`:?", rf"- Command [\1]({DOCS}/reference/cli.html#\1):"), (r"### [^\n]+\n+(?=### )", ""), # remove empty sections ] order_bullet_points = partial( diff --git a/test/test_release.py b/test/test_release.py index 39b6b63fc1..005fe8de07 100644 --- a/test/test_release.py +++ b/test/test_release.py @@ -69,8 +69,8 @@ def rst_changelog(): def md_changelog(): return r"""### New features -- Command **`list`**: Update. -- Plugin **`substitute`**: Some substitute multi-line change. :bug: (\#5467) +- Command [list](https://beets.readthedocs.io/en/stable/reference/cli.html#list): Update. +- Plugin [substitute](https://beets.readthedocs.io/en/stable/plugins/substitute.html): Some substitute multi-line change. :bug: (\#5467) You can do something with this command: From 0f45791f3a6df9d02cfab64778b80ee168bee3f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0ar=C5=ABnas=20Nejus?= Date: Wed, 4 Dec 2024 04:51:38 +0000 Subject: [PATCH 16/18] Fix Unreleased changelog template --- extra/release.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/extra/release.py b/extra/release.py index 25dc940f5c..65547d7f1f 100755 --- a/extra/release.py +++ b/extra/release.py @@ -64,8 +64,11 @@ def update_changelog(text: str, new: Version) -> str: ---------- New features: + Bug fixes: + For packagers: + Other changes: {new_header} From 555cf322dbd11194057e1505e7381dcf5c9024d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0ar=C5=ABnas=20Nejus?= Date: Wed, 4 Dec 2024 06:21:25 +0000 Subject: [PATCH 17/18] Remove the temporary testing workflow --- .github/workflows/test-changelog.yaml | 49 --------------------------- 1 file changed, 49 deletions(-) delete mode 100644 .github/workflows/test-changelog.yaml diff --git a/.github/workflows/test-changelog.yaml b/.github/workflows/test-changelog.yaml deleted file mode 100644 index 1ff642f91f..0000000000 --- a/.github/workflows/test-changelog.yaml +++ /dev/null @@ -1,49 +0,0 @@ -name: Temporarily test changelog formatting -on: - pull_request: - -jobs: - build: - name: Get changelog - runs-on: ubuntu-latest - outputs: - changelog: ${{ steps.generate_changelog.outputs.changelog }} - steps: - - uses: actions/checkout@v4 - - - name: Install Python tools - uses: BrandonLWhite/pipx-install-action@v0.1.1 - - uses: actions/setup-python@v5 - with: - python-version: "3.9" - cache: poetry - - - name: Install dependencies - run: poetry install --with=release --extras=docs - - - name: Install pandoc - run: sudo apt update && sudo apt install pandoc -y - - - name: Obtain the changelog - id: generate_changelog - run: | - { - echo 'changelog<> "$GITHUB_OUTPUT" - test: - runs-on: ubuntu-latest - permissions: - pull-requests: write - needs: build - env: - CHANGELOG: ${{ needs.build.outputs.changelog }} - steps: - - uses: mshick/add-pr-comment@v2 - with: - message: |- - ### Commit: ${{ github.sha }} - ### Changelog: - - ${{ env.CHANGELOG }} From eb557f720d6ca4b412f641779e9561e6ab32e794 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0ar=C5=ABnas=20Nejus?= Date: Thu, 5 Dec 2024 08:41:02 +0000 Subject: [PATCH 18/18] Resolve all URLs for markdown --- .github/workflows/ci.yaml | 3 +- .github/workflows/make_release.yaml | 1 + extra/release.py | 132 ++++++++++++++++++++++++---- test/test_release.py | 9 +- 4 files changed, 123 insertions(+), 22 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6261c26d68..2ed4548cee 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -34,7 +34,8 @@ jobs: run: | sudo apt update sudo apt install ffmpeg gobject-introspection libgirepository1.0-dev pandoc - poetry install --with=release --extras=replaygain --extras=reflink + poetry install --with=release --extras=docs --extras=replaygain --extras=reflink + poe docs - name: Install Python dependencies run: poetry install --only=main,test --extras=autobpm diff --git a/.github/workflows/make_release.yaml b/.github/workflows/make_release.yaml index aeb4c0a329..248755703b 100644 --- a/.github/workflows/make_release.yaml +++ b/.github/workflows/make_release.yaml @@ -65,6 +65,7 @@ jobs: - name: Obtain the changelog id: generate_changelog run: | + poe docs { echo 'changelog< Ref: + """Create Ref from a Sphinx objects.inv line. + + Each line has the following structure: + [optional title : ] + + """ + if len(line_parts := line.split(" ", 1)) == 1: + return cls(line, None, None) + + id, path_with_name = line_parts + parts = [p.strip() for p in path_with_name.split(":", 1)] + + if len(parts) == 1: + path, name = parts[0], None + else: + name, path = parts + + return cls(id, path, name) + + @property + def url(self) -> str: + """Full documentation URL.""" + return f"{DOCS}/{self.path}" + + @property + def name(self) -> str: + """Display name (title if available, otherwise ID).""" + return self.title or self.id + + +def get_refs() -> dict[str, Ref]: + """Parse Sphinx objects.inv and return dict of documentation references.""" + objects_filepath = Path("docs/_build/html/objects.inv") + if not objects_filepath.exists(): + raise ValueError("Documentation does not exist. Run 'poe docs' first.") + + captured_output = StringIO() + + with redirect_stdout(captured_output): + intersphinx.inspect_main([str(objects_filepath)]) + + return { + r.id: r + for ln in captured_output.getvalue().split("\n") + if ln.startswith("\t") and (r := Ref.from_line(ln.strip())) + } + + +def create_rst_replacements() -> list[Replacement]: + """Generate list of pattern replacements for RST changelog.""" + refs = get_refs() + + def make_ref_link(ref_id: str, name: str | None = None) -> str: + ref = refs[ref_id] + return rf"`{name or ref.name} <{ref.url}>`_" + + commands = "|".join(r.split("-")[0] for r in refs if r.endswith("-cmd")) + plugins = "|".join( + r.split("/")[-1] for r in refs if r.startswith("plugins/") + ) + return [ + # Fix nested bullet points indent: use 2 spaces consistently + (r"(?<=\n) {3,4}(?=\*)", " "), + # Fix nested text indent: use 4 spaces consistently + (r"(?<=\n) {5,6}(?=[\w:`])", " "), + # Replace Sphinx :ref: and :doc: directives by documentation URLs + # :ref:`/plugins/autobpm` -> [AutoBPM Plugin](DOCS/plugins/autobpm.html) + ( + r":(?:ref|doc):`+(?:([^`<]+)<)?/?([\w./_-]+)>?`+", + lambda m: make_ref_link(m[2], m[1]), + ), + # Convert command references to documentation URLs + # `beet move` or `move` command -> [import](DOCS/reference/cli.html#import) + ( + rf"`+beet ({commands})`+|`+({commands})`+(?= command)", + lambda m: make_ref_link(f"{m[1] or m[2]}-cmd"), + ), + # Convert plugin references to documentation URLs + # `fetchart` plugin -> [fetchart](DOCS/plugins/fetchart.html) + (rf"`+({plugins})`+", lambda m: make_ref_link(f"plugins/{m[1]}")), + # Add additional backticks around existing backticked text to ensure it + # is rendered as inline code in Markdown + (r"(?<=[\s])(`[^`]+`)(?!_)", r"`\1`"), + # Convert bug references to GitHub issue links + (r":bug:`(\d+)`", r":bug: (#\1)"), + # Convert user references to GitHub @mentions + (r":user:`(\w+)`", r"\@\1"), + ] + + MD_REPLACEMENTS: list[Replacement] = [ (r"^ (- )", r"\1"), # remove indent from top-level bullet points (r"^ +( - )", r"\1"), # adjust nested bullet points indent (r"^(\w[^\n]{,80}):(?=\n\n[^ ])", r"### \1"), # format section headers (r"^(\w[^\n]{81,}):(?=\n\n[^ ])", r"**\1**"), # and bolden too long ones - (r"^- `/?plugins/(\w+)`:?", rf"- Plugin [\1]({DOCS}/plugins/\1.html):"), - (r"^- `(\w+)-cmd`:?", rf"- Command [\1]({DOCS}/reference/cli.html#\1):"), (r"### [^\n]+\n+(?=### )", ""), # remove empty sections ] order_bullet_points = partial( @@ -123,7 +219,7 @@ def rst2md(text: str) -> str: """Use Pandoc to convert text from ReST to Markdown.""" return ( subprocess.check_output( - ["pandoc", "--from=rst", "--to=gfm", "--wrap=none"], + ["pandoc", "--from=rst", "--to=gfm+hard_line_breaks"], input=text.encode(), ) .decode() @@ -132,7 +228,6 @@ def rst2md(text: str) -> str: def get_changelog_contents() -> str | None: - return CHANGELOG.read_text() if m := RST_LATEST_CHANGES.search(CHANGELOG.read_text()): return m.group(1) @@ -141,8 +236,8 @@ def get_changelog_contents() -> str | None: def changelog_as_markdown(rst: str) -> str: """Get the latest changelog entry as hacked up Markdown.""" - for pattern, repl in RST_REPLACEMENTS: - rst = re.sub(pattern, repl, rst, flags=re.M) + for pattern, repl in create_rst_replacements(): + rst = re.sub(pattern, repl, rst, flags=re.M | re.DOTALL) md = rst2md(rst) @@ -170,7 +265,10 @@ def bump(version: Version) -> None: def changelog(): """Get the most recent version's changelog as Markdown.""" if changelog := get_changelog_contents(): - print(changelog_as_markdown(changelog)) + try: + print(changelog_as_markdown(changelog)) + except ValueError as e: + raise click.exceptions.UsageError(str(e)) if __name__ == "__main__": diff --git a/test/test_release.py b/test/test_release.py index 005fe8de07..4b3f37113b 100644 --- a/test/test_release.py +++ b/test/test_release.py @@ -6,7 +6,8 @@ import pytest -from extra.release import changelog_as_markdown +release = pytest.importorskip("extra.release") + pytestmark = pytest.mark.skipif( not ( @@ -69,8 +70,8 @@ def rst_changelog(): def md_changelog(): return r"""### New features -- Command [list](https://beets.readthedocs.io/en/stable/reference/cli.html#list): Update. -- Plugin [substitute](https://beets.readthedocs.io/en/stable/plugins/substitute.html): Some substitute multi-line change. :bug: (\#5467) +- [Substitute Plugin](https://beets.readthedocs.io/en/stable/plugins/substitute.html): Some substitute multi-line change. :bug: (\#5467) +- [list](https://beets.readthedocs.io/en/stable/reference/cli.html#list-cmd) Update. You can do something with this command: @@ -102,6 +103,6 @@ def md_changelog(): def test_convert_rst_to_md(rst_changelog, md_changelog): - actual = changelog_as_markdown(rst_changelog) + actual = release.changelog_as_markdown(rst_changelog) assert actual == md_changelog