From a41717af348d2eac53d4af7a113e57b54d003ce1 Mon Sep 17 00:00:00 2001 From: galargh Date: Wed, 27 Nov 2024 12:52:01 +0100 Subject: [PATCH 01/18] feat: enforce dependency on released versions of packages --- .github/workflows/dependency-check.yml | 91 ++++++++++++++++++++++++++ CONTRIBUTING.md | 10 ++- 2 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/dependency-check.yml diff --git a/.github/workflows/dependency-check.yml b/.github/workflows/dependency-check.yml new file mode 100644 index 00000000000..4bf88fd6aa7 --- /dev/null +++ b/.github/workflows/dependency-check.yml @@ -0,0 +1,91 @@ +name: Dependency Check + +on: + pull_request: + paths: + - 'go.mod' + - 'go.sum' + - '.github/workflows/dependency-check.yml' + +jobs: + dependency-check: + runs-on: ubuntu-latest + name: Dependency Check + steps: + - uses: actions/checkout@v3 + - id: dependencies + env: + ALLOWED: | + [ + { + "Path": "github.com/filecoin-project/go-data-transfer/v2", + "Version": "v2.0.0-rc7", + "Reason": "unknown" + }, + { + "Path": "github.com/filecoin-project/go-state-types", + "Version": "v0.16.0-dev", + "Reason": "unknown" + }, + { + "Path": "github.com/syndtr/goleveldb", + "Version": "v1.0.1-0.20210819022825-2ae1ddf74ef7", + "Reason": "unknown" + }, + { + "Path": "github.com/xorcare/golden", + "Version": "v0.6.1-0.20191112154924-b87f686d7542", + "Reason": "unknown" + }, + { + "Path": "github.com/xordataexchange/crypt", + "Version": "v0.0.3-0.20170626215501-b2862e3d0a77", + "Reason": "unknown" + }, + { + "Path": "github.com/yugabyte/pgx/v5", + "Version": "v5.5.3-yb-2", + "Reason": "unknown" + }, + { + "Path": "go.dedis.ch/kyber/v4", + "Version": "v4.0.0-pre2.0.20240924132404-4de33740016e", + "Reason": "unknown" + }, + { + "Path": "gopkg.in/check.v1", + "Version": "v1.0.0-20201130134442-10cb98267c6c", + "Reason": "unknown" + }, + { + "Path": "gopkg.in/tomb.v1", + "Version": "v1.0.0-20141024135613-dd632973f1e7", + "Reason": "unknown" + }, + { + "Path": "honnef.co/go/tools", + "Version": "v0.0.1-2020.1.4", + "Reason": "unknown" + } + ] + run: | + echo "unreleased<> $GITHUB_OUTPUT + go list -m -json all | + jq -s ' + ( + map({Path: .Path, Version: .Version}) | + map(select(.Version | test("^v\\d+\\.\\d+\\.\\d+-")?)) | + map(select(.Version | test("^v0\\.0\\.0-")? | not)) + ) - ( + env.ALLOWED | fromjson | + map({Path: .Path, Version: .Version}) + ) + ' | tee -a $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + - if: steps.dependencies.outputs.unreleased != '[]' + env: + MESSAGE: | + A new unreleased dependency was discovered in this PR. Please do one of the options in [changelog management conventions](https://github.com/filecoin-project/lotus/blob/master/CONTRIBUTING.md#changelog-management) + run: | + echo "::error::${MESSAGE//$'\n'/%0A}" + exit 1 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 66c0003c330..0c45ca4d125 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -49,7 +49,7 @@ Note that this is enforced with https://github.com/filecoin-project/lotus/blob/m ## CHANGELOG Management -To expedite the release process, the CHANGELOG is built-up incrementally. +To expedite the release process, the CHANGELOG is built-up incrementally. We enforce that each PR updates CHANGELOG.md or signals that the change doesn't need it. If the PR affects users (e.g., new feature, bug fix, system requirements change), update the CHANGELOG.md and add details to the UNRELEASED section. If the change does not require a CHANGELOG.md entry, do one of the following: @@ -58,6 +58,14 @@ If the change does not require a CHANGELOG.md entry, do one of the following: Note that this is enforced with https://github.com/filecoin-project/lotus/blob/master/.github/workflows/changelog.yml +## Dependency Management + +We enforce that each dependency on an unreleased version of a package, as long as said package has any released versions (i.e. it is not a `v0.0.0`), is explicitly documented in the `ALLOWED` list stored in [.github/workflows/dependency-check.yml](.github/workflows/dependency-check.yml). +If you are adding such a dependency, please add it to the `ALLOWED` list. +Please note that this requirement applies both to direct and indirect dependencies. + +Note that this is enforced with https://github.com/filecoin-project/lotus/blob/master/.github/workflows/dependency-check.yml + ## Markdown Conventions We optimize our markdown files for viewing on GitHub. That isn't to say other syntaxes can't be used, but that is the flavor we focus on and at the minimum don't want to break. From dd55bbe2f99d51192fdc1993afae73f834bdaffd Mon Sep 17 00:00:00 2001 From: galargh Date: Thu, 28 Nov 2024 18:56:58 +0100 Subject: [PATCH 02/18] test: intentionally break the dependency check --- .github/workflows/dependency-check.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/dependency-check.yml b/.github/workflows/dependency-check.yml index 4bf88fd6aa7..b68ab521a06 100644 --- a/.github/workflows/dependency-check.yml +++ b/.github/workflows/dependency-check.yml @@ -61,11 +61,6 @@ jobs: "Path": "gopkg.in/tomb.v1", "Version": "v1.0.0-20141024135613-dd632973f1e7", "Reason": "unknown" - }, - { - "Path": "honnef.co/go/tools", - "Version": "v0.0.1-2020.1.4", - "Reason": "unknown" } ] run: | From 3105ba844ca814e78e257951b8702f9cc5be9e52 Mon Sep 17 00:00:00 2001 From: galargh Date: Thu, 28 Nov 2024 18:59:17 +0100 Subject: [PATCH 03/18] fix: check out the submodules recursively --- .github/workflows/dependency-check.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/dependency-check.yml b/.github/workflows/dependency-check.yml index b68ab521a06..9fdb4acaa8b 100644 --- a/.github/workflows/dependency-check.yml +++ b/.github/workflows/dependency-check.yml @@ -13,6 +13,8 @@ jobs: name: Dependency Check steps: - uses: actions/checkout@v3 + with: + submodules: 'recursive' - id: dependencies env: ALLOWED: | From ee01e18ec2e113aadfecf1406fb4c444b51c30c2 Mon Sep 17 00:00:00 2001 From: galargh Date: Thu, 28 Nov 2024 19:05:30 +0100 Subject: [PATCH 04/18] fix: the error message provided by the dependency checker --- .github/workflows/dependency-check.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dependency-check.yml b/.github/workflows/dependency-check.yml index 9fdb4acaa8b..1eea9b5cc03 100644 --- a/.github/workflows/dependency-check.yml +++ b/.github/workflows/dependency-check.yml @@ -82,7 +82,10 @@ jobs: - if: steps.dependencies.outputs.unreleased != '[]' env: MESSAGE: | - A new unreleased dependency was discovered in this PR. Please do one of the options in [changelog management conventions](https://github.com/filecoin-project/lotus/blob/master/CONTRIBUTING.md#changelog-management) + A new unreleased dependency was discovered in this PR. Please do one of the options in [dependency management conventions](https://github.com/filecoin-project/lotus/blob/master/CONTRIBUTING.md#dependency-management) + + Unreleased dependencies: + ${{steps.dependencies.outputs.unreleased}} run: | echo "::error::${MESSAGE//$'\n'/%0A}" exit 1 From e5aa00970bf389f7bd20f1760e08ce908b786766 Mon Sep 17 00:00:00 2001 From: galargh Date: Thu, 28 Nov 2024 19:07:17 +0100 Subject: [PATCH 05/18] Revert "test: intentionally break the dependency check" This reverts commit dd55bbe2f99d51192fdc1993afae73f834bdaffd. --- .github/workflows/dependency-check.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/dependency-check.yml b/.github/workflows/dependency-check.yml index 1eea9b5cc03..4816548ad69 100644 --- a/.github/workflows/dependency-check.yml +++ b/.github/workflows/dependency-check.yml @@ -63,6 +63,11 @@ jobs: "Path": "gopkg.in/tomb.v1", "Version": "v1.0.0-20141024135613-dd632973f1e7", "Reason": "unknown" + }, + { + "Path": "honnef.co/go/tools", + "Version": "v0.0.1-2020.1.4", + "Reason": "unknown" } ] run: | From 36da6130e408a99c6026b44367fa57c1f03fbde5 Mon Sep 17 00:00:00 2001 From: galargh Date: Thu, 28 Nov 2024 19:08:14 +0100 Subject: [PATCH 06/18] chore: add github.com/quic-go/webtransport-go to allowed unreleased dependencies --- .github/workflows/dependency-check.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/dependency-check.yml b/.github/workflows/dependency-check.yml index 4816548ad69..364b271f594 100644 --- a/.github/workflows/dependency-check.yml +++ b/.github/workflows/dependency-check.yml @@ -68,6 +68,11 @@ jobs: "Path": "honnef.co/go/tools", "Version": "v0.0.1-2020.1.4", "Reason": "unknown" + }, + { + "Path": "github.com/quic-go/webtransport-go", + "Version": "v0.8.1-0.20241018022711-4ac2c9250e66", + "Reason": "unknown" } ] run: | From e18cc37e68a8a292367f41c39ff0cc18c2f2c097 Mon Sep 17 00:00:00 2001 From: galargh Date: Mon, 9 Dec 2024 15:07:27 +0100 Subject: [PATCH 07/18] chore: make the dependency checker operate on go.mod --- .github/workflows/dependency-check.yml | 94 ++++++-------------------- CONTRIBUTING.md | 4 +- 2 files changed, 23 insertions(+), 75 deletions(-) diff --git a/.github/workflows/dependency-check.yml b/.github/workflows/dependency-check.yml index 364b271f594..e873372dcc0 100644 --- a/.github/workflows/dependency-check.yml +++ b/.github/workflows/dependency-check.yml @@ -15,87 +15,35 @@ jobs: - uses: actions/checkout@v3 with: submodules: 'recursive' - - id: dependencies + - id: all + run: | + echo "dependencies<> $GITHUB_OUTPUT + # Extract all dependencies from go.mod (include indirect dependencies and comments) + sed -n '/require (/,/)/{/require (/!{/)/!p;};}' go.mod | sed 's/^[[:space:]]*//' | tee -a $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + - id: unreleased + env: + DEPENDENCIES: ${{ steps.all.outputs.dependencies }} + run: | + echo "dependencies<> $GITHUB_OUTPUT + # Find all dependencies that use prerelease versions (exclude v0.0.0 versions) + grep -Pv 'v0\.0\.0|v[0-9]+\.[0-9]+\.[0-9]+([^-]|$)' <<< "$DEPENDENCIES" | tee -a $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + - id: unexplained env: - ALLOWED: | - [ - { - "Path": "github.com/filecoin-project/go-data-transfer/v2", - "Version": "v2.0.0-rc7", - "Reason": "unknown" - }, - { - "Path": "github.com/filecoin-project/go-state-types", - "Version": "v0.16.0-dev", - "Reason": "unknown" - }, - { - "Path": "github.com/syndtr/goleveldb", - "Version": "v1.0.1-0.20210819022825-2ae1ddf74ef7", - "Reason": "unknown" - }, - { - "Path": "github.com/xorcare/golden", - "Version": "v0.6.1-0.20191112154924-b87f686d7542", - "Reason": "unknown" - }, - { - "Path": "github.com/xordataexchange/crypt", - "Version": "v0.0.3-0.20170626215501-b2862e3d0a77", - "Reason": "unknown" - }, - { - "Path": "github.com/yugabyte/pgx/v5", - "Version": "v5.5.3-yb-2", - "Reason": "unknown" - }, - { - "Path": "go.dedis.ch/kyber/v4", - "Version": "v4.0.0-pre2.0.20240924132404-4de33740016e", - "Reason": "unknown" - }, - { - "Path": "gopkg.in/check.v1", - "Version": "v1.0.0-20201130134442-10cb98267c6c", - "Reason": "unknown" - }, - { - "Path": "gopkg.in/tomb.v1", - "Version": "v1.0.0-20141024135613-dd632973f1e7", - "Reason": "unknown" - }, - { - "Path": "honnef.co/go/tools", - "Version": "v0.0.1-2020.1.4", - "Reason": "unknown" - }, - { - "Path": "github.com/quic-go/webtransport-go", - "Version": "v0.8.1-0.20241018022711-4ac2c9250e66", - "Reason": "unknown" - } - ] + DEPENDENCIES: ${{ steps.unreleased.outputs.dependencies }} run: | - echo "unreleased<> $GITHUB_OUTPUT - go list -m -json all | - jq -s ' - ( - map({Path: .Path, Version: .Version}) | - map(select(.Version | test("^v\\d+\\.\\d+\\.\\d+-")?)) | - map(select(.Version | test("^v0\\.0\\.0-")? | not)) - ) - ( - env.ALLOWED | fromjson | - map({Path: .Path, Version: .Version}) - ) - ' | tee -a $GITHUB_OUTPUT + echo "dependencies<> $GITHUB_OUTPUT + # Find all unreleased dependencies without a dependency-check-ignore comment + grep -Pv 'dependency-check-ignore:\s' <<< "$DEPENDENCIES" | tee -a $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT - - if: steps.dependencies.outputs.unreleased != '[]' + - if: steps.unexplained.outputs.dependencies != '' env: MESSAGE: | A new unreleased dependency was discovered in this PR. Please do one of the options in [dependency management conventions](https://github.com/filecoin-project/lotus/blob/master/CONTRIBUTING.md#dependency-management) Unreleased dependencies: - ${{steps.dependencies.outputs.unreleased}} + ${{steps.unexplained.outputs.dependencies}} run: | echo "::error::${MESSAGE//$'\n'/%0A}" exit 1 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0c45ca4d125..32a413ccabd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -60,8 +60,8 @@ Note that this is enforced with https://github.com/filecoin-project/lotus/blob/m ## Dependency Management -We enforce that each dependency on an unreleased version of a package, as long as said package has any released versions (i.e. it is not a `v0.0.0`), is explicitly documented in the `ALLOWED` list stored in [.github/workflows/dependency-check.yml](.github/workflows/dependency-check.yml). -If you are adding such a dependency, please add it to the `ALLOWED` list. +We enforce that each dependency on an unreleased version of a package, as long as said package has any released versions (i.e. it is not a `v0.0.0`), is explicitly documented in the `go.mod` file via an inline comment of the form `dependency-check-ignore: `. +If you are adding such a dependency, please add a suitable comment to the `go.mod` file as well. Please note that this requirement applies both to direct and indirect dependencies. Note that this is enforced with https://github.com/filecoin-project/lotus/blob/master/.github/workflows/dependency-check.yml From 01b45ef1636ed22769875c8f4b012117c7ba346f Mon Sep 17 00:00:00 2001 From: galargh Date: Mon, 9 Dec 2024 15:15:04 +0100 Subject: [PATCH 08/18] chore: document dependencies ignored in the checker --- go.mod | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index edace0bfb97..0ade66e9813 100644 --- a/go.mod +++ b/go.mod @@ -52,7 +52,7 @@ require ( github.com/filecoin-project/go-jsonrpc v0.7.0 github.com/filecoin-project/go-padreader v0.0.1 github.com/filecoin-project/go-paramfetch v0.0.4 - github.com/filecoin-project/go-state-types v0.16.0-rc1 + github.com/filecoin-project/go-state-types v0.16.0-rc1 // dependency-check-ignore: unknown github.com/filecoin-project/go-statemachine v1.0.3 github.com/filecoin-project/go-statestore v0.2.0 github.com/filecoin-project/go-storedcounter v0.1.0 @@ -135,15 +135,15 @@ require ( github.com/samber/lo v1.39.0 github.com/sirupsen/logrus v1.9.2 github.com/stretchr/testify v1.10.0 - github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 + github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // dependency-check-ignore: unknown github.com/triplewz/poseidon v0.0.2 github.com/urfave/cli/v2 v2.25.5 github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba github.com/whyrusleeping/cbor-gen v0.2.0 github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 github.com/xeipuuv/gojsonschema v1.2.0 - github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542 - github.com/yugabyte/pgx/v5 v5.5.3-yb-2 + github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542 // dependency-check-ignore: unknown + github.com/yugabyte/pgx/v5 v5.5.3-yb-2 // dependency-check-ignore: unknown github.com/zondax/ledger-filecoin-go v0.11.1 github.com/zyedidia/generic v1.2.1 go.opencensus.io v0.24.0 @@ -316,7 +316,7 @@ require ( github.com/prometheus/statsd_exporter v0.22.7 // indirect github.com/quic-go/qpack v0.5.1 // indirect github.com/quic-go/quic-go v0.48.2 // indirect - github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 // indirect + github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 // indirect; dependency-check-ignore: unknown github.com/rivo/uniseg v0.4.7 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/shirou/gopsutil v2.18.12+incompatible // indirect @@ -337,7 +337,7 @@ require ( github.com/zondax/ledger-go v0.14.3 // indirect gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b // indirect gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02 // indirect - go.dedis.ch/kyber/v4 v4.0.0-pre2.0.20240924132404-4de33740016e // indirect + go.dedis.ch/kyber/v4 v4.0.0-pre2.0.20240924132404-4de33740016e // indirect; dependency-check-ignore: unknown go.opentelemetry.io/otel/trace v1.28.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/dig v1.18.0 // indirect From 6e0bc3517c5cd68fd7c633ed57bbbf1e69dee404 Mon Sep 17 00:00:00 2001 From: galargh Date: Mon, 9 Dec 2024 15:23:15 +0100 Subject: [PATCH 09/18] docs: update the depednency conventions section --- CONTRIBUTING.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 32a413ccabd..96615995904 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -60,11 +60,16 @@ Note that this is enforced with https://github.com/filecoin-project/lotus/blob/m ## Dependency Management -We enforce that each dependency on an unreleased version of a package, as long as said package has any released versions (i.e. it is not a `v0.0.0`), is explicitly documented in the `go.mod` file via an inline comment of the form `dependency-check-ignore: `. -If you are adding such a dependency, please add a suitable comment to the `go.mod` file as well. -Please note that this requirement applies both to direct and indirect dependencies. - -Note that this is enforced with https://github.com/filecoin-project/lotus/blob/master/.github/workflows/dependency-check.yml +We strive to use release dependencies because: +1. Security / reliability - While there's no guarantee that a released version doesn't have bugs or issues, it seems fair to assume that non-released versions have even more. For example, https://github.com/filecoin-project/lotus/issues/12467 was triggered because of a bug in non-released library that lotus was depending on when the latest released version didn't have the bug. +2. Faster builds +3. Makes Lotus a better citizen when it's imported by other projects. +We enforce that each dependency on an unreleased version of a package is explicitly documented in the `go.mod` file via an inline comment of the form `dependency-check-ignore: `. +* If you are adding such a dependency, please add a suitable comment to the `go.mod` file as well. +* This requirement applies both to direct and indirect dependencies. +* This requirement applies to packages that have released versions (i.e., is not a `v0.0.0`), +* This is enforced with https://github.com/filecoin-project/lotus/blob/master/.github/workflows/dependency-check.yml +* This enforcement was initially done per [#7131](https://github.com/filecoin-project/lotus/issues/7131) ## Markdown Conventions We optimize our markdown files for viewing on GitHub. That isn't to say other syntaxes can't be used, but that is the flavor we focus on and at the minimum don't want to break. From c119ac0dac4798dc739540204204c695f3af2bf6 Mon Sep 17 00:00:00 2001 From: Piotr Galar Date: Thu, 12 Dec 2024 11:26:17 +0000 Subject: [PATCH 10/18] Update dependency-check.yml Co-authored-by: Steve Loeppky --- .github/workflows/dependency-check.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dependency-check.yml b/.github/workflows/dependency-check.yml index e873372dcc0..c7dba41da97 100644 --- a/.github/workflows/dependency-check.yml +++ b/.github/workflows/dependency-check.yml @@ -18,7 +18,10 @@ jobs: - id: all run: | echo "dependencies<> $GITHUB_OUTPUT - # Extract all dependencies from go.mod (include indirect dependencies and comments) + # Extract all dependencies from go.mod (include indirect dependencies and comments). + # `go list` isn't used because: + # 1. it lists ALL the transitive dependencies, even those that are unused and don't make it to the go.mod file + # 2. It doesn't extract the inline `dependency-check-ignore` comments. sed -n '/require (/,/)/{/require (/!{/)/!p;};}' go.mod | sed 's/^[[:space:]]*//' | tee -a $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT - id: unreleased From ffbecb5867379d40aa697bf399045c42947a36db Mon Sep 17 00:00:00 2001 From: Piotr Galar Date: Thu, 12 Dec 2024 11:27:33 +0000 Subject: [PATCH 11/18] Update dependency-check.yml Co-authored-by: Steve Loeppky --- .github/workflows/dependency-check.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dependency-check.yml b/.github/workflows/dependency-check.yml index c7dba41da97..df411198fcc 100644 --- a/.github/workflows/dependency-check.yml +++ b/.github/workflows/dependency-check.yml @@ -22,7 +22,20 @@ jobs: # `go list` isn't used because: # 1. it lists ALL the transitive dependencies, even those that are unused and don't make it to the go.mod file # 2. It doesn't extract the inline `dependency-check-ignore` comments. - sed -n '/require (/,/)/{/require (/!{/)/!p;};}' go.mod | sed 's/^[[:space:]]*//' | tee -a $GITHUB_OUTPUT + # Extract the lines from 'require (' to the first ')' including those lines in the go.mod file. + sed -n '/require (/,/)/p' go.mod | + + # Remove the 'require (' line. + sed '/require (/d' | + + # Remove the ')' line. + sed '/^)/d' | + + # Remove leading whitespace from each line. + sed 's/^[[:space:]]*//' | + + # Append the result to the file specified by the GITHUB_OUTPUT environment variable. + tee -a $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT - id: unreleased env: From 2fda23ec45a166d52fceb8b9bbe053537eb710c2 Mon Sep 17 00:00:00 2001 From: Piotr Galar Date: Thu, 12 Dec 2024 11:28:13 +0000 Subject: [PATCH 12/18] Update dependency-check.yml Co-authored-by: Steve Loeppky --- .github/workflows/dependency-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dependency-check.yml b/.github/workflows/dependency-check.yml index df411198fcc..a7c70d66111 100644 --- a/.github/workflows/dependency-check.yml +++ b/.github/workflows/dependency-check.yml @@ -56,7 +56,7 @@ jobs: - if: steps.unexplained.outputs.dependencies != '' env: MESSAGE: | - A new unreleased dependency was discovered in this PR. Please do one of the options in [dependency management conventions](https://github.com/filecoin-project/lotus/blob/master/CONTRIBUTING.md#dependency-management) + A new unreleased and unexplained dependency was discovered in this PR. Please do one of the options in [dependency management conventions](https://github.com/filecoin-project/lotus/blob/master/CONTRIBUTING.md#dependency-management). Unreleased dependencies: ${{steps.unexplained.outputs.dependencies}} From 942049f82981422c63f21143769b0278123d80d8 Mon Sep 17 00:00:00 2001 From: Piotr Galar Date: Thu, 12 Dec 2024 11:28:22 +0000 Subject: [PATCH 13/18] Update dependency-check.yml Co-authored-by: Steve Loeppky --- .github/workflows/dependency-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dependency-check.yml b/.github/workflows/dependency-check.yml index a7c70d66111..5a2ee30fa86 100644 --- a/.github/workflows/dependency-check.yml +++ b/.github/workflows/dependency-check.yml @@ -58,7 +58,7 @@ jobs: MESSAGE: | A new unreleased and unexplained dependency was discovered in this PR. Please do one of the options in [dependency management conventions](https://github.com/filecoin-project/lotus/blob/master/CONTRIBUTING.md#dependency-management). - Unreleased dependencies: + Unexplained unreleased dependencies: ${{steps.unexplained.outputs.dependencies}} run: | echo "::error::${MESSAGE//$'\n'/%0A}" From 55b0578f00a1db988a72b15f531d18ef283889b0 Mon Sep 17 00:00:00 2001 From: Piotr Galar Date: Thu, 12 Dec 2024 11:28:30 +0000 Subject: [PATCH 14/18] Update CONTRIBUTING.md Co-authored-by: Steve Loeppky --- CONTRIBUTING.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 96615995904..1149a8e17bf 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -64,12 +64,13 @@ We strive to use release dependencies because: 1. Security / reliability - While there's no guarantee that a released version doesn't have bugs or issues, it seems fair to assume that non-released versions have even more. For example, https://github.com/filecoin-project/lotus/issues/12467 was triggered because of a bug in non-released library that lotus was depending on when the latest released version didn't have the bug. 2. Faster builds 3. Makes Lotus a better citizen when it's imported by other projects. + We enforce that each dependency on an unreleased version of a package is explicitly documented in the `go.mod` file via an inline comment of the form `dependency-check-ignore: `. * If you are adding such a dependency, please add a suitable comment to the `go.mod` file as well. * This requirement applies both to direct and indirect dependencies. -* This requirement applies to packages that have released versions (i.e., is not a `v0.0.0`), +* This requirement applies to packages that have released versions (i.e., is not a `v0.0.0`). * This is enforced with https://github.com/filecoin-project/lotus/blob/master/.github/workflows/dependency-check.yml -* This enforcement was initially done per [#7131](https://github.com/filecoin-project/lotus/issues/7131) +* This enforcement was initially done per [#7131](https://github.com/filecoin-project/lotus/issues/7131). ## Markdown Conventions We optimize our markdown files for viewing on GitHub. That isn't to say other syntaxes can't be used, but that is the flavor we focus on and at the minimum don't want to break. From c937dc017d78e01bd5c77a765a9d8a67d7e413f9 Mon Sep 17 00:00:00 2001 From: Piotr Galar Date: Thu, 12 Dec 2024 11:32:03 +0000 Subject: [PATCH 15/18] Update dependency-check.yml Co-authored-by: Steve Loeppky --- .github/workflows/dependency-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dependency-check.yml b/.github/workflows/dependency-check.yml index 5a2ee30fa86..fc892eb35c7 100644 --- a/.github/workflows/dependency-check.yml +++ b/.github/workflows/dependency-check.yml @@ -42,7 +42,7 @@ jobs: DEPENDENCIES: ${{ steps.all.outputs.dependencies }} run: | echo "dependencies<> $GITHUB_OUTPUT - # Find all dependencies that use prerelease versions (exclude v0.0.0 versions) + # Find all dependencies that use prerelease versions (i.e., exclude vX.Y.Z and v0.0.0 versions). grep -Pv 'v0\.0\.0|v[0-9]+\.[0-9]+\.[0-9]+([^-]|$)' <<< "$DEPENDENCIES" | tee -a $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT - id: unexplained From 688eae17b6a3d7a335ede3ff2d69fa5fe2d375d7 Mon Sep 17 00:00:00 2001 From: Piotr Galar Date: Thu, 12 Dec 2024 11:34:28 +0000 Subject: [PATCH 16/18] Update dependency-check.yml Co-authored-by: Rod Vagg --- .github/workflows/dependency-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dependency-check.yml b/.github/workflows/dependency-check.yml index fc892eb35c7..48f99183ea0 100644 --- a/.github/workflows/dependency-check.yml +++ b/.github/workflows/dependency-check.yml @@ -43,7 +43,7 @@ jobs: run: | echo "dependencies<> $GITHUB_OUTPUT # Find all dependencies that use prerelease versions (i.e., exclude vX.Y.Z and v0.0.0 versions). - grep -Pv 'v0\.0\.0|v[0-9]+\.[0-9]+\.[0-9]+([^-]|$)' <<< "$DEPENDENCIES" | tee -a $GITHUB_OUTPUT + grep -Pv 'v0\.0\.0-[0-9]{14}-[0-9a-f]{7,}$|v[0-9]+\.[0-9]+\.[0-9]+(\+incompatible)?$' <<< "$DEPENDENCIES" | tee -a $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT - id: unexplained env: From 42fc4b4802d9260b630e10a5554e647821ca9c1b Mon Sep 17 00:00:00 2001 From: Piotr Galar Date: Thu, 12 Dec 2024 11:41:05 +0000 Subject: [PATCH 17/18] chore: add step names in the dependency checker --- .github/workflows/dependency-check.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dependency-check.yml b/.github/workflows/dependency-check.yml index 48f99183ea0..b28b978d85b 100644 --- a/.github/workflows/dependency-check.yml +++ b/.github/workflows/dependency-check.yml @@ -13,12 +13,13 @@ jobs: name: Dependency Check steps: - uses: actions/checkout@v3 + name: Check out the repository with: submodules: 'recursive' - id: all + name: Extract all dependencies from go.mod (include indirect dependencies and comments) run: | echo "dependencies<> $GITHUB_OUTPUT - # Extract all dependencies from go.mod (include indirect dependencies and comments). # `go list` isn't used because: # 1. it lists ALL the transitive dependencies, even those that are unused and don't make it to the go.mod file # 2. It doesn't extract the inline `dependency-check-ignore` comments. @@ -38,22 +39,23 @@ jobs: tee -a $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT - id: unreleased + name: Find all dependencies that use prerelease versions (i.e., exclude vX.Y.Z and v0.0.0 versions) env: DEPENDENCIES: ${{ steps.all.outputs.dependencies }} run: | echo "dependencies<> $GITHUB_OUTPUT - # Find all dependencies that use prerelease versions (i.e., exclude vX.Y.Z and v0.0.0 versions). grep -Pv 'v0\.0\.0-[0-9]{14}-[0-9a-f]{7,}$|v[0-9]+\.[0-9]+\.[0-9]+(\+incompatible)?$' <<< "$DEPENDENCIES" | tee -a $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT - id: unexplained + name: Find all unreleased dependencies without a dependency-check-ignore comment env: DEPENDENCIES: ${{ steps.unreleased.outputs.dependencies }} run: | echo "dependencies<> $GITHUB_OUTPUT - # Find all unreleased dependencies without a dependency-check-ignore comment grep -Pv 'dependency-check-ignore:\s' <<< "$DEPENDENCIES" | tee -a $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT - if: steps.unexplained.outputs.dependencies != '' + name: Throw if any unreleased unexplained dependencies exist env: MESSAGE: | A new unreleased and unexplained dependency was discovered in this PR. Please do one of the options in [dependency management conventions](https://github.com/filecoin-project/lotus/blob/master/CONTRIBUTING.md#dependency-management). From b6c46d4a3a6129704718ad49ea41f42dbe97e4f7 Mon Sep 17 00:00:00 2001 From: galargh Date: Fri, 13 Dec 2024 10:44:11 +0000 Subject: [PATCH 18/18] fix: the v0.0.0 or vX.Y.Z regex --- .github/workflows/dependency-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dependency-check.yml b/.github/workflows/dependency-check.yml index b28b978d85b..5d64cc20ffa 100644 --- a/.github/workflows/dependency-check.yml +++ b/.github/workflows/dependency-check.yml @@ -44,7 +44,7 @@ jobs: DEPENDENCIES: ${{ steps.all.outputs.dependencies }} run: | echo "dependencies<> $GITHUB_OUTPUT - grep -Pv 'v0\.0\.0-[0-9]{14}-[0-9a-f]{7,}$|v[0-9]+\.[0-9]+\.[0-9]+(\+incompatible)?$' <<< "$DEPENDENCIES" | tee -a $GITHUB_OUTPUT + grep -Pv '(v0\.0\.0-[0-9]{14}-[0-9a-f]{7,}|v[0-9]+\.[0-9]+\.[0-9]+(\+incompatible)?)( |$)' <<< "$DEPENDENCIES" | tee -a $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT - id: unexplained name: Find all unreleased dependencies without a dependency-check-ignore comment