diff --git a/lib/LuxTestUtils/.buildkite/pipeline.yml b/lib/LuxTestUtils/.buildkite/pipeline.yml new file mode 100644 index 000000000..d6f1131fe --- /dev/null +++ b/lib/LuxTestUtils/.buildkite/pipeline.yml @@ -0,0 +1,115 @@ +steps: + - group: ":telescope: Downstream CUDA" + steps: + - label: ":julia: {{matrix.repo}} (Julia {{matrix.julia}} + CUDA GPU)" + plugins: + - JuliaCI/julia#v1: + version: "{{matrix.julia}}" + - JuliaCI/julia-coverage#v1: + codecov: true + dirs: + - src + command: | + julia --code-coverage=user --color=yes --project -e ' + using Pkg + + repo = ENV["DOWNSTREAM_TEST_REPO"] + if contains(repo, "#") + repo, group = split(repo, "#") + else + group = "CUDA" + end + + println("--- :julia: Instantiating project") + withenv("JULIA_PKG_PRECOMPILE_AUTO" => 0, "GROUP" => group) do + Pkg.instantiate() + + try + Pkg.develop(repo) + println("+++ :julia: Running tests") + Pkg.test("$(repo)"; coverage=true) + catch err + err isa Pkg.Resolve.ResolverError || rethrow() + @info "Not compatible with this release. No problem." exception=err + exit(0) + end + end + + println("+++ :julia: Finished Downstream Test")' + agents: + queue: "juliagpu" + cuda: "*" + env: + DOWNSTREAM_TEST_REPO: "{{matrix.repo}}" + if: build.message !~ /\[skip tests\]/ || build.message !~ /\[skip downstream\]/ + timeout_in_minutes: 240 + matrix: + setup: + julia: + - "1" + repo: + - "Lux" + - "LuxLib" + + - group: ":telescope: Downstream AMD GPU" + steps: + - label: ":julia: {{matrix.repo}} (Julia {{matrix.julia}} + AMD GPU)" + plugins: + - JuliaCI/julia#v1: + version: "{{matrix.julia}}" + - JuliaCI/julia-coverage#v1: + codecov: true + dirs: + - src + command: | + julia --code-coverage=user --color=yes --project -e ' + using Pkg + + repo = ENV["DOWNSTREAM_TEST_REPO"] + if contains(repo, "#") + repo, group = split(repo, "#") + else + group = "AMDGPU" + end + + println("--- :julia: Instantiating project") + withenv("JULIA_PKG_PRECOMPILE_AUTO" => 0, "GROUP" => group) do + Pkg.instantiate() + + try + Pkg.develop(repo) + println("+++ :julia: Running tests") + Pkg.test("$(repo)"; coverage=true) + catch err + err isa Pkg.Resolve.ResolverError || rethrow() + @info "Not compatible with this release. No problem." exception=err + exit(0) + end + end + + println("+++ :julia: Finished Downstream Test")' + agents: + queue: "juliagpu" + rocm: "*" + rocmgpu: "*" + env: + JULIA_AMDGPU_CORE_MUST_LOAD: "1" + JULIA_AMDGPU_HIP_MUST_LOAD: "1" + JULIA_AMDGPU_DISABLE_ARTIFACTS: "1" + DOWNSTREAM_TEST_REPO: "{{matrix.repo}}" + if: build.message !~ /\[skip tests\]/ || build.message !~ /\[skip downstream\]/ + timeout_in_minutes: 60 + matrix: + setup: + julia: + - "1" + repo: + - "Lux" + - "LuxLib" + +env: + RETESTITEMS_NWORKERS: 2 + RETESTITEMS_NWORKER_THREADS: 2 + JULIA_AMDGPU_LOGGING_ENABLED: true + RETESTITEMS_TESTITEM_TIMEOUT: 10000 + SECRET_CODECOV_TOKEN: "jQ0BMTQgyZx7QGyU0Q2Ec7qB9mtE2q/tDu0FsfxvEG7/zOAGvXkyXrzIFFOQxvDoFcP+K2+hYZKMxicYdNqzr5wcxu505aNGN2GM3wyegAr+hO6q12bCFYx6qXzU9FLCCdeqINqn9gUSSOlGtWNFrbAlrTyz/D4Yo66TqBDzvaLL63FMnhCLaXW/zJt3hNuEAJaPY2O6Ze1rX2WZ3Y+i+s3uQ8aLImtoCJhPe8CRx+OhuYiTzGhynFfGntZ0738/1RN4gNM0S/hTC4gLE7XMVBanJpGh32rFaiDwW4zAyXKBrDkL3QA3MS1RvLTJxGJ085S16hCk0C4ddAhZCvIM9Q==;U2FsdGVkX1+bXdFeKMs5G79catOCyby2n07A2fg0FjVAvrjQLZ0yfvDS4paJiFikLkodho0khz2YALKb2Y0K6w==" diff --git a/lib/LuxTestUtils/.github/workflows/CI.yml b/lib/LuxTestUtils/.github/workflows/CI.yml index d35ff3c77..1ae67fbbe 100644 --- a/lib/LuxTestUtils/.github/workflows/CI.yml +++ b/lib/LuxTestUtils/.github/workflows/CI.yml @@ -36,3 +36,10 @@ jobs: ${{ runner.os }}- - uses: julia-actions/julia-buildpkg@v1 - uses: julia-actions/julia-runtest@v1 + - uses: julia-actions/julia-processcoverage@v1 + - uses: codecov/codecov-action@v4 + with: + files: lcov.info + token: ${{ secrets.CODECOV_TOKEN }} + verbose: true + fail_ci_if_error: true diff --git a/lib/LuxTestUtils/.github/workflows/Downgrade.yml b/lib/LuxTestUtils/.github/workflows/Downgrade.yml new file mode 100644 index 000000000..59922aae5 --- /dev/null +++ b/lib/LuxTestUtils/.github/workflows/Downgrade.yml @@ -0,0 +1,39 @@ +name: Downgrade +on: + pull_request: + branches: + - main + paths-ignore: + - 'docs/**' + push: + branches: + - master + paths-ignore: + - 'docs/**' +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + version: ['1'] + steps: + - uses: actions/checkout@v4 + - uses: julia-actions/setup-julia@v2 + with: + version: ${{ matrix.version }} + - uses: cjdoris/julia-downgrade-compat-action@v1 + with: + skip: Pkg,TOML + - uses: julia-actions/julia-buildpkg@v1 + - uses: julia-actions/julia-runtest@v1 + env: + GROUP: "CPU" + RETESTITEMS_NWORKERS: 4 + RETESTITEMS_NWORKER_THREADS: 2 + - uses: julia-actions/julia-processcoverage@v1 + - uses: codecov/codecov-action@v4 + with: + files: lcov.info + token: ${{ secrets.CODECOV_TOKEN }} + verbose: true + fail_ci_if_error: true \ No newline at end of file diff --git a/lib/LuxTestUtils/.github/workflows/Downstream.yml b/lib/LuxTestUtils/.github/workflows/Downstream.yml index ddc4197e0..5f479344b 100644 --- a/lib/LuxTestUtils/.github/workflows/Downstream.yml +++ b/lib/LuxTestUtils/.github/workflows/Downstream.yml @@ -54,7 +54,13 @@ jobs: @info "Not compatible with this release. No problem." exception=err exit(0) # Exit immediately, as a success end + env: + RETESTITEMS_NWORKERS: 2 + RETESTITEMS_NWORKER_THREADS: 2 - uses: julia-actions/julia-processcoverage@v1 - uses: codecov/codecov-action@v4 with: files: lcov.info + token: ${{ secrets.CODECOV_TOKEN }} + verbose: true + fail_ci_if_error: true diff --git a/lib/LuxTestUtils/Project.toml b/lib/LuxTestUtils/Project.toml index d92bf9457..495b536d1 100644 --- a/lib/LuxTestUtils/Project.toml +++ b/lib/LuxTestUtils/Project.toml @@ -1,7 +1,7 @@ name = "LuxTestUtils" uuid = "ac9de150-d08f-4546-94fb-7472b5760531" authors = ["Avik Pal "] -version = "0.1.15" +version = "0.1.16" [deps] ComponentArrays = "b0b7db55-cfe3-40fc-9ded-d10e2dbeff66" @@ -21,7 +21,7 @@ Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c" Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" [compat] -ComponentArrays = "0.13, 0.14, 0.15" +ComponentArrays = "0.15" FiniteDifferences = "0.12" ForwardDiff = "0.10" Functors = "0.4" diff --git a/lib/LuxTestUtils/README.md b/lib/LuxTestUtils/README.md index b98926622..b2a823afd 100644 --- a/lib/LuxTestUtils/README.md +++ b/lib/LuxTestUtils/README.md @@ -1,10 +1,11 @@ # LuxTestUtils.jl [![Join the chat at https://julialang.zulipchat.com #machine-learning](https://img.shields.io/static/v1?label=Zulip&message=chat&color=9558b2&labelColor=389826)](https://julialang.zulipchat.com/#narrow/stream/machine-learning) -[![Latest Docs](https://img.shields.io/badge/docs-latest-blue.svg)](http://lux.csail.mit.edu/dev/api/) -[![Stable Docs](https://img.shields.io/badge/docs-stable-blue.svg)](http://lux.csail.mit.edu/stable/api/) +[![Latest Docs](https://img.shields.io/badge/docs-latest-blue.svg)](https://lux.csail.mit.edu/dev/api/Testing_Functionality/LuxTestUtils) +[![Stable Docs](https://img.shields.io/badge/docs-stable-blue.svg)](https://lux.csail.mit.edu/stable/api/Testing_Functionality/LuxTestUtils) [![CI](https://github.com/LuxDL/LuxTestUtils.jl/actions/workflows/CI.yml/badge.svg)](https://github.com/LuxDL/LuxTestUtils.jl/actions/workflows/CI.yml) +[![Build status](https://img.shields.io/buildkite/e788fcafd7f48b654ded5b39d5ca119ee82f76274d2edb1bc9/main.svg?label=gpu&branch=master)](https://buildkite.com/julialang/lux-dot-jl) [![ColPrac: Contributor's Guide on Collaborative Practices for Community Packages](https://img.shields.io/badge/ColPrac-Contributor's%20Guide-blueviolet)](https://github.com/SciML/ColPrac) [![SciML Code Style](https://img.shields.io/static/v1?label=code%20style&message=SciML&color=9558b2&labelColor=389826)](https://github.com/SciML/SciMLStyle) @@ -22,129 +23,6 @@ Utilities for testing [Lux.jl](http://lux.csail.mit.edu/stable). load times. It is recommended that you exclusively use this package for testing and not add a dependency to it in your main package Project.toml. -## Exported Functions - -### Testing using [JET.jl](https://github.com/aviatesk/JET.jl) - -We export a simple macro `@jet` to allow testing your code using JET - -```julia -help> @jet - - @jet f(args...) call_broken=false opt_broken=false - - - Run JET tests on the function `f` with the arguments `args`. If JET fails to compile or - julia version is < 1.7, then the macro will be a no-op. - - Keyword Arguments - =================== - - • `call_broken`: Marks the test_call as broken. - - • `opt_broken`: Marks the test_opt as broken. - - All additional arguments will be forwarded to @JET.test_call and @JET.test_opt. - - │ Note - │ - │ Instead of specifying target_modules with every call, you can set preferences for - │ target_modules using Preferences.jl. For example, to set `target_modules` to - │ (Lux, LuxLib) we can run: - │ - │ using Preferences - │ - │ set_preferences!(Base.UUID("ac9de150-d08f-4546-94fb-7472b5760531"), - │ "target_modules" => ["Lux", "LuxLib"]) - - Example - ========= - - @jet sum([1, 2, 3]) target_modules=(Base, Core) - - @jet sum(1, 1) target_modules=(Base, Core) opt_broken=true -``` - -### Gradient Correctness - -```julia -help?> @test_gradients - @test_gradients f args... [kwargs...] - - - Compare the gradients computed by `Zygote.jl` (Reverse Mode AD) against: - - • `Tracker.jl` (Reverse Mode AD) - - • `ReverseDiff.jl` (Reverse Mode AD) - - • `ForwardDiff.jl` (Forward Mode AD) - - • `FiniteDifferences.jl` (Finite Differences) - - │ Tip - │ - │ This function is completely compatible with `Test.jl` - - Arguments - =========== - - • `f`: The function to test. - - • `args`...: Inputs to f wrt which the gradients are computed. - - Keyword Arguments - =================== - - • `gpu_testing`: Disables ForwardDiff, ReverseDiff and FiniteDifferences tests. - (Default: `false`) - - • `soft_fail`: If `true`, the test will not fail if any of the gradients are incorrect, - instead it will show up as broken. (Default: `false`) - - • `skip_(tracker|reverse_diff|forward_diff|finite_differences)`: Skip the corresponding - gradient computation and check. (Default: `false`) - - • `large_arrays_skip_(forward_diff|finite_differences)`: Skip the corresponding - gradient computation and check for large arrays. (Forward Mode and Finite Differences - are not efficient for large arrays.) (Default: `true`) - - • `large_array_length`: The length of the array above which the gradient computation is - considered large. (Default: `25`) - - • `max_total_array_size`: Treat as large array if the total size of all arrays is - greater than this value. (Default: `100`) - - • `(tracker|reverse_diff|forward_diff|finite_differences)_broken`: Mark the - corresponding gradient test as broken. (Default: `false`) - - Keyword Arguments for check_approx - ==================================== - - • `atol`: Absolute tolerance for gradient comparisons. (Default: `0.0`) - - • `rtol`: Relative tolerance for gradient comparisons. (Default: - `atol > 0 ? 0.0 : √eps(typeof(atol))`) - - • `nans`: Whether or not NaNs are considered equal. (Default: `false`) - - Example - ========= - - using LuxTestUtils, Test - - x = randn(10) - - @testset "Showcase Gradient Testing" begin - @test_gradients sum abs2 x - - @test_gradients prod x - end -``` - -Internally, it uses `check_approx` which extends `Base.isapprox` for more common cases. It -follows the exact same function call as `isapprox`. - ## Passing Runtime Variables to Macro Macros operate on the syntax and hence can't directly take variable inputs. To get around diff --git a/lib/LuxTestUtils/src/LuxTestUtils.jl b/lib/LuxTestUtils/src/LuxTestUtils.jl index 32be24eea..30ff26d77 100644 --- a/lib/LuxTestUtils/src/LuxTestUtils.jl +++ b/lib/LuxTestUtils/src/LuxTestUtils.jl @@ -2,7 +2,6 @@ module LuxTestUtils using ComponentArrays, Optimisers, Preferences, LuxCore, LuxDeviceUtils, Test using ForwardDiff, ReverseDiff, Tracker, Zygote, FiniteDifferences -# TODO: Yota, Enzyme const JET_TARGET_MODULES = @load_preference("target_modules", nothing) @@ -32,20 +31,18 @@ or julia version is < 1.7, then the macro will be a no-op. All additional arguments will be forwarded to `@JET.test_call` and `@JET.test_opt`. -:::tip +!!! tip -Instead of specifying `target_modules` with every call, you can set preferences for -`target_modules` using `Preferences.jl`. For example, to set `target_modules` to -`(Lux, LuxLib)` we can run: + Instead of specifying `target_modules` with every call, you can set preferences for + `target_modules` using `Preferences.jl`. For example, to set `target_modules` to + `(Lux, LuxLib)` we can run: -```julia -using Preferences - -set_preferences!(Base.UUID("ac9de150-d08f-4546-94fb-7472b5760531"), - "target_modules" => ["Lux", "LuxLib"]) -``` + ```julia + using Preferences -::: + set_preferences!(Base.UUID("ac9de150-d08f-4546-94fb-7472b5760531"), + "target_modules" => ["Lux", "LuxLib"]) + ``` ## Example @@ -163,11 +160,9 @@ Compare the gradients computed by Zygote.jl (Reverse Mode AD) against: - ForwardDiff.jl (Forward Mode AD) - FiniteDifferences.jl (Finite Differences) -:::tip - -This function is completely compatible with Test.jl +!!! tip -::: + This function is completely compatible with Test.jl ## Arguments