Skip to content

Commit

Permalink
Merge pull request #19 from LuxDL/ap/testing
Browse files Browse the repository at this point in the history
Update the workflows
  • Loading branch information
avik-pal authored Apr 12, 2024
2 parents bf9fe3c + 6f0574a commit 6eb61e5
Show file tree
Hide file tree
Showing 7 changed files with 183 additions and 143 deletions.
115 changes: 115 additions & 0 deletions lib/LuxTestUtils/.buildkite/pipeline.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
steps:
- group: ":telescope: Downstream CUDA"
steps:
- label: ":julia: {{matrix.repo}} (Julia {{matrix.julia}} + CUDA GPU)"
plugins:
- JuliaCI/julia#v1:
version: "{{matrix.julia}}"
- JuliaCI/julia-coverage#v1:
codecov: true
dirs:
- src
command: |
julia --code-coverage=user --color=yes --project -e '
using Pkg
repo = ENV["DOWNSTREAM_TEST_REPO"]
if contains(repo, "#")
repo, group = split(repo, "#")
else
group = "CUDA"
end
println("--- :julia: Instantiating project")
withenv("JULIA_PKG_PRECOMPILE_AUTO" => 0, "GROUP" => group) do
Pkg.instantiate()
try
Pkg.develop(repo)
println("+++ :julia: Running tests")
Pkg.test("$(repo)"; coverage=true)
catch err
err isa Pkg.Resolve.ResolverError || rethrow()
@info "Not compatible with this release. No problem." exception=err
exit(0)
end
end
println("+++ :julia: Finished Downstream Test")'
agents:
queue: "juliagpu"
cuda: "*"
env:
DOWNSTREAM_TEST_REPO: "{{matrix.repo}}"
if: build.message !~ /\[skip tests\]/ || build.message !~ /\[skip downstream\]/
timeout_in_minutes: 240
matrix:
setup:
julia:
- "1"
repo:
- "Lux"
- "LuxLib"

- group: ":telescope: Downstream AMD GPU"
steps:
- label: ":julia: {{matrix.repo}} (Julia {{matrix.julia}} + AMD GPU)"
plugins:
- JuliaCI/julia#v1:
version: "{{matrix.julia}}"
- JuliaCI/julia-coverage#v1:
codecov: true
dirs:
- src
command: |
julia --code-coverage=user --color=yes --project -e '
using Pkg
repo = ENV["DOWNSTREAM_TEST_REPO"]
if contains(repo, "#")
repo, group = split(repo, "#")
else
group = "AMDGPU"
end
println("--- :julia: Instantiating project")
withenv("JULIA_PKG_PRECOMPILE_AUTO" => 0, "GROUP" => group) do
Pkg.instantiate()
try
Pkg.develop(repo)
println("+++ :julia: Running tests")
Pkg.test("$(repo)"; coverage=true)
catch err
err isa Pkg.Resolve.ResolverError || rethrow()
@info "Not compatible with this release. No problem." exception=err
exit(0)
end
end
println("+++ :julia: Finished Downstream Test")'
agents:
queue: "juliagpu"
rocm: "*"
rocmgpu: "*"
env:
JULIA_AMDGPU_CORE_MUST_LOAD: "1"
JULIA_AMDGPU_HIP_MUST_LOAD: "1"
JULIA_AMDGPU_DISABLE_ARTIFACTS: "1"
DOWNSTREAM_TEST_REPO: "{{matrix.repo}}"
if: build.message !~ /\[skip tests\]/ || build.message !~ /\[skip downstream\]/
timeout_in_minutes: 60
matrix:
setup:
julia:
- "1"
repo:
- "Lux"
- "LuxLib"

env:
RETESTITEMS_NWORKERS: 2
RETESTITEMS_NWORKER_THREADS: 2
JULIA_AMDGPU_LOGGING_ENABLED: true
RETESTITEMS_TESTITEM_TIMEOUT: 10000
SECRET_CODECOV_TOKEN: "jQ0BMTQgyZx7QGyU0Q2Ec7qB9mtE2q/tDu0FsfxvEG7/zOAGvXkyXrzIFFOQxvDoFcP+K2+hYZKMxicYdNqzr5wcxu505aNGN2GM3wyegAr+hO6q12bCFYx6qXzU9FLCCdeqINqn9gUSSOlGtWNFrbAlrTyz/D4Yo66TqBDzvaLL63FMnhCLaXW/zJt3hNuEAJaPY2O6Ze1rX2WZ3Y+i+s3uQ8aLImtoCJhPe8CRx+OhuYiTzGhynFfGntZ0738/1RN4gNM0S/hTC4gLE7XMVBanJpGh32rFaiDwW4zAyXKBrDkL3QA3MS1RvLTJxGJ085S16hCk0C4ddAhZCvIM9Q==;U2FsdGVkX1+bXdFeKMs5G79catOCyby2n07A2fg0FjVAvrjQLZ0yfvDS4paJiFikLkodho0khz2YALKb2Y0K6w=="
7 changes: 7 additions & 0 deletions lib/LuxTestUtils/.github/workflows/CI.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,3 +36,10 @@ jobs:
${{ runner.os }}-
- uses: julia-actions/julia-buildpkg@v1
- uses: julia-actions/julia-runtest@v1
- uses: julia-actions/julia-processcoverage@v1
- uses: codecov/codecov-action@v4
with:
files: lcov.info
token: ${{ secrets.CODECOV_TOKEN }}
verbose: true
fail_ci_if_error: true
39 changes: 39 additions & 0 deletions lib/LuxTestUtils/.github/workflows/Downgrade.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
name: Downgrade
on:
pull_request:
branches:
- main
paths-ignore:
- 'docs/**'
push:
branches:
- master
paths-ignore:
- 'docs/**'
jobs:
test:
runs-on: ubuntu-latest
strategy:
matrix:
version: ['1']
steps:
- uses: actions/checkout@v4
- uses: julia-actions/setup-julia@v2
with:
version: ${{ matrix.version }}
- uses: cjdoris/julia-downgrade-compat-action@v1
with:
skip: Pkg,TOML
- uses: julia-actions/julia-buildpkg@v1
- uses: julia-actions/julia-runtest@v1
env:
GROUP: "CPU"
RETESTITEMS_NWORKERS: 4
RETESTITEMS_NWORKER_THREADS: 2
- uses: julia-actions/julia-processcoverage@v1
- uses: codecov/codecov-action@v4
with:
files: lcov.info
token: ${{ secrets.CODECOV_TOKEN }}
verbose: true
fail_ci_if_error: true
6 changes: 6 additions & 0 deletions lib/LuxTestUtils/.github/workflows/Downstream.yml
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,13 @@ jobs:
@info "Not compatible with this release. No problem." exception=err
exit(0) # Exit immediately, as a success
end
env:
RETESTITEMS_NWORKERS: 2
RETESTITEMS_NWORKER_THREADS: 2
- uses: julia-actions/julia-processcoverage@v1
- uses: codecov/codecov-action@v4
with:
files: lcov.info
token: ${{ secrets.CODECOV_TOKEN }}
verbose: true
fail_ci_if_error: true
4 changes: 2 additions & 2 deletions lib/LuxTestUtils/Project.toml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
name = "LuxTestUtils"
uuid = "ac9de150-d08f-4546-94fb-7472b5760531"
authors = ["Avik Pal <avikpal@mit.edu>"]
version = "0.1.15"
version = "0.1.16"

[deps]
ComponentArrays = "b0b7db55-cfe3-40fc-9ded-d10e2dbeff66"
Expand All @@ -21,7 +21,7 @@ Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c"
Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"

[compat]
ComponentArrays = "0.13, 0.14, 0.15"
ComponentArrays = "0.15"
FiniteDifferences = "0.12"
ForwardDiff = "0.10"
Functors = "0.4"
Expand Down
128 changes: 3 additions & 125 deletions lib/LuxTestUtils/README.md
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
# LuxTestUtils.jl

[![Join the chat at https://julialang.zulipchat.com #machine-learning](https://img.shields.io/static/v1?label=Zulip&message=chat&color=9558b2&labelColor=389826)](https://julialang.zulipchat.com/#narrow/stream/machine-learning)
[![Latest Docs](https://img.shields.io/badge/docs-latest-blue.svg)](http://lux.csail.mit.edu/dev/api/)
[![Stable Docs](https://img.shields.io/badge/docs-stable-blue.svg)](http://lux.csail.mit.edu/stable/api/)
[![Latest Docs](https://img.shields.io/badge/docs-latest-blue.svg)](https://lux.csail.mit.edu/dev/api/Testing_Functionality/LuxTestUtils)
[![Stable Docs](https://img.shields.io/badge/docs-stable-blue.svg)](https://lux.csail.mit.edu/stable/api/Testing_Functionality/LuxTestUtils)

[![CI](https://github.com/LuxDL/LuxTestUtils.jl/actions/workflows/CI.yml/badge.svg)](https://github.com/LuxDL/LuxTestUtils.jl/actions/workflows/CI.yml)
[![Build status](https://img.shields.io/buildkite/e788fcafd7f48b654ded5b39d5ca119ee82f76274d2edb1bc9/main.svg?label=gpu&branch=master)](https://buildkite.com/julialang/lux-dot-jl)

[![ColPrac: Contributor's Guide on Collaborative Practices for Community Packages](https://img.shields.io/badge/ColPrac-Contributor's%20Guide-blueviolet)](https://github.com/SciML/ColPrac)
[![SciML Code Style](https://img.shields.io/static/v1?label=code%20style&message=SciML&color=9558b2&labelColor=389826)](https://github.com/SciML/SciMLStyle)
Expand All @@ -22,129 +23,6 @@ Utilities for testing [Lux.jl](http://lux.csail.mit.edu/stable).
load times. It is recommended that you exclusively use this package for testing and not
add a dependency to it in your main package Project.toml.

## Exported Functions

### Testing using [JET.jl](https://github.com/aviatesk/JET.jl)

We export a simple macro `@jet` to allow testing your code using JET

```julia
help> @jet

@jet f(args...) call_broken=false opt_broken=false


Run JET tests on the function `f` with the arguments `args`. If JET fails to compile or
julia version is < 1.7, then the macro will be a no-op.

Keyword Arguments
===================

`call_broken`: Marks the test_call as broken.

`opt_broken`: Marks the test_opt as broken.

All additional arguments will be forwarded to @JET.test_call and @JET.test_opt.

│ Note
│ Instead of specifying target_modules with every call, you can set preferences for
│ target_modules using Preferences.jl. For example, to set `target_modules` to
│ (Lux, LuxLib) we can run:
using Preferences
set_preferences!(Base.UUID("ac9de150-d08f-4546-94fb-7472b5760531"),
"target_modules" => ["Lux", "LuxLib"])

Example
=========

@jet sum([1, 2, 3]) target_modules=(Base, Core)

@jet sum(1, 1) target_modules=(Base, Core) opt_broken=true
```

### Gradient Correctness

```julia
help?> @test_gradients
@test_gradients f args... [kwargs...]


Compare the gradients computed by `Zygote.jl` (Reverse Mode AD) against:

`Tracker.jl` (Reverse Mode AD)

`ReverseDiff.jl` (Reverse Mode AD)

`ForwardDiff.jl` (Forward Mode AD)

`FiniteDifferences.jl` (Finite Differences)

│ Tip
│ This function is completely compatible with `Test.jl`

Arguments
===========

`f`: The function to test.

`args`...: Inputs to f wrt which the gradients are computed.

Keyword Arguments
===================

`gpu_testing`: Disables ForwardDiff, ReverseDiff and FiniteDifferences tests.
(Default: `false`)

`soft_fail`: If `true`, the test will not fail if any of the gradients are incorrect,
instead it will show up as broken. (Default: `false`)

`skip_(tracker|reverse_diff|forward_diff|finite_differences)`: Skip the corresponding
gradient computation and check. (Default: `false`)

`large_arrays_skip_(forward_diff|finite_differences)`: Skip the corresponding
gradient computation and check for large arrays. (Forward Mode and Finite Differences
are not efficient for large arrays.) (Default: `true`)

`large_array_length`: The length of the array above which the gradient computation is
considered large. (Default: `25`)

`max_total_array_size`: Treat as large array if the total size of all arrays is
greater than this value. (Default: `100`)

`(tracker|reverse_diff|forward_diff|finite_differences)_broken`: Mark the
corresponding gradient test as broken. (Default: `false`)

Keyword Arguments for check_approx
====================================

`atol`: Absolute tolerance for gradient comparisons. (Default: `0.0`)

`rtol`: Relative tolerance for gradient comparisons. (Default:
`atol > 0 ? 0.0 : √eps(typeof(atol))`)

`nans`: Whether or not NaNs are considered equal. (Default: `false`)

Example
=========

using LuxTestUtils, Test

x = randn(10)

@testset "Showcase Gradient Testing" begin
@test_gradients sum abs2 x

@test_gradients prod x
end
```

Internally, it uses `check_approx` which extends `Base.isapprox` for more common cases. It
follows the exact same function call as `isapprox`.

## Passing Runtime Variables to Macro

Macros operate on the syntax and hence can't directly take variable inputs. To get around
Expand Down
27 changes: 11 additions & 16 deletions lib/LuxTestUtils/src/LuxTestUtils.jl
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ module LuxTestUtils

using ComponentArrays, Optimisers, Preferences, LuxCore, LuxDeviceUtils, Test
using ForwardDiff, ReverseDiff, Tracker, Zygote, FiniteDifferences
# TODO: Yota, Enzyme

const JET_TARGET_MODULES = @load_preference("target_modules", nothing)

Expand Down Expand Up @@ -32,20 +31,18 @@ or julia version is < 1.7, then the macro will be a no-op.
All additional arguments will be forwarded to `@JET.test_call` and `@JET.test_opt`.
:::tip
!!! tip
Instead of specifying `target_modules` with every call, you can set preferences for
`target_modules` using `Preferences.jl`. For example, to set `target_modules` to
`(Lux, LuxLib)` we can run:
Instead of specifying `target_modules` with every call, you can set preferences for
`target_modules` using `Preferences.jl`. For example, to set `target_modules` to
`(Lux, LuxLib)` we can run:
```julia
using Preferences
set_preferences!(Base.UUID("ac9de150-d08f-4546-94fb-7472b5760531"),
"target_modules" => ["Lux", "LuxLib"])
```
```julia
using Preferences
:::
set_preferences!(Base.UUID("ac9de150-d08f-4546-94fb-7472b5760531"),
"target_modules" => ["Lux", "LuxLib"])
```
## Example
Expand Down Expand Up @@ -163,11 +160,9 @@ Compare the gradients computed by Zygote.jl (Reverse Mode AD) against:
- ForwardDiff.jl (Forward Mode AD)
- FiniteDifferences.jl (Finite Differences)
:::tip
This function is completely compatible with Test.jl
!!! tip
:::
This function is completely compatible with Test.jl
## Arguments
Expand Down

0 comments on commit 6eb61e5

Please sign in to comment.