refactor: move all subpackages into a mono-repo

.buildkite/benchmarks.yml

@@ -15,7 +15,11 @@ steps:
         command: |
           julia --project=benchmarks -e 'println("--- :julia: Instantiating project")
               using Pkg
-              Pkg.develop([PackageSpec(path=pwd())])'
+              Pkg.develop([
+                PackageSpec(path=pwd()),
+                PackageSpec(path="lib/LuxLib"),
+                PackageSpec(path="lib/MLDataDevices"),
+              ])'
 
           julia --project=benchmarks -e 'println("--- :julia: Run Benchmarks")
               include("benchmarks/runbenchmarks.jl")'
@@ -36,12 +40,16 @@ steps:
               version: "1"
         command: |
           julia --project=benchmarks -e 'println("--- :julia: Instantiating project")
-              using Pkg
-              Pkg.develop([PackageSpec(path=pwd())])'
+              using Pkg;
+              Pkg.develop([
+                PackageSpec(path=pwd()),
+                PackageSpec(path="lib/LuxLib"),
+                PackageSpec(path="lib/MLDataDevices"),
+              ])'
 
           julia --project=benchmarks -e 'println("--- :julia: Add CUDA to benchmarks environment")
               using Pkg
-              Pkg.add("LuxCUDA")'
+              Pkg.develop([PackageSpec(path="lib/LuxCUDA")])'
 
           julia --project=benchmarks -e 'println("--- :julia: Run Benchmarks")
               include("benchmarks/runbenchmarks.jl")'

.buildkite/documentation.yml

@@ -69,7 +69,11 @@ steps:
           julia --code-coverage=user --color=yes --project=docs -e '
             println("--- :julia: Instantiating project")
             using Pkg
-            Pkg.develop(PackageSpec(path=pwd()))
+            dev_pkgs = Pkg.PackageSpec[]
+            for pkg in ("lib/LuxLib", "lib/LuxCore", "lib/MLDataDevices", "lib/LuxTestUtils", "lib/WeightInitializers", ".")
+              push!(dev_pkgs, Pkg.PackageSpec(path=pkg));
+            end
+            Pkg.develop(dev_pkgs)
             Pkg.instantiate()
             println("+++ :julia: Building documentation")
             include("docs/make.jl")'

.buildkite/downstream.yml

@@ -0,0 +1,68 @@
+steps:
+  - group: ":telescope: Downstream CUDA"
+    steps:
+      - label: ":julia: {{matrix.repo}} (Julia 1 + CUDA GPU)"
+        plugins:
+          - JuliaCI/julia#v1:
+              version: "1.10"
+          - JuliaCI/julia-coverage#v1:
+              codecov: true
+              dirs:
+                - src
+                - ext
+                - lib/LuxCore/src
+                - lib/LuxCore/ext
+                - lib/LuxLib/src
+                - lib/LuxLib/ext
+                - lib/MLDataDevices/src
+                - lib/MLDataDevices/ext
+                - lib/WeightInitializers/src
+                - lib/WeightInitializers/ext
+        command: julia --code-coverage=user --color=yes --project .buildkite/scripts/downstream.jl "{{matrix.repo}}" "CUDA"
+        agents:
+          queue: "juliagpu"
+          cuda: "*"
+        if: build.message !~ /\[skip tests\]/ && build.message !~ /\[skip downstream\]/ && build.message !~ /\[skip ci\]/ && build.pull_request.labels includes "run downstream test"
+        timeout_in_minutes: 60
+        matrix:
+          setup:
+            repo:
+              - "Boltz"
+              - "NeuralPDE"
+              - "DeepEquilibriumNetworks"
+              - "NeuralOperators"
+
+  - group: ":telescope: Downstream AMD GPU"
+    steps:
+      - label: ":julia: {{matrix.repo}} (Julia 1 + AMD GPU)"
+        plugins:
+          - JuliaCI/julia#v1:
+              version: "1.10"
+          - JuliaCI/julia-coverage#v1:
+              codecov: true
+              dirs:
+                - src
+                - ext
+                - lib/LuxCore/src
+                - lib/LuxCore/ext
+                - lib/LuxLib/src
+                - lib/LuxLib/ext
+                - lib/MLDataDevices/src
+                - lib/MLDataDevices/ext
+                - lib/WeightInitializers/src
+                - lib/WeightInitializers/ext
+        command: julia --code-coverage=user --color=yes --project .buildkite/scripts/downstream.jl "{{matrix.repo}}" "AMDGPU"
+        agents:
+          queue: "juliagpu"
+          rocm: "*"
+          rocmgpu: "*"
+        if: build.message !~ /\[skip tests\]/ && build.message !~ /\[skip downstream\]/ && build.message !~ /\[skip ci\]/ && build.pull_request.labels includes "run downstream test"
+        timeout_in_minutes: 60
+        matrix:
+          setup:
+            repo:
+              - "Boltz"
+              - "NeuralOperators"
+
+env:
+  SECRET_CODECOV_TOKEN: "jQ0BMTQgyZx7QGyU0Q2Ec7qB9mtE2q/tDu0FsfxvEG7/zOAGvXkyXrzIFFOQxvDoFcP+K2+hYZKMxicYdNqzr5wcxu505aNGN2GM3wyegAr+hO6q12bCFYx6qXzU9FLCCdeqINqn9gUSSOlGtWNFrbAlrTyz/D4Yo66TqBDzvaLL63FMnhCLaXW/zJt3hNuEAJaPY2O6Ze1rX2WZ3Y+i+s3uQ8aLImtoCJhPe8CRx+OhuYiTzGhynFfGntZ0738/1RN4gNM0S/hTC4gLE7XMVBanJpGh32rFaiDwW4zAyXKBrDkL3QA3MS1RvLTJxGJ085S16hCk0C4ddAhZCvIM9Q==;U2FsdGVkX1+bXdFeKMs5G79catOCyby2n07A2fg0FjVAvrjQLZ0yfvDS4paJiFikLkodho0khz2YALKb2Y0K6w=="

.buildkite/pipeline.yml

@@ -8,6 +8,7 @@ steps:
           diff: ".buildkite/scripts/diff.sh $BUILDKITE_COMMIT"
           interpolation: false
           watch:
+            # Core Lux Testing
             - path:
                 - "src/"
                 - "ext/"
@@ -18,6 +19,70 @@ steps:
                 command: "buildkite-agent pipeline upload .buildkite/testing.yml"
                 agents:
                   queue: "juliagpu"
+
+            # LuxCUDA Testing
+            - path:
+                - "lib/LuxCUDA/"
+                - ".buildkite/"
+              config:
+                command: "buildkite-agent pipeline upload .buildkite/testing_luxcuda.yml"
+                agents:
+                  queue: "juliagpu"
+
+            # WeightInitializers Testing
+            - path:
+                - "lib/WeightInitializers/"
+                - ".buildkite/"
+              config:
+                command: "buildkite-agent pipeline upload .buildkite/testing_weightinitializers.yml"
+                agents:
+                  queue: "juliagpu"
+
+            # MLDataDevices Testing
+            - path:
+                - "lib/MLDataDevices/"
+                - ".buildkite/"
+              config:
+                command: "buildkite-agent pipeline upload .buildkite/testing_mldatadevices.yml"
+                agents:
+                  queue: "juliagpu"
+
+            # LuxLib Testing
+            - path:
+                - "lib/LuxLib/"
+                - ".buildkite/"
+                - "lib/LuxTestUtils/"
+                - "lib/LuxCore/"
+                - "lib/MLDataDevices/"
+              config:
+                command: "buildkite-agent pipeline upload .buildkite/testing_luxlib.yml"
+                agents:
+                  queue: "juliagpu"
+
+            # LuxTestUtils Testing
+            - path:
+                - "lib/LuxTestUtils/"
+                - ".buildkite/"
+              config:
+                command: "buildkite-agent pipeline upload .buildkite/testing_luxtestutils.yml"
+                agents:
+                  queue: "juliagpu"
+
+            # Benchmarks
+            - path:
+                - "src/"
+                - "ext/"
+                - "test/"
+                - "Project.toml"
+                - ".buildkite/"
+                - "benchmarks/"
+              if: build.pull_request.labels includes "run benchmarks"
+              config:
+                command: "buildkite-agent pipeline upload .buildkite/benchmarks.yml"
+                agents:
+                  queue: "juliagpu"
+
+            # Documentation
             - path:
                 - "src/"
                 - "ext/"
@@ -26,20 +91,22 @@ steps:
                 - "docs/"
                 - "examples/"
                 - ".buildkite/"
+                - "lib"
               config:
                 command: "buildkite-agent pipeline upload .buildkite/documentation.yml"
                 agents:
                   queue: "juliagpu"
+
+            # Downstream
             - path:
                 - "src/"
                 - "ext/"
-                - "test/"
+                - "lib/"
                 - "Project.toml"
                 - ".buildkite/"
-                - "benchmarks/"
-              if: build.pull_request.labels includes "run benchmarks"
+              if: build.pull_request.labels includes "run downstream test"
               config:
-                command: "buildkite-agent pipeline upload .buildkite/benchmarks.yml"
+                command: "buildkite-agent pipeline upload .buildkite/downstream.yml"
                 agents:
                   queue: "juliagpu"
 
@@ -48,6 +115,18 @@ steps:
     agents:
       queue: "juliagpu"
     command: |
+      # Core Lux Testing
       buildkite-agent pipeline upload .buildkite/testing.yml
+
+      # Subpackage testing
+      buildkite-agent pipeline upload .buildkite/testing_luxcuda.yml
+      buildkite-agent pipeline upload .buildkite/testing_weightinitializers.yml
+      buildkite-agent pipeline upload .buildkite/testing_luxlib.yml
+      buildkite-agent pipeline upload .buildkite/testing_mldatadevices.yml
+      buildkite-agent pipeline upload .buildkite/testing_luxtestutils.yml
+
+      # Documentation
       buildkite-agent pipeline upload .buildkite/documentation.yml
+
+      # Benchmarks
       buildkite-agent pipeline upload .buildkite/benchmarks.yml

.buildkite/testing.yml

@@ -1,20 +1,48 @@
 steps:
-  - group: ":julia: CUDA GPU"
+  - group: ":julia: (Lux) CUDA GPU"
     steps:
       - label: ":julia: Julia {{matrix.julia}} + CUDA GPU"
         plugins:
           - JuliaCI/julia#v1:
               version: "{{matrix.julia}}"
-          - JuliaCI/julia-test#v1:
-              test_args: "--quickfail"
           - JuliaCI/julia-coverage#v1:
               codecov: true
               dirs:
                 - src
                 - ext
+                - lib/LuxCore/src
+                - lib/LuxCore/ext
+                - lib/MLDataDevices/src
+                - lib/MLDataDevices/ext
+                - lib/WeightInitializers/src
+                - lib/WeightInitializers/ext
+                - lib/LuxLib/src
+                - lib/LuxLib/ext
+                - lib/LuxTestUtils/src
         agents:
           queue: "juliagpu"
           cuda: "*"
+        command: |
+          julia --color=yes --code-coverage=user --depwarn=yes --project=. -e '
+            import Pkg;
+            dev_pkgs = Pkg.PackageSpec[];
+            for pkg in ("lib/LuxCore", "lib/MLDataDevices", "lib/WeightInitializers", "lib/LuxLib",)
+              push!(dev_pkgs, Pkg.PackageSpec(path=pkg));
+            end
+            Pkg.develop(dev_pkgs);
+            Pkg.Registry.update();
+            Pkg.instantiate();
+            Pkg.activate("test");
+            dev_pkgs = Pkg.PackageSpec[];
+            for pkg in ("lib/LuxTestUtils", "lib/LuxLib", "lib/MLDataDevices", "lib/LuxCore", ".")
+              push!(dev_pkgs, Pkg.PackageSpec(path=pkg))
+            end
+            Pkg.develop(dev_pkgs);
+            Pkg.instantiate();'
+            julia --color=yes --code-coverage=user --depwarn=yes --project=test -e '
+              import Pkg, Lux;
+              dir = dirname(pathof(Lux));
+              include(joinpath(dir, "../test/runtests.jl"))'
         env:
           BACKEND_GROUP: "CUDA"
         if: build.message !~ /\[skip tests\]/ && build.message !~ /\[skip ci\]/
@@ -23,44 +51,49 @@ steps:
           setup:
             julia:
               - "1.10"
+              - "1"
 
-  - group: ":telescope: Downstream CUDA"
-    steps:
-      - label: ":julia: {{matrix.repo}} (Julia 1 + CUDA GPU)"
-        plugins:
-          - JuliaCI/julia#v1:
-              version: "1.10"
-          - JuliaCI/julia-coverage#v1:
-              codecov: true
-              dirs:
-                - src
-                - ext
-        command: julia --code-coverage=user --color=yes --project .buildkite/scripts/downstream.jl "{{matrix.repo}}" "CUDA"
-        agents:
-          queue: "juliagpu"
-          cuda: "*"
-        if: build.message !~ /\[skip tests\]/ && build.message !~ /\[skip downstream\]/ && build.message !~ /\[skip ci\]/ && build.pull_request.labels includes "run downstream test"
-        timeout_in_minutes: 60
-        matrix:
-          setup:
-            repo:
-              - "Boltz"
-              - "NeuralPDE#GPU"
-              - "DeepEquilibriumNetworks"
-
-  - group: ":julia: AMD GPU"
+  - group: ":julia: (Lux) AMD GPU"
     steps:
       - label: ":julia: Julia: {{matrix.julia}} + AMD GPU"
         plugins:
           - JuliaCI/julia#v1:
               version: "{{matrix.julia}}"
-          - JuliaCI/julia-test#v1:
-              test_args: "--quickfail"
           - JuliaCI/julia-coverage#v1:
               codecov: true
               dirs:
                 - src
                 - ext
+                - lib/LuxCore/src
+                - lib/LuxCore/ext
+                - lib/MLDataDevices/src
+                - lib/MLDataDevices/ext
+                - lib/WeightInitializers/src
+                - lib/WeightInitializers/ext
+                - lib/LuxLib/src
+                - lib/LuxLib/ext
+                - lib/LuxTestUtils/src
+        command: |
+          julia --color=yes --code-coverage=user --depwarn=yes --project=. -e '
+            import Pkg;
+            dev_pkgs = Pkg.PackageSpec[];
+            for pkg in ("lib/LuxCore", "lib/MLDataDevices", "lib/WeightInitializers", "lib/LuxLib",)
+              push!(dev_pkgs, Pkg.PackageSpec(path=pkg));
+            end
+            Pkg.develop(dev_pkgs);
+            Pkg.Registry.update();
+            Pkg.instantiate();
+            Pkg.activate("test");
+            dev_pkgs = Pkg.PackageSpec[];
+            for pkg in ("lib/LuxTestUtils", "lib/LuxLib", "lib/MLDataDevices", "lib/LuxCore", ".")
+              push!(dev_pkgs, Pkg.PackageSpec(path=pkg))
+            end
+            Pkg.develop(dev_pkgs);
+            Pkg.instantiate();'
+            julia --color=yes --code-coverage=user --depwarn=yes --project=test -e '
+              import Pkg, Lux;
+              dir = dirname(pathof(Lux));
+              include(joinpath(dir, "../test/runtests.jl"))'
         env:
           BACKEND_GROUP: "AMDGPU"
         agents:
@@ -73,29 +106,7 @@ steps:
           setup:
             julia:
               - "1.10"
-
-  - group: ":telescope: Downstream AMD GPU"
-    steps:
-      - label: ":julia: {{matrix.repo}} (Julia 1 + AMD GPU)"
-        plugins:
-          - JuliaCI/julia#v1:
-              version: "1.10"
-          - JuliaCI/julia-coverage#v1:
-              codecov: true
-              dirs:
-                - src
-                - ext
-        command: julia --code-coverage=user --color=yes --project .buildkite/scripts/downstream.jl "{{matrix.repo}}" "AMDGPU"
-        agents:
-          queue: "juliagpu"
-          rocm: "*"
-          rocmgpu: "*"
-        if: build.message !~ /\[skip tests\]/ && build.message !~ /\[skip downstream\]/ && build.message !~ /\[skip ci\]/ && build.pull_request.labels includes "run downstream test"
-        timeout_in_minutes: 60
-        matrix:
-          setup:
-            repo:
-              - "Boltz"
+              - "1"
 
 env:
   SECRET_CODECOV_TOKEN: "jQ0BMTQgyZx7QGyU0Q2Ec7qB9mtE2q/tDu0FsfxvEG7/zOAGvXkyXrzIFFOQxvDoFcP+K2+hYZKMxicYdNqzr5wcxu505aNGN2GM3wyegAr+hO6q12bCFYx6qXzU9FLCCdeqINqn9gUSSOlGtWNFrbAlrTyz/D4Yo66TqBDzvaLL63FMnhCLaXW/zJt3hNuEAJaPY2O6Ze1rX2WZ3Y+i+s3uQ8aLImtoCJhPe8CRx+OhuYiTzGhynFfGntZ0738/1RN4gNM0S/hTC4gLE7XMVBanJpGh32rFaiDwW4zAyXKBrDkL3QA3MS1RvLTJxGJ085S16hCk0C4ddAhZCvIM9Q==;U2FsdGVkX1+bXdFeKMs5G79catOCyby2n07A2fg0FjVAvrjQLZ0yfvDS4paJiFikLkodho0khz2YALKb2Y0K6w=="