Merge pull request #196 from FluxML/julia-1-point-7-issue

Use StableRNGs more comprehensively.

test/classifier.jl

@@ -47,11 +47,13 @@ losses = []
 
     # check flux model is an improvement on predicting constant
     # distribution:
+    stable_rng = StableRNGs.StableRNG(123)
     model = MLJFlux.NeuralNetworkClassifier(epochs=50,
                                             builder=builder,
                                             optimiser=optimiser,
                                             acceleration=accel,
-                                            batch_size=10)
+                                            batch_size=10,
+                                            rng=stable_rng)
     @time mach = fit!(machine(model, X, y), rows=train, verbosity=0)
     first_last_training_loss = MLJBase.report(mach)[1][[1, end]]
     push!(losses, first_last_training_loss[2])

test/core.jl

@@ -1,24 +1,25 @@
 Random.seed!(123)
+stable_rng = StableRNGs.StableRNG(123)
 
 rowvec(y) = y
 rowvec(y::Vector) = reshape(y, 1, length(y))
 
 @test MLJFlux.MLJModelInterface.istransparent(Flux.ADAM(0.1))
 
 @testset "nrows" begin
-    Xmatrix = rand(10, 3)
+    Xmatrix = rand(stable_rng, 10, 3)
     X = MLJBase.table(Xmatrix)
     @test MLJFlux.nrows(X) == 10
     @test MLJFlux.nrows(Tables.columntable(X)) == 10
 end
 
 @testset "collate" begin
     # NeuralNetworRegressor:
-    Xmatrix = broadcast(x->round(x, sigdigits=2), rand(10, 3))
+    Xmatrix = broadcast(x->round(x, sigdigits=2), rand(stable_rng, 10, 3))
     # convert to a column table:
     X = MLJBase.table(Xmatrix)
 
-    y = rand(10)
+    y = rand(stable_rng, 10)
     model = MLJFlux.NeuralNetworkRegressor()
     model.batch_size= 3
     @test MLJFlux.collate(model, X, y) ==
@@ -37,7 +38,7 @@ end
                             reshape([1; 0], (2,1))]))
 
     # MultitargetNeuralNetworRegressor:
-    ymatrix = rand(10, 2)
+    ymatrix = rand(stable_rng, 10, 2)
     y = MLJBase.table(ymatrix) # a rowaccess table
     model = MLJFlux.NeuralNetworkRegressor()
     model.batch_size= 3
@@ -53,7 +54,7 @@ end
                   ymatrix'[:,7:9], ymatrix'[:,10:10]]))
 
     # ImageClassifier
-    Xmatrix = coerce(rand(6, 6, 1, 10), GrayImage)
+    Xmatrix = coerce(rand(stable_rng, 6, 6, 1, 10), GrayImage)
     y = categorical(['a', 'b', 'a', 'a', 'b', 'a', 'a', 'a', 'b', 'a'])
     model = MLJFlux.ImageClassifier(batch_size=2)
 
@@ -68,7 +69,7 @@ end
 
 end
 
-Xmatrix = rand(100, 5)
+Xmatrix = rand(stable_rng, 100, 5)
 X = MLJBase.table(Xmatrix)
 y = Xmatrix[:, 1] + Xmatrix[:, 2] + Xmatrix[:, 3] +
     Xmatrix[:, 4] + Xmatrix[:, 5]
@@ -95,7 +96,7 @@ model = MLJFlux.NeuralNetworkRegressor() # any model will do here
 chain_no_drop = deepcopy(chain_yes_drop)
 chain_no_drop.layers[2].p = 1.0
 
-test_input = rand(Float32, 5, 1)
+test_input = rand(stable_rng, Float32, 5, 1)
 
 # check both chains have same behaviour before training:
 @test chain_yes_drop(test_input) == chain_no_drop(test_input)

test/image.jl

@@ -1,6 +1,7 @@
 ## BASIC IMAGE TESTS GREY
 
 Random.seed!(123)
+stable_rng = StableRNGs.StableRNG(123)
 
 mutable struct MyNeuralNetwork <: MLJFlux.Builder
     kernel1
@@ -14,26 +15,28 @@ function MLJFlux.build(model::MyNeuralNetwork, rng, ip, op, n_channels)
         Flux.Conv(model.kernel2, 2=>1, init=init),
         x->reshape(x, :, size(x)[end]),
         Flux.Dense(16, op, init=init))
-end 
+end
 
 builder = MyNeuralNetwork((2,2), (2,2))
 
 # collection of gray images as a 4D array in WHCN format:
-raw_images = rand(Float32, 6, 6, 1, 50);
+raw_images = rand(stable_rng, Float32, 6, 6, 1, 50);
 
 # as a vector of Matrix{<:AbstractRGB}
 images = coerce(raw_images, GrayImage);
 @test scitype(images) == AbstractVector{GrayImage{6,6}}
-labels = categorical(rand(1:5, 50));
+labels = categorical(rand(stable_rng, 1:5, 50));
 
 losses = []
 @testset_accelerated "ImageClassifier basic tests" accel begin
 
     Random.seed!(123)
+    stable_rng = StableRNGs.StableRNG(123)
 
     model = MLJFlux.ImageClassifier(builder=builder,
                                     epochs=10,
-                                    acceleration=accel)
+                                    acceleration=accel,
+                                    rng=stable_rng)
 
     fitresult, cache, _report = MLJBase.fit(model, 0, images, labels)
 
@@ -45,8 +48,12 @@ losses = []
     pred = MLJBase.predict(model, fitresult, images[1:6])
 
     # try with batch_size > 1:
-    model = MLJFlux.ImageClassifier(builder=builder, epochs=10, batch_size=2,
-                                    acceleration=accel)
+    model = MLJFlux.ImageClassifier(builder=builder,
+                                    epochs=10,
+                                    batch_size=2,
+                                    acceleration=accel,
+                                    rng=stable_rng)
+    model.optimiser.eta = 0.005
     @time fitresult, cache, _report = MLJBase.fit(model, 0, images, labels);
     first_last_training_loss = _report[1][[1, end]]
     push!(losses, first_last_training_loss[2])
@@ -61,9 +68,10 @@ losses = []
 
 end
 
-# check different resources (CPU1, CUDALibs, etc)) give about the same loss:
+# check different resources (CPU1, CUDALibs) give about the same loss:
 reference = losses[1]
-@test all(x->abs(x - reference)/reference < 1e-5, losses[2:end])
+@info "Losses for each computational resource: $losses"
+@test all(x->abs(x - reference)/reference < 5e-4, losses[2:end])
 
 
 ## MNIST IMAGES TEST
@@ -100,10 +108,12 @@ losses = []
 @testset_accelerated "Image MNIST" accel begin
 
     Random.seed!(123)
+    stable_rng = StableRNGs.StableRNG(123)
 
     model = MLJFlux.ImageClassifier(builder=MyConvBuilder(),
                                     acceleration=accel,
-                                    batch_size=50)
+                                    batch_size=50,
+                                    rng=stable_rng)
 
     @time fitresult, cache, _report =
         MLJBase.fit(model, 0, images[1:500], labels[1:500]);
@@ -119,6 +129,7 @@ end
 
 # check different resources (CPU1, CUDALibs, etc)) give about the same loss:
 reference = losses[1]
+@info "Losses for each computational resource: $losses"
 @test all(x->abs(x - reference)/reference < 0.05, losses[2:end])
 
 
@@ -127,7 +138,7 @@ reference = losses[1]
 builder = MyNeuralNetwork((2,2), (2,2))
 
 # collection of color images as a 4D array in WHCN format:
-raw_images = rand(Float32, 6, 6, 3, 50);
+raw_images = rand(stable_rng, Float32, 6, 6, 3, 50);
 
 images = coerce(raw_images, ColorImage);
 @test scitype(images) == AbstractVector{ColorImage{6,6}}
@@ -138,10 +149,12 @@ losses = []
 @testset_accelerated "ColorImages" accel begin
 
     Random.seed!(123)
+    stable_rng = StableRNGs.StableRNG(123)
 
     model = MLJFlux.ImageClassifier(builder=builder,
                                     epochs=10,
-                                    acceleration=accel)
+                                    acceleration=accel,
+                                    rng=stable_rng)
 
     # tests update logic, etc (see test_utililites.jl):
     @test basictest(MLJFlux.ImageClassifier, images, labels,
@@ -157,7 +170,8 @@ losses = []
     model = MLJFlux.ImageClassifier(builder=builder,
                                     epochs=10,
                                     batch_size=2,
-                                    acceleration=accel)
+                                    acceleration=accel,
+                                    rng=stable_rng)
     fitresult, cache, _report = MLJBase.fit(model, 0, images, labels);
 
     @test optimisertest(MLJFlux.ImageClassifier, images, labels,
@@ -167,6 +181,7 @@ end
 
 # check different resources (CPU1, CUDALibs, etc)) give about the same loss:
 reference = losses[1]
+@info "Losses for each computational resource: $losses"
 @test all(x->abs(x - reference)/reference < 1e-5, losses[2:end])
 
 true

test/regressor.jl

@@ -27,8 +27,10 @@ train, test = MLJBase.partition(1:N, 0.7)
               accel)
 
     # test model is a bit better than constant predictor:
+    stable_rng = StableRNGs.StableRNG(123)
     model = MLJFlux.NeuralNetworkRegressor(builder=builder,
-                                           acceleration=accel)
+                                           acceleration=accel,
+                                           rng=stable_rng)
     @time fitresult, _, rpt =
         fit(model, 0, MLJBase.selectrows(X, train), y[train])
     first_last_training_loss = rpt[1][[1, end]]

test/test_utils.jl

@@ -57,9 +57,12 @@ function basictest(ModelType, X, y, builder, optimiser, threshold, accel)
 
     eval(quote
 
+         stable_rng = StableRNGs.StableRNG(123)
+
          model = $ModelType_ex(builder=$builder,
                                optimiser=$optimiser,
-                               acceleration=$accel_ex)
+                               acceleration=$accel_ex,
+                               rng=stable_rng)
 
          fitresult, cache, _report = MLJBase.fit(model, 0, $X, $y);
 
@@ -89,7 +92,8 @@ function basictest(ModelType, X, y, builder, optimiser, threshold, accel)
          model = $ModelType_ex(builder=$builder,
                                optimiser=$optimiser,
                                epochs=2,
-                               acceleration=$accel_ex)
+                               acceleration=$accel_ex,
+                               rng=stable_rng)
 
          fitresult, cache, _report = MLJBase.fit(model, 0, $X, $y);
 
@@ -127,6 +131,7 @@ function optimisertest(ModelType, X, y, builder, optimiser, accel)
     optimiser = deepcopy(optimiser)
 
     eval(quote
+
          model = $ModelType_ex(builder=$builder,
                                optimiser=$optimiser,
                                acceleration=$accel_ex,