diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 70d1fd4828d4e9..2fa91dd6bb3318 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -1554,12 +1554,16 @@ function (==)(A::AbstractArray, B::AbstractArray) if isa(A,AbstractRange) != isa(B,AbstractRange) return false end + anymissing = false for (a, b) in zip(A, B) - if !(a == b) + eq = (a == b) + if ismissing(eq) + anymissing = true + elseif !eq return false end end - return true + return anymissing ? missing : true end # sub2ind and ind2sub diff --git a/base/essentials.jl b/base/essentials.jl index cc4e01f647709c..0a9a57373cc374 100644 --- a/base/essentials.jl +++ b/base/essentials.jl @@ -751,3 +751,26 @@ This function simply returns its argument by default, since the elements of a general iterator are normally considered its "values". """ values(itr) = itr + +""" + Missing + +A type with no fields whose singleton instance [`missing`](@ref) is used +to represent missing values. +""" +struct Missing end + +""" + missing + +The singleton instance of type [`Missing`](@ref) representing a missing value. +""" +const missing = Missing() + +""" + ismissing(x) + +Indicate whether `x` is [`missing`](@ref). +""" +ismissing(::Any) = false +ismissing(::Missing) = true \ No newline at end of file diff --git a/base/exports.jl b/base/exports.jl index 84e7e3ed29790d..1fa5198a8ab471 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -76,6 +76,7 @@ export Irrational, Matrix, MergeSort, + Missing, NTuple, Nullable, ObjectIdDict, @@ -150,6 +151,7 @@ export EOFError, InvalidStateException, KeyError, + MissingException, NullException, ParseError, SystemError, @@ -882,6 +884,10 @@ export isready, fetch, +# missing values + ismissing, + missing, + # time sleep, time, diff --git a/base/missing.jl b/base/missing.jl new file mode 100644 index 00000000000000..744f2e5f149b08 --- /dev/null +++ b/base/missing.jl @@ -0,0 +1,115 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +# Missing, missing and ismissing are defined in essentials.jl + +show(io::IO, x::Missing) = print(io, "missing") + +""" + MissingException(msg) + +Exception thrown when a [`missing`](@ref) value is encountered in a situation +where it is not supported. The error message, in the `msg` field +may provide more specific details. +""" +struct MissingException <: Exception + msg::AbstractString +end + +showerror(io::IO, ex::MissingException) = + print(io, "MissingException: ", ex.msg) + +promote_rule(::Type{Missing}, ::Type{T}) where {T} = Union{T, Missing} +promote_rule(::Type{Union{S,Missing}}, ::Type{T}) where {T,S} = Union{promote_type(T, S), Missing} +promote_rule(::Type{Any}, ::Type{T}) where {T} = Any +promote_rule(::Type{Any}, ::Type{Missing}) = Any +promote_rule(::Type{Missing}, ::Type{Any}) = Any +promote_rule(::Type{Missing}, ::Type{Missing}) = Missing + +convert(::Type{Union{T, Missing}}, x) where {T} = convert(T, x) +# To print more appropriate message than "T not defined" +convert(::Type{Missing}, x) = throw(MethodError(convert, (Missing, x))) +convert(::Type{Missing}, ::Missing) = missing + +# Comparison operators +==(::Missing, ::Missing) = missing +==(::Missing, ::Any) = missing +==(::Any, ::Missing) = missing +# To fix ambiguity +==(::Missing, ::WeakRef) = missing +==(::WeakRef, ::Missing) = missing +isequal(::Missing, ::Missing) = true +isequal(::Missing, ::Any) = false +isequal(::Any, ::Missing) = false +<(::Missing, ::Missing) = missing +<(::Missing, ::Any) = missing +<(::Any, ::Missing) = missing +isless(::Missing, ::Missing) = false +isless(::Missing, ::Any) = false +isless(::Any, ::Missing) = true + +# Unary operators/functions +for f in (:(!), :(+), :(-), :(identity), :(zero), + :(abs), :(abs2), :(sign), + :(acos), :(acosh), :(asin), :(asinh), :(atan), :(atanh), + :(sin), :(sinh), :(cos), :(cosh), :(tan), :(tanh), + :(exp), :(exp2), :(expm1), :(log), :(log10), :(log1p), + :(log2), :(exponent), :(sqrt), :(gamma), :(lgamma), + :(iseven), :(ispow2), :(isfinite), :(isinf), :(isodd), + :(isinteger), :(isreal), :(isnan), :(isempty), + :(iszero), :(transpose), :(float)) + @eval Math.$(f)(::Missing) = missing +end + +zero(::Type{Union{T, Missing}}) where {T} = zero(T) +# To prevent StackOverflowError +zero(::Type{Any}) = throw(MethodError(zero, (Any,))) + +# Binary operators/functions +for f in (:(+), :(-), :(*), :(/), :(^), + :(div), :(mod), :(fld), :(rem), :(min), :(max)) + @eval begin + # Scalar with missing + ($f)(::Missing, ::Missing) = missing + ($f)(d::Missing, x::Number) = missing + ($f)(d::Number, x::Missing) = missing + end +end + +# Rounding and related functions +for f in (:(ceil), :(floor), :(round), :(trunc)) + @eval begin + ($f)(::Missing, digits::Integer=0, base::Integer=0) = missing + ($f)(::Type{>:Missing}, ::Missing) = missing + ($f)(::Type{T}, ::Missing) where {T} = + throw(MissingException("cannot convert a missing value to type $T")) + end +end + +# to avoid ambiguity warnings +(^)(::Missing, ::Integer) = missing + +# Bit operators +(&)(::Missing, ::Missing) = missing +(&)(a::Missing, b::Bool) = ifelse(b, missing, false) +(&)(b::Bool, a::Missing) = ifelse(b, missing, false) +(&)(::Missing, ::Integer) = missing +(&)(::Integer, ::Missing) = missing +(|)(::Missing, ::Missing) = missing +(|)(a::Missing, b::Bool) = ifelse(b, true, missing) +(|)(b::Bool, a::Missing) = ifelse(b, true, missing) +(|)(::Missing, ::Integer) = missing +(|)(::Integer, ::Missing) = missing +xor(::Missing, ::Missing) = missing +xor(a::Missing, b::Bool) = missing +xor(b::Bool, a::Missing) = missing +xor(::Missing, ::Integer) = missing +xor(::Integer, ::Missing) = missing + +*(d::Missing, x::AbstractString) = missing +*(d::AbstractString, x::Missing) = missing + +function float(A::AbstractArray{Union{T, Missing}}) where {T} + U = typeof(float(zero(T))) + convert(AbstractArray{Union{U, Missing}}, A) +end +float(A::AbstractArray{Missing}) = A \ No newline at end of file diff --git a/base/operators.jl b/base/operators.jl index 60860e71b3d08c..d40cf227410e18 100644 --- a/base/operators.jl +++ b/base/operators.jl @@ -155,7 +155,7 @@ julia> "foo" ≠ "foo" false ``` """ -!=(x, y) = !(x == y)::Bool +!=(x, y) = !(x == y) const ≠ = != """ diff --git a/base/reduce.jl b/base/reduce.jl index 0cc67be13e55a0..4e52ccfcaf1893 100644 --- a/base/reduce.jl +++ b/base/reduce.jl @@ -610,10 +610,16 @@ true ``` """ function any(f, itr) + anymissing = false for x in itr - f(x) && return true + v = f(x) + if ismissing(v) + anymissing = true + elseif v + return true + end end - return false + return anymissing ? missing : false end """ @@ -635,12 +641,19 @@ false ``` """ function all(f, itr) + anymissing = false for x in itr - f(x) || return false + v = f(x) + if ismissing(v) + anymissing = true + elseif !v + return false + end end - return true + return anymissing ? missing : true end + ## in & contains """ diff --git a/base/sysimg.jl b/base/sysimg.jl index 0a3963d4872a26..233b2ea7d790b1 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -394,6 +394,9 @@ const × = cross # statistics include("statistics.jl") +# missing values +include("missing.jl") + # libgit2 support include("libgit2/libgit2.jl") diff --git a/test/ambiguous.jl b/test/ambiguous.jl index 6f7f8fc13f7fd1..fa3c7f7112c3a6 100644 --- a/test/ambiguous.jl +++ b/test/ambiguous.jl @@ -280,6 +280,10 @@ end pop!(need_to_handle_undef_sparam, which(Base.SparseArrays._absspvec_vcat, (AbstractSparseArray{Tv, Ti, 1} where {Tv, Ti},))) pop!(need_to_handle_undef_sparam, which(Base.SparseArrays._absspvec_hcat, (AbstractSparseArray{Tv, Ti, 1} where {Tv, Ti},))) pop!(need_to_handle_undef_sparam, which(Base.cat, (Any, Base.SparseArrays._TypedDenseConcatGroup{T} where T))) + pop!(need_to_handle_undef_sparam, which(Base.float, Tuple{AbstractArray{Union{Missing, T},N} where {T, N}})) + pop!(need_to_handle_undef_sparam, which(Base.convert, Tuple{Type{Union{Missing, T}} where T, Any})) + pop!(need_to_handle_undef_sparam, which(Base.promote_rule, Tuple{Type{Union{Missing, S}} where S, Type{T} where T})) + pop!(need_to_handle_undef_sparam, which(Base.zero, Tuple{Type{Union{Missing, T}} where T})) @test need_to_handle_undef_sparam == Set() end end diff --git a/test/choosetests.jl b/test/choosetests.jl index cfb0a3d08a6f0a..ddd956e982e2ae 100644 --- a/test/choosetests.jl +++ b/test/choosetests.jl @@ -49,7 +49,7 @@ function choosetests(choices = []) "checked", "bitset", "floatfuncs", "compile", "distributed", "inline", "boundscheck", "error", "ambiguous", "cartesian", "asmvariant", "osutils", "channels", "iostream", "specificity", "codegen", "codevalidation", - "reinterpretarray", "syntax" + "reinterpretarray", "syntax", "missing" ] if isdir(joinpath(JULIA_HOME, Base.DOCDIR, "examples")) diff --git a/test/missing.jl b/test/missing.jl new file mode 100644 index 00000000000000..c4f3695d1b658c --- /dev/null +++ b/test/missing.jl @@ -0,0 +1,225 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +@testset "MissingException" begin + @test sprint(showerror, MissingException("test")) == "MissingException: test" +end + +@testset "convert" begin + @test convert(Union{Int, Missing}, 1) === 1 + @test convert(Union{Int, Missing}, 1.0) === 1 + @test_throws MethodError convert(Missing, 1) + @test_throws MethodError convert(Union{Int, Missing}, "a") +end + +@testset "promote rules" begin + @test promote_type(Missing, Missing) == Missing + @test promote_type(Missing, Int) == Union{Missing, Int} + @test promote_type(Int, Missing) == Union{Missing, Int} + @test promote_type(Int, Any) == Any + @test promote_type(Any, Any) == Any + @test promote_type(Missing, Any) == Any + @test promote_type(Any, Missing) == Any + @test promote_type(Union{Int, Missing}, Missing) == Union{Int, Missing} + @test promote_type(Missing, Union{Int, Missing}) == Union{Int, Missing} + @test promote_type(Union{Int, Missing}, Int) == Union{Int, Missing} + @test promote_type(Int, Union{Int, Missing}) == Union{Int, Missing} + @test promote_type(Any, Union{Int, Missing}) == Any + @test promote_type(Union{Int, Missing}, Union{Int, Missing}) == Union{Int, Missing} + @test promote_type(Union{Float64, Missing}, Union{String, Missing}) == Any + @test promote_type(Union{Float64, Missing}, Union{Int, Missing}) == Union{Float64, Missing} + @test_broken promote_type(Union{Void, Missing, Int}, Float64) == Any +end + +@testset "comparison operators" begin + @test (missing == missing) === missing + @test (1 == missing) === missing + @test (missing == 1) === missing + @test (missing != missing) === missing + @test (1 != missing) === missing + @test (missing != 1) === missing + @test isequal(missing, missing) + @test !isequal(1, missing) + @test !isequal(missing, 1) + @test (missing < missing) === missing + @test (missing < 1) === missing + @test (1 < missing) === missing + @test (missing <= missing) === missing + @test (missing <= 1) === missing + @test (1 <= missing) === missing + @test !isless(missing, missing) + @test !isless(missing, 1) + @test isless(1, missing) +end + +@testset "arithmetic operators" begin + arithmetic_operators = [+, -, *, /, ^, Base.div, Base.mod, Base.fld, Base.rem] + + # All unary operators return missing when evaluating missing + for f in [!, +, -] + @test ismissing(f(missing)) + end + + # All arithmetic operators return missing when operating on two missing's + # All arithmetic operators return missing when operating on a scalar and an missing + # All arithmetic operators return missing when operating on an missing and a scalar + for f in arithmetic_operators + @test ismissing(f(missing, missing)) + @test ismissing(f(1, missing)) + @test ismissing(f(missing, 1)) + end +end + +@testset "bit operators" begin + bit_operators = [&, |, ⊻] + + # All bit operators return missing when operating on two missing's + for f in bit_operators + @test ismissing(f(missing, missing)) + end +end + +@testset "boolean operators" begin + @test ismissing(missing & true) + @test ismissing(true & missing) + @test !(missing & false) + @test !(false & missing) + @test ismissing(missing | false) + @test ismissing(false | missing) + @test missing | true + @test true | missing + @test ismissing(xor(missing, true)) + @test ismissing(xor(true, missing)) + @test ismissing(xor(missing, false)) + @test ismissing(xor(false, missing)) + + @test ismissing(missing & 1) + @test ismissing(1 & missing) + @test ismissing(missing | 1) + @test ismissing(1 | missing) + @test ismissing(xor(missing, 1)) + @test ismissing(xor(1, missing)) +end + +@testset "* string concatenation" begin + @test ismissing("a" * missing) + @test ismissing(missing * "a") +end + +@testset "elementary functions" begin + elementary_functions = [abs, abs2, sign, + acos, acosh, asin, asinh, atan, atanh, sin, sinh, + conj, cos, cosh, tan, tanh, + exp, exp2, expm1, log, log10, log1p, log2, + exponent, sqrt, gamma, lgamma, + identity, zero, + iseven, isodd, ispow2, + isfinite, isinf, isnan, iszero, + isinteger, isreal, isimag, + isempty, transpose, ctranspose, float] + + # All elementary functions return missing when evaluating missing + for f in elementary_functions + @test ismissing(f(missing)) + end + + @test zero(Union{Int, Missing}) === 0 + @test zero(Union{Float64, Missing}) === 0.0 + @test_throws MethodError zero(Any) + @test_throws MethodError zero(String) + @test_throws MethodError zero(Union{String, Missing}) +end + +@testset "rounding functions" begin + rounding_functions = [ceil, floor, round, trunc] + + # All rounding functions return missing when evaluating missing as first argument + for f in rounding_functions + @test ismissing(f(missing)) + @test ismissing(f(missing, 1)) + @test ismissing(f(missing, 1, 1)) + @test ismissing(f(Union{Int, Missing}, missing)) + @test_throws MissingException f(Int, missing) + end +end + +@testset "printing" begin + @test sprint(show, missing) == "missing" + @test sprint(showcompact, missing) == "missing" + @test sprint(show, [missing]) == "$Missing[missing]" + @test sprint(show, [1 missing]) == "$(Union{Int, Missing})[1 missing]" + b = IOBuffer() + display(TextDisplay(b), [missing]) + @test String(take!(b)) == "1-element Array{$Missing,1}:\n missing" + b = IOBuffer() + display(TextDisplay(b), [1 missing]) + @test String(take!(b)) == "1×2 Array{$(Union{Int, Missing}),2}:\n 1 missing" +end + +@testset "arrays with missing values" begin + x = convert(Vector{Union{Int, Missing}}, [1.0, missing]) + @test isa(x, Vector{Union{Int, Missing}}) + @test isequal(x, [1, missing]) + x = convert(Vector{Union{Int, Missing}}, [1.0]) + @test isa(x, Vector{Union{Int, Missing}}) + @test x == [1] + x = convert(Vector{Union{Int, Missing}}, [missing]) + @test isa(x, Vector{Union{Int, Missing}}) + @test isequal(x, [missing]) +end + +@testset "== and != on arrays" begin + @test ismissing([1, missing] == [1, missing]) + @test ismissing(["a", missing] == ["a", missing]) + @test ismissing(Any[1, missing] == Any[1, missing]) + @test ismissing(Any[missing] == Any[missing]) + @test ismissing([missing] == [missing]) + @test ismissing(Any[missing, 2] == Any[1, missing]) + @test ismissing([missing, false] == BitArray([true, false])) + @test ismissing(Any[missing, false] == BitArray([true, false])) + @test Union{Int, Missing}[1] == Union{Float64, Missing}[1.0] + @test Union{Int, Missing}[1] == [1.0] + @test Union{Bool, Missing}[true] == BitArray([true]) + @test !(Union{Int, Missing}[1] == [2]) + @test !([1] == Union{Int, Missing}[2]) + @test !(Union{Int, Missing}[1] == Union{Int, Missing}[2]) + + @test ismissing([1, missing] != [1, missing]) + @test ismissing(["a", missing] != ["a", missing]) + @test ismissing(Any[1, missing] != Any[1, missing]) + @test ismissing(Any[missing] != Any[missing]) + @test ismissing([missing] != [missing]) + @test ismissing(Any[missing, 2] != Any[1, missing]) + @test ismissing([missing, false] != BitArray([true, false])) + @test ismissing(Any[missing, false] != BitArray([true, false])) + @test !(Union{Int, Missing}[1] != Union{Float64, Missing}[1.0]) + @test !(Union{Int, Missing}[1] != [1.0]) + @test !(Union{Bool, Missing}[true] != BitArray([true])) + @test Union{Int, Missing}[1] != [2] + @test [1] != Union{Int, Missing}[2] + @test Union{Int, Missing}[1] != Union{Int, Missing}[2] +end + +@testset "any & all" begin + @test any([true, missing]) + @test any(x -> x == 1, [1, missing]) + @test ismissing(any([false, missing])) + @test ismissing(any(x -> x == 1, [2, missing])) + @test ismissing(all([true, missing])) + @test ismissing(all(x -> x == 1, [1, missing])) + @test !all([false, missing]) + @test !all(x -> x == 1, [2, missing]) + @test 1 in [1, missing] + @test ismissing(2 in [1, missing]) + @test ismissing(missing in [1, missing]) +end + +@testset "float" begin + @test isequal(float([1, missing]), [1, missing]) + @test float([1, missing]) isa Vector{Union{Float64, Missing}} + @test isequal(float(Union{Int, Missing}[missing]), [missing]) + @test float(Union{Int, Missing}[missing]) isa Vector{Union{Float64, Missing}} + @test float(Union{Int, Missing}[1]) == [1] + @test float(Union{Int, Missing}[1]) isa Vector{Union{Float64, Missing}} + @test isequal(float([missing]), [missing]) + @test float([missing]) isa Vector{Missing} +end diff --git a/test/reduce.jl b/test/reduce.jl index e3c3273247dcf4..ae18cb1b7a7aee 100644 --- a/test/reduce.jl +++ b/test/reduce.jl @@ -238,39 +238,39 @@ A = circshift(reshape(1:24,2,3,4), (0,1,1)) # any & all -@test any([]) == false -@test any(Bool[]) == false -@test any([true]) == true -@test any([false, false]) == false -@test any([false, true]) == true -@test any([true, false]) == true -@test any([true, true]) == true -@test any([true, true, true]) == true -@test any([true, false, true]) == true -@test any([false, false, false]) == false - -@test all([]) == true -@test all(Bool[]) == true -@test all([true]) == true -@test all([false, false]) == false -@test all([false, true]) == false -@test all([true, false]) == false -@test all([true, true]) == true -@test all([true, true, true]) == true -@test all([true, false, true]) == false -@test all([false, false, false]) == false - -@test any(x->x>0, []) == false -@test any(x->x>0, Int[]) == false -@test any(x->x>0, [-3]) == false -@test any(x->x>0, [4]) == true -@test any(x->x>0, [-3, 4, 5]) == true - -@test all(x->x>0, []) == true -@test all(x->x>0, Int[]) == true -@test all(x->x>0, [-3]) == false -@test all(x->x>0, [4]) == true -@test all(x->x>0, [-3, 4, 5]) == false +@test @inferred any([]) == false +@test @inferred any(Bool[]) == false +@test @inferred any([true]) == true +@test @inferred any([false, false]) == false +@test @inferred any([false, true]) == true +@test @inferred any([true, false]) == true +@test @inferred any([true, true]) == true +@test @inferred any([true, true, true]) == true +@test @inferred any([true, false, true]) == true +@test @inferred any([false, false, false]) == false + +@test @inferred all([]) == true +@test @inferred all(Bool[]) == true +@test @inferred all([true]) == true +@test @inferred all([false, false]) == false +@test @inferred all([false, true]) == false +@test @inferred all([true, false]) == false +@test @inferred all([true, true]) == true +@test @inferred all([true, true, true]) == true +@test @inferred all([true, false, true]) == false +@test @inferred all([false, false, false]) == false + +@test @inferred any(x->x>0, []) == false +@test @inferred any(x->x>0, Int[]) == false +@test @inferred any(x->x>0, [-3]) == false +@test @inferred any(x->x>0, [4]) == true +@test @inferred any(x->x>0, [-3, 4, 5]) == true + +@test @inferred all(x->x>0, []) == true +@test @inferred all(x->x>0, Int[]) == true +@test @inferred all(x->x>0, [-3]) == false +@test @inferred all(x->x>0, [4]) == true +@test @inferred all(x->x>0, [-3, 4, 5]) == false @test reduce((a, b) -> a .| b, fill(trues(5), 24)) == trues(5) @test reduce((a, b) -> a .| b, fill(falses(5), 24)) == falses(5) @@ -301,13 +301,13 @@ end # any/all with non-boolean collections let f(x) = x == 1 ? true : x == 2 ? false : 1 - @test any(Any[false,true,false]) - @test any(map(f, [2,1,2])) - @test any([f(x) for x in [2,1,2]]) + @test @inferred any(Any[false,true,false]) + @test @inferred any(map(f, [2,1,2])) + @test @inferred any([f(x) for x in [2,1,2]]) - @test all(Any[true,true,true]) - @test all(map(f, [1,1,1])) - @test all([f(x) for x in [1,1,1]]) + @test @inferred all(Any[true,true,true]) + @test @inferred all(map(f, [1,1,1])) + @test @inferred all([f(x) for x in [1,1,1]]) @test_throws TypeError any([1,true]) @test_throws TypeError all([true,1]) @@ -320,8 +320,8 @@ end struct SomeFunctor end (::SomeFunctor)(x) = true -@test any(SomeFunctor(), 1:10) -@test all(SomeFunctor(), 1:10) +@test @inferred any(SomeFunctor(), 1:10) +@test @inferred all(SomeFunctor(), 1:10) # in