Add skipmissing() to skip missing values

Author: nalimilan

base/exports.jl

@@ -886,6 +886,7 @@ export
 # missing values
     ismissing,
     missing,
+    skipmissing,
 
 # time
     sleep,

base/missing.jl

@@ -18,6 +18,11 @@ end
 showerror(io::IO, ex::MissingException) =
     print(io, "MissingException: ", ex.msg)
 
+nonmissingtype(::Type{Union{T, Missing}}) where {T} = T
+nonmissingtype(::Type{Missing}) = Union{}
+nonmissingtype(::Type{T}) where {T} = T
+nonmissingtype(::Type{Any}) = Any
+
 promote_rule(::Type{Missing}, ::Type{T}) where {T} = Union{T, Missing}
 promote_rule(::Type{Union{S,Missing}}, ::Type{T}) where {T,S} = Union{promote_type(T, S), Missing}
 promote_rule(::Type{Any}, ::Type{T}) where {T} = Any
@@ -116,3 +121,81 @@ function float(A::AbstractArray{Union{T, Missing}}) where {T}
     convert(AbstractArray{Union{U, Missing}}, A)
 end
 float(A::AbstractArray{Missing}) = A
+
+"""
+    skipmissing(itr)
+
+Return an iterator over the elements in `itr` skipping [`missing`](@ref) values.
+
+Use [`collect`](@ref) to obtain an `Array` containing the non-`missing` values in
+`itr`. Note that even if `itr` is a multidimensional array, the result will always
+be a `Vector` since it is not possible to remove missings while preserving dimensions
+of the input.
+
+# Examples
+```jldoctest
+julia> sum(skipmissing([1, missing, 2]))
+3
+
+julia> collect(skipmissing([1, missing, 2]))
+2-element Array{Int64,1}:
+1
+2
+
+julia> collect(skipmissing([1 missing; 2 missing]))
+2-element Array{Int64,1}:
+1
+2
+
+```
+"""
+skipmissing(itr) = SkipMissing(itr)
+
+struct SkipMissing{T}
+    x::T
+end
+iteratorsize(::Type{<:SkipMissing}) = SizeUnknown()
+iteratoreltype(::Type{SkipMissing{T}}) where {T} = iteratoreltype(T)
+eltype(itr::SkipMissing) = nonmissingtype(eltype(itr.x))
+# Fallback implementation for general iterables: we cannot access a value twice,
+# so after finding the next non-missing element in start() or next(), we have to
+# pass it in the iterator state, which introduces a type instability since the value
+# is missing if the input does not contain any non-missing element.
+@inline function Base.start(itr::SkipMissing)
+    s = start(itr.x)
+    v = missing
+    @inbounds while !done(itr.x, s) && v isa Missing
+        v, s = next(itr.x, s)
+    end
+    (v, s)
+end
+@inline Base.done(itr::SkipMissing, state) = ismissing(state[1]) && done(itr.x, state[2])
+@inline function Base.next(itr::SkipMissing, state)
+    v1, s = state
+    v2 = missing
+    @inbounds while !done(itr.x, s) && v2 isa Missing
+        v2, s = next(itr.x, s)
+    end
+    (v1, (v2, s))
+end
+# Optimized implementation for AbstractArray, relying on the ability to access x[i] twice:
+# once in done() to find the next non-missing entry, and once in next() to return it.
+# This works around the type instability problem of the generic fallback.
+@inline function _next_nonmissing_ind(x::AbstractArray, s)
+    idx = eachindex(x)
+    @inbounds while !done(idx, s)
+        i, new_s = next(idx, s)
+        x[i] isa Missing || break
+        s = new_s
+    end
+    s
+end
+@inline Base.start(itr::SkipMissing{<:AbstractArray}) =
+    _next_nonmissing_ind(itr.x, start(eachindex(itr.x)))
+@inline Base.done(itr::SkipMissing{<:AbstractArray}, state) =
+    done(eachindex(itr.x), state)
+@inline function Base.next(itr::SkipMissing{<:AbstractArray}, state)
+    i, state = next(eachindex(itr.x), state)
+    @inbounds v = itr.x[i]::eltype(itr)
+    (v, _next_nonmissing_ind(itr.x, state))
+end

doc/src/manual/missing.md

@@ -296,6 +296,46 @@ since type constructors fall back to convert methods.
 Stacktrace:
 [...]
 ```
+## Skipping Missing Values
+
+Since `missing` values propagate with standard mathematical operators, reduction
+functions return `missing` when called on arrays which contain missing values:
+```doctest
+julia> sum([1, missing])
+missing
+
+```
+
+In this situation, use the [`skipmissing`](@ref) function to skip missing values:
+```doctest
+julia> sum(skipmissing([1, missing]))
+1
+
+```
+
+This convenience function returns an iterator which filters out `missing` values
+efficiently. It can therefore be used with any function which supports iterators:
+```doctest
+julia> maximum(skipmissing([3, missing, 2, 1]))
+3
+
+julia> mean(skipmissing([3, missing, 2, 1]))
+2.0
+
+julia> mapreduce(sqrt, +, skipmissing([3, missing, 2, 1]))
+4.146264369941973
+
+```
+
+Use [`collect`](@ref) to extract non-`missing` values and store them in an array:
+```doctest
+julia> collect(skipmissing([3, missing, 2, 1]))
+3-element Array{Int64,1}:
+ 3
+ 2
+ 1
+
+```
 
 ## Logical Operations on Arrays
 

doc/src/manual/noteworthy-differences.md

@@ -183,7 +183,9 @@ For users coming to Julia from R, these are some noteworthy differences:
   * Julia does not support the `NULL` type. The closest equivalent is [`nothing`](@ref), but it
     behaves like a scalar value rather than like a list. Use `x == nothing` instead of `is.null(x)`.
   * In Julia, missing values are represented by the [`missing`](@ref) object rather than by `NA`.
-    Use [`ismissing(x)`](@ref) instead of `isna(x)`.
+    Use [`ismissing(x)`](@ref) instead of `isna(x)`. The [`skipmissing`](@ref) function is generally
+    used instead of `na.rm=TRUE` (though in some particular cases functions take a `skipmissing`
+    argument).
   * Julia lacks the equivalent of R's `assign` or `get`.
   * In Julia, `return` does not require parentheses.
   * In R, an idiomatic way to remove unwanted values is to use logical indexing, like in the expression

doc/src/stdlib/base.md

@@ -226,6 +226,7 @@ Base.unsafe_get
 Base.Missing
 Base.missing
 Base.ismissing
+Base.skipmissing
 ```
 
 ## System

test/ambiguous.jl

@@ -287,6 +287,7 @@ end
         pop!(need_to_handle_undef_sparam, which(Base.zero, Tuple{Type{Union{Missing, T}} where T}))
         pop!(need_to_handle_undef_sparam, which(Base.one, Tuple{Type{Union{Missing, T}} where T}))
         pop!(need_to_handle_undef_sparam, which(Base.oneunit, Tuple{Type{Union{Missing, T}} where T}))
+        pop!(need_to_handle_undef_sparam, which(Base.nonmissingtype, Tuple{Type{Union{Missing, T}} where T}))
         @test need_to_handle_undef_sparam == Set()
     end
 end

test/missing.jl

@@ -4,6 +4,12 @@
     @test sprint(showerror, MissingException("test")) == "MissingException: test"
 end
 
+@testset "nonmissingtype" begin
+    @test Base.nonmissingtype(Union{Int, Missing}) == Int
+    @test Base.nonmissingtype(Any) == Any
+    @test Base.nonmissingtype(Missing) == Union{}
+end
+
 @testset "convert" begin
     @test convert(Union{Int, Missing}, 1) === 1
     @test convert(Union{Int, Missing}, 1.0) === 1
@@ -246,3 +252,35 @@ end
     @test isequal(float([missing]), [missing])
     @test float([missing]) isa Vector{Missing}
 end
+
+@testset "skipmissing" begin
+    x = skipmissing([1, 2, missing, 4])
+    @test eltype(x) === Int
+    @test collect(x) == [1, 2, 4]
+    @test collect(x) isa Vector{Int}
+
+    x = skipmissing([1  2; missing 4])
+    @test eltype(x) === Int
+    @test collect(x) == [1, 2, 4]
+    @test collect(x) isa Vector{Int}
+
+    x = collect(skipmissing([missing]))
+    @test eltype(x) === Union{}
+    @test isempty(collect(x))
+    @test collect(x) isa Vector{Union{}}
+
+    x = collect(skipmissing(Union{Int, Missing}[]))
+    @test eltype(x) === Int
+    @test isempty(collect(x))
+    @test collect(x) isa Vector{Int}
+
+    x = skipmissing([missing, missing, 1, 2, missing, 4, missing, missing])
+    @test eltype(x) === Int
+    @test collect(x) == [1, 2, 4]
+    @test collect(x) isa Vector{Int}
+
+    x = skipmissing(v for v in [missing, 1, missing, 2, 4])
+    @test eltype(x) === Any
+    @test collect(x) == [1, 2, 4]
+    @test collect(x) isa Vector{Int}
+end