Add ability to ignore repeated delimiters

Author: quinnj

src/Parsers.jl

@@ -283,30 +283,47 @@ end
     Parsing on a `Parsers.Delimited` will first call `Parsers.parse!(d.next, io, result; kwargs...)`, then expect the next bytes to be one of the expected `delims` arguments.
     If one of `delims` is not found, the result is `Parsers.INVALID`, but parsing will continue until a valid `delims` is found. An `eof(io)` is _always_ considered a valid termination state in place of a delimiter.
 """
-struct Delimited{I, T <: Trie} <: Layer
+struct Delimited{IR, I, T <: Trie} <: Layer
     next::I
     delims::T
 end
-Delimited(next, delims::Union{Char, String}...=',') = Delimited(next, Trie(String[string(d) for d in delims], DELIMITED))
-Delimited(delims::Union{Char, String}...=',') = Delimited(defaultparser, Trie(String[string(d) for d in delims], DELIMITED))
+Delimited(ignore_repeated::Bool, next::I, delims::T) where {I, T <: Trie} = Delimited{ignore_repeated, I, T}(next, delims)
+Delimited(next::Union{Layer, Base.Callable}=defaultparser, delims::Union{Char, String}...; ignore_repeated::Bool=false) = Delimited(ignore_repeated, next, Trie(String[string(d) for d in (isempty(delims) ? (",",) : delims)], DELIMITED))
+Delimited(delims::Union{Char, String}...; ignore_repeated::Bool=false) = Delimited(ignore_repeated, defaultparser, Trie(String[string(d) for d in (isempty(delims) ? (",",) : delims)], DELIMITED))
 
-@inline function parse!(d::Delimited, io::IO, r::Result{T}; kwargs...) where {T}
+@inline function parse!(d::Delimited{ignore_repeated}, io::IO, r::Result{T}; kwargs...) where {ignore_repeated, T}
     # @debug "xparse Delimited - $T"
     parse!(d.next, io, r; kwargs...)
     # @debug "Delimited - $T: r.code=$(r.code), r.result=$(r.result)"
     if eof(io)
         r.code |= EOF
         return r
     end
-    match!(d.delims, io, r, false) && return r
+    if ignore_repeated
+        matched = false
+        while match!(d.delims, io, r, false)
+            matched = true
+        end
+        matched && return r
+    else
+        match!(d.delims, io, r, false) && return r
+    end
     # @debug "didn't find delimiters at expected location; result is invalid, parsing until delimiter is found"
     while true
         b = readbyte(io)
         if eof(io)
             r.code |= EOF
             break
         end
-        match!(d.delims, io, r, false) && break
+        if ignore_repeated
+            matched = false
+            while match!(d.delims, io, r, false)
+                matched = true
+            end
+            matched && break
+        else
+            match!(d.delims, io, r, false) && break
+        end
     end
     r.code |= INVALID_DELIMITER
     return r

test/runtests.jl

@@ -557,6 +557,25 @@ let io=IOBuffer("1,2,null,4"), layers=Parsers.Delimited(Parsers.Quoted(Parsers.S
     @test r.pos == 9
 end
 
+let io=IOBuffer("1,,,2,null,4"), layers=Parsers.Delimited(Parsers.Quoted(Parsers.Sentinel(["null"])); ignore_repeated=true)
+    r = Parsers.parse(layers, io, Int)
+    @test r.result === 1
+    @test r.code === OK | DELIMITED
+    @test r.pos == 0
+    r = Parsers.parse(layers, io, Int)
+    @test r.result === 2
+    @test r.code === OK | DELIMITED
+    @test r.pos == 4
+    r = Parsers.parse(layers, io, Int)
+    @test r.result === missing
+    @test r.code === SENTINEL | DELIMITED
+    @test r.pos == 6
+    r = Parsers.parse(layers, io, Int)
+    @test r.result === 4
+    @test r.code === OK | EOF
+    @test r.pos == 11
+end
+
 end # @testset
 
 end # @testset