Merge pull request #25021 from JuliaLang/nl/unicode

Move Unicode-related functions to new Unicode stdlib package

Author: nalimilan

CONTRIBUTING.md

@@ -174,7 +174,7 @@ The steps required to add a new docstring are listed below:
 Examples written within docstrings can be used as testcases known as "doctests" by annotating code blocks with `jldoctest`.
 
     ```jldoctest
-    julia> uppercase("Docstring test")
+    julia> Unicode.uppercase("Docstring test")
     "DOCSTRING TEST"
     ```
 

NEWS.md

@@ -738,6 +738,16 @@ Deprecated or removed
   * The `sum_kbn` and `cumsum_kbn` functions have been moved to the
     [KahanSummation](https://github.com/JuliaMath/KahanSummation.jl) package ([#24869]).
 
+  * Unicode-related string functions have been moved to the new `Unicode` standard
+    library module ([#25021]). This applies to `normalize_string`, `graphemes`,
+    `is_assigned_char`, `textwidth`, `isascii`, `islower`, `isupper`, `isalpha`,
+    `isdigit`, `isxdigit`, `isnumber`, `isalnum`, `iscntrl`, `ispunct`, `isspace`,
+    `isprint`, `isgraph`, `lowercase`, `uppercase`, `titlecase`, `lcfirst` and `ucfirst`.
+
+  * `isnumber` has been deprecated in favor of `isnumeric`, `is_assigned_char`
+    in favor of `isassigned` and `normalize_string` in favor of `normalize`, all three
+    in the new `Unicode` standard library module ([#25021]).
+
 Command-line option changes
 ---------------------------
 
@@ -1711,3 +1721,4 @@ Command-line option changes
 [#24413]: https://github.com/JuliaLang/julia/issues/24413
 [#24653]: https://github.com/JuliaLang/julia/issues/24653
 [#24869]: https://github.com/JuliaLang/julia/issues/24869
+[#25021]: https://github.com/JuliaLang/julia/issues/25021

base/arrayshow.jl

@@ -166,7 +166,7 @@ function print_matrix(io::IO, X::AbstractVecOrMat,
     screenwidth -= length(pre) + length(post)
     presp = repeat(" ", length(pre))  # indent each row to match pre string
     postsp = ""
-    @assert textwidth(hdots) == textwidth(ddots)
+    @assert Unicode.textwidth(hdots) == Unicode.textwidth(ddots)
     sepsize = length(sep)
     rowsA, colsA = indices(X,1), indices(X,2)
     m, n = length(rowsA), length(colsA)

base/char.jl

@@ -64,7 +64,7 @@ function show(io::IO, c::Char)
             return
         end
     end
-    if isprint(c)
+    if Unicode.isprint(c)
         write(io, 0x27, c, 0x27)
     else
         u = UInt32(c)
@@ -81,6 +81,6 @@ end
 function show(io::IO, ::MIME"text/plain", c::Char)
     show(io, c)
     u = UInt32(c)
-    print(io, ": ", isascii(c) ? "ASCII/" : "", "Unicode U+", hex(u, u > 0xffff ? 6 : 4))
-    print(io, " (category ", UTF8proc.category_abbrev(c), ": ", UTF8proc.category_string(c), ")")
+    print(io, ": ", Unicode.isascii(c) ? "ASCII/" : "", "Unicode U+", hex(u, u > 0xffff ? 6 : 4))
+    print(io, " (category ", Unicode.category_abbrev(c), ": ", Unicode.category_string(c), ")")
 end

base/client.jl

@@ -359,7 +359,7 @@ function load_machine_file(path::AbstractString)
         s = split(line, '*'; keep = false)
         map!(strip, s, s)
         if length(s) > 1
-            cnt = isnumber(s[1]) ? parse(Int,s[1]) : Symbol(s[1])
+            cnt = all(isdigit, s[1]) ? parse(Int,s[1]) : Symbol(s[1])
             push!(machines,(s[2], cnt))
         else
             push!(machines,line)

base/deprecated.jl

@@ -1072,13 +1072,6 @@ function Matrix()
     return Matrix(uninitialized, 0, 0)
 end
 
-for name in ("alnum", "alpha", "cntrl", "digit", "number", "graph",
-             "lower", "print", "punct", "space", "upper", "xdigit")
-    f = Symbol("is",name)
-    @eval import .UTF8proc: $f
-    @eval @deprecate ($f)(s::AbstractString) all($f, s)
-end
-
 # TODO: remove warning for using `_` in parse_input_line in base/client.jl
 
 # Special functions have been moved to a package
@@ -1512,7 +1505,7 @@ export hex2num
 @deprecate convert(::Type{Symbol}, s::AbstractString)         Symbol(s)
 @deprecate convert(::Type{String}, s::Symbol)                 String(s)
 @deprecate convert(::Type{String}, v::Vector{UInt8})          String(v)
-@deprecate convert(::Type{S}, g::UTF8proc.GraphemeIterator) where {S<:AbstractString}  convert(S, g.s)
+@deprecate convert(::Type{S}, g::Unicode.GraphemeIterator) where {S<:AbstractString}  convert(S, g.s)
 
 # Issue #19923
 @deprecate ror                  circshift
@@ -2972,6 +2965,28 @@ end
 @deprecate_moved sum_kbn "KahanSummation"
 @deprecate_moved cumsum_kbn "KahanSummation"
 
+# PR #25021
+@deprecate_moved normalize_string "Unicode" true true
+@deprecate_moved graphemes "Unicode" true true
+@deprecate_moved is_assigned_char "Unicode" true true
+@deprecate_moved textwidth "Unicode" true true
+@deprecate_moved islower "Unicode" true true
+@deprecate_moved isupper "Unicode" true true
+@deprecate_moved isalpha "Unicode" true true
+@deprecate_moved isdigit "Unicode" true true
+@deprecate_moved isnumber "Unicode" true true
+@deprecate_moved isalnum "Unicode" true true
+@deprecate_moved iscntrl "Unicode" true true
+@deprecate_moved ispunct "Unicode" true true
+@deprecate_moved isspace "Unicode" true true
+@deprecate_moved isprint "Unicode" true true
+@deprecate_moved isgraph "Unicode" true true
+@deprecate_moved lowercase "Unicode" true true
+@deprecate_moved uppercase "Unicode" true true
+@deprecate_moved titlecase "Unicode" true true
+@deprecate_moved lcfirst "Unicode" true true
+@deprecate_moved ucfirst "Unicode" true true
+
 # END 0.7 deprecations
 
 # BEGIN 1.0 deprecations

base/dict.jl

@@ -1,15 +1,15 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
 function _truncate_at_width_or_chars(str, width, chars="", truncmark="…")
-    truncwidth = textwidth(truncmark)
+    truncwidth = Unicode.textwidth(truncmark)
     (width <= 0 || width < truncwidth) && return ""
 
     wid = truncidx = lastidx = 0
     idx = start(str)
     while !done(str, idx)
         lastidx = idx
         c, idx = next(str, idx)
-        wid += textwidth(c)
+        wid += Unicode.textwidth(c)
         wid >= width - truncwidth && truncidx == 0 && (truncidx = lastidx)
         (wid >= width || c in chars) && break
     end

base/distributed/Distributed.jl

@@ -15,6 +15,7 @@ using Base: Process, Semaphore, JLOptions, AnyDict, buffer_writes, wait_connecte
             binding_module, notify_error, atexit, julia_exename, julia_cmd,
             AsyncGenerator, display_error, acquire, release, invokelatest, warn_once,
             shell_escape_posixly, uv_error
+using Base.Unicode: isascii, isdigit, isnumeric
 
 # NOTE: clusterserialize.jl imports additional symbols from Base.Serializer for use
 

base/docs/utils.jl

@@ -3,6 +3,7 @@
 # Text / HTML objects
 
 import Base: print, show, ==, hash
+using Base.Unicode
 
 export HTML, @html_str
 
@@ -231,7 +232,8 @@ function matchinds(needle, haystack; acronym = false)
     for (i, char) in enumerate(haystack)
         isempty(chars) && break
         while chars[1] == ' ' shift!(chars) end # skip spaces
-        if lowercase(char) == lowercase(chars[1]) && (!acronym || !isalpha(lastc))
+        if Unicode.lowercase(char) == Unicode.lowercase(chars[1]) &&
+           (!acronym || !Unicode.isalpha(lastc))
             push!(is, i)
             shift!(chars)
         end

base/exports.jl

@@ -725,40 +725,22 @@ export
     eachmatch,
     endswith,
     escape_string,
-    graphemes,
     hex,
     hex2bytes,
     hex2bytes!,
     ind2chr,
     info,
-    is_assigned_char,
-    isalnum,
-    isalpha,
-    isascii,
-    iscntrl,
-    isdigit,
-    isgraph,
-    islower,
     ismatch,
-    isnumber,
-    isprint,
-    ispunct,
-    isspace,
-    isupper,
     isvalid,
-    isxdigit,
     join,
-    lcfirst,
     logging,
-    lowercase,
     lpad,
     lstrip,
     match,
     matchall,
     ncodeunits,
     ndigits,
     nextind,
-    normalize_string,
     oct,
     prevind,
     print,
@@ -785,13 +767,9 @@ export
     string,
     strip,
     summary,
-    textwidth,
     thisind,
-    titlecase,
     transcode,
-    ucfirst,
     unescape_string,
-    uppercase,
     warn,
 
 # random numbers