fix some issues with color configuration (#36689)

- `--color=no` did not remove all color in the REPL
- color was not fully disabled by default when stdout was a pipe
- `--color=yes` did not enable color when stdout was a pipe (fixes #30703)

Author: JeffBezanson

base/client.jl

@@ -371,13 +371,12 @@ function run_main_repl(interactive::Bool, quiet::Bool, banner::Bool, history_fil
         invokelatest(REPL_MODULE_REF[]) do REPL
             term_env = get(ENV, "TERM", @static Sys.iswindows() ? "" : "dumb")
             term = REPL.Terminals.TTYTerminal(term_env, stdin, stdout, stderr)
-            color_set || (global have_color = REPL.Terminals.hascolor(term))
             banner && Base.banner(term)
             if term.term_type == "dumb"
                 active_repl = REPL.BasicREPL(term)
                 quiet || @warn "Terminal not fully functional"
             else
-                active_repl = REPL.LineEditREPL(term, have_color, true)
+                active_repl = REPL.LineEditREPL(term, get(stdout, :color, false), true)
                 active_repl.history_file = history_file
             end
             # Make sure any displays pushed in .julia/config/startup.jl ends up above the
@@ -487,7 +486,7 @@ function _start()
         invokelatest(display_error, catch_stack())
         exit(1)
     end
-    if is_interactive && have_color === true
+    if is_interactive && get(stdout, :color, false)
         print(color_normal)
     end
 end

base/libuv.jl

@@ -115,6 +115,16 @@ function reinit_stdio()
     global stdin = init_stdio(ccall(:jl_stdin_stream, Ptr{Cvoid}, ()))
     global stdout = init_stdio(ccall(:jl_stdout_stream, Ptr{Cvoid}, ()))
     global stderr = init_stdio(ccall(:jl_stderr_stream, Ptr{Cvoid}, ()))
+    opts = JLOptions()
+    if opts.color != 0
+        have_color = (opts.color == 1)
+        if !isa(stdout, TTY)
+            global stdout = IOContext(stdout, :color => have_color)
+        end
+        if !isa(stderr, TTY)
+            global stderr = IOContext(stderr, :color => have_color)
+        end
+    end
     nothing
 end
 

stdlib/REPL/src/LineEdit.jl

@@ -395,7 +395,7 @@ function refresh_multi_line(termbuf::TerminalBuffer, terminal::UnixTerminal, buf
     regstart, regstop = region(buf)
     written = 0
     # Write out the prompt string
-    lindent = write_prompt(termbuf, prompt)
+    lindent = write_prompt(termbuf, prompt, hascolor(terminal))
     # Count the '\n' at the end of the line if the terminal emulator does (specific to DOS cmd prompt)
     miscountnl = @static Sys.iswindows() ? (isa(Terminals.pipe_reader(terminal), Base.TTY) && !Base.ispty(Terminals.pipe_reader(terminal))) : false
 
@@ -1259,15 +1259,15 @@ refresh_line(s, termbuf) = refresh_multi_line(termbuf, s)
 default_completion_cb(::IOBuffer) = []
 default_enter_cb(_) = true
 
-write_prompt(terminal, s::PromptState) = write_prompt(terminal, s.p)
+write_prompt(terminal, s::PromptState, color::Bool) = write_prompt(terminal, s.p, color)
 
-function write_prompt(terminal, p::Prompt)
+function write_prompt(terminal, p::Prompt, color::Bool)
     prefix = prompt_string(p.prompt_prefix)
     suffix = prompt_string(p.prompt_suffix)
     write(terminal, prefix)
-    write(terminal, Base.text_colors[:bold])
-    width = write_prompt(terminal, p.prompt)
-    write(terminal, Base.text_colors[:normal])
+    color && write(terminal, Base.text_colors[:bold])
+    width = write_prompt(terminal, p.prompt, color)
+    color && write(terminal, Base.text_colors[:normal])
     write(terminal, suffix)
     return width
 end
@@ -1303,7 +1303,7 @@ end
 end
 
 # returns the width of the written prompt
-function write_prompt(terminal, s::Union{AbstractString,Function})
+function write_prompt(terminal, s::Union{AbstractString,Function}, color::Bool)
     @static Sys.iswindows() && _reset_console_mode()
     promptstr = prompt_string(s)
     write(terminal, promptstr)
@@ -1740,7 +1740,7 @@ end
 
 input_string(s::PrefixSearchState) = String(take!(copy(s.response_buffer)))
 
-write_prompt(terminal, s::PrefixSearchState) = write_prompt(terminal, s.histprompt.parent_prompt)
+write_prompt(terminal, s::PrefixSearchState, color::Bool) = write_prompt(terminal, s.histprompt.parent_prompt, color)
 prompt_string(s::PrefixSearchState) = prompt_string(s.histprompt.parent_prompt.prompt)
 
 terminal(s::PrefixSearchState) = s.terminal

stdlib/REPL/src/REPL.jl

@@ -436,9 +436,15 @@ mutable struct LineEditREPL <: AbstractREPL
     last_shown_line_infos::Vector{Tuple{String,Int}}
     interface::ModalInterface
     backendref::REPLBackendRef
-    LineEditREPL(t,hascolor,prompt_color,input_color,answer_color,shell_color,help_color,history_file,in_shell,in_help,envcolors) =
+    function LineEditREPL(t,hascolor,prompt_color,input_color,answer_color,shell_color,help_color,history_file,in_shell,in_help,envcolors)
+        opts = Options()
+        opts.hascolor = hascolor
+        if !hascolor
+            opts.beep_colors = [""]
+        end
         new(t,hascolor,prompt_color,input_color,answer_color,shell_color,help_color,history_file,in_shell,
-            in_help,envcolors,false,nothing, Options(), nothing, Tuple{String,Int}[])
+            in_help,envcolors,false,nothing, opts, nothing, Tuple{String,Int}[])
+    end
 end
 outstream(r::LineEditREPL) = r.t
 specialdisplay(r::LineEditREPL) = r.specialdisplay
@@ -849,7 +855,8 @@ end
 
 function reset(repl::LineEditREPL)
     raw!(repl.t, false)
-    print(repl.t, Base.text_colors[:normal])
+    hascolor(repl) && print(repl.t, Base.text_colors[:normal])
+    nothing
 end
 
 function prepare_next(repl::LineEditREPL)

stdlib/REPL/src/Terminals.jl

@@ -152,7 +152,7 @@ beep(t::UnixTerminal) = write(t.err_stream,"\x7")
 
 Base.displaysize(t::UnixTerminal) = displaysize(t.out_stream)
 
-hascolor(t::TTYTerminal) = Base.ttyhascolor(t.term_type)
+hascolor(t::TTYTerminal) = get(t.out_stream, :color, false)::Bool
 
 # use cached value of have_color
 Base.in(key_value::Pair, t::TTYTerminal) = in(key_value, pipe_writer(t))

stdlib/REPL/test/repl.jl

@@ -90,7 +90,7 @@ end
 # in the mix. If verification needs to be done, keep it to the bare minimum. Basically
 # this should make sure nothing crashes without depending on how exactly the control
 # characters are being used.
-fake_repl(options = REPL.Options(confirm_exit=false,hascolor=false)) do stdin_write, stdout_read, repl
+fake_repl(options = REPL.Options(confirm_exit=false,hascolor=true)) do stdin_write, stdout_read, repl
     repl.specialdisplay = REPL.REPLDisplay(repl)
     repl.history_file = false
 
@@ -1279,7 +1279,7 @@ end
     repl.backendref = REPL.REPLBackendRef(Channel(1), Channel(1))
     put!(repl.backendref.response_channel, (bt, true))
 
-    REPL.print_response(repl, (err, true), true, false)
+    REPL.print_response(repl, (bt, true), true, false)
     seekstart(out_stream)
     @test count(
         contains(