Merge branch 'master' into tkf/tsan-ci

tkf · tkf · commit b92db8b0f0c7 · 2021-10-03T17:04:12.000-04:00
diff --git a/.buildkite/pipelines/experimental/launch_unsigned_builders.yml b/.buildkite/pipelines/experimental/launch_unsigned_builders.yml
@@ -1,6 +1,6 @@
 steps:
   - label: ":buildkite: Launch unsigned pipelines"
     commands: |
-      buildkite-agent pipeline upload .buildkite/pipelines/experimental/misc/sanitizers.yml
+      true
     agents:
       queue: julia
diff --git a/.buildkite/pipelines/main/launch_unsigned_builders.yml b/.buildkite/pipelines/main/launch_unsigned_builders.yml
@@ -22,6 +22,7 @@ steps:
       buildkite-agent pipeline upload .buildkite/pipelines/main/misc/doctest.yml
       buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml
       buildkite-agent pipeline upload .buildkite/pipelines/main/misc/llvmpasses.yml
+      buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers.yml
 
       # Launch all of the platform builders.
       bash .buildkite/pipelines/main/platforms/platforms.sh package_linux
diff --git a/.buildkite/pipelines/main/misc/doctest.yml b/.buildkite/pipelines/main/misc/doctest.yml
@@ -8,7 +8,9 @@ steps:
     key: doctest
     plugins:
       - JuliaCI/julia#v1:
-          version: 1.6
+          # Drop default "registries" directory, so it is not persisted from execution to execution
+          persist_depot_dirs: packages,artifacts,compiled
+          version: '1.6'
       - staticfloat/sandbox#v1:
           rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.1/package_linux.x86_64.tar.gz
           rootfs_treehash: "8c33c341a864852629b8aac01a6eb6a79b73570e"
diff --git a/.buildkite/pipelines/main/misc/embedding.yml b/.buildkite/pipelines/main/misc/embedding.yml
@@ -8,7 +8,9 @@ steps:
     key: "embedding"
     plugins:
       - JuliaCI/julia#v1:
-          version: 1.6
+          # Drop default "registries" directory, so it is not persisted from execution to execution
+          persist_depot_dirs: packages,artifacts,compiled
+          version: '1.6'
       - staticfloat/sandbox#v1:
           rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.1/package_linux.x86_64.tar.gz
           rootfs_treehash: "8c33c341a864852629b8aac01a6eb6a79b73570e"
diff --git a/.buildkite/pipelines/main/misc/llvmpasses.yml b/.buildkite/pipelines/main/misc/llvmpasses.yml
@@ -8,7 +8,9 @@ steps:
     key: "analyzegc"
     plugins:
       - JuliaCI/julia#v1:
-          version: 1.6
+          # Drop default "registries" directory, so it is not persisted from execution to execution
+          persist_depot_dirs: packages,artifacts,compiled
+          version: '1.6'
       - staticfloat/sandbox#v1:
           rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.1/llvm_passes.x86_64.tar.gz
           rootfs_treehash: "9dd715500b117a16fcfa419ea0bca0c0ca902cee"
@@ -26,7 +28,9 @@ steps:
     key: "llvmpasses"
     plugins:
       - JuliaCI/julia#v1:
-          version: 1.6
+          # Drop default "registries" directory, so it is not persisted from execution to execution
+          persist_depot_dirs: packages,artifacts,compiled
+          version: '1.6'
       - staticfloat/sandbox#v1:
           rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.8/package_linux.x86_64.tar.gz
           rootfs_treehash: "84a323ae8fcc724f8ea5aca5901bbbf4bda3e519"
diff --git a/.buildkite/pipelines/main/misc/sanitizers.yml b/.buildkite/pipelines/main/misc/sanitizers.yml
@@ -8,18 +8,19 @@ steps:
     key: "asan"
     plugins:
       - JuliaCI/julia#v1:
-          version: 1.6
+          # Drop default "registries" directory, so it is not persisted from execution to execution
+          persist_depot_dirs: packages,artifacts,compiled
+          version: '1.6'
       - staticfloat/sandbox#v1:
           rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.1/llvm_passes.x86_64.tar.gz
           rootfs_treehash: "9dd715500b117a16fcfa419ea0bca0c0ca902cee"
           uid: 1000
           gid: 1000
           workspaces:
             - "/cache/repos:/cache/repos"
-      # `contrib/check-asan.jl` needs a `julia` binary inside the inner sandbox:
-      - JuliaCI/julia#v1:
-          version: 1.6
     timeout_in_minutes: 120
+    if: | # We only run the `asan` job on Julia 1.8 and later.
+      (pipeline.slug != "julia-release-1-dot-6") && (pipeline.slug != "julia-release-1-dot-7")
     commands: |
       echo "--- Build julia-debug with ASAN"
       contrib/asan/build.sh ./tmp/test-asan -j$${JULIA_CPU_THREADS:?} debug
diff --git a/.buildkite/pipelines/main/misc/whitespace.yml b/.buildkite/pipelines/main/misc/whitespace.yml
@@ -8,7 +8,9 @@ steps:
     key: "whitespace"
     plugins:
       - JuliaCI/julia#v1:
-          version: 1.6
+          # Drop default "registries" directory, so it is not persisted from execution to execution
+          persist_depot_dirs: packages,artifacts,compiled
+          version: '1.6'
       - staticfloat/sandbox#v1:
           rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.1/package_linux.x86_64.tar.gz
           rootfs_treehash: "8c33c341a864852629b8aac01a6eb6a79b73570e"
diff --git a/.buildkite/pipelines/main/platforms/package_linux.yml b/.buildkite/pipelines/main/platforms/package_linux.yml
@@ -10,7 +10,7 @@ steps:
       - JuliaCI/julia#v1:
           # Drop default "registries" directory, so it is not persisted from execution to execution
           persist_depot_dirs: packages,artifacts,compiled
-          version: 1.6
+          version: '1.6'
       - staticfloat/sandbox#v1:
           rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/package_linux.${ROOTFS_ARCH?}.tar.gz
           rootfs_treehash: "${ROOTFS_TREE?}"
diff --git a/.buildkite/pipelines/main/platforms/tester_linux.yml b/.buildkite/pipelines/main/platforms/tester_linux.yml
@@ -11,7 +11,7 @@ steps:
       - JuliaCI/julia#v1:
           # Drop default "registries" directory, so it is not persisted from execution to execution
           persist_depot_dirs: packages,artifacts,compiled
-          version: 1.6
+          version: '1.6'
       - staticfloat/sandbox#v1:
           rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/package_linux.${ROOTFS_ARCH?}.tar.gz
           rootfs_treehash: "${ROOTFS_TREE?}"
@@ -53,6 +53,7 @@ steps:
       echo "--- Run the Julia test suite"
       unset JULIA_DEPOT_PATH
       export JULIA_UNDER_RR="$${JULIA_BINARY:?} .buildkite/utilities/rr/rr_capture.jl $${JULIA_BINARY:?}"
+      export OPENBLAS_NUM_THREADS=8
 
       if [[ "$${BUILDKITE_STEP_KEY:?}"   == "tester_linux64_rr" ]]; then
         # For the `rr` job, we disable multi-threading.
diff --git a/.buildkite/pipelines/scheduled/coverage/coverage_linux64.yml b/.buildkite/pipelines/scheduled/coverage/coverage_linux64.yml
@@ -11,7 +11,9 @@ steps:
             - CODECOV_TOKEN="U2FsdGVkX19l0fhdBabbuiEdysyEabkJLRHfxm7CNRkuGbnwPV365sxxC7Czs/CVcws0N1oB4pVwALRRMe36oA=="
             - COVERALLS_TOKEN="U2FsdGVkX19zopI0hMNzzi2UUOvNVFD8Y0iisFnO/ryVxU7Tit8ZEaeN+gxodRx4CosUUh192F1+q3dTMWRIvw=="
       - JuliaCI/julia#v1:
-          version: 1.6
+          # Drop default "registries" directory, so it is not persisted from execution to execution
+          persist_depot_dirs: packages,artifacts,compiled
+          version: '1.6'
       - staticfloat/sandbox#v1:
           rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.1/package_linux.x86_64.tar.gz
           rootfs_treehash: "8c33c341a864852629b8aac01a6eb6a79b73570e"
diff --git a/README.md b/README.md
@@ -97,7 +97,7 @@ Then, acquire the source code by cloning the git repository:
     git clone https://github.com/JuliaLang/julia.git
 
 By default you will be building the latest unstable version of
-Julia. However, most users should use the most recent stable version
+Julia. However, most users should use the [most recent stable version](https://github.com/JuliaLang/julia/releases)
 of Julia. You can get this version by changing to the Julia directory
 and running:
 
diff --git a/base/Base.jl b/base/Base.jl
@@ -462,18 +462,9 @@ end
 if is_primary_base_module
 function __init__()
     # try to ensuremake sure OpenBLAS does not set CPU affinity (#1070, #9639)
-    if !haskey(ENV, "OPENBLAS_MAIN_FREE") && !haskey(ENV, "GOTOBLAS_MAIN_FREE")
+    if !haskey(ENV, "OPENBLAS_MAIN_FREE")
         ENV["OPENBLAS_MAIN_FREE"] = "1"
     end
-    # And try to prevent openblas from starting too many threads, unless/until specifically requested
-    if !haskey(ENV, "OPENBLAS_NUM_THREADS") && !haskey(ENV, "OMP_NUM_THREADS")
-        cpu_threads = Sys.CPU_THREADS::Int
-        if cpu_threads > 8 # always at most 8
-            ENV["OPENBLAS_NUM_THREADS"] = "8"
-        elseif haskey(ENV, "JULIA_CPU_THREADS") # or exactly as specified
-            ENV["OPENBLAS_NUM_THREADS"] = cpu_threads
-        end # otherwise, trust that openblas will pick CPU_THREADS anyways, without any intervention
-    end
     # for the few uses of Libc.rand in Base:
     Libc.srand()
     # Base library init
diff --git a/base/compiler/abstractinterpretation.jl b/base/compiler/abstractinterpretation.jl
@@ -1086,16 +1086,7 @@ function abstract_call_builtin(interp::AbstractInterpreter, f::Builtin, fargs::U
         end
     end
     rt = builtin_tfunction(interp, f, argtypes[2:end], sv)
-    if f === getfield && isa(fargs, Vector{Any}) && la == 3 &&
-       (a3 = argtypes[3]; isa(a3, Const)) && (idx = a3.val; isa(idx, Int)) &&
-       (a2 = argtypes[2]; a2 ⊑ Tuple)
-        # TODO: why doesn't this use the getfield_tfunc?
-        cti_info = precise_container_type(interp, iterate, a2, sv)
-        cti = cti_info[1]::Vector{Any}
-        if 1 <= idx <= length(cti)
-            rt = unwrapva(cti[idx])
-        end
-    elseif (rt === Bool || (isa(rt, Const) && isa(rt.val, Bool))) && isa(fargs, Vector{Any})
+    if (rt === Bool || (isa(rt, Const) && isa(rt.val, Bool))) && isa(fargs, Vector{Any})
         # perform very limited back-propagation of type information for `is` and `isa`
         if f === isa
             a = ssa_def_slot(fargs[2], sv)
diff --git a/base/compiler/inferenceresult.jl b/base/compiler/inferenceresult.jl
@@ -71,10 +71,8 @@ function most_general_argtypes(method::Union{Method, Nothing}, @nospecialize(spe
                 va = linfo_argtypes[linfo_argtypes_length]
                 if isvarargtype(va)
                     new_va = rewrap_unionall(unconstrain_vararg_length(va), specTypes)
-                    vargtype_elements = Any[new_va]
                     vargtype = Tuple{new_va}
                 else
-                    vargtype_elements = Any[]
                     vargtype = Tuple{}
                 end
             else
diff --git a/base/compiler/ssair/inlining.jl b/base/compiler/ssair/inlining.jl
@@ -842,51 +842,6 @@ function InliningTodo(mi::MethodInstance, src::Union{CodeInfo, Array{UInt8, 1}})
     end
 end
 
-# Neither the product iterator not CartesianIndices are available
-# here, so use this poor man's version
-struct SimpleCartesian
-    ranges::Vector{UnitRange{Int}}
-end
-function iterate(s::SimpleCartesian, state::Vector{Int}=Int[1 for _ in 1:length(s.ranges)])
-    state[end] > last(s.ranges[end]) && return nothing
-    vals = copy(state)
-    any = false
-    for i = 1:length(s.ranges)
-        if state[i] < last(s.ranges[i])
-            for j = 1:(i-1)
-                state[j] = first(s.ranges[j])
-            end
-            state[i] += 1
-            any = true
-            break
-        end
-    end
-    if !any
-        state[end] += 1
-    end
-    (vals, state)
-end
-
-# Given a signure, iterate over the signatures to union split over
-struct UnionSplitSignature
-    it::SimpleCartesian
-    typs::Vector{Any}
-end
-
-function UnionSplitSignature(atypes::Vector{Any})
-    typs = Any[uniontypes(widenconst(atypes[i])) for i = 1:length(atypes)]
-    ranges = UnitRange{Int}[1:length(typs[i]) for i = 1:length(typs)]
-    return UnionSplitSignature(SimpleCartesian(ranges), typs)
-end
-
-function iterate(split::UnionSplitSignature, state::Vector{Int}...)
-    y = iterate(split.it, state...)
-    y === nothing && return nothing
-    idxs, state = y
-    sig = Any[split.typs[i][j] for (i, j) in enumerate(idxs)]
-    return (sig, state)
-end
-
 function handle_single_case!(ir::IRCode, stmt::Expr, idx::Int, @nospecialize(case), isinvoke::Bool, todo::Vector{Pair{Int, Any}})
     if isa(case, ConstantCase)
         ir[SSAValue(idx)] = case.val
diff --git a/base/compiler/typeinfer.jl b/base/compiler/typeinfer.jl
@@ -785,11 +785,13 @@ function typeinf_edge(interp::AbstractInterpreter, method::Method, @nospecialize
             rettype = code.rettype
             if isdefined(code, :rettype_const)
                 rettype_const = code.rettype_const
+                # the second subtyping conditions are necessary to distinguish usual cases
+                # from rare cases when `Const` wrapped those extended lattice type objects
                 if isa(rettype_const, Vector{Any}) && !(Vector{Any} <: rettype)
                     return PartialStruct(rettype, rettype_const), mi
-                elseif rettype <: Core.OpaqueClosure && isa(rettype_const, PartialOpaque)
+                elseif isa(rettype_const, PartialOpaque) && rettype <: Core.OpaqueClosure
                     return rettype_const, mi
-                elseif isa(rettype_const, InterConditional)
+                elseif isa(rettype_const, InterConditional) && !(InterConditional <: rettype)
                     return rettype_const, mi
                 else
                     return Const(rettype_const), mi
diff --git a/base/strings/util.jl b/base/strings/util.jl
@@ -399,6 +399,9 @@ The default of `limit=0` implies no maximum.
 
 See also [`split`](@ref).
 
+!!! compat "Julia 1.8"
+    The `eachsplit` function requires at least Julia 1.8.
+
 # Examples
 ```jldoctest
 julia> a = "Ma.rch"
diff --git a/base/tuple.jl b/base/tuple.jl
@@ -354,6 +354,23 @@ _totuple(::Type{Tuple}, itr::NamedTuple) = (itr...,)
 
 end
 
+## find ##
+
+_findfirst_rec(f, i::Int, ::Tuple{}) = nothing
+_findfirst_rec(f, i::Int, t::Tuple) = (@inline; f(first(t)) ? i : _findfirst_rec(f, i+1, tail(t)))
+function _findfirst_loop(f::Function, t)
+    for i in 1:length(t)
+        f(t[i]) && return i
+    end
+    return nothing
+end
+findfirst(f::Function, t::Tuple) = length(t) < 32 ? _findfirst_rec(f, 1, t) : _findfirst_loop(f, t)
+
+function findlast(f::Function, x::Tuple)
+    r = findfirst(f, reverse(x))
+    return isnothing(r) ? r : length(x) - r + 1
+end
+
 ## filter ##
 
 filter_rec(f, xs::Tuple) = afoldl((ys, x) -> f(x) ? (ys..., x) : ys, (), xs...)
diff --git a/cli/loader_exe.c b/cli/loader_exe.c
@@ -36,8 +36,8 @@ int main(int argc, char * argv[])
 {
 #endif
 
-#ifdef _COMPILER_ASAN_ENABLED_
-    // ASAN does not support RTLD_DEEPBIND
+#if defined(_COMPILER_ASAN_ENABLED_) || defined(_COMPILER_TSAN_ENABLED_)
+    // ASAN/TSAN do not support RTLD_DEEPBIND
     // https://github.com/google/sanitizers/issues/611
     putenv("LBT_USE_RTLD_DEEPBIND=0");
 #endif
diff --git a/deps/llvm-options.mk b/deps/llvm-options.mk
@@ -15,9 +15,13 @@ LLVM_FLAVOR := $(LLVM_BUILDTYPE)
 ifeq ($(LLVM_SANITIZE),1)
 ifeq ($(SANITIZE_MEMORY),1)
 LLVM_BUILDTYPE := $(LLVM_BUILDTYPE)+MSAN
-else
+endif
+ifeq ($(SANITIZE_ADDRESS),1)
 LLVM_BUILDTYPE := $(LLVM_BUILDTYPE)+ASAN
 endif
+ifeq ($(SANITIZE_THREAD),1)
+LLVM_BUILDTYPE := $(LLVM_BUILDTYPE)+TSAN
+endif
 endif
 
 
diff --git a/deps/llvm.mk b/deps/llvm.mk
@@ -177,12 +177,19 @@ LLVM_CFLAGS += -fsanitize=memory -fsanitize-memory-track-origins
 LLVM_LDFLAGS += -fsanitize=memory -fsanitize-memory-track-origins
 LLVM_CXXFLAGS += -fsanitize=memory -fsanitize-memory-track-origins
 LLVM_CMAKE += -DLLVM_USE_SANITIZER="MemoryWithOrigins"
-else
+endif
+ifeq ($(SANITIZE_ADDRESS),1)
 LLVM_CFLAGS += -fsanitize=address
 LLVM_LDFLAGS += -fsanitize=address
 LLVM_CXXFLAGS += -fsanitize=address
 LLVM_CMAKE += -DLLVM_USE_SANITIZER="Address"
 endif
+ifeq ($(SANITIZE_THREAD),1)
+LLVM_CFLAGS += -fsanitize=thread
+LLVM_LDFLAGS += -fsanitize=thread
+LLVM_CXXFLAGS += -fsanitize=thread
+LLVM_CMAKE += -DLLVM_USE_SANITIZER="Thread"
+endif
 endif # LLVM_SANITIZE
 
 ifeq ($(LLVM_LTO),1)
diff --git a/deps/openblas.mk b/deps/openblas.mk
@@ -105,7 +105,7 @@ $(BUILDDIR)/$(OPENBLAS_SRC_DIR)/openblas-ofast-power.patch-applied: $(BUILDDIR)/
 
 $(BUILDDIR)/$(OPENBLAS_SRC_DIR)/neoverse-generic-kernels.patch-applied: $(BUILDDIR)/$(OPENBLAS_SRC_DIR)/openblas-ofast-power.patch-applied
 	cd $(BUILDDIR)/$(OPENBLAS_SRC_DIR) && \
-		patch -p1 -f < $(SRCDIR)/patches/openblas-exshift.patch
+		patch -p1 -f < $(SRCDIR)/patches/neoverse-generic-kernels.patch
 	echo 1 > $@
 
 $(BUILDDIR)/$(OPENBLAS_SRC_DIR)/build-configured: $(BUILDDIR)/$(OPENBLAS_SRC_DIR)/neoverse-generic-kernels.patch-applied
diff --git a/src/llvm-final-gc-lowering.cpp b/src/llvm-final-gc-lowering.cpp
@@ -218,10 +218,11 @@ bool FinalLowerGC::doInitialization(Module &M) {
 
 bool FinalLowerGC::doFinalization(Module &M)
 {
+    GlobalValue *functionList[] = {queueRootFunc, poolAllocFunc, bigAllocFunc};
+    queueRootFunc = poolAllocFunc = bigAllocFunc = nullptr;
     auto used = M.getGlobalVariable("llvm.compiler.used");
     if (!used)
         return false;
-    GlobalValue *functionList[] = {queueRootFunc, poolAllocFunc, bigAllocFunc};
     SmallPtrSet<Constant*, 16> InitAsSet(
         functionList,
         functionList + sizeof(functionList) / sizeof(void*));
diff --git a/src/llvm-late-gc-lowering.cpp b/src/llvm-late-gc-lowering.cpp
diff --git a/stdlib/Distributed/test/distributed_exec.jl b/stdlib/Distributed/test/distributed_exec.jl
diff --git a/test/compiler/inference.jl b/test/compiler/inference.jl
diff --git a/test/tuple.jl b/test/tuple.jl
