From b2f4e525e351d9ece4553eaf549f9b250438f4a2 Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Tue, 6 Jul 2021 20:24:45 -0700 Subject: [PATCH 01/73] Add embedding and whitespace jobs to buildkite (#41321) * Add "Check whitespace" buildkite job * Add `key` values for later `wait` blocks * Add "embedding tests" to buildkite configuration * Eliminate unnecessary work in buildkite builds Don't bother to precompile the Julia system image like we normally would want to if we're just going to run things once. Also use `JULIA_NUM_CORES` instead of hard-coding `-j 6` into the buildsystem. * Update embedding.yml * Update whitespace.yml (cherry picked from commit a2af84564cb96d36b37c632bf02b45ec5b7fc116) --- .buildkite/embedding.yml | 34 ++++++++++++++++++++++++++++++++++ .buildkite/llvm_passes.yml | 14 ++++++++------ .buildkite/pipeline.yml | 2 ++ .buildkite/whitespace.yml | 24 ++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 6 deletions(-) create mode 100644 .buildkite/embedding.yml create mode 100644 .buildkite/whitespace.yml diff --git a/.buildkite/embedding.yml b/.buildkite/embedding.yml new file mode 100644 index 0000000000000..a7cd2611033e5 --- /dev/null +++ b/.buildkite/embedding.yml @@ -0,0 +1,34 @@ +# These steps should only run on `sandbox.jl` machines, not `docker`-isolated ones +# since we need nestable sandboxing. The rootfs images being used here are built from +# the `.buildkite/rootfs_images/llvm-passes.jl` file. +agents: + queue: "julia" + # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing + sandbox.jl: "true" + os: "linux" + +steps: + - label: "Run embedding tests" + key: embedding + plugins: + - JuliaCI/julia#v1: + version: 1.6 + - staticfloat/sandbox#v1: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v1/llvm-passes.tar.gz + rootfs_treehash: "f3ed53f159e8f13edfba8b20ebdb8ece73c1b8a8" + uid: 1000 + gid: 1000 + commands: | + prefix="/tmp/prefix" + echo "+++ Build julia, deploy to $${prefix}" + make -j$${JULIA_NUM_CORES} JULIA_PRECOMPILE=0 prefix=$${prefix} install + + embedding_output="/tmp/embedding-test" + echo "+++ Run embedding tests, deploy to $${embedding_output}" + mkdir -p "$${embedding_output}" + make -j$${JULIA_NUM_CORES} -C test/embedding JULIA="$${prefix}/bin/julia" BIN="$${embedding_output}" + + timeout_in_minutes: 60 + notify: + - github_commit_status: + context: "embedding" diff --git a/.buildkite/llvm_passes.yml b/.buildkite/llvm_passes.yml index 862f748c18499..7beb30e418bb2 100644 --- a/.buildkite/llvm_passes.yml +++ b/.buildkite/llvm_passes.yml @@ -9,6 +9,7 @@ agents: steps: - label: "analyzegc" + key: analyzegc plugins: - JuliaCI/julia#v1: version: 1.6 @@ -17,13 +18,14 @@ steps: rootfs_treehash: "f3ed53f159e8f13edfba8b20ebdb8ece73c1b8a8" commands: | echo "--- Install in-tree LLVM dependencies" - make -j 6 -C deps install-llvm install-clang install-llvm-tools install-libuv install-utf8proc install-unwind + make -j$${JULIA_NUM_CORES} -C deps install-llvm install-clang install-llvm-tools install-libuv install-utf8proc install-unwind echo "+++ run clangsa/analyzegc" - make -j 6 -C test/clangsa - make -j 6 -C src analyzegc + make -j$${JULIA_NUM_CORES} -C test/clangsa + make -j$${JULIA_NUM_CORES} -C src analyzegc timeout_in_minutes: 60 - label: "llvmpasses" + key: llvmpasses plugins: - JuliaCI/julia#v1: version: 1.6 @@ -34,7 +36,7 @@ steps: gid: 1000 commands: | echo "+++ run llvmpasses" - make -j 6 release - make -j 6 -C src install-analysis-deps - make -j 6 -C test/llvmpasses + make -j$${JULIA_NUM_CORES} release JULIA_PRECOMPILE=0 + make -j$${JULIA_NUM_CORES} -C src install-analysis-deps + make -j$${JULIA_NUM_CORES} -C test/llvmpasses timeout_in_minutes: 60 diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index d76f3fd77bd4f..0dee5f2aaa3f7 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -14,6 +14,8 @@ steps: - label: ":buildkite: Launch unsigned pipelines" commands: | + buildkite-agent pipeline upload .buildkite/whitespace.yml buildkite-agent pipeline upload .buildkite/llvm_passes.yml + buildkite-agent pipeline upload .buildkite/embedding.yml agents: queue: julia diff --git a/.buildkite/whitespace.yml b/.buildkite/whitespace.yml new file mode 100644 index 0000000000000..94dcac590a8ca --- /dev/null +++ b/.buildkite/whitespace.yml @@ -0,0 +1,24 @@ +# These steps should only run on `sandbox.jl` machines, not `docker`-isolated ones +# since we need nestable sandboxing. The rootfs images being used here are built from +# the `.buildkite/rootfs_images/llvm-passes.jl` file. +agents: + queue: "julia" + # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing + sandbox.jl: "true" + os: "linux" + +steps: + - label: "Check whitespace" + key: whitespace + plugins: + - JuliaCI/julia#v1: + version: 1.6 + - staticfloat/sandbox#v1: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v1/llvm-passes.tar.gz + rootfs_treehash: "f3ed53f159e8f13edfba8b20ebdb8ece73c1b8a8" + commands: | + make -j$${JULIA_NUM_CORES} check-whitespace + timeout_in_minutes: 10 + notify: + - github_commit_status: + context: "whitespace" From 537a1d274716bda6c1467b1789ec8b9adfc253ca Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Tue, 6 Jul 2021 20:30:34 -0700 Subject: [PATCH 02/73] Comment out signed pipeline test (#41493) * Comment out signed pipeline test This is confirmed working, so let's comment it out until it's actually used by a codesigning step or similar. * Specifically notify llvm passes (cherry picked from commit f2d73006fbffbe91d00f9c32f5c9a1082e255648) --- .buildkite/0_webui.yml | 6 +++--- .buildkite/llvm_passes.yml | 6 ++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.buildkite/0_webui.yml b/.buildkite/0_webui.yml index d5ba4e0ea7cf9..aed6a5e59d9c3 100644 --- a/.buildkite/0_webui.yml +++ b/.buildkite/0_webui.yml @@ -19,6 +19,6 @@ steps: # Our signed pipelines must have a `signature` or `signature_file` parameter that # verifies the treehash of the pipeline itself and the inputs listed in `inputs` - signed_pipelines: - - pipeline: .buildkite/signed_pipeline_test.yml - signature: "U2FsdGVkX18aZgryp6AJTArgD2uOnVWyFFGVOP5qsY4WbGQ/LVAcYiMEp9cweV+2iht+vmEF949CuuGTeQPA1fKlhPwkG3nZ688752DUB6en9oM2nuL31NoDKWHhpygZ" + #signed_pipelines: + # - pipeline: .buildkite/signed_pipeline_test.yml + # signature: "U2FsdGVkX18aZgryp6AJTArgD2uOnVWyFFGVOP5qsY4WbGQ/LVAcYiMEp9cweV+2iht+vmEF949CuuGTeQPA1fKlhPwkG3nZ688752DUB6en9oM2nuL31NoDKWHhpygZ" diff --git a/.buildkite/llvm_passes.yml b/.buildkite/llvm_passes.yml index 7beb30e418bb2..27611a02b8391 100644 --- a/.buildkite/llvm_passes.yml +++ b/.buildkite/llvm_passes.yml @@ -23,6 +23,9 @@ steps: make -j$${JULIA_NUM_CORES} -C test/clangsa make -j$${JULIA_NUM_CORES} -C src analyzegc timeout_in_minutes: 60 + notify: + - github_commit_status: + context: "analyzegc" - label: "llvmpasses" key: llvmpasses @@ -40,3 +43,6 @@ steps: make -j$${JULIA_NUM_CORES} -C src install-analysis-deps make -j$${JULIA_NUM_CORES} -C test/llvmpasses timeout_in_minutes: 60 + notify: + - github_commit_status: + context: "llvm passes" From 63c02571676b63570d3bae01f7e5e39f264b93f2 Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Tue, 6 Jul 2021 23:02:58 -0700 Subject: [PATCH 03/73] Streamline buildkite configuration a bit (#41494) * Add `/cache/repos` as a mapping into the CI sandbox This should allow `git` to find its cached objects properly, which should silence the warnings on CI, and also give us the proper git version info within buildkite builds * Break up `llvmpasses` output a bit * Provide `/cache/repos` for `whitespace` as well * Give a positive message if whitespace check passes It's a little unnerving to have a silent command block in buildkite, so let's output a success message if everything is on the up-and-up (cherry picked from commit 460e981de68169acea23cbe768ea4f8be50d7b85) --- .buildkite/embedding.yml | 3 +++ .buildkite/llvm_passes.yml | 9 ++++++++- .buildkite/whitespace.yml | 2 ++ contrib/check-whitespace.sh | 2 ++ 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/.buildkite/embedding.yml b/.buildkite/embedding.yml index a7cd2611033e5..faffeb1f7cc87 100644 --- a/.buildkite/embedding.yml +++ b/.buildkite/embedding.yml @@ -18,6 +18,9 @@ steps: rootfs_treehash: "f3ed53f159e8f13edfba8b20ebdb8ece73c1b8a8" uid: 1000 gid: 1000 + workspaces: + # Include `/cache/repos` so that our `git` version introspection works. + - "/cache/repos:/cache/repos" commands: | prefix="/tmp/prefix" echo "+++ Build julia, deploy to $${prefix}" diff --git a/.buildkite/llvm_passes.yml b/.buildkite/llvm_passes.yml index 27611a02b8391..b4311f689d089 100644 --- a/.buildkite/llvm_passes.yml +++ b/.buildkite/llvm_passes.yml @@ -16,6 +16,9 @@ steps: - staticfloat/sandbox#v1: rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v1/llvm-passes.tar.gz rootfs_treehash: "f3ed53f159e8f13edfba8b20ebdb8ece73c1b8a8" + workspaces: + # Include `/cache/repos` so that our `git` version introspection works. + - "/cache/repos:/cache/repos" commands: | echo "--- Install in-tree LLVM dependencies" make -j$${JULIA_NUM_CORES} -C deps install-llvm install-clang install-llvm-tools install-libuv install-utf8proc install-unwind @@ -37,10 +40,14 @@ steps: rootfs_treehash: "f3ed53f159e8f13edfba8b20ebdb8ece73c1b8a8" uid: 1000 gid: 1000 + workspaces: + - "/cache/repos:/cache/repos" commands: | - echo "+++ run llvmpasses" + echo "--- make release" make -j$${JULIA_NUM_CORES} release JULIA_PRECOMPILE=0 + echo "--- make src/install-analysis-deps" make -j$${JULIA_NUM_CORES} -C src install-analysis-deps + echo "+++ make test/llvmpasses" make -j$${JULIA_NUM_CORES} -C test/llvmpasses timeout_in_minutes: 60 notify: diff --git a/.buildkite/whitespace.yml b/.buildkite/whitespace.yml index 94dcac590a8ca..37fa66f75a3d9 100644 --- a/.buildkite/whitespace.yml +++ b/.buildkite/whitespace.yml @@ -16,6 +16,8 @@ steps: - staticfloat/sandbox#v1: rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v1/llvm-passes.tar.gz rootfs_treehash: "f3ed53f159e8f13edfba8b20ebdb8ece73c1b8a8" + workspaces: + - "/cache/repos:/cache/repos" commands: | make -j$${JULIA_NUM_CORES} check-whitespace timeout_in_minutes: 10 diff --git a/contrib/check-whitespace.sh b/contrib/check-whitespace.sh index c380d7bdd2969..ff5bd24ab2cbe 100755 --- a/contrib/check-whitespace.sh +++ b/contrib/check-whitespace.sh @@ -35,3 +35,5 @@ if git --no-pager grep --color -n --full-name -e ' $' -- $file_patterns; then echo "and then a forced push of the correct branch" exit 1 fi + +echo "Whitespace check found no issues" From 2916959c78c704aa563eb746d4b9522518c65fc5 Mon Sep 17 00:00:00 2001 From: Pete Vilter <7341+vilterp@users.noreply.github.com> Date: Wed, 7 Jul 2021 18:58:21 -0400 Subject: [PATCH 04/73] add a GC safepoint in Task.wait (#41441) (cherry picked from commit 97f817a379b0c3c5f9bb803427fe88a018ebfe18) --- base/task.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/base/task.jl b/base/task.jl index 0d37e1fe9eae2..db586d59d02ca 100644 --- a/base/task.jl +++ b/base/task.jl @@ -764,6 +764,7 @@ end end function wait() + GC.safepoint() W = Workqueues[Threads.threadid()] poptask(W) result = try_yieldto(ensure_rescheduled) From e4b9fd9da0686a7ffc5da95f37a21710aa2244f1 Mon Sep 17 00:00:00 2001 From: Simeon Schaub Date: Fri, 12 Feb 2021 22:25:57 +0100 Subject: [PATCH 05/73] improve constraint propagation with multiple || (#39618) fixes #39611 (cherry picked from commit abd56cdfbdd3ecf3806f4b4117479cdeff26a3c2) --- src/julia-syntax.scm | 40 +++++++++++++++++++++++++------------- test/broadcast.jl | 19 +++++------------- test/compiler/inference.jl | 8 ++++++++ 3 files changed, 39 insertions(+), 28 deletions(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 9b4b6abc27f71..4debe7324598e 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -1926,8 +1926,8 @@ (blk? (and (pair? test) (eq? (car test) 'block))) (stmts (if blk? (cdr (butlast test)) '())) (test (if blk? (last test) test))) - (if (and (pair? test) (eq? (car test) '&&)) - (let ((clauses `(&& ,@(map expand-forms (cdr (flatten-ex '&& test)))))) + (if (and (pair? test) (memq (car test) '(&& |\|\||))) + (let ((clauses `(,(car test) ,@(map expand-forms (cdr (flatten-ex (car test) test)))))) `(if ,(if blk? `(block ,@(map expand-forms stmts) ,clauses) clauses) @@ -4086,18 +4086,30 @@ f(x) = yt(x) (compile (cadr e) break-labels value tail) #f)) ((if elseif) - (let ((tests (let* ((cond (cadr e)) - (cond (if (and (pair? cond) (eq? (car cond) 'block)) - (begin (if (length> cond 2) (compile (butlast cond) break-labels #f #f)) - (last cond)) - cond))) - (map (lambda (clause) - (emit `(gotoifnot ,(compile-cond clause break-labels) _))) - (if (and (pair? cond) (eq? (car cond) '&&)) - (cdr cond) - (list cond))))) - (end-jump `(goto _)) - (val (if (and value (not tail)) (new-mutable-var) #f))) + (let* ((cnd (cadr e)) + (cnd (if (and (pair? cnd) (eq? (car cnd) 'block)) + (begin (if (length> cnd 2) (compile (butlast cnd) break-labels #f #f)) + (last cnd)) + cnd)) + (or? (and (pair? cnd) (eq? (car cnd) '|\|\||))) + (tests (if or? + (let ((short-circuit `(goto _))) + (for-each + (lambda (clause) + (let ((jmp (emit `(gotoifnot ,(compile-cond clause break-labels) _)))) + (emit short-circuit) + (set-car! (cddr jmp) (make&mark-label)))) + (butlast (cdr cnd))) + (let ((last-jmp (emit `(gotoifnot ,(compile-cond (last (cdr cnd)) break-labels) _)))) + (set-car! (cdr short-circuit) (make&mark-label)) + (list last-jmp))) + (map (lambda (clause) + (emit `(gotoifnot ,(compile-cond clause break-labels) _))) + (if (and (pair? cnd) (eq? (car cnd) '&&)) + (cdr cnd) + (list cnd))))) + (end-jump `(goto _)) + (val (if (and value (not tail)) (new-mutable-var) #f))) (let ((v1 (compile (caddr e) break-labels value tail))) (if val (emit-assignment val v1)) (if (and (not tail) (or (length> e 3) val)) diff --git a/test/broadcast.jl b/test/broadcast.jl index dff306ee27c11..0cfede78afadf 100644 --- a/test/broadcast.jl +++ b/test/broadcast.jl @@ -516,7 +516,7 @@ Base.BroadcastStyle(::Type{T}) where {T<:AD2Dim} = AD2DimStyle() @test a .+ 1 .* 2 == @inferred(fadd2(aa)) @test a .* a' == @inferred(fprod(aa)) @test isequal(a .+ [missing; 1:9], fadd3(aa)) - @test_broken Core.Compiler.return_type(fadd3, (typeof(aa),)) <: Array19745{<:Union{Float64, Missing}} + @test Core.Compiler.return_type(fadd3, (typeof(aa),)) <: Array19745{<:Union{Float64, Missing}} @test isa(aa .+ 1, Array19745) @test isa(aa .+ 1 .* 2, Array19745) @test isa(aa .* aa', Array19745) @@ -953,29 +953,20 @@ p0 = copy(p) @testset "Issue #28382: inferrability of broadcast with Union eltype" begin @test isequal([1, 2] .+ [3.0, missing], [4.0, missing]) - @test_broken Core.Compiler.return_type(broadcast, Tuple{typeof(+), Vector{Int}, - Vector{Union{Float64, Missing}}}) == - Vector{<:Union{Float64, Missing}} @test Core.Compiler.return_type(broadcast, Tuple{typeof(+), Vector{Int}, Vector{Union{Float64, Missing}}}) == - AbstractVector{<:Union{Float64, Missing}} + Vector{<:Union{Float64, Missing}} @test isequal([1, 2] + [3.0, missing], [4.0, missing]) - @test_broken Core.Compiler.return_type(+, Tuple{Vector{Int}, - Vector{Union{Float64, Missing}}}) == + @test Core.Compiler.return_type(+, Tuple{Vector{Int}, + Vector{Union{Float64, Missing}}}) == Vector{<:Union{Float64, Missing}} @test Core.Compiler.return_type(+, Tuple{Vector{Int}, Vector{Union{Float64, Missing}}}) == - AbstractVector{<:Union{Float64, Missing}} - @test_broken Core.Compiler.return_type(+, Tuple{Vector{Int}, - Vector{Union{Float64, Missing}}}) == Vector{<:Union{Float64, Missing}} @test isequal(tuple.([1, 2], [3.0, missing]), [(1, 3.0), (2, missing)]) - @test_broken Core.Compiler.return_type(broadcast, Tuple{typeof(tuple), Vector{Int}, - Vector{Union{Float64, Missing}}}) == - Vector{<:Tuple{Int, Any}} @test Core.Compiler.return_type(broadcast, Tuple{typeof(tuple), Vector{Int}, Vector{Union{Float64, Missing}}}) == - AbstractVector{<:Tuple{Int, Any}} + Vector{<:Tuple{Int, Any}} # Check that corner cases do not throw an error @test isequal(broadcast(x -> x === 1 ? nothing : x, [1, 2, missing]), [nothing, 2, missing]) diff --git a/test/compiler/inference.jl b/test/compiler/inference.jl index a2cc39e98e9b2..1cbf36810f4d5 100644 --- a/test/compiler/inference.jl +++ b/test/compiler/inference.jl @@ -2994,3 +2994,11 @@ end # issue #40804 @test Base.return_types(()) do; ===(); end == Any[Union{}] @test Base.return_types(()) do; typeassert(); end == Any[Union{}] + +# issue #39611 +Base.return_types((Union{Int,Nothing},)) do x + if x === nothing || x < 0 + return 0 + end + x +end == [Int] From dede9df0425828926ac6a0e81547244ede64dbc0 Mon Sep 17 00:00:00 2001 From: Simeon Schaub Date: Wed, 17 Feb 2021 21:33:33 +0100 Subject: [PATCH 06/73] fix #39705: lowering of Expr(:||) (#39709) (cherry picked from commit 0858864409596d158d2ab7f5bc49308c1f718bdc) --- src/julia-syntax.scm | 5 ++++- test/cartesian.jl | 4 ++++ test/syntax.jl | 4 ++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 4debe7324598e..48b175680c535 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -1927,7 +1927,10 @@ (stmts (if blk? (cdr (butlast test)) '())) (test (if blk? (last test) test))) (if (and (pair? test) (memq (car test) '(&& |\|\||))) - (let ((clauses `(,(car test) ,@(map expand-forms (cdr (flatten-ex (car test) test)))))) + (let* ((clauses `(,(car test) ,@(map expand-forms (cdr (flatten-ex (car test) test))))) + (clauses (if (null? (cdr clauses)) + (if (eq? (car clauses) '&&) '(true) '(false)) + clauses))) `(if ,(if blk? `(block ,@(map expand-forms stmts) ,clauses) clauses) diff --git a/test/cartesian.jl b/test/cartesian.jl index 32c3e1fb0a4e6..af0bc466e3d04 100644 --- a/test/cartesian.jl +++ b/test/cartesian.jl @@ -405,3 +405,7 @@ end @test @inferred(intersect(I, J)) == CartesianIndices((2:3, 4:5)) end + +# issue #39705 +f39705() = Base.Cartesian.@nany 0 _ -> true +@test f39705() === false diff --git a/test/syntax.jl b/test/syntax.jl index 68f32b600b79d..b032283523c4f 100644 --- a/test/syntax.jl +++ b/test/syntax.jl @@ -2700,3 +2700,7 @@ end @test eval(Expr(:string, "a", Expr(:string, "b", "c"))) == "abc" @test eval(Expr(:string, "a", Expr(:string, "b", Expr(:string, "c")))) == "abc" + +# issue #39705 +@eval f39705(x) = $(Expr(:||)) && x +@test f39705(1) === false From 3f36bc7be45b4b090f41f60c38f100f89f457a08 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Sun, 11 Jul 2021 01:19:12 -0400 Subject: [PATCH 07/73] CI (Buildkite): use a consistent naming scheme for labels, keys, and commit statuses (#41543) (cherry picked from commit ffc340c07ed15e1b1960ab7c900ef5203e3a0fad) --- .buildkite/embedding.yml | 4 ++-- .buildkite/llvm_passes.yml | 6 +++--- .buildkite/whitespace.yml | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.buildkite/embedding.yml b/.buildkite/embedding.yml index faffeb1f7cc87..5cf6b985573f5 100644 --- a/.buildkite/embedding.yml +++ b/.buildkite/embedding.yml @@ -8,8 +8,8 @@ agents: os: "linux" steps: - - label: "Run embedding tests" - key: embedding + - label: "embedding" + key: "embedding" plugins: - JuliaCI/julia#v1: version: 1.6 diff --git a/.buildkite/llvm_passes.yml b/.buildkite/llvm_passes.yml index b4311f689d089..7453af82c1e73 100644 --- a/.buildkite/llvm_passes.yml +++ b/.buildkite/llvm_passes.yml @@ -9,7 +9,7 @@ agents: steps: - label: "analyzegc" - key: analyzegc + key: "analyzegc" plugins: - JuliaCI/julia#v1: version: 1.6 @@ -31,7 +31,7 @@ steps: context: "analyzegc" - label: "llvmpasses" - key: llvmpasses + key: "llvmpasses" plugins: - JuliaCI/julia#v1: version: 1.6 @@ -52,4 +52,4 @@ steps: timeout_in_minutes: 60 notify: - github_commit_status: - context: "llvm passes" + context: "llvmpasses" diff --git a/.buildkite/whitespace.yml b/.buildkite/whitespace.yml index 37fa66f75a3d9..bd625e855c5b5 100644 --- a/.buildkite/whitespace.yml +++ b/.buildkite/whitespace.yml @@ -8,8 +8,8 @@ agents: os: "linux" steps: - - label: "Check whitespace" - key: whitespace + - label: "whitespace" + key: "whitespace" plugins: - JuliaCI/julia#v1: version: 1.6 From 1c93dc488b3334c5bbef90be41f47dc219e76c27 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Sun, 11 Jul 2021 17:01:58 -0400 Subject: [PATCH 08/73] CI: Add more Debian packages to the `llvm-passes` rootfs image (#41544) * Add `less` to the `llvm-passes` rootfs image Also, sort the entries of `packages` in alphabetical order. * Update llvm-passes.jl (cherry picked from commit 516af1b182e38aefd4223b740e4c6fef7aea4797) --- .buildkite/rootfs_images/llvm-passes.jl | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.buildkite/rootfs_images/llvm-passes.jl b/.buildkite/rootfs_images/llvm-passes.jl index 17c9588f75c9e..2d9e47ba4f0fc 100755 --- a/.buildkite/rootfs_images/llvm-passes.jl +++ b/.buildkite/rootfs_images/llvm-passes.jl @@ -8,18 +8,20 @@ include("rootfs_utils.jl") # Build debian-based image with the following extra packages: packages = [ + "bash", "build-essential", + "cmake", + "curl", + "gfortran", + "git", + "less", "libatomic1", + "m4", + "perl", + "pkg-config", "python", "python3", - "gfortran", - "perl", "wget", - "m4", - "cmake", - "pkg-config", - "curl", - "git", ] tarball_path = debootstrap("llvm-passes"; packages) From 04e722581ad7ec8ac89826cd600b2481b08fd572 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Mon, 12 Jul 2021 13:53:43 -0400 Subject: [PATCH 09/73] CI (Buildkite): rootfs_images: allow user to override the default value of `tag_name` (#41549) (cherry picked from commit a9232d67203e6f1aca4934318d86f85cad50550b) --- .buildkite/rootfs_images/llvm-passes.jl | 2 +- .buildkite/rootfs_images/rootfs_utils.jl | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.buildkite/rootfs_images/llvm-passes.jl b/.buildkite/rootfs_images/llvm-passes.jl index 2d9e47ba4f0fc..3df2de49522c9 100755 --- a/.buildkite/rootfs_images/llvm-passes.jl +++ b/.buildkite/rootfs_images/llvm-passes.jl @@ -26,4 +26,4 @@ packages = [ tarball_path = debootstrap("llvm-passes"; packages) # Upload it -upload_rootfs_image(tarball_path) +upload_rootfs_image(tarball_path; tag_name = "v1") diff --git a/.buildkite/rootfs_images/rootfs_utils.jl b/.buildkite/rootfs_images/rootfs_utils.jl index 7df224a31f740..3412fbcc03f56 100644 --- a/.buildkite/rootfs_images/rootfs_utils.jl +++ b/.buildkite/rootfs_images/rootfs_utils.jl @@ -82,9 +82,10 @@ function debootstrap(name::String; release::String="buster", variant::String="mi return tarball_path end -function upload_rootfs_image(tarball_path::String; github_repo::String="JuliaCI/rootfs-images") +function upload_rootfs_image(tarball_path::String; + github_repo::String="JuliaCI/rootfs-images", + tag_name::String="v1") # Upload it to `github_repo` - tag_name = "v1" tarball_url = "https://github.com/$(github_repo)/releases/download/$(tag_name)/$(basename(tarball_path))" @info("Uploading to $(github_repo)@$(tag_name)", tarball_url) run(`$(ghr_jll.ghr()) -u $(dirname(github_repo)) -r $(basename(github_repo)) -replace $(tag_name) $(tarball_path)`) From 667c8832e50e2c02d2f000e2bdef617dbb42e122 Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Mon, 12 Jul 2021 16:17:15 -0700 Subject: [PATCH 10/73] Fix `get-task-allow` entitlement (#41559) We had apparently misspelled this entitlement, not allowing `lldb` to attach to official macOS builds. (cherry picked from commit ad6740b4f27ddc5a32bc3ee7496383b2fa3edfea) --- contrib/mac/app/Entitlements.plist | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/mac/app/Entitlements.plist b/contrib/mac/app/Entitlements.plist index b84dccb00f95c..95c1a02d58958 100644 --- a/contrib/mac/app/Entitlements.plist +++ b/contrib/mac/app/Entitlements.plist @@ -4,7 +4,7 @@ com.apple.security.automation.apple-events - com.apple.security.cs.get-task-allow + com.apple.security.get-task-allow com.apple.security.cs.allow-dyld-environment-variables From 92765ad3eaf2f06749867a20edd75c947a43c9c0 Mon Sep 17 00:00:00 2001 From: Sukera <11753998+Seelengrab@users.noreply.github.com> Date: Thu, 15 Jul 2021 12:14:27 +0200 Subject: [PATCH 11/73] REPL doc lookup assumed ASCII for the given string, fixes #41589 (#41594) (cherry picked from commit be443ac05451202fc92ca4fd8ad991c6057e0516) --- stdlib/REPL/src/docview.jl | 2 +- stdlib/REPL/test/docview.jl | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/stdlib/REPL/src/docview.jl b/stdlib/REPL/src/docview.jl index 4e805a6369c3c..cac5927ba6352 100644 --- a/stdlib/REPL/src/docview.jl +++ b/stdlib/REPL/src/docview.jl @@ -221,7 +221,7 @@ function lookup_doc(ex) str = string(ex) isdotted = startswith(str, ".") if endswith(str, "=") && Base.operator_precedence(ex) == Base.prec_assignment && ex !== :(:=) - op = str[1:end-1] + op = chop(str) eq = isdotted ? ".=" : "=" return Markdown.parse("`x $op= y` is a synonym for `x $eq x $op y`") elseif isdotted && ex !== :(..) diff --git a/stdlib/REPL/test/docview.jl b/stdlib/REPL/test/docview.jl index a5935a0426434..269086917f861 100644 --- a/stdlib/REPL/test/docview.jl +++ b/stdlib/REPL/test/docview.jl @@ -27,3 +27,7 @@ end # https://github.com/JuliaLang/julia/issues/37757 @test REPL.insert_hlines(IOBuffer(), nothing) === nothing end + +@testset "Unicode doc lookup (#41589)" begin + @test REPL.lookup_doc(:(÷=)) isa Markdown.MD +end From ca236c307d08aadf40931979590a7ff0fbb8308c Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Wed, 14 Jul 2021 13:09:54 -0400 Subject: [PATCH 12/73] CI (rootfs_utils): Require that the user explicitly specify the `tag_name` (#41571) (cherry picked from commit 23dbf169f0e4b831adfd20d666bedd1f669527ca) --- .buildkite/rootfs_images/llvm-passes.jl | 7 ++++++- .buildkite/rootfs_images/rootfs_utils.jl | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.buildkite/rootfs_images/llvm-passes.jl b/.buildkite/rootfs_images/llvm-passes.jl index 3df2de49522c9..9555abe183368 100755 --- a/.buildkite/rootfs_images/llvm-passes.jl +++ b/.buildkite/rootfs_images/llvm-passes.jl @@ -4,6 +4,11 @@ ## Eventually, this image will probably be replaced with the actual builder image, ## as that will have the necessary toolchains as well, but that image is not built yet. +if length(ARGS) != 1 + throw(ArgumentError("Usage: llvm-passes.jl [tag_name]")) +end +const tag_name = convert(String, strip(ARGS[1]))::String + include("rootfs_utils.jl") # Build debian-based image with the following extra packages: @@ -26,4 +31,4 @@ packages = [ tarball_path = debootstrap("llvm-passes"; packages) # Upload it -upload_rootfs_image(tarball_path; tag_name = "v1") +upload_rootfs_image(tarball_path; tag_name) diff --git a/.buildkite/rootfs_images/rootfs_utils.jl b/.buildkite/rootfs_images/rootfs_utils.jl index 3412fbcc03f56..0079b13ae4026 100644 --- a/.buildkite/rootfs_images/rootfs_utils.jl +++ b/.buildkite/rootfs_images/rootfs_utils.jl @@ -84,7 +84,7 @@ end function upload_rootfs_image(tarball_path::String; github_repo::String="JuliaCI/rootfs-images", - tag_name::String="v1") + tag_name::String) # Upload it to `github_repo` tarball_url = "https://github.com/$(github_repo)/releases/download/$(tag_name)/$(basename(tarball_path))" @info("Uploading to $(github_repo)@$(tag_name)", tarball_url) From 5480733cd4bf2051befe38b4580734e6b3763d5c Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Thu, 15 Jul 2021 18:55:37 -0400 Subject: [PATCH 13/73] CI (rootfs images): when uploading a tarball, don't allow it to override an existing tarball (unless the user provides the `--force-overwrite` command-line flag) (#41591) Co-authored-by: Elliot Saba Co-authored-by: Elliot Saba (cherry picked from commit 3ac7c383a1d5825866664939e6e2de694c55d19d) --- .buildkite/rootfs_images/llvm-passes.jl | 9 +++---- .buildkite/rootfs_images/rootfs_utils.jl | 31 ++++++++++++++++++++++-- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/.buildkite/rootfs_images/llvm-passes.jl b/.buildkite/rootfs_images/llvm-passes.jl index 9555abe183368..bc6d57eb2f87e 100755 --- a/.buildkite/rootfs_images/llvm-passes.jl +++ b/.buildkite/rootfs_images/llvm-passes.jl @@ -4,13 +4,10 @@ ## Eventually, this image will probably be replaced with the actual builder image, ## as that will have the necessary toolchains as well, but that image is not built yet. -if length(ARGS) != 1 - throw(ArgumentError("Usage: llvm-passes.jl [tag_name]")) -end -const tag_name = convert(String, strip(ARGS[1]))::String - include("rootfs_utils.jl") +const tag_name, force_overwrite = get_arguments(ARGS, @__FILE__) + # Build debian-based image with the following extra packages: packages = [ "bash", @@ -31,4 +28,4 @@ packages = [ tarball_path = debootstrap("llvm-passes"; packages) # Upload it -upload_rootfs_image(tarball_path; tag_name) +upload_rootfs_image(tarball_path; tag_name, force_overwrite) diff --git a/.buildkite/rootfs_images/rootfs_utils.jl b/.buildkite/rootfs_images/rootfs_utils.jl index 0079b13ae4026..3d31747e0e6f8 100644 --- a/.buildkite/rootfs_images/rootfs_utils.jl +++ b/.buildkite/rootfs_images/rootfs_utils.jl @@ -84,10 +84,37 @@ end function upload_rootfs_image(tarball_path::String; github_repo::String="JuliaCI/rootfs-images", - tag_name::String) + tag_name::String, + force_overwrite::Bool) # Upload it to `github_repo` tarball_url = "https://github.com/$(github_repo)/releases/download/$(tag_name)/$(basename(tarball_path))" @info("Uploading to $(github_repo)@$(tag_name)", tarball_url) - run(`$(ghr_jll.ghr()) -u $(dirname(github_repo)) -r $(basename(github_repo)) -replace $(tag_name) $(tarball_path)`) + replace_flag = force_overwrite ? "-replace" : "" + run(`$(ghr_jll.ghr()) -u $(dirname(github_repo)) -r $(basename(github_repo)) $(replace_flag) $(tag_name) $(tarball_path)`) return tarball_url end + +# process command-line arguments + +function get_arguments(args::AbstractVector, script_file::AbstractString) + usage = "Usage: $(basename(script_file)) [--force-overwrite]" + length(args) < 1 && throw(ArgumentError(usage)) + length(args) > 2 && throw(ArgumentError(usage)) + tag_name = get_tag_name(args; usage) + force_overwrite = get_force_overwrite(args; usage) + return (; tag_name, force_overwrite) +end + +function get_tag_name(args::AbstractVector; usage::AbstractString) + tag_name = convert(String, strip(args[1]))::String + isempty(tag_name) && throw(ArgumentError(usage)) + startswith(tag_name, "--") && throw(ArgumentError(usage)) + return tag_name +end + +function get_force_overwrite(args::AbstractVector; usage::AbstractString) + force_overwrite_string = strip(get(args, 2, "")) + force_overwrite_string == "" && return false + force_overwrite_string == "--force-overwrite" && return true + throw(ArgumentError(usage)) +end From 56fff1008282dba59f0ad1840f71171efb9b1f87 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Fri, 16 Jul 2021 16:19:50 -0400 Subject: [PATCH 14/73] CI (rootfs images): fix a bug in the `upload_rootfs_image` function (#41610) (cherry picked from commit e1aeb8a95ede8212ea09bdb49bdd805f66dfb9d5) --- .buildkite/rootfs_images/rootfs_utils.jl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.buildkite/rootfs_images/rootfs_utils.jl b/.buildkite/rootfs_images/rootfs_utils.jl index 3d31747e0e6f8..82baeec7a0933 100644 --- a/.buildkite/rootfs_images/rootfs_utils.jl +++ b/.buildkite/rootfs_images/rootfs_utils.jl @@ -89,8 +89,11 @@ function upload_rootfs_image(tarball_path::String; # Upload it to `github_repo` tarball_url = "https://github.com/$(github_repo)/releases/download/$(tag_name)/$(basename(tarball_path))" @info("Uploading to $(github_repo)@$(tag_name)", tarball_url) - replace_flag = force_overwrite ? "-replace" : "" - run(`$(ghr_jll.ghr()) -u $(dirname(github_repo)) -r $(basename(github_repo)) $(replace_flag) $(tag_name) $(tarball_path)`) + cmd = ghr_jll.ghr() + append!(cmd.exec, ["-u", dirname(github_repo), "-r", basename(github_repo)]) + force_overwrite && push!(cmd.exec, "-replace") + append!(cmd.exec, [tag_name, tarball_path]) + run(cmd) return tarball_url end From 2fab90bf8e7a5c7a4210ab82e0467b4c1a20605a Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Tue, 13 Jul 2021 08:54:20 -0400 Subject: [PATCH 15/73] add default "auto" as option to --check-bounds (#41551) * add default "auto" as option to --check-bounds * add to NEWS (cherry picked from commit 7d0f7693fd9999f88623dc435e0f83dc9681fbd1) --- NEWS.md | 1 + base/util.jl | 2 +- doc/man/julia.1 | 4 ++-- doc/src/devdocs/boundscheck.md | 4 ++++ doc/src/manual/command-line-options.md | 2 +- src/jloptions.c | 7 +++++-- test/cmdlineargs.jl | 2 ++ 7 files changed, 16 insertions(+), 6 deletions(-) diff --git a/NEWS.md b/NEWS.md index 0ce4a5cc458d5..916781b397dbe 100644 --- a/NEWS.md +++ b/NEWS.md @@ -30,6 +30,7 @@ New language features for example `a, b... = [1, 2, 3]`. This syntax is implemented using `Base.rest`, which can be overloaded to customize its behavior for different collection types ([#37410]). +* The default behavior of observing `@inbounds` declarations is now an option via `auto` in `--check-bounds=yes|no|auto` ([#41551]) Language changes ---------------- diff --git a/base/util.jl b/base/util.jl index 81399720c84a3..0466614278d5e 100644 --- a/base/util.jl +++ b/base/util.jl @@ -154,7 +154,7 @@ function julia_cmd(julia=joinpath(Sys.BINDIR::String, julia_exename())) elseif opts.check_bounds == 2 "no" # off else - "" # "default" + "" # default = "auto" end isempty(check_bounds) || push!(addflags, "--check-bounds=$check_bounds") end diff --git a/doc/man/julia.1 b/doc/man/julia.1 index 49ee30e0af90e..726e478362e4c 100644 --- a/doc/man/julia.1 +++ b/doc/man/julia.1 @@ -158,8 +158,8 @@ Set the level of debug info generation to Control whether inlining is permitted (overrides functions declared as @inline) .TP ---check-bounds={yes|no} -Emit bounds checks always or never (ignoring declarations) +--check-bounds={yes|no|auto} +Emit bounds checks always, never, or respect @inbounds declarations .TP --math-mode={ieee|user} diff --git a/doc/src/devdocs/boundscheck.md b/doc/src/devdocs/boundscheck.md index a9fb3baaae447..c4894469e4092 100644 --- a/doc/src/devdocs/boundscheck.md +++ b/doc/src/devdocs/boundscheck.md @@ -89,3 +89,7 @@ Note this hierarchy has been designed to reduce the likelihood of method ambigui to make `checkbounds` the place to specialize on array type, and try to avoid specializations on index types; conversely, `checkindex` is intended to be specialized only on index type (especially, the last argument). + +## Emit bounds checks + +Julia can be launched with `--check-bounds={yes|no|auto}` to emit bounds checks always, never, or respect @inbounds declarations. diff --git a/doc/src/manual/command-line-options.md b/doc/src/manual/command-line-options.md index b6dd1b6b02722..0efef2fa14ca2 100644 --- a/doc/src/manual/command-line-options.md +++ b/doc/src/manual/command-line-options.md @@ -30,7 +30,7 @@ The following is a complete list of command-line switches available when launchi |`-O`, `--optimize={0,1,2,3}` |Set the optimization level (default level is 2 if unspecified or 3 if used without a level)| |`-g`, `-g ` |Enable / Set the level of debug info generation (default level is 1 if unspecified or 2 if used without a level)| |`--inline={yes\|no}` |Control whether inlining is permitted, including overriding `@inline` declarations| -|`--check-bounds={yes\|no}` |Emit bounds checks always or never (ignoring declarations)| +|`--check-bounds={yes\|no\|auto}` |Emit bounds checks always, never, or respect @inbounds declarations| |`--math-mode={ieee,fast}` |Disallow or enable unsafe floating point optimizations (overrides @fastmath declaration)| |`--code-coverage={none\|user\|all}` |Count executions of source lines| |`--code-coverage` |equivalent to `--code-coverage=user`| diff --git a/src/jloptions.c b/src/jloptions.c index 92affe7679098..1beea6f30f5c3 100644 --- a/src/jloptions.c +++ b/src/jloptions.c @@ -129,7 +129,8 @@ static const char opts[] = " (default level is 1 if unspecified or 2 if used without a level)\n" #endif " --inline={yes|no} Control whether inlining is permitted, including overriding @inline declarations\n" - " --check-bounds={yes|no} Emit bounds checks always or never (ignoring declarations)\n" + " --check-bounds={yes|no|auto}\n" + " Emit bounds checks always, never, or respect @inbounds declarations\n" #ifdef USE_POLLY " --polly={yes|no} Enable or disable the polyhedral optimizer Polly (overrides @polly declaration)\n" #endif @@ -544,8 +545,10 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) jl_options.check_bounds = JL_OPTIONS_CHECK_BOUNDS_ON; else if (!strcmp(optarg,"no")) jl_options.check_bounds = JL_OPTIONS_CHECK_BOUNDS_OFF; + else if (!strcmp(optarg,"auto")) + jl_options.check_bounds = JL_OPTIONS_CHECK_BOUNDS_DEFAULT; else - jl_errorf("julia: invalid argument to --check-bounds={yes|no} (%s)", optarg); + jl_errorf("julia: invalid argument to --check-bounds={yes|no|auto} (%s)", optarg); break; case opt_output_bc: jl_options.outputbc = optarg; diff --git a/test/cmdlineargs.jl b/test/cmdlineargs.jl index 6b600ad40f934..4c69bce4253f8 100644 --- a/test/cmdlineargs.jl +++ b/test/cmdlineargs.jl @@ -378,6 +378,8 @@ let exename = `$(Base.julia_cmd()) --startup-file=no` filter!(a -> !startswith(a, "--check-bounds="), exename_default_checkbounds.exec) @test parse(Int, readchomp(`$exename_default_checkbounds -E "Int(Base.JLOptions().check_bounds)"`)) == JL_OPTIONS_CHECK_BOUNDS_DEFAULT + @test parse(Int, readchomp(`$exename -E "Int(Base.JLOptions().check_bounds)" + --check-bounds=auto`)) == JL_OPTIONS_CHECK_BOUNDS_DEFAULT @test parse(Int, readchomp(`$exename -E "Int(Base.JLOptions().check_bounds)" --check-bounds=yes`)) == JL_OPTIONS_CHECK_BOUNDS_ON @test parse(Int, readchomp(`$exename -E "Int(Base.JLOptions().check_bounds)" From fd6d2941ee8c8c96a7bd83cda573594c7f741029 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Sun, 20 Jun 2021 03:09:58 -0400 Subject: [PATCH 16/73] Transition the `coverage-linux64` pipeline to Buildkite (#41238) * Transition the `coverage-linux64` pipeline to Buildkite * Simplify, run inside of a sandbox * Upload coverage reports to Codecov and Coveralls * Add `COVERALLS_TOKEN` Co-authored-by: Elliot Saba (cherry picked from commit 9d5f31e9231c1d77e24ee820908e32f559e23057) --- .buildkite/coverage-linux64/0_webui.yml | 16 +++ .buildkite/coverage-linux64/README.md | 6 + .buildkite/coverage-linux64/pipeline.yml | 44 +++++++ .buildkite/coverage-linux64/run_tests_base.jl | 36 ++++++ .../coverage-linux64/upload_coverage.jl | 115 ++++++++++++++++++ 5 files changed, 217 insertions(+) create mode 100644 .buildkite/coverage-linux64/0_webui.yml create mode 100644 .buildkite/coverage-linux64/README.md create mode 100644 .buildkite/coverage-linux64/pipeline.yml create mode 100644 .buildkite/coverage-linux64/run_tests_base.jl create mode 100644 .buildkite/coverage-linux64/upload_coverage.jl diff --git a/.buildkite/coverage-linux64/0_webui.yml b/.buildkite/coverage-linux64/0_webui.yml new file mode 100644 index 0000000000000..6dafcd86f6bcd --- /dev/null +++ b/.buildkite/coverage-linux64/0_webui.yml @@ -0,0 +1,16 @@ +# This file represents what is put into the webUI. +# It is purely for keeping track of the changes we make to the webUI configuration; modifying this file has no effect. +# We use the `cryptic` buildkite plugin to provide secrets management, which requires some integration into the WebUI's steps. +agents: + queue: "julia" + sandbox.jl: "true" + +steps: + - label: ":unlock: Unlock secrets, launch pipelines" + plugins: + - staticfloat/cryptic: + # Our signed pipelines must have a `signature` or `signature_file` parameter that + # verifies the treehash of the pipeline itself and the inputs listed in `inputs` + signed_pipelines: + - pipeline: .buildkite/coverage-linux64/pipeline.yml + signature: "U2FsdGVkX18eQWpd3hMYLO5Kd+6K+oBoLk1I6J3qIw7lc6g5/jaeWyq/wralosZCfTzyjS4NstNKFvhQf3KDPEBVElipNvTxoWOjVLRVOrfBqqvTkQN4xVosY/r026Gy" diff --git a/.buildkite/coverage-linux64/README.md b/.buildkite/coverage-linux64/README.md new file mode 100644 index 0000000000000..8cfb31d698225 --- /dev/null +++ b/.buildkite/coverage-linux64/README.md @@ -0,0 +1,6 @@ +# Coverage pipeline + +We run coverage on a separate pipeline, that uses a scheduled build rather than webhooks. +The pipeline is here: https://buildkite.com/julialang/julia-coverage-linux64 + +It contains [its own webui steps](0_webuiy.ml) (listed here in this repository for clarity) and its own [pipeline.yml](pipeline.yml). diff --git a/.buildkite/coverage-linux64/pipeline.yml b/.buildkite/coverage-linux64/pipeline.yml new file mode 100644 index 0000000000000..078b00254c4f9 --- /dev/null +++ b/.buildkite/coverage-linux64/pipeline.yml @@ -0,0 +1,44 @@ +# These steps should only run on `sandbox.jl` machines, not `docker`-isolated ones +# since we need nestable sandboxing. The rootfs images being used here are built from +# the `.buildkite/rootfs_images/llvm-passes.jl` file. +agents: + queue: "julia" + # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing + sandbox.jl: "true" + os: "linux" + +steps: + - label: ":unlock: :coverage: Run coverage test" + plugins: + - staticfloat/cryptic: + variables: + - CODECOV_TOKEN="U2FsdGVkX19l0fhdBabbuiEdysyEabkJLRHfxm7CNRkuGbnwPV365sxxC7Czs/CVcws0N1oB4pVwALRRMe36oA==" + - COVERALLS_TOKEN="U2FsdGVkX19zopI0hMNzzi2UUOvNVFD8Y0iisFnO/ryVxU7Tit8ZEaeN+gxodRx4CosUUh192F1+q3dTMWRIvw==" + - JuliaCI/julia#v1: + version: 1.6 + - staticfloat/sandbox#v1: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v1/llvm-passes.tar.gz + rootfs_treehash: "f3ed53f159e8f13edfba8b20ebdb8ece73c1b8a8" + uid: 1000 + gid: 1000 + commands: | + echo "--- Build Julia from source" + make -j 6 + + echo "--- Print Julia version info" + ./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()' + ./julia -e '@info "" Sys.CPU_THREADS' + # this is necessary to make sure that the LibGit2 tests passes + git config --global init.defaultBranch master + + echo "--- Run Julia tests with code coverage enabled" + # Run the actual tests + ./julia --code-coverage=all --sysimage-native-code=no .buildkite/coverage-linux64/run_tests_base.jl + + echo "--- Process and upload coverage information" + ./julia .buildkite/coverage-linux64/upload_coverage.jl + timeout_in_minutes: 600 # 600 minutes = 10 hours + +# We must accept the signed job id secret in order to propagate secrets +env: + BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} diff --git a/.buildkite/coverage-linux64/run_tests_base.jl b/.buildkite/coverage-linux64/run_tests_base.jl new file mode 100644 index 0000000000000..5386828aa4e14 --- /dev/null +++ b/.buildkite/coverage-linux64/run_tests_base.jl @@ -0,0 +1,36 @@ +# When running this file, make sure to set the `--code-coverage=all` command-line flag. + +# Important note: even if one or more tests fail, we will still exit with status code 0. + +# The reason for this is that we always want to upload code coverage, even if some of the +# tests fail. Therefore, even if the `coverage-linux64` pipeline passes, you should not +# assume that all of the tests passed. If you want to know if all of the tests are passing, +# please look at the status of the `tester_linux64` pipeline. + +const include_tests = String[] + +const exclude_tests = String[] + +empty!(Base.DEPOT_PATH) +push!(Base.DEPOT_PATH, mktempdir(; cleanup = true)) + +module ChooseTests + include(joinpath(dirname(dirname(@__DIR__)), "test", "choosetests.jl")) +end + +const tests = ChooseTests.choosetests() |> + first |> + x -> setdiff(x, exclude_tests) |> + x -> vcat(x, include_tests) |> + unique |> + sort + +const ncores = min(Sys.CPU_THREADS, Threads.nthreads()) + +@info "" ncores Sys.CPU_THREADS Threads.nthreads() + +try + Base.runtests(tests; ncores) +catch ex + @error "" exception=(ex, catch_backtrace()) +end diff --git a/.buildkite/coverage-linux64/upload_coverage.jl b/.buildkite/coverage-linux64/upload_coverage.jl new file mode 100644 index 0000000000000..04ddd9651861e --- /dev/null +++ b/.buildkite/coverage-linux64/upload_coverage.jl @@ -0,0 +1,115 @@ +empty!(Base.DEPOT_PATH) +push!(Base.DEPOT_PATH, mktempdir(; cleanup = true)) + +import Pkg +import Logging +import TOML + +Pkg.add(; name = "Coverage", uuid = "a2441757-f6aa-5fb2-8edb-039e3f45d037", version = "1") +Pkg.precompile() + +import Coverage + +function get_external_stdlib_names(stdlib_dir::AbstractString) + filename_list = filter(x -> isfile(joinpath(stdlib_dir, x)), readdir(stdlib_dir)) + # find all of the files like `Pkg.version`, `Statistics.version`, etc. + regex_matches_or_nothing = match.(Ref(r"^([\w].*?)\.version$"), filename_list) + regex_matches = filter(x -> x !== nothing, regex_matches_or_nothing) + # get the names of the external stdlibs, like `Pkg`, `Statistics`, etc. + external_stdlib_names = only.(regex_matches) + unique!(external_stdlib_names) + sort!(external_stdlib_names) + @info "# Begin list of external stdlibs" + for (i, x) in enumerate(external_stdlib_names) + @info "$(i). $(x)" + end + @info "# End list of external stdlibs" + return external_stdlib_names +end + +function get_external_stdlib_prefixes(stdlib_dir::AbstractString) + external_stdlib_names = get_external_stdlib_names(stdlib_dir) + prefixes_1 = joinpath.(Ref(stdlib_dir), external_stdlib_names, Ref("")) + prefixes_2 = joinpath.(Ref(stdlib_dir), string.(external_stdlib_names, Ref("-"))) + prefixes = vcat(prefixes_1, prefixes_2) + unique!(prefixes) + sort!(prefixes) + # example of what `prefixes` might look like: + # 4-element Vector{String}: + # "stdlib/Pkg-" + # "stdlib/Pkg/" + # "stdlib/Statistics-" + # "stdlib/Statistics/" + return prefixes +end + +function print_coverage_summary(fc::Coverage.FileCoverage) + cov_lines, tot_lines = Coverage.get_summary(fc) + if cov_lines == tot_lines == 0 + cov_pct = 0 + else + cov_pct = floor(Int, cov_lines/tot_lines * 100) + end + pad_1 = 71 + pad_2 = 15 + pad_3 = 15 + col_1 = rpad(fc.filename, pad_1) + col_2 = rpad(string(cov_pct, " %"), pad_2) + col_3 = string( + rpad(string(cov_lines), pad_3), + string(tot_lines), + ) + @info "$(col_1) $(col_2) $(col_3)" + return nothing +end + +function print_coverage_summary( + fcs::Vector{Coverage.FileCoverage}, description::AbstractString, + ) + cov_lines, tot_lines = Coverage.get_summary(fcs) + if cov_lines == tot_lines == 0 + cov_pct = 0 + else + cov_pct = floor(Int, cov_lines/tot_lines * 100) + end + @info "$(description): $(cov_pct)% ($(cov_lines)/$(tot_lines))" + return nothing +end + +# `Coverage.process_folder` will have a LOT of `@info` statements that will make the log +# way too long. So before we run `Coverage.process_folder`, we disable logging for `@info` +# statements. After we run `Coverage.process_folder`, we re-enable logging for `@info` +# statements. +Logging.disable_logging(Logging.Info) +const fcs = Coverage.merge_coverage_counts( + Coverage.process_folder("base"), + Coverage.process_folder("stdlib"), +); +Logging.disable_logging(Logging.Debug) + +# Only include source code files. Exclude test files, benchmarking files, etc. +filter!(fcs) do fc + occursin(r"^base\/", fc.filename) || occursin("/src/", fc.filename) +end; + +# Exclude all external stdlibs (stdlibs that live in external repos). +const external_stdlib_prefixes = get_external_stdlib_prefixes("stdlib") +filter!(fcs) do fc + all(x -> !startswith(fc.filename, x), external_stdlib_prefixes) +end; + +# Exclude all stdlib JLLs (stdlibs of the form `stdlib/*_jll/`). +filter!(fcs) do fc + !occursin(r"^stdlib\/[A-Za-z0-9]*?_jll\/", fc.filename) +end + +sort!(fcs; by = fc -> fc.filename) + +print_coverage_summary.(fcs); +print_coverage_summary(fcs, "Total") + +# In order to upload to Codecov, you need to have the `CODECOV_TOKEN` environment variable defined. +Coverage.Codecov.submit_local(fcs) + +# In order to upload to Coveralls, you need to have the `COVERALLS_TOKEN` environment variable defined. +Coverage.Coveralls.submit_local(fcs) From 570cd0d534c0aba0026f43bd45a77878ffef3310 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Sat, 26 Jun 2021 11:25:24 -0400 Subject: [PATCH 17/73] Code coverage: fix the code coverage pipeline (#41375) * Code coverage: fix the code coverage pipeline * Run all tests (cherry picked from commit 05e41385de563a2e8aeea9000a7e381fd9bb3318) --- .../coverage-linux64/upload_coverage.jl | 136 +++++++++++++++--- 1 file changed, 120 insertions(+), 16 deletions(-) diff --git a/.buildkite/coverage-linux64/upload_coverage.jl b/.buildkite/coverage-linux64/upload_coverage.jl index 04ddd9651861e..8d14cded56140 100644 --- a/.buildkite/coverage-linux64/upload_coverage.jl +++ b/.buildkite/coverage-linux64/upload_coverage.jl @@ -10,6 +10,24 @@ Pkg.precompile() import Coverage +function process_folders() + # `Coverage.process_folder` will have a LOT of `@info` statements that will make the log + # way too long. So before we run `Coverage.process_folder`, we disable logging for `@info` + # statements. After we run `Coverage.process_folder`, we re-enable logging for `@info` + # statements. + Logging.disable_logging(Logging.Info) + fcs_base = Coverage.process_folder("base"); + fcs_stdlib = Coverage.process_folder("stdlib"); + Logging.disable_logging(Logging.Debug) + + fcs = Coverage.merge_coverage_counts( + fcs_base, + fcs_stdlib, + ); + + return fcs +end + function get_external_stdlib_names(stdlib_dir::AbstractString) filename_list = filter(x -> isfile(joinpath(stdlib_dir, x)), readdir(stdlib_dir)) # find all of the files like `Pkg.version`, `Statistics.version`, etc. @@ -76,16 +94,90 @@ function print_coverage_summary( return nothing end -# `Coverage.process_folder` will have a LOT of `@info` statements that will make the log -# way too long. So before we run `Coverage.process_folder`, we disable logging for `@info` -# statements. After we run `Coverage.process_folder`, we re-enable logging for `@info` -# statements. -Logging.disable_logging(Logging.Info) -const fcs = Coverage.merge_coverage_counts( - Coverage.process_folder("base"), - Coverage.process_folder("stdlib"), -); -Logging.disable_logging(Logging.Debug) +function buildkite_env(name::String) + value = String(strip(ENV[name])) + if isempty(value) + throw(ErrorException("environment variable $(name) is empty")) + end + return value +end + +function buildkite_env(name_1::String, name_2::String, default::String) + value_1 = String(strip(ENV[name_1])) + value_2 = String(strip(ENV[name_2])) + !isempty(value_1) && return value_1 + !isempty(value_2) && return value_2 + return default +end + +function buildkite_branch_and_commit() + branch = buildkite_env("BUILDKITE_BRANCH") + commit = buildkite_env("BUILDKITE_COMMIT") + head_rev_parse = String(strip(read(`git rev-parse HEAD`, String))) + if strip(commit) == "HEAD" + commit = head_rev_parse + end + if commit !== head_rev_parse + msg = "mismatch" + @error msg commit head_rev_parse + throw(ErrorException(msg)) + end + if !occursin(r"^[a-f0-9]{40}$", commit) + msg = "BUILDKITE_COMMIT does not look like a long commit SHA" + @error msg commit + throw(ErrorException(msg)) + end + return (; branch, commit) +end + +function codecov_buildkite_add_local_to_kwargs() + branch, commit = buildkite_branch_and_commit() + kwargs = Coverage.Codecov.set_defaults( + Dict(); + branch, + commit, + ) + return kwargs +end + +function coveralls_buildkite_query_git_info() + branch, commit = buildkite_branch_and_commit() + remote_name = "origin" + remote = buildkite_env("BUILDKITE_REPO") + message = buildkite_env("BUILDKITE_MESSAGE") + author_name = buildkite_env( + "BUILDKITE_BUILD_AUTHOR", + "BUILDKITE_BUILD_CREATOR", + "", + ) + author_email = buildkite_env( + "BUILDKITE_BUILD_AUTHOR_EMAIL", + "BUILDKITE_BUILD_CREATOR_EMAIL", + "", + ) + remotes = [ + Dict( + "name" => remote_name, + "url" => remote, + ) + ] + head = Dict( + "id" => commit, + "author_name" => author_name, + "author_email" => author_email, + "committer_name" => author_name, + "committer_email" => author_email, + "message" => message, + ) + git_info = Dict( + "branch" => branch, + "remotes" => remotes, + "head" => head, + ) + return git_info +end + +const fcs = process_folders() # Only include source code files. Exclude test files, benchmarking files, etc. filter!(fcs) do fc @@ -101,15 +193,27 @@ end; # Exclude all stdlib JLLs (stdlibs of the form `stdlib/*_jll/`). filter!(fcs) do fc !occursin(r"^stdlib\/[A-Za-z0-9]*?_jll\/", fc.filename) -end +end; -sort!(fcs; by = fc -> fc.filename) +sort!(fcs; by = fc -> fc.filename); print_coverage_summary.(fcs); print_coverage_summary(fcs, "Total") -# In order to upload to Codecov, you need to have the `CODECOV_TOKEN` environment variable defined. -Coverage.Codecov.submit_local(fcs) +let + git_info = coveralls_buildkite_query_git_info() + @info "" git_info + @info "" git_info["branch"] + @info "" git_info["head"] -# In order to upload to Coveralls, you need to have the `COVERALLS_TOKEN` environment variable defined. -Coverage.Coveralls.submit_local(fcs) + # In order to upload to Coveralls, you need to have the `COVERALLS_TOKEN` environment variable defined. + Coverage.Coveralls.submit_local(fcs, git_info) +end + +let + kwargs = codecov_buildkite_add_local_to_kwargs() + @info "" kwargs + + # In order to upload to Codecov, you need to have the `CODECOV_TOKEN` environment variable defined. + Coverage.Codecov.submit_generic(fcs, kwargs) +end From 248ccdda8080fc02f03d247029c8543b8a842566 Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Wed, 7 Jul 2021 14:43:45 -0700 Subject: [PATCH 18/73] Add signed pipeline test back (cherry picked from commit 480ff81e78b2d7b139ec34f0f2d70cfdc51895d2) --- .buildkite/0_webui.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.buildkite/0_webui.yml b/.buildkite/0_webui.yml index aed6a5e59d9c3..d5ba4e0ea7cf9 100644 --- a/.buildkite/0_webui.yml +++ b/.buildkite/0_webui.yml @@ -19,6 +19,6 @@ steps: # Our signed pipelines must have a `signature` or `signature_file` parameter that # verifies the treehash of the pipeline itself and the inputs listed in `inputs` - #signed_pipelines: - # - pipeline: .buildkite/signed_pipeline_test.yml - # signature: "U2FsdGVkX18aZgryp6AJTArgD2uOnVWyFFGVOP5qsY4WbGQ/LVAcYiMEp9cweV+2iht+vmEF949CuuGTeQPA1fKlhPwkG3nZ688752DUB6en9oM2nuL31NoDKWHhpygZ" + signed_pipelines: + - pipeline: .buildkite/signed_pipeline_test.yml + signature: "U2FsdGVkX18aZgryp6AJTArgD2uOnVWyFFGVOP5qsY4WbGQ/LVAcYiMEp9cweV+2iht+vmEF949CuuGTeQPA1fKlhPwkG3nZ688752DUB6en9oM2nuL31NoDKWHhpygZ" From ebbe1c786aabbe72b8c43920392cb0d318aa8f3f Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Thu, 8 Jul 2021 11:44:44 -0400 Subject: [PATCH 19/73] CI (code coverage): run some tests in serial (#41507) (cherry picked from commit 9564bfbd6cc0254bb6ee900e09034e7291ca7e19) --- .buildkite/0_webui.yml | 2 +- .buildkite/coverage-linux64/0_webui.yml | 2 +- .buildkite/coverage-linux64/pipeline.yml | 15 ++++++++++++--- .../{run_tests_base.jl => run_tests_parallel.jl} | 0 .buildkite/coverage-linux64/run_tests_serial.jl | 16 ++++++++++++++++ 5 files changed, 30 insertions(+), 5 deletions(-) rename .buildkite/coverage-linux64/{run_tests_base.jl => run_tests_parallel.jl} (100%) create mode 100644 .buildkite/coverage-linux64/run_tests_serial.jl diff --git a/.buildkite/0_webui.yml b/.buildkite/0_webui.yml index d5ba4e0ea7cf9..440d2d443ce7e 100644 --- a/.buildkite/0_webui.yml +++ b/.buildkite/0_webui.yml @@ -21,4 +21,4 @@ steps: # verifies the treehash of the pipeline itself and the inputs listed in `inputs` signed_pipelines: - pipeline: .buildkite/signed_pipeline_test.yml - signature: "U2FsdGVkX18aZgryp6AJTArgD2uOnVWyFFGVOP5qsY4WbGQ/LVAcYiMEp9cweV+2iht+vmEF949CuuGTeQPA1fKlhPwkG3nZ688752DUB6en9oM2nuL31NoDKWHhpygZ" + signature: "U2FsdGVkX18ZrMlORSIi0TvW99eZ3JyUEhntMyXjrOSJ9jAtUohgHc8+dMnWUc6qWjYt8k63wfPrth33SGWDiF814Bm1W5Zk3+R6HgVu6UCrQEI5sLm+SJPbrvET+Jkn" diff --git a/.buildkite/coverage-linux64/0_webui.yml b/.buildkite/coverage-linux64/0_webui.yml index 6dafcd86f6bcd..b0df8ec41de68 100644 --- a/.buildkite/coverage-linux64/0_webui.yml +++ b/.buildkite/coverage-linux64/0_webui.yml @@ -13,4 +13,4 @@ steps: # verifies the treehash of the pipeline itself and the inputs listed in `inputs` signed_pipelines: - pipeline: .buildkite/coverage-linux64/pipeline.yml - signature: "U2FsdGVkX18eQWpd3hMYLO5Kd+6K+oBoLk1I6J3qIw7lc6g5/jaeWyq/wralosZCfTzyjS4NstNKFvhQf3KDPEBVElipNvTxoWOjVLRVOrfBqqvTkQN4xVosY/r026Gy" + signature: U2FsdGVkX190BiaBGCTT6GNBDe5cHX3ZAP0IXbBfOWo7ys+1IuD5ymf4ImJbRIeE8NQac+iqud+bqCPHjii6DKmqzX+Jz6vax1NY3AxROhYlO5spUClvKr1wdngUCQON diff --git a/.buildkite/coverage-linux64/pipeline.yml b/.buildkite/coverage-linux64/pipeline.yml index 078b00254c4f9..218212d24ac2a 100644 --- a/.buildkite/coverage-linux64/pipeline.yml +++ b/.buildkite/coverage-linux64/pipeline.yml @@ -31,9 +31,18 @@ steps: # this is necessary to make sure that the LibGit2 tests passes git config --global init.defaultBranch master - echo "--- Run Julia tests with code coverage enabled" - # Run the actual tests - ./julia --code-coverage=all --sysimage-native-code=no .buildkite/coverage-linux64/run_tests_base.jl + echo "--- Run some Julia tests in serial" + ./julia --code-coverage=all --sysimage-native-code=no .buildkite/coverage-linux64/run_tests_serial.jl test/compiler/codegen.jl + ./julia --code-coverage=all --sysimage-native-code=no .buildkite/coverage-linux64/run_tests_serial.jl test/compiler/contextual.jl + ./julia --code-coverage=all --sysimage-native-code=no .buildkite/coverage-linux64/run_tests_serial.jl test/compiler/inference.jl + ./julia --code-coverage=all --sysimage-native-code=no .buildkite/coverage-linux64/run_tests_serial.jl test/compiler/inline.jl + ./julia --code-coverage=all --sysimage-native-code=no .buildkite/coverage-linux64/run_tests_serial.jl test/compiler/interpreter_exec.jl + ./julia --code-coverage=all --sysimage-native-code=no .buildkite/coverage-linux64/run_tests_serial.jl test/compiler/irpasses.jl + ./julia --code-coverage=all --sysimage-native-code=no .buildkite/coverage-linux64/run_tests_serial.jl test/compiler/ssair.jl + ./julia --code-coverage=all --sysimage-native-code=no .buildkite/coverage-linux64/run_tests_serial.jl test/compiler/validation.jl + + echo "--- Run Julia tests in parallel with code coverage enabled" + ./julia --code-coverage=all --sysimage-native-code=no .buildkite/coverage-linux64/run_tests_parallel.jl echo "--- Process and upload coverage information" ./julia .buildkite/coverage-linux64/upload_coverage.jl diff --git a/.buildkite/coverage-linux64/run_tests_base.jl b/.buildkite/coverage-linux64/run_tests_parallel.jl similarity index 100% rename from .buildkite/coverage-linux64/run_tests_base.jl rename to .buildkite/coverage-linux64/run_tests_parallel.jl diff --git a/.buildkite/coverage-linux64/run_tests_serial.jl b/.buildkite/coverage-linux64/run_tests_serial.jl new file mode 100644 index 0000000000000..6d7380a55e402 --- /dev/null +++ b/.buildkite/coverage-linux64/run_tests_serial.jl @@ -0,0 +1,16 @@ +using Test + +const repository_root = dirname(dirname(@__DIR__)) + +for filename in ARGS + path = joinpath(repository_root, filename) + @info "Starting $(filename)" + try + @testset "$(filename)" begin + include(path) + end + catch ex + @error "" exception=(ex, catch_backtrace()) + end + @info "Finished $(filename)" +end From f1d821473ffd605d8ecc7527c2f5997e0abec33b Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Mon, 19 Jul 2021 07:27:37 -0400 Subject: [PATCH 20/73] Move compile timing to inside `@time`'s main timing block. Fixes >100% compilation time reports (#41286) * ensure `@time`'s inner timing functions are compiled (cherry picked from commit 0dbd3f77d975edcc7e03e167f3f60aca186aff6c) --- base/timing.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/base/timing.jl b/base/timing.jl index 7af6f038ba6ea..ab7af23048305 100644 --- a/base/timing.jl +++ b/base/timing.jl @@ -205,11 +205,11 @@ macro time(ex) quote while false; end # compiler heuristic: compile this block (alter this if the heuristic changes) local stats = gc_num() - local compile_elapsedtime = cumulative_compile_time_ns_before() local elapsedtime = time_ns() + local compile_elapsedtime = cumulative_compile_time_ns_before() local val = $(esc(ex)) - elapsedtime = time_ns() - elapsedtime compile_elapsedtime = cumulative_compile_time_ns_after() - compile_elapsedtime + elapsedtime = time_ns() - elapsedtime local diff = GC_Diff(gc_num(), stats) time_print(elapsedtime, diff.allocd, diff.total_time, gc_alloc_count(diff), compile_elapsedtime, true) val @@ -251,11 +251,11 @@ macro timev(ex) quote while false; end # compiler heuristic: compile this block (alter this if the heuristic changes) local stats = gc_num() - local compile_elapsedtime = cumulative_compile_time_ns_before() local elapsedtime = time_ns() + local compile_elapsedtime = cumulative_compile_time_ns_before() local val = $(esc(ex)) - elapsedtime = time_ns() - elapsedtime compile_elapsedtime = cumulative_compile_time_ns_after() - compile_elapsedtime + elapsedtime = time_ns() - elapsedtime local diff = GC_Diff(gc_num(), stats) timev_print(elapsedtime, diff, compile_elapsedtime) val From 09c4ad1d88e1bb3b83d427b9dcb5cae79a5799bb Mon Sep 17 00:00:00 2001 From: Jakob Nybo Nissen Date: Mon, 19 Jul 2021 13:28:03 +0200 Subject: [PATCH 21/73] Do not error when showing invalid enums (#40042) (#41596) (cherry picked from commit 02807b279a5e6d5acaeb7095e4c0527e2a5c190e) --- base/Enums.jl | 10 ++++++++-- test/enums.jl | 7 +++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/base/Enums.jl b/base/Enums.jl index 06860402fbcb1..7b5e9587d5f6c 100644 --- a/base/Enums.jl +++ b/base/Enums.jl @@ -25,10 +25,16 @@ Base.isless(x::T, y::T) where {T<:Enum} = isless(basetype(T)(x), basetype(T)(y)) Base.Symbol(x::Enum) = namemap(typeof(x))[Integer(x)]::Symbol -Base.print(io::IO, x::Enum) = print(io, Symbol(x)) +function _symbol(x::Enum) + names = namemap(typeof(x)) + x = Integer(x) + get(() -> Symbol(""), names, x)::Symbol +end + +Base.print(io::IO, x::Enum) = print(io, _symbol(x)) function Base.show(io::IO, x::Enum) - sym = Symbol(x) + sym = _symbol(x) if !(get(io, :compact, false)::Bool) from = get(io, :module, Main) def = typeof(x).name.module diff --git a/test/enums.jl b/test/enums.jl index d3c585678c572..5a83e1b4dfa42 100644 --- a/test/enums.jl +++ b/test/enums.jl @@ -143,6 +143,10 @@ let io = IOBuffer() @test String(take!(io)) == sprint(print, Fruit) end +# Test printing of invalid enums +@test repr("text/plain", reinterpret(Fruit, Int32(11))) == "::Fruit = 11" +@test repr("text/plain", reinterpret(Fruit, Int32(-5))) == "::Fruit = -5" + @enum LogLevel DEBUG INFO WARN ERROR CRITICAL @test DEBUG < CRITICAL @@ -160,6 +164,9 @@ end @test repr("text/plain", sevn) == "$(string(sevn))::UI8 = 0x07" @test repr("text/plain", fiftn) == "$(string(fiftn))::UI8 = 0xf0" +@test repr("text/plain", reinterpret(UI8, 0x01)) == "::UI8 = 0x01" +@test repr("text/plain", reinterpret(UI8, 0xff)) == "::UI8 = 0xff" + # test block form @enum BritishFood begin blackpudding = 1 From 7e6136513baaf8f34ae88ddc7f200b8d6b1275ee Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 13 Jul 2021 14:28:29 -0700 Subject: [PATCH 22/73] CI: add the sanitizers pipelines (e.g. ASAN) to Buildkite (#41530) * Setup CI for ASAN * Launch the `sanitizers.yml` unsigned pipeline * Use a workspace directory in ./tmp * Add some log group headers to make the logs easier to navigate * Install `julia` binary inside sandbox * Double timeout * More descriptive message from sanitizer CI * Fix the path to the binary * Use addenv * Apply suggestions from code review Co-authored-by: Elliot Saba * Group ASAN related files under contrib/asan/ * Remove redundant JULIA_PRECOMPILE=1 Co-authored-by: Dilum Aluthge Co-authored-by: Elliot Saba (cherry picked from commit 84934e66db2ed0c3b888270380b4c34541d9ffc8) --- .buildkite/pipeline.yml | 5 +- .buildkite/sanitizers.yml | 34 +++++++++++++ contrib/asan/Make.user.asan | 24 ++++++++++ contrib/asan/Make.user.tools | 2 + contrib/asan/build.sh | 53 +++++++++++++++++++++ contrib/asan/check.jl | 92 ++++++++++++++++++++++++++++++++++++ 6 files changed, 209 insertions(+), 1 deletion(-) create mode 100644 .buildkite/sanitizers.yml create mode 100644 contrib/asan/Make.user.asan create mode 100644 contrib/asan/Make.user.tools create mode 100755 contrib/asan/build.sh create mode 100755 contrib/asan/check.jl diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 0dee5f2aaa3f7..99473055f2eeb 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -14,8 +14,11 @@ steps: - label: ":buildkite: Launch unsigned pipelines" commands: | + # We launch whitespace first, because we want that pipeline to finish as quickly as possible. + # The remaining unsigned pipelines are launched in alphabetical order. buildkite-agent pipeline upload .buildkite/whitespace.yml - buildkite-agent pipeline upload .buildkite/llvm_passes.yml buildkite-agent pipeline upload .buildkite/embedding.yml + buildkite-agent pipeline upload .buildkite/llvm_passes.yml + buildkite-agent pipeline upload .buildkite/sanitizers.yml agents: queue: julia diff --git a/.buildkite/sanitizers.yml b/.buildkite/sanitizers.yml new file mode 100644 index 0000000000000..f29ed4e42a4a6 --- /dev/null +++ b/.buildkite/sanitizers.yml @@ -0,0 +1,34 @@ +# These steps should only run on `sandbox.jl` machines, not `docker`-isolated ones +# since we need nestable sandboxing. The rootfs images being used here are built from +# the `.buildkite/rootfs_images/llvm-passes.jl` file. +agents: + queue: "julia" + # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing + sandbox.jl: "true" + os: "linux" + +steps: + - label: "asan" + key: asan + plugins: + - JuliaCI/julia#v1: + version: 1.6 + - staticfloat/sandbox#v1: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v1/llvm-passes.tar.gz + rootfs_treehash: "f3ed53f159e8f13edfba8b20ebdb8ece73c1b8a8" + uid: 1000 + gid: 1000 + workspaces: + - "/cache/repos:/cache/repos" + # `contrib/check-asan.jl` needs a `julia` binary: + - JuliaCI/julia#v1: + version: 1.6 + commands: | + echo "--- Build julia-debug with ASAN" + contrib/asan/build.sh ./tmp/test-asan -j$${JULIA_NUM_CORES} debug + echo "--- Test that ASAN is enabled" + contrib/asan/check.jl ./tmp/test-asan/asan/usr/bin/julia-debug + timeout_in_minutes: 120 + notify: + - github_commit_status: + context: "asan" diff --git a/contrib/asan/Make.user.asan b/contrib/asan/Make.user.asan new file mode 100644 index 0000000000000..3bcc34df68323 --- /dev/null +++ b/contrib/asan/Make.user.asan @@ -0,0 +1,24 @@ +TOOLCHAIN=$(BUILDROOT)/../toolchain/usr/tools + +# use our new toolchain +USECLANG=1 +override CC=$(TOOLCHAIN)/clang +override CXX=$(TOOLCHAIN)/clang++ +export ASAN_SYMBOLIZER_PATH=$(TOOLCHAIN)/llvm-symbolizer + +USE_BINARYBUILDER_LLVM=1 + +override SANITIZE=1 +override SANITIZE_ADDRESS=1 + +# make the GC use regular malloc/frees, which are hooked by ASAN +override WITH_GC_DEBUG_ENV=1 + +# default to a debug build for better line number reporting +override JULIA_BUILD_MODE=debug + +# make ASAN consume less memory +export ASAN_OPTIONS=detect_leaks=0:fast_unwind_on_malloc=0:allow_user_segv_handler=1:malloc_context_size=2 + +# tell libblastrampoline to not use RTLD_DEEPBIND +export LBT_USE_RTLD_DEEPBIND=0 diff --git a/contrib/asan/Make.user.tools b/contrib/asan/Make.user.tools new file mode 100644 index 0000000000000..1bd6f97e39111 --- /dev/null +++ b/contrib/asan/Make.user.tools @@ -0,0 +1,2 @@ +USE_BINARYBUILDER_LLVM=1 +BUILD_LLVM_CLANG=1 diff --git a/contrib/asan/build.sh b/contrib/asan/build.sh new file mode 100755 index 0000000000000..d124e0a92f1e0 --- /dev/null +++ b/contrib/asan/build.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# This file is a part of Julia. License is MIT: https://julialang.org/license +# +# Usage: +# contrib/asan/build.sh [...] +# +# Build ASAN-enabled julia. Given a workspace directory , build +# ASAN-enabled julia in /asan. Required toolss are install under +# /toolchain. This scripts also takes optional arguments +# which are passed to `make`. The default make target is `debug`. + +set -ue + +# `$WORKSPACE` is a directory in which we create `toolchain` and `asan` +# sub-directories. +WORKSPACE="$1" +shift +if [ "$WORKSPACE" = "" ]; then + echo "Workspace directory must be specified as the first argument" >&2 + exit 2 +fi + +mkdir -pv "$WORKSPACE" +WORKSPACE="$(cd "$WORKSPACE" && pwd)" +if [ "$WORKSPACE" = "" ]; then + echo "Failed to create the workspace directory." >&2 + exit 2 +fi + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +JULIA_HOME="$HERE/../../" + +echo +echo "Installing toolchain..." + +TOOLCHAIN="$WORKSPACE/toolchain" +if [ ! -d "$TOOLCHAIN" ]; then + make -C "$JULIA_HOME" configure O=$TOOLCHAIN + cp "$HERE/Make.user.tools" "$TOOLCHAIN/Make.user" +fi + +make -C "$TOOLCHAIN/deps" install-clang install-llvm-tools + +echo +echo "Building Julia..." + +BUILD="$WORKSPACE/asan" +if [ ! -d "$BUILD" ]; then + make -C "$JULIA_HOME" configure O="$BUILD" + cp "$HERE/Make.user.asan" "$BUILD/Make.user" +fi + +make -C "$BUILD" "$@" diff --git a/contrib/asan/check.jl b/contrib/asan/check.jl new file mode 100755 index 0000000000000..0c1e12f7f471a --- /dev/null +++ b/contrib/asan/check.jl @@ -0,0 +1,92 @@ +#!/bin/bash +# -*- mode: julia -*- +# This file is a part of Julia. License is MIT: https://julialang.org/license +# +# Usage: +# contrib/asan/check.jl +# +# Check that is built with ASAN. +# +#= +JULIA="${JULIA:-julia}" +exec "$JULIA" --startup-file=no --compile=min "${BASH_SOURCE[0]}" "$@" +=# + +function main(args = ARGS)::Int + if length(args) != 1 + @error "Expect a single argument" args + return 2 + end + julia, = args + + # It looks like double-free is easy to robustly trigger. + code = """ + @info "Testing a pattern that would trigger ASAN" + write(ARGS[1], "started") + + ptr = ccall(:malloc, Ptr{UInt}, (Csize_t,), 256) + ccall(:free, Cvoid, (Ptr{UInt},), ptr) + ccall(:free, Cvoid, (Ptr{UInt},), ptr) + + @error "Failed to trigger ASAN" + """ + + local proc + timeout = Threads.Atomic{Bool}(false) + isstarted = false + mktemp() do tmppath, tmpio + cmd = addenv( + `$julia -e $code $tmppath`, + "ASAN_OPTIONS" => + "detect_leaks=0:fast_unwind_on_malloc=0:allow_user_segv_handler=1:malloc_context_size=2", + "LBT_USE_RTLD_DEEPBIND" => "0", + ) + # Note: Ideally, we set ASAN_SYMBOLIZER_PATH here. But there is no easy + # way to find out the path from just a Julia binary. + + @debug "Starting a process" cmd + proc = run(pipeline(cmd; stdout, stderr); wait = false) + timer = Timer(10) + @sync try + @async begin + try + wait(timer) + true + catch err + err isa EOFError || rethrow() + false + end && begin + timeout[] = true + kill(proc) + end + end + wait(proc) + finally + close(timer) + end + + # At the very beginning of the process, the `julia` subprocess put a + # marker that it is successfully started. This is to avoid mixing + # non-functional `julia` binary (or even non-`julia` command) and + # correctly working `julia` with ASAN: + isstarted = read(tmpio, String) == "started" + end + + if timeout[] + @error "Timeout waiting for the subprocess" + return 1 + elseif success(proc) + @error "ASAN was not triggered" + return 1 + elseif !isstarted + @error "Failed to start the process" + return 1 + else + @info "ASAN is functional in the Julia binary `$julia`" + return 0 + end +end + +if abspath(PROGRAM_FILE) == @__FILE__ + exit(main()) +end From 5990b8d12767d9c9725cb786dcf281237517db4d Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Tue, 20 Jul 2021 09:21:52 +0200 Subject: [PATCH 23/73] make precompile files writable (#41614) (cherry picked from commit 6d58068541720a11eede10edfa24aae47ab2f5ab) --- base/loading.jl | 4 ++-- test/precompile.jl | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/base/loading.jl b/base/loading.jl index 37e5111bfbf5b..a6145398946b8 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -1354,8 +1354,8 @@ function compilecache(pkg::PkgId, path::String, internal_stderr::IO = stderr, in open(tmppath, "a+") do f write(f, _crc32c(seekstart(f))) end - # inherit permission from the source file - chmod(tmppath, filemode(path) & 0o777) + # inherit permission from the source file (and make them writable) + chmod(tmppath, filemode(path) & 0o777 | 0o200) # Read preferences hash back from .ji file (we can't precompute because # we don't actually know what the list of compile-time preferences are without compiling) diff --git a/test/precompile.jl b/test/precompile.jl index 2fcad7f64113c..08e945e1e3339 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -826,6 +826,10 @@ precompile_test_harness("Issue #25971") do load_path chmod(sourcefile, 0o600) cachefile = Base.compilecache(Base.PkgId("Foo25971")) @test filemode(sourcefile) == filemode(cachefile) + chmod(sourcefile, 0o444) + cachefile = Base.compilecache(Base.PkgId("Foo25971")) + # Check writable + @test touch(cachefile) == cachefile end precompile_test_harness("Issue #38312") do load_path From 0aa183597f94c732cfae48b5fad891737b7f11a5 Mon Sep 17 00:00:00 2001 From: Fred Callaway Date: Tue, 20 Jul 2021 00:25:43 -0700 Subject: [PATCH 24/73] while loop -> for loop in _simplify_include_frames (fixes #41566) (#41622) * while loop -> for loop in _simplify_include_frames (fixes #41566) prevents the possibility of a bounds error when i = 0 (cherry picked from commit ed4f3169573903f1767871243375d2407db959a0) --- base/errorshow.jl | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/base/errorshow.jl b/base/errorshow.jl index 49fe9680e180d..e8153cd191e86 100644 --- a/base/errorshow.jl +++ b/base/errorshow.jl @@ -788,10 +788,9 @@ end # For improved user experience, filter out frames for include() implementation # - see #33065. See also #35371 for extended discussion of internal frames. function _simplify_include_frames(trace) - i = length(trace) - kept_frames = trues(i) + kept_frames = trues(length(trace)) first_ignored = nothing - while i >= 1 + for i in length(trace):-1:1 frame::StackFrame, _ = trace[i] mod = parentmodule(frame) if first_ignored === nothing @@ -813,10 +812,9 @@ function _simplify_include_frames(trace) first_ignored = nothing end end - i -= 1 end if first_ignored !== nothing - kept_frames[i:first_ignored] .= false + kept_frames[1:first_ignored] .= false end return trace[kept_frames] end From 15cb1fdb1049718b45b290ed2d0093c5f7f34784 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Tue, 20 Jul 2021 05:58:28 -0400 Subject: [PATCH 25/73] CI (Buildkite): move the `sanitizers` builders (e.g. `asan`) to the experimental pipeline (#41606) (cherry picked from commit c95448fb988441ecd7fb0604606ac08c5014abd3) --- .buildkite/experimental/0_webui.yml | 18 ++++++++++++++++++ .buildkite/experimental/pipeline.yml | 19 +++++++++++++++++++ .buildkite/{ => experimental}/sanitizers.yml | 10 +++++----- .buildkite/pipeline.yml | 1 - 4 files changed, 42 insertions(+), 6 deletions(-) create mode 100644 .buildkite/experimental/0_webui.yml create mode 100644 .buildkite/experimental/pipeline.yml rename .buildkite/{ => experimental}/sanitizers.yml (85%) diff --git a/.buildkite/experimental/0_webui.yml b/.buildkite/experimental/0_webui.yml new file mode 100644 index 0000000000000..ab3f68db4c393 --- /dev/null +++ b/.buildkite/experimental/0_webui.yml @@ -0,0 +1,18 @@ +# This file represents what is put into the webUI. +# It is purely for keeping track of the changes we make to the webUI configuration; modifying this file has no effect. +# We use the `cryptic` buildkite plugin to provide secrets management, which requires some integration into the WebUI's steps. +agents: + queue: "julia" + sandbox.jl: "true" + +steps: + - label: ":unlock: Unlock secrets, launch pipelines" + plugins: + - staticfloat/cryptic: + # Our list of pipelines that should be launched (but don't require a signature) + # These pipelines can be modified by any contributor and CI will still run. + # Build secrets will not be available in these pipelines (or their children) + # but some of our signed pipelines can wait upon the completion of these unsigned + # pipelines. + unsigned_pipelines: + - .buildkite/experimental/pipeline.yml diff --git a/.buildkite/experimental/pipeline.yml b/.buildkite/experimental/pipeline.yml new file mode 100644 index 0000000000000..6f86f7ef243e5 --- /dev/null +++ b/.buildkite/experimental/pipeline.yml @@ -0,0 +1,19 @@ +# This file launches all the build jobs that _don't_ require secrets access. +# These jobs can pass their output off to jobs that do require secrets access, +# but those privileged steps require signing before they can be run. +# +# Yes, this is creating another layer of indirection; the flow now looks like: +# +# [webui] -> pipeline.yml -> llvm_passes.yml +# +# when we could theoretically just have the `webui` launch `llvm_passes.yml`, +# however this raises the bar for contributors to add new (unsigned) steps to +# our CI configuration, so I'd rather live with an extra layer of indirection +# and only need to touch the webui configuration when we need to alter +# something about the privileged steps. +steps: + - label: ":buildkite: Launch unsigned pipelines" + commands: | + buildkite-agent pipeline upload .buildkite/experimental/sanitizers.yml + agents: + queue: julia diff --git a/.buildkite/sanitizers.yml b/.buildkite/experimental/sanitizers.yml similarity index 85% rename from .buildkite/sanitizers.yml rename to .buildkite/experimental/sanitizers.yml index f29ed4e42a4a6..534dac81e53f3 100644 --- a/.buildkite/sanitizers.yml +++ b/.buildkite/experimental/sanitizers.yml @@ -14,8 +14,8 @@ steps: - JuliaCI/julia#v1: version: 1.6 - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v1/llvm-passes.tar.gz - rootfs_treehash: "f3ed53f159e8f13edfba8b20ebdb8ece73c1b8a8" + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v2.0/llvm_passes.tar.gz + rootfs_treehash: "0b5a80c1d0ab110a57fbdb7f4bc042a5ede310f3" uid: 1000 gid: 1000 workspaces: @@ -29,6 +29,6 @@ steps: echo "--- Test that ASAN is enabled" contrib/asan/check.jl ./tmp/test-asan/asan/usr/bin/julia-debug timeout_in_minutes: 120 - notify: - - github_commit_status: - context: "asan" + # notify: + # - github_commit_status: + # context: "asan" diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 99473055f2eeb..097b8976eea70 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -19,6 +19,5 @@ steps: buildkite-agent pipeline upload .buildkite/whitespace.yml buildkite-agent pipeline upload .buildkite/embedding.yml buildkite-agent pipeline upload .buildkite/llvm_passes.yml - buildkite-agent pipeline upload .buildkite/sanitizers.yml agents: queue: julia From a6005e6905892cfccae5a6c747f29fb25b5c76d5 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 21 Jul 2021 12:15:12 -0400 Subject: [PATCH 26/73] fix memory corruption in `deleteat!` (#41646) n.b. `n == a->nrows + dec` Fixes https://github.com/JuliaData/DataFrames.jl/issues/2819 (cherry picked from commit 2e06a016fdad70736dd6c25c5a5286809e442b35) --- src/array.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/array.c b/src/array.c index 20c6cf7706880..3218731d5a20b 100644 --- a/src/array.c +++ b/src/array.c @@ -1095,7 +1095,7 @@ STATIC_INLINE void jl_array_del_at_beg(jl_array_t *a, size_t idx, size_t dec, // Move the rest of the data if the offset changed if (newoffs != offset) { memmove_safe(a->flags.hasptr, newdata + nb1, olddata + nb1 + nbdec, nbtotal - nb1); - if (isbitsunion) memmove(newtypetagdata + idx, typetagdata + idx + dec, n - idx); + if (isbitsunion) memmove(newtypetagdata + idx, typetagdata + idx + dec, a->nrows - idx); } a->data = newdata; } From d32c44d5a5249ec3d26d559d5973b31a99fe0742 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Wed, 21 Jul 2021 13:10:03 -0400 Subject: [PATCH 27/73] CI: delete the `.buildkite/rootfs_images` directory (#41648) (cherry picked from commit 61b6994884642f8fc7e18844d37dc70b9dfdb9ab) --- .buildkite/rootfs_images/Manifest.toml | 134 ----------------------- .buildkite/rootfs_images/Project.toml | 5 - .buildkite/rootfs_images/README.md | 5 - .buildkite/rootfs_images/llvm-passes.jl | 31 ------ .buildkite/rootfs_images/rootfs_utils.jl | 123 --------------------- 5 files changed, 298 deletions(-) delete mode 100644 .buildkite/rootfs_images/Manifest.toml delete mode 100644 .buildkite/rootfs_images/Project.toml delete mode 100644 .buildkite/rootfs_images/README.md delete mode 100755 .buildkite/rootfs_images/llvm-passes.jl delete mode 100644 .buildkite/rootfs_images/rootfs_utils.jl diff --git a/.buildkite/rootfs_images/Manifest.toml b/.buildkite/rootfs_images/Manifest.toml deleted file mode 100644 index d24e9a4ee166e..0000000000000 --- a/.buildkite/rootfs_images/Manifest.toml +++ /dev/null @@ -1,134 +0,0 @@ -# This file is machine-generated - editing it directly is not advised - -[[ArgTools]] -uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" - -[[Artifacts]] -uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" - -[[Base64]] -uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" - -[[Dates]] -deps = ["Printf"] -uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" - -[[Downloads]] -deps = ["ArgTools", "LibCURL", "NetworkOptions"] -uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" - -[[InteractiveUtils]] -deps = ["Markdown"] -uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" - -[[JLLWrappers]] -deps = ["Preferences"] -git-tree-sha1 = "642a199af8b68253517b80bd3bfd17eb4e84df6e" -uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" -version = "1.3.0" - -[[LibCURL]] -deps = ["LibCURL_jll", "MozillaCACerts_jll"] -uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" - -[[LibCURL_jll]] -deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] -uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" - -[[LibGit2]] -deps = ["Base64", "NetworkOptions", "Printf", "SHA"] -uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" - -[[LibSSH2_jll]] -deps = ["Artifacts", "Libdl", "MbedTLS_jll"] -uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" - -[[Libdl]] -uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" - -[[Logging]] -uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" - -[[Markdown]] -deps = ["Base64"] -uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" - -[[MbedTLS_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" - -[[MozillaCACerts_jll]] -uuid = "14a3606d-f60d-562e-9121-12d972cd8159" - -[[NetworkOptions]] -uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" - -[[Pkg]] -deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] -uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" - -[[Preferences]] -deps = ["TOML"] -git-tree-sha1 = "00cfd92944ca9c760982747e9a1d0d5d86ab1e5a" -uuid = "21216c6a-2e73-6563-6e65-726566657250" -version = "1.2.2" - -[[Printf]] -deps = ["Unicode"] -uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" - -[[REPL]] -deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] -uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" - -[[Random]] -deps = ["Serialization"] -uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" - -[[SHA]] -uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" - -[[Scratch]] -deps = ["Dates"] -git-tree-sha1 = "0b4b7f1393cff97c33891da2a0bf69c6ed241fda" -uuid = "6c6a2e73-6563-6170-7368-637461726353" -version = "1.1.0" - -[[Serialization]] -uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" - -[[Sockets]] -uuid = "6462fe0b-24de-5631-8697-dd941f90decc" - -[[TOML]] -deps = ["Dates"] -uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" - -[[Tar]] -deps = ["ArgTools", "SHA"] -uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" - -[[UUIDs]] -deps = ["Random", "SHA"] -uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" - -[[Unicode]] -uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" - -[[Zlib_jll]] -deps = ["Libdl"] -uuid = "83775a58-1f1d-513f-b197-d71354ab007a" - -[[ghr_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "f5c8cb306d4fe2d1fff90443a088fc5ba536c134" -uuid = "07c12ed4-43bc-5495-8a2a-d5838ef8d533" -version = "0.13.0+1" - -[[nghttp2_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" - -[[p7zip_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" diff --git a/.buildkite/rootfs_images/Project.toml b/.buildkite/rootfs_images/Project.toml deleted file mode 100644 index 1dbde5ed9df66..0000000000000 --- a/.buildkite/rootfs_images/Project.toml +++ /dev/null @@ -1,5 +0,0 @@ -[deps] -Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" -SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce" -Scratch = "6c6a2e73-6563-6170-7368-637461726353" -ghr_jll = "07c12ed4-43bc-5495-8a2a-d5838ef8d533" diff --git a/.buildkite/rootfs_images/README.md b/.buildkite/rootfs_images/README.md deleted file mode 100644 index 1d3962c2bee3e..0000000000000 --- a/.buildkite/rootfs_images/README.md +++ /dev/null @@ -1,5 +0,0 @@ -## Rootfs images - -Our CI setup makes use of rootfs images that contain our build tools. -These rootfs images are built using the fairly simple scripts held within this directory. -Most images are based on Debian, making use of `debootstrap` to provide a quick and easy rootfs with packages installed through an initial `apt` invocation. diff --git a/.buildkite/rootfs_images/llvm-passes.jl b/.buildkite/rootfs_images/llvm-passes.jl deleted file mode 100755 index bc6d57eb2f87e..0000000000000 --- a/.buildkite/rootfs_images/llvm-passes.jl +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env julia - -## This rootfs includes enough of a host toolchain to build the LLVM passes. -## Eventually, this image will probably be replaced with the actual builder image, -## as that will have the necessary toolchains as well, but that image is not built yet. - -include("rootfs_utils.jl") - -const tag_name, force_overwrite = get_arguments(ARGS, @__FILE__) - -# Build debian-based image with the following extra packages: -packages = [ - "bash", - "build-essential", - "cmake", - "curl", - "gfortran", - "git", - "less", - "libatomic1", - "m4", - "perl", - "pkg-config", - "python", - "python3", - "wget", -] -tarball_path = debootstrap("llvm-passes"; packages) - -# Upload it -upload_rootfs_image(tarball_path; tag_name, force_overwrite) diff --git a/.buildkite/rootfs_images/rootfs_utils.jl b/.buildkite/rootfs_images/rootfs_utils.jl deleted file mode 100644 index 82baeec7a0933..0000000000000 --- a/.buildkite/rootfs_images/rootfs_utils.jl +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env julia - -# This is an example invocation of `debootstrap` to generate a Debian/Ubuntu-based rootfs -using Scratch, Pkg, Pkg.Artifacts, ghr_jll, SHA, Dates - -# Utility functions -getuid() = ccall(:getuid, Cint, ()) -getgid() = ccall(:getgid, Cint, ()) - -function debootstrap(name::String; release::String="buster", variant::String="minbase", - packages::Vector{String}=String[], force::Bool=false) - if Sys.which("debootstrap") === nothing - error("Must install `debootstrap`!") - end - - tarball_path = joinpath(@get_scratch!("rootfs-images"), "$(name).tar.gz") - if !force && isfile(tarball_path) - @error("Refusing to overwrite tarball without `force` set", tarball_path) - error() - end - - artifact_hash = create_artifact() do rootfs - packages_string = join(push!(packages, "locales"), ",") - @info("Running debootstrap", release, variant, packages) - run(`sudo debootstrap --variant=$(variant) --include=$(packages_string) $(release) "$(rootfs)"`) - - # Remove special `dev` files - @info("Cleaning up `/dev`") - for f in readdir(joinpath(rootfs, "dev"); join=true) - # Keep the symlinks around (such as `/dev/fd`), as they're useful - if !islink(f) - run(`sudo rm -rf "$(f)"`) - end - end - - # take ownership of the entire rootfs - @info("Chown'ing rootfs") - run(`sudo chown $(getuid()):$(getgid()) -R "$(rootfs)"`) - - # Write out rootfs-info to contain a minimally-identifying string - open(joinpath(rootfs, "etc", "rootfs-info"), write=true) do io - write(io, """ - rootfs_type=debootstrap - release=$(release) - variant=$(variant) - packages=$(packages_string) - build_date=$(Dates.now()) - """) - end - - # Write out a reasonable default resolv.conf - open(joinpath(rootfs, "etc", "resolv.conf"), write=true) do io - write(io, """ - nameserver 1.1.1.1 - nameserver 8.8.8.8 - """) - end - - # Remove `_apt` user so that `apt` doesn't try to `setgroups()` - @info("Removing `_apt` user") - open(joinpath(rootfs, "etc", "passwd"), write=true, read=true) do io - filtered_lines = filter(l -> !startswith(l, "_apt:"), readlines(io)) - truncate(io, 0) - seek(io, 0) - for l in filtered_lines - println(io, l) - end - end - - # Set up the one true locale - @info("Setting up UTF-8 locale") - open(joinpath(rootfs, "etc", "locale.gen"), "a") do io - println(io, "en_US.UTF-8 UTF-8") - end - run(`sudo chroot --userspec=$(getuid()):$(getgid()) $(rootfs) locale-gen`) - end - - # Archive it into a `.tar.gz` file - @info("Archiving", tarball_path, artifact_hash) - archive_artifact(artifact_hash, tarball_path) - - return tarball_path -end - -function upload_rootfs_image(tarball_path::String; - github_repo::String="JuliaCI/rootfs-images", - tag_name::String, - force_overwrite::Bool) - # Upload it to `github_repo` - tarball_url = "https://github.com/$(github_repo)/releases/download/$(tag_name)/$(basename(tarball_path))" - @info("Uploading to $(github_repo)@$(tag_name)", tarball_url) - cmd = ghr_jll.ghr() - append!(cmd.exec, ["-u", dirname(github_repo), "-r", basename(github_repo)]) - force_overwrite && push!(cmd.exec, "-replace") - append!(cmd.exec, [tag_name, tarball_path]) - run(cmd) - return tarball_url -end - -# process command-line arguments - -function get_arguments(args::AbstractVector, script_file::AbstractString) - usage = "Usage: $(basename(script_file)) [--force-overwrite]" - length(args) < 1 && throw(ArgumentError(usage)) - length(args) > 2 && throw(ArgumentError(usage)) - tag_name = get_tag_name(args; usage) - force_overwrite = get_force_overwrite(args; usage) - return (; tag_name, force_overwrite) -end - -function get_tag_name(args::AbstractVector; usage::AbstractString) - tag_name = convert(String, strip(args[1]))::String - isempty(tag_name) && throw(ArgumentError(usage)) - startswith(tag_name, "--") && throw(ArgumentError(usage)) - return tag_name -end - -function get_force_overwrite(args::AbstractVector; usage::AbstractString) - force_overwrite_string = strip(get(args, 2, "")) - force_overwrite_string == "" && return false - force_overwrite_string == "--force-overwrite" && return true - throw(ArgumentError(usage)) -end From 3e8eb9e8deb72f5bf231dbde3a48eae69c558947 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Thu, 22 Jul 2021 20:21:27 -0400 Subject: [PATCH 28/73] Update the `file.jl` tests to allow for both `EPERM` and `EINVAL` in the non-root CHOWN tests (#41682) Co-authored-by: Elliot Saba Co-authored-by: Elliot Saba (cherry picked from commit 114ee174cff04b7ecf3514d58e310384fa1bbf17) --- test/file.jl | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/test/file.jl b/test/file.jl index 0f39bc7c140f6..4f93417deb112 100644 --- a/test/file.jl +++ b/test/file.jl @@ -491,6 +491,29 @@ rm(c_tmpdir, recursive=true) @test_throws Base._UVError("unlink($(repr(c_tmpdir)))", Base.UV_ENOENT) rm(c_tmpdir, recursive=true) @test rm(c_tmpdir, force=true, recursive=true) === nothing +# Some operations can return multiple different error codes depending on the system environment. +function throws_matching_exception(f::Function, acceptable_exceptions::AbstractVector) + try + f() + @error "No exception was thrown." + return false + catch ex + if ex in acceptable_exceptions + return true + else + @error "The thrown exception is not in the list of acceptable exceptions" acceptable_exceptions exception=(ex, catch_backtrace()) + return false + end + end +end +function throws_matching_uv_error(f::Function, pfx::AbstractString, codes::AbstractVector{<:Integer}) + acceptable_exceptions = multiple_uv_errors(pfx, codes) + return throws_matching_exception(f, acceptable_exceptions) +end +function multiple_uv_errors(pfx::AbstractString, codes::AbstractVector{<:Integer}) + return [Base._UVError(pfx, code) for code in codes] +end + if !Sys.iswindows() # chown will give an error if the user does not have permissions to change files if get(ENV, "USER", "") == "root" || get(ENV, "HOME", "") == "/root" @@ -503,8 +526,12 @@ if !Sys.iswindows() @test stat(file).gid == 0 @test stat(file).uid == 0 else - @test_throws Base._UVError("chown($(repr(file)), -2, -1)", Base.UV_EPERM) chown(file, -2, -1) # Non-root user cannot change ownership to another user - @test_throws Base._UVError("chown($(repr(file)), -1, -2)", Base.UV_EPERM) chown(file, -1, -2) # Non-root user cannot change group to a group they are not a member of (eg: nogroup) + @test throws_matching_uv_error("chown($(repr(file)), -2, -1)", [Base.UV_EPERM, Base.UV_EINVAL]) do + chown(file, -2, -1) # Non-root user cannot change ownership to another user + end + @test throws_matching_uv_error("chown($(repr(file)), -1, -2)", [Base.UV_EPERM, Base.UV_EINVAL]) do + chown(file, -1, -2) # Non-root user cannot change group to a group they are not a member of (eg: nogroup) + end end else # test that chown doesn't cause any errors for Windows From f0d2edbbe4bc33c8d5c67456883abe547d1b7d34 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Sat, 24 Jul 2021 22:58:54 -0400 Subject: [PATCH 29/73] CI (Buildkite): Add the `package_linux64` and `doctest` builders (#41541) Co-authored-by: Elliot Saba (cherry picked from commit c7097c5793545658aef20d6bb83eb1af7bc81310) --- .buildkite/misc/doctest.yml | 39 +++++ .buildkite/{ => misc}/embedding.yml | 4 +- .../{llvm_passes.yml => misc/llvmpasses.yml} | 8 +- .buildkite/{ => misc}/whitespace.yml | 4 +- .buildkite/pipeline.yml | 19 ++- .buildkite/platforms/linux64.yml | 80 +++++++++++ .buildkite/rr_capture.jl | 134 ++++++++++++++++++ 7 files changed, 273 insertions(+), 15 deletions(-) create mode 100644 .buildkite/misc/doctest.yml rename .buildkite/{ => misc}/embedding.yml (91%) rename .buildkite/{llvm_passes.yml => misc/llvmpasses.yml} (88%) rename .buildkite/{ => misc}/whitespace.yml (87%) create mode 100644 .buildkite/platforms/linux64.yml create mode 100644 .buildkite/rr_capture.jl diff --git a/.buildkite/misc/doctest.yml b/.buildkite/misc/doctest.yml new file mode 100644 index 0000000000000..234e49f5dcf53 --- /dev/null +++ b/.buildkite/misc/doctest.yml @@ -0,0 +1,39 @@ +# These steps should only run on `sandbox.jl` machines, not `docker`-isolated ones +# since we need nestable sandboxing. The rootfs images being used here are built from +# the `.buildkite/rootfs_images/llvm-passes.jl` file. +agents: + queue: "julia" + # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing + sandbox.jl: "true" + os: "linux" + +steps: + - label: "doctest" + key: doctest + plugins: + - JuliaCI/julia#v1: + version: 1.6 + - staticfloat/sandbox#v1: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.0/package_linux.x86_64.tar.gz + rootfs_treehash: "d5722d586b93eb307bb6340d275afdbf7578a756" + uid: 1000 + gid: 1000 + workspaces: + # Include `/cache/repos` so that our `git` version introspection works. + - "/cache/repos:/cache/repos" + commands: | + echo "--- Build Julia from source" + make -j 6 + + echo "--- Print Julia version info" + ./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()' + + echo "--- Build Julia docs" + make docs + + echo "--- Run Julia doctests" + JULIA_NUM_THREADS=1 make -C doc doctest=true + timeout_in_minutes: 120 + notify: + - github_commit_status: + context: "doctest" diff --git a/.buildkite/embedding.yml b/.buildkite/misc/embedding.yml similarity index 91% rename from .buildkite/embedding.yml rename to .buildkite/misc/embedding.yml index 5cf6b985573f5..31f85633479a4 100644 --- a/.buildkite/embedding.yml +++ b/.buildkite/misc/embedding.yml @@ -14,8 +14,8 @@ steps: - JuliaCI/julia#v1: version: 1.6 - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v1/llvm-passes.tar.gz - rootfs_treehash: "f3ed53f159e8f13edfba8b20ebdb8ece73c1b8a8" + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.0/package_linux.x86_64.tar.gz + rootfs_treehash: "d5722d586b93eb307bb6340d275afdbf7578a756" uid: 1000 gid: 1000 workspaces: diff --git a/.buildkite/llvm_passes.yml b/.buildkite/misc/llvmpasses.yml similarity index 88% rename from .buildkite/llvm_passes.yml rename to .buildkite/misc/llvmpasses.yml index 7453af82c1e73..7660b3978e967 100644 --- a/.buildkite/llvm_passes.yml +++ b/.buildkite/misc/llvmpasses.yml @@ -14,8 +14,8 @@ steps: - JuliaCI/julia#v1: version: 1.6 - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v1/llvm-passes.tar.gz - rootfs_treehash: "f3ed53f159e8f13edfba8b20ebdb8ece73c1b8a8" + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.1/llvm_passes.x86_64.tar.gz + rootfs_treehash: "9dd715500b117a16fcfa419ea0bca0c0ca902cee" workspaces: # Include `/cache/repos` so that our `git` version introspection works. - "/cache/repos:/cache/repos" @@ -36,8 +36,8 @@ steps: - JuliaCI/julia#v1: version: 1.6 - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v1/llvm-passes.tar.gz - rootfs_treehash: "f3ed53f159e8f13edfba8b20ebdb8ece73c1b8a8" + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.0/package_linux.x86_64.tar.gz + rootfs_treehash: "d5722d586b93eb307bb6340d275afdbf7578a756" uid: 1000 gid: 1000 workspaces: diff --git a/.buildkite/whitespace.yml b/.buildkite/misc/whitespace.yml similarity index 87% rename from .buildkite/whitespace.yml rename to .buildkite/misc/whitespace.yml index bd625e855c5b5..01b4c79e7822b 100644 --- a/.buildkite/whitespace.yml +++ b/.buildkite/misc/whitespace.yml @@ -14,8 +14,8 @@ steps: - JuliaCI/julia#v1: version: 1.6 - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v1/llvm-passes.tar.gz - rootfs_treehash: "f3ed53f159e8f13edfba8b20ebdb8ece73c1b8a8" + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.0/package_linux.x86_64.tar.gz + rootfs_treehash: "d5722d586b93eb307bb6340d275afdbf7578a756" workspaces: - "/cache/repos:/cache/repos" commands: | diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 097b8976eea70..3a3ebc718cbaa 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -4,9 +4,9 @@ # # Yes, this is creating another layer of indirection; the flow now looks like: # -# [webui] -> pipeline.yml -> llvm_passes.yml +# [webui] -> pipeline.yml -> misc/whitespace.yml # -# when we could theoretically just have the `webui` launch `llvm_passes.yml`, +# when we could theoretically just have the `webui` launch `misc/whitespace.yml`, # however this raises the bar for contributors to add new (unsigned) steps to # our CI configuration, so I'd rather live with an extra layer of indirection # and only need to touch the webui configuration when we need to alter @@ -14,10 +14,15 @@ steps: - label: ":buildkite: Launch unsigned pipelines" commands: | - # We launch whitespace first, because we want that pipeline to finish as quickly as possible. - # The remaining unsigned pipelines are launched in alphabetical order. - buildkite-agent pipeline upload .buildkite/whitespace.yml - buildkite-agent pipeline upload .buildkite/embedding.yml - buildkite-agent pipeline upload .buildkite/llvm_passes.yml + # First, we launch whitespace, because we want that pipeline to finish as quickly as possible. + buildkite-agent pipeline upload .buildkite/misc/whitespace.yml + + # Next, we launch the miscellaneous pipelines in alphabetical order. + buildkite-agent pipeline upload .buildkite/misc/doctest.yml + buildkite-agent pipeline upload .buildkite/misc/embedding.yml + buildkite-agent pipeline upload .buildkite/misc/llvmpasses.yml + + # Finally, we launch the platform pipelines in alphabetical order. + buildkite-agent pipeline upload .buildkite/platforms/linux64.yml agents: queue: julia diff --git a/.buildkite/platforms/linux64.yml b/.buildkite/platforms/linux64.yml new file mode 100644 index 0000000000000..69befe1064df7 --- /dev/null +++ b/.buildkite/platforms/linux64.yml @@ -0,0 +1,80 @@ +# These steps should only run on `sandbox.jl` machines, not `docker`-isolated ones +# since we need nestable sandboxing. The rootfs images being used here are built from +# the `.buildkite/rootfs_images/llvm-passes.jl` file. +agents: + queue: "julia" + # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing + sandbox.jl: "true" + os: "linux" + +steps: + - label: "package_linux64" + key: package_linux64 + plugins: + - JuliaCI/julia#v1: + version: 1.6 + - staticfloat/sandbox#v1: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.0/package_linux.x86_64.tar.gz + rootfs_treehash: "d5722d586b93eb307bb6340d275afdbf7578a756" + uid: 1000 + gid: 1000 + workspaces: + # Include `/cache/repos` so that our `git` version introspection works. + - "/cache/repos:/cache/repos" + commands: | + echo "--- Build Julia from source" + make -j 6 + make release + make install + + echo "--- Print Julia version info" + ./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()' + + echo "--- Compress build artifacts" + mv julia-* julia-artifact + rm -rf julia-linux64.tar.gz + tar czf julia-linux64.tar.gz julia-artifact/ + + echo "--- Upload build artifacts" + buildkite-agent artifact upload julia-linux64.tar.gz + timeout_in_minutes: 60 + notify: + - github_commit_status: + context: "package_linux64" + + # - label: "tester_linux64" + # key: tester_linux64 + # depends_on: package_linux64 + # plugins: + # - JuliaCI/julia#v1: + # version: 1.6 + # - staticfloat/sandbox#v1: + # # TODO: use a separate `tester_linux` image, instead of using the `package_linux` image. + # rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.0/package_linux.x86_64.tar.gz + # rootfs_treehash: "d5722d586b93eb307bb6340d275afdbf7578a756" + # uid: 1000 + # gid: 1000 + # workspaces: + # # Include `/cache/repos` so that our `git` version introspection works. + # - "/cache/repos:/cache/repos" + # env: + # JULIA_SHELL: "/bin/bash" + # commands: | + # echo "--- Download build artifacts" + # rm -rf julia-linux64.tar.gz + # buildkite-agent artifact download julia-linux64.tar.gz . + # + # echo "--- Extract build artifacts" + # rm -rf julia-artifact/ + # tar xzf julia-linux64.tar.gz julia-artifact/ + # + # echo "--- Print Julia version info" + # julia-artifact/bin/julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()' + # + # echo "--- Run the Julia test suite" + # unset JULIA_DEPOT_PATH + # julia-artifact/bin/julia .buildkite/rr_capture.jl julia-artifact/bin/julia -e 'Base.runtests(["all"]; ncores = Sys.CPU_THREADS)' + # timeout_in_minutes: 120 + # notify: + # - github_commit_status: + # context: "tester_linux64" diff --git a/.buildkite/rr_capture.jl b/.buildkite/rr_capture.jl new file mode 100644 index 0000000000000..07d57f31ff29c --- /dev/null +++ b/.buildkite/rr_capture.jl @@ -0,0 +1,134 @@ +using Dates +using Pkg +using Tar + +if Base.VERSION < v"1.6" + throw(ErrorException("The `rr_capture.jl` script requires Julia 1.6 or greater")) +end + +if length(ARGS) < 1 + throw(ErrorException("Usage: rr_capture.jl [command...]")) +end + +const TIMEOUT = 2 * 60 * 60 # timeout in seconds + +# We only use `rr` on the `tester_linux64` builder +const use_rr_if_builder_is = "tester_linux64" + +const run_id = get(ENV, "BUILDKITE_JOB_ID", "unknown") +const shortcommit = get(ENV, "BUILDKITE_COMMIT", "unknown") +const builder = get(ENV, "BUILDKITE_STEP_KEY", use_rr_if_builder_is) +const use_rr = builder == use_rr_if_builder_is + +@info "" run_id shortcommit builder use_rr +@info "" ARGS + +# if !use_rr # TODO: uncomment this line +if true # TODO: delete this line + @info "We will not run the tests under rr" + p = run(`$ARGS`) + exit(p.exitcode) +end + +@info "We will run the tests under rr" + +const num_cores = min(Sys.CPU_THREADS, 8, parse(Int, get(ENV, "JULIA_TEST_NUM_CORES", "8")) + 1) +@info "" num_cores + +proc = nothing + +new_env = copy(ENV) +mktempdir() do dir + Pkg.activate(dir) + Pkg.add("rr_jll") + Pkg.add("Zstd_jll") + + rr_jll = Base.require(Base.PkgId(Base.UUID((0xe86bdf43_55f7_5ea2_9fd0_e7daa2c0f2b4)), "rr_jll")) + zstd_jll = Base.require(Base.PkgId(Base.UUID((0x3161d3a3_bdf6_5164_811a_617609db77b4)), "Zstd_jll")) + rr(func) = Base.invokelatest(rr_jll.rr, func; adjust_LIBPATH=false) + rr() do rr_path + capture_script_path = joinpath(dir, "capture_output.sh") + loader = Sys.WORD_SIZE == 64 ? "/lib64/ld-linux-x86-64.so.2" : "/lib/ld-linux.so.2" + open(capture_script_path, "w") do io + write(io, """ + #!/bin/bash + + $(rr_path) record --nested=detach "\$@" > >(tee -a $(dir)/stdout.log) 2> >(tee -a $(dir)/stderr.log >&2) + """) + end + chmod(capture_script_path, 0o755) + + new_env = copy(ENV) + new_env["_RR_TRACE_DIR"] = joinpath(dir, "rr_traces") + new_env["RR_LOG"]="all:debug" + new_env["RR_LOG_BUFFER"]="100000" + new_env["JULIA_RR"] = capture_script_path + t_start = time() + global proc = run(setenv(`$(rr_path) record --num-cores=$(num_cores) $ARGS`, new_env), (stdin, stdout, stderr); wait=false) + + # Start asynchronous timer that will kill `rr` + @async begin + sleep(TIMEOUT) + + # If we've exceeded the timeout and `rr` is still running, kill it. + if isopen(proc) + println(stderr, "\n\nProcess timed out. Signalling `rr` for force-cleanup!") + kill(proc, Base.SIGTERM) + + # Give `rr` a chance to cleanup + sleep(60) + + if isopen(proc) + println(stderr, "\n\n`rr` failed to cleanup within one minute, killing and exiting immediately!") + kill(proc, Base.SIGKILL) + exit(1) + end + end + end + + # Wait for `rr` to finish, either through naturally finishing its run, or `SIGTERM`. + # If we have to `SIGKILL` + wait(proc) + + # On a non-zero exit code, upload the `rr` trace + if !success(proc) + println(stderr, "`rr` returned $(proc.exitcode), packing and uploading traces...") + + if !isdir(joinpath(dir, "rr_traces")) + println(stderr, "No `rr_traces` directory! Did `rr` itself fail?") + exit(1) + end + + # Clean up non-traces + rm(joinpath(dir, "rr_traces", "latest-trace")) + rm(joinpath(dir, "rr_traces", "cpu_lock")) + + # Create a directory for the pack files to go + pack_dir = joinpath(dir, "pack") + mkdir(pack_dir) + + # Pack all traces + trace_dirs = [joinpath(dir, "rr_traces", f) for f in readdir(joinpath(dir, "rr_traces"))] + filter!(isdir, trace_dirs) + run(ignorestatus(`$(rr_path) pack --pack-dir=$pack_dir $(trace_dirs)`)) + + # Tar it up + mkpath("dumps") + datestr = Dates.format(now(), dateformat"yyyy-mm-dd_HH_MM_SS") + dst_path = "dumps/rr-run_$(run_id)-gitsha_$(shortcommit)-$(datestr).tar.zst" + zstd_jll.zstdmt() do zstdp + tarproc = open(`$zstdp -o $dst_path`, "w") + Tar.create(dir, tarproc) + close(tarproc.in) + end + end + end +end + +# Pass the exit code back up to Buildkite +if proc.termsignal != 0 + ccall(:raise, Cvoid, (Cint,), proc.termsignal) + exit(1) # Just in case the signal did not cause an exit +else + exit(proc.exitcode) +end From 551c1027b99556ec60aad58b5f99552d1e2477c1 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Sun, 25 Jul 2021 05:16:24 -0400 Subject: [PATCH 30/73] CI (Buildkite): Update all rootfs images to version v3.1, add some documentation, and reorganize/rename/move a bunch of configuration files (#41698) (cherry picked from commit e6e79f7ab4549ab4d6ffbad22038ebfce1368f93) --- .buildkite/README.md | 5 ++ .buildkite/coverage-linux64/0_webui.yml | 16 ------ .buildkite/coverage-linux64/README.md | 6 --- .buildkite/coverage-linux64/pipeline.yml | 53 ------------------- .../coverage-linux64/run_tests_parallel.jl | 36 ------------- .../coverage-linux64/run_tests_serial.jl | 16 ------ .buildkite/experimental/pipeline.yml | 19 ------- .buildkite/pipeline.yml | 28 ---------- .../{ => pipelines}/experimental/0_webui.yml | 8 ++- .buildkite/pipelines/experimental/README.md | 7 +++ .../experimental/launch_unsigned_builders.yml | 6 +++ .../experimental/misc}/sanitizers.yml | 13 ++--- .buildkite/{ => pipelines/main}/0_webui.yml | 4 +- .buildkite/pipelines/main/README.md | 15 ++++++ .../main/launch_unsigned_builders.yml | 29 ++++++++++ .../{ => pipelines/main}/misc/doctest.yml | 9 ++-- .../{ => pipelines/main}/misc/embedding.yml | 7 +-- .../{ => pipelines/main}/misc/llvmpasses.yml | 7 +-- .../main/misc}/signed_pipeline_test.yml | 0 .../{ => pipelines/main}/misc/whitespace.yml | 7 +-- .../main}/platforms/linux64.yml | 14 +++-- .buildkite/pipelines/scheduled/0_webui.yml | 24 +++++++++ .buildkite/pipelines/scheduled/README.md | 5 ++ .../scheduled/coverage/coverage_linux64.yml | 40 ++++++++++++++ .../scheduled/coverage/run_tests_parallel.jl | 25 +++++++++ .../scheduled/coverage}/upload_coverage.jl | 0 .../scheduled/launch_unsigned_builders.yml | 0 .buildkite/{ => utilities/rr}/rr_capture.jl | 0 28 files changed, 185 insertions(+), 214 deletions(-) create mode 100644 .buildkite/README.md delete mode 100644 .buildkite/coverage-linux64/0_webui.yml delete mode 100644 .buildkite/coverage-linux64/README.md delete mode 100644 .buildkite/coverage-linux64/pipeline.yml delete mode 100644 .buildkite/coverage-linux64/run_tests_parallel.jl delete mode 100644 .buildkite/coverage-linux64/run_tests_serial.jl delete mode 100644 .buildkite/experimental/pipeline.yml delete mode 100644 .buildkite/pipeline.yml rename .buildkite/{ => pipelines}/experimental/0_webui.yml (67%) create mode 100644 .buildkite/pipelines/experimental/README.md create mode 100644 .buildkite/pipelines/experimental/launch_unsigned_builders.yml rename .buildkite/{experimental => pipelines/experimental/misc}/sanitizers.yml (67%) rename .buildkite/{ => pipelines/main}/0_webui.yml (89%) create mode 100644 .buildkite/pipelines/main/README.md create mode 100644 .buildkite/pipelines/main/launch_unsigned_builders.yml rename .buildkite/{ => pipelines/main}/misc/doctest.yml (71%) rename .buildkite/{ => pipelines/main}/misc/embedding.yml (76%) rename .buildkite/{ => pipelines/main}/misc/llvmpasses.yml (83%) rename .buildkite/{ => pipelines/main/misc}/signed_pipeline_test.yml (100%) rename .buildkite/{ => pipelines/main}/misc/whitespace.yml (63%) rename .buildkite/{ => pipelines/main}/platforms/linux64.yml (79%) create mode 100644 .buildkite/pipelines/scheduled/0_webui.yml create mode 100644 .buildkite/pipelines/scheduled/README.md create mode 100644 .buildkite/pipelines/scheduled/coverage/coverage_linux64.yml create mode 100644 .buildkite/pipelines/scheduled/coverage/run_tests_parallel.jl rename .buildkite/{coverage-linux64 => pipelines/scheduled/coverage}/upload_coverage.jl (100%) create mode 100644 .buildkite/pipelines/scheduled/launch_unsigned_builders.yml rename .buildkite/{ => utilities/rr}/rr_capture.jl (100%) diff --git a/.buildkite/README.md b/.buildkite/README.md new file mode 100644 index 0000000000000..12887536b90c1 --- /dev/null +++ b/.buildkite/README.md @@ -0,0 +1,5 @@ +# Buildkite + +This directory contains the Buildkite configuration files for Base Julia CI. + +The rootfs image definitions are located in the [rootfs-images](https://github.com/JuliaCI/rootfs-images) repository. diff --git a/.buildkite/coverage-linux64/0_webui.yml b/.buildkite/coverage-linux64/0_webui.yml deleted file mode 100644 index b0df8ec41de68..0000000000000 --- a/.buildkite/coverage-linux64/0_webui.yml +++ /dev/null @@ -1,16 +0,0 @@ -# This file represents what is put into the webUI. -# It is purely for keeping track of the changes we make to the webUI configuration; modifying this file has no effect. -# We use the `cryptic` buildkite plugin to provide secrets management, which requires some integration into the WebUI's steps. -agents: - queue: "julia" - sandbox.jl: "true" - -steps: - - label: ":unlock: Unlock secrets, launch pipelines" - plugins: - - staticfloat/cryptic: - # Our signed pipelines must have a `signature` or `signature_file` parameter that - # verifies the treehash of the pipeline itself and the inputs listed in `inputs` - signed_pipelines: - - pipeline: .buildkite/coverage-linux64/pipeline.yml - signature: U2FsdGVkX190BiaBGCTT6GNBDe5cHX3ZAP0IXbBfOWo7ys+1IuD5ymf4ImJbRIeE8NQac+iqud+bqCPHjii6DKmqzX+Jz6vax1NY3AxROhYlO5spUClvKr1wdngUCQON diff --git a/.buildkite/coverage-linux64/README.md b/.buildkite/coverage-linux64/README.md deleted file mode 100644 index 8cfb31d698225..0000000000000 --- a/.buildkite/coverage-linux64/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# Coverage pipeline - -We run coverage on a separate pipeline, that uses a scheduled build rather than webhooks. -The pipeline is here: https://buildkite.com/julialang/julia-coverage-linux64 - -It contains [its own webui steps](0_webuiy.ml) (listed here in this repository for clarity) and its own [pipeline.yml](pipeline.yml). diff --git a/.buildkite/coverage-linux64/pipeline.yml b/.buildkite/coverage-linux64/pipeline.yml deleted file mode 100644 index 218212d24ac2a..0000000000000 --- a/.buildkite/coverage-linux64/pipeline.yml +++ /dev/null @@ -1,53 +0,0 @@ -# These steps should only run on `sandbox.jl` machines, not `docker`-isolated ones -# since we need nestable sandboxing. The rootfs images being used here are built from -# the `.buildkite/rootfs_images/llvm-passes.jl` file. -agents: - queue: "julia" - # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing - sandbox.jl: "true" - os: "linux" - -steps: - - label: ":unlock: :coverage: Run coverage test" - plugins: - - staticfloat/cryptic: - variables: - - CODECOV_TOKEN="U2FsdGVkX19l0fhdBabbuiEdysyEabkJLRHfxm7CNRkuGbnwPV365sxxC7Czs/CVcws0N1oB4pVwALRRMe36oA==" - - COVERALLS_TOKEN="U2FsdGVkX19zopI0hMNzzi2UUOvNVFD8Y0iisFnO/ryVxU7Tit8ZEaeN+gxodRx4CosUUh192F1+q3dTMWRIvw==" - - JuliaCI/julia#v1: - version: 1.6 - - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v1/llvm-passes.tar.gz - rootfs_treehash: "f3ed53f159e8f13edfba8b20ebdb8ece73c1b8a8" - uid: 1000 - gid: 1000 - commands: | - echo "--- Build Julia from source" - make -j 6 - - echo "--- Print Julia version info" - ./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()' - ./julia -e '@info "" Sys.CPU_THREADS' - # this is necessary to make sure that the LibGit2 tests passes - git config --global init.defaultBranch master - - echo "--- Run some Julia tests in serial" - ./julia --code-coverage=all --sysimage-native-code=no .buildkite/coverage-linux64/run_tests_serial.jl test/compiler/codegen.jl - ./julia --code-coverage=all --sysimage-native-code=no .buildkite/coverage-linux64/run_tests_serial.jl test/compiler/contextual.jl - ./julia --code-coverage=all --sysimage-native-code=no .buildkite/coverage-linux64/run_tests_serial.jl test/compiler/inference.jl - ./julia --code-coverage=all --sysimage-native-code=no .buildkite/coverage-linux64/run_tests_serial.jl test/compiler/inline.jl - ./julia --code-coverage=all --sysimage-native-code=no .buildkite/coverage-linux64/run_tests_serial.jl test/compiler/interpreter_exec.jl - ./julia --code-coverage=all --sysimage-native-code=no .buildkite/coverage-linux64/run_tests_serial.jl test/compiler/irpasses.jl - ./julia --code-coverage=all --sysimage-native-code=no .buildkite/coverage-linux64/run_tests_serial.jl test/compiler/ssair.jl - ./julia --code-coverage=all --sysimage-native-code=no .buildkite/coverage-linux64/run_tests_serial.jl test/compiler/validation.jl - - echo "--- Run Julia tests in parallel with code coverage enabled" - ./julia --code-coverage=all --sysimage-native-code=no .buildkite/coverage-linux64/run_tests_parallel.jl - - echo "--- Process and upload coverage information" - ./julia .buildkite/coverage-linux64/upload_coverage.jl - timeout_in_minutes: 600 # 600 minutes = 10 hours - -# We must accept the signed job id secret in order to propagate secrets -env: - BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} diff --git a/.buildkite/coverage-linux64/run_tests_parallel.jl b/.buildkite/coverage-linux64/run_tests_parallel.jl deleted file mode 100644 index 5386828aa4e14..0000000000000 --- a/.buildkite/coverage-linux64/run_tests_parallel.jl +++ /dev/null @@ -1,36 +0,0 @@ -# When running this file, make sure to set the `--code-coverage=all` command-line flag. - -# Important note: even if one or more tests fail, we will still exit with status code 0. - -# The reason for this is that we always want to upload code coverage, even if some of the -# tests fail. Therefore, even if the `coverage-linux64` pipeline passes, you should not -# assume that all of the tests passed. If you want to know if all of the tests are passing, -# please look at the status of the `tester_linux64` pipeline. - -const include_tests = String[] - -const exclude_tests = String[] - -empty!(Base.DEPOT_PATH) -push!(Base.DEPOT_PATH, mktempdir(; cleanup = true)) - -module ChooseTests - include(joinpath(dirname(dirname(@__DIR__)), "test", "choosetests.jl")) -end - -const tests = ChooseTests.choosetests() |> - first |> - x -> setdiff(x, exclude_tests) |> - x -> vcat(x, include_tests) |> - unique |> - sort - -const ncores = min(Sys.CPU_THREADS, Threads.nthreads()) - -@info "" ncores Sys.CPU_THREADS Threads.nthreads() - -try - Base.runtests(tests; ncores) -catch ex - @error "" exception=(ex, catch_backtrace()) -end diff --git a/.buildkite/coverage-linux64/run_tests_serial.jl b/.buildkite/coverage-linux64/run_tests_serial.jl deleted file mode 100644 index 6d7380a55e402..0000000000000 --- a/.buildkite/coverage-linux64/run_tests_serial.jl +++ /dev/null @@ -1,16 +0,0 @@ -using Test - -const repository_root = dirname(dirname(@__DIR__)) - -for filename in ARGS - path = joinpath(repository_root, filename) - @info "Starting $(filename)" - try - @testset "$(filename)" begin - include(path) - end - catch ex - @error "" exception=(ex, catch_backtrace()) - end - @info "Finished $(filename)" -end diff --git a/.buildkite/experimental/pipeline.yml b/.buildkite/experimental/pipeline.yml deleted file mode 100644 index 6f86f7ef243e5..0000000000000 --- a/.buildkite/experimental/pipeline.yml +++ /dev/null @@ -1,19 +0,0 @@ -# This file launches all the build jobs that _don't_ require secrets access. -# These jobs can pass their output off to jobs that do require secrets access, -# but those privileged steps require signing before they can be run. -# -# Yes, this is creating another layer of indirection; the flow now looks like: -# -# [webui] -> pipeline.yml -> llvm_passes.yml -# -# when we could theoretically just have the `webui` launch `llvm_passes.yml`, -# however this raises the bar for contributors to add new (unsigned) steps to -# our CI configuration, so I'd rather live with an extra layer of indirection -# and only need to touch the webui configuration when we need to alter -# something about the privileged steps. -steps: - - label: ":buildkite: Launch unsigned pipelines" - commands: | - buildkite-agent pipeline upload .buildkite/experimental/sanitizers.yml - agents: - queue: julia diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml deleted file mode 100644 index 3a3ebc718cbaa..0000000000000 --- a/.buildkite/pipeline.yml +++ /dev/null @@ -1,28 +0,0 @@ -# This file launches all the build jobs that _don't_ require secrets access. -# These jobs can pass their output off to jobs that do require secrets access, -# but those privileged steps require signing before they can be run. -# -# Yes, this is creating another layer of indirection; the flow now looks like: -# -# [webui] -> pipeline.yml -> misc/whitespace.yml -# -# when we could theoretically just have the `webui` launch `misc/whitespace.yml`, -# however this raises the bar for contributors to add new (unsigned) steps to -# our CI configuration, so I'd rather live with an extra layer of indirection -# and only need to touch the webui configuration when we need to alter -# something about the privileged steps. -steps: - - label: ":buildkite: Launch unsigned pipelines" - commands: | - # First, we launch whitespace, because we want that pipeline to finish as quickly as possible. - buildkite-agent pipeline upload .buildkite/misc/whitespace.yml - - # Next, we launch the miscellaneous pipelines in alphabetical order. - buildkite-agent pipeline upload .buildkite/misc/doctest.yml - buildkite-agent pipeline upload .buildkite/misc/embedding.yml - buildkite-agent pipeline upload .buildkite/misc/llvmpasses.yml - - # Finally, we launch the platform pipelines in alphabetical order. - buildkite-agent pipeline upload .buildkite/platforms/linux64.yml - agents: - queue: julia diff --git a/.buildkite/experimental/0_webui.yml b/.buildkite/pipelines/experimental/0_webui.yml similarity index 67% rename from .buildkite/experimental/0_webui.yml rename to .buildkite/pipelines/experimental/0_webui.yml index ab3f68db4c393..54dbbc59d4256 100644 --- a/.buildkite/experimental/0_webui.yml +++ b/.buildkite/pipelines/experimental/0_webui.yml @@ -15,4 +15,10 @@ steps: # but some of our signed pipelines can wait upon the completion of these unsigned # pipelines. unsigned_pipelines: - - .buildkite/experimental/pipeline.yml + - .buildkite/pipelines/experimental/launch_unsigned_builders.yml + + # Our signed pipelines must have a `signature` or `signature_file` parameter that + # verifies the treehash of the pipeline itself and the inputs listed in `inputs` + # signed_pipelines: + # - pipeline: .buildkite/pipelines/experimental/misc/foo_bar_baz.yml + # signature: "my_signature" diff --git a/.buildkite/pipelines/experimental/README.md b/.buildkite/pipelines/experimental/README.md new file mode 100644 index 0000000000000..f92aac7a1af02 --- /dev/null +++ b/.buildkite/pipelines/experimental/README.md @@ -0,0 +1,7 @@ +## Experimental pipeline (`master` branch only) + +This is the [`julia-master->experimental`](https://buildkite.com/julialang/julia-master-experimental) pipeline. + +We use this pipeline for builders that are not yet stable enough to go into the main pipeline. + +These builders are triggered by GitHub webhook events, such as pushes and pull requests. diff --git a/.buildkite/pipelines/experimental/launch_unsigned_builders.yml b/.buildkite/pipelines/experimental/launch_unsigned_builders.yml new file mode 100644 index 0000000000000..f023e19a5c940 --- /dev/null +++ b/.buildkite/pipelines/experimental/launch_unsigned_builders.yml @@ -0,0 +1,6 @@ +steps: + - label: ":buildkite: Launch unsigned pipelines" + commands: | + buildkite-agent pipeline upload .buildkite/pipelines/experimental/misc/sanitizers.yml + agents: + queue: julia diff --git a/.buildkite/experimental/sanitizers.yml b/.buildkite/pipelines/experimental/misc/sanitizers.yml similarity index 67% rename from .buildkite/experimental/sanitizers.yml rename to .buildkite/pipelines/experimental/misc/sanitizers.yml index 534dac81e53f3..67c0b547d4b20 100644 --- a/.buildkite/experimental/sanitizers.yml +++ b/.buildkite/pipelines/experimental/misc/sanitizers.yml @@ -1,6 +1,3 @@ -# These steps should only run on `sandbox.jl` machines, not `docker`-isolated ones -# since we need nestable sandboxing. The rootfs images being used here are built from -# the `.buildkite/rootfs_images/llvm-passes.jl` file. agents: queue: "julia" # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing @@ -14,8 +11,8 @@ steps: - JuliaCI/julia#v1: version: 1.6 - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v2.0/llvm_passes.tar.gz - rootfs_treehash: "0b5a80c1d0ab110a57fbdb7f4bc042a5ede310f3" + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.1/llvm_passes.x86_64.tar.gz + rootfs_treehash: "9dd715500b117a16fcfa419ea0bca0c0ca902cee" uid: 1000 gid: 1000 workspaces: @@ -29,6 +26,6 @@ steps: echo "--- Test that ASAN is enabled" contrib/asan/check.jl ./tmp/test-asan/asan/usr/bin/julia-debug timeout_in_minutes: 120 - # notify: - # - github_commit_status: - # context: "asan" + # notify: # TODO: uncomment this line + # - github_commit_status: # TODO: uncomment this line + # context: "asan" # TODO: uncomment this line diff --git a/.buildkite/0_webui.yml b/.buildkite/pipelines/main/0_webui.yml similarity index 89% rename from .buildkite/0_webui.yml rename to .buildkite/pipelines/main/0_webui.yml index 440d2d443ce7e..96dc81cef36fe 100644 --- a/.buildkite/0_webui.yml +++ b/.buildkite/pipelines/main/0_webui.yml @@ -15,10 +15,10 @@ steps: # but some of our signed pipelines can wait upon the completion of these unsigned # pipelines. unsigned_pipelines: - - .buildkite/pipeline.yml + - .buildkite/pipelines/main/launch_unsigned_builders.yml # Our signed pipelines must have a `signature` or `signature_file` parameter that # verifies the treehash of the pipeline itself and the inputs listed in `inputs` signed_pipelines: - - pipeline: .buildkite/signed_pipeline_test.yml + - pipeline: .buildkite/pipelines/main/misc/signed_pipeline_test.yml signature: "U2FsdGVkX18ZrMlORSIi0TvW99eZ3JyUEhntMyXjrOSJ9jAtUohgHc8+dMnWUc6qWjYt8k63wfPrth33SGWDiF814Bm1W5Zk3+R6HgVu6UCrQEI5sLm+SJPbrvET+Jkn" diff --git a/.buildkite/pipelines/main/README.md b/.buildkite/pipelines/main/README.md new file mode 100644 index 0000000000000..6b9d67bd7cc3a --- /dev/null +++ b/.buildkite/pipelines/main/README.md @@ -0,0 +1,15 @@ +## Main pipeline + +This is the main pipeline. It contains most of the builders. These builders are triggered by GitHub webhook events, such as pushes and pull requests. + +We have a different main pipeline for each permanent branch. + +For example: + +| Permanent Branch | Pipeline | +| ---------------- | -------------------------------------------------------------------------------- | +| `master` | [`julia-master`](https://buildkite.com/julialang/julia-master) | +| `release-1.6` | [`julia-release-1.6`](https://buildkite.com/julialang/julia-release-1-dot-6) | +| `release-1.7` | [`julia-release-1.7`](https://buildkite.com/julialang/julia-release-1-dot-7) | + +(This is not a complete list.) diff --git a/.buildkite/pipelines/main/launch_unsigned_builders.yml b/.buildkite/pipelines/main/launch_unsigned_builders.yml new file mode 100644 index 0000000000000..6e9f0f0d8fa23 --- /dev/null +++ b/.buildkite/pipelines/main/launch_unsigned_builders.yml @@ -0,0 +1,29 @@ +# This file launches all the build jobs that _don't_ require secrets access. +# These jobs can pass their output off to jobs that do require secrets access, +# but those privileged steps require signing before they can be run. +# +# Yes, this is creating another layer of indirection; the flow now looks like: +# +# [webui] -> launch_unsigned_builders.yml -> misc/whitespace.yml +# +# when we could theoretically just have the `webui` launch `misc/whitespace.yml`, +# however this raises the bar for contributors to add new (unsigned) steps to +# our CI configuration, so I'd rather live with an extra layer of indirection +# and only need to touch the webui configuration when we need to alter +# something about the privileged steps. + +steps: + - label: ":buildkite: Launch unsigned builders" + commands: | + # First, we launch the `whitespace` builder, because we want that builder to finish as quickly as possible. + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/whitespace.yml + + # Next, we launch the miscellaneous builders in alphabetical order. + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/doctest.yml + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/llvmpasses.yml + + # Finally, we launch the platform builders (`package_*`) and (`tester_*`) in alphabetical order. + buildkite-agent pipeline upload .buildkite/pipelines/main/platforms/linux64.yml + agents: + queue: julia diff --git a/.buildkite/misc/doctest.yml b/.buildkite/pipelines/main/misc/doctest.yml similarity index 71% rename from .buildkite/misc/doctest.yml rename to .buildkite/pipelines/main/misc/doctest.yml index 234e49f5dcf53..0a5dc29bcb1c7 100644 --- a/.buildkite/misc/doctest.yml +++ b/.buildkite/pipelines/main/misc/doctest.yml @@ -1,6 +1,3 @@ -# These steps should only run on `sandbox.jl` machines, not `docker`-isolated ones -# since we need nestable sandboxing. The rootfs images being used here are built from -# the `.buildkite/rootfs_images/llvm-passes.jl` file. agents: queue: "julia" # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing @@ -14,8 +11,8 @@ steps: - JuliaCI/julia#v1: version: 1.6 - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.0/package_linux.x86_64.tar.gz - rootfs_treehash: "d5722d586b93eb307bb6340d275afdbf7578a756" + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.1/package_linux.x86_64.tar.gz + rootfs_treehash: "8c33c341a864852629b8aac01a6eb6a79b73570e" uid: 1000 gid: 1000 workspaces: @@ -33,7 +30,7 @@ steps: echo "--- Run Julia doctests" JULIA_NUM_THREADS=1 make -C doc doctest=true - timeout_in_minutes: 120 + timeout_in_minutes: 45 notify: - github_commit_status: context: "doctest" diff --git a/.buildkite/misc/embedding.yml b/.buildkite/pipelines/main/misc/embedding.yml similarity index 76% rename from .buildkite/misc/embedding.yml rename to .buildkite/pipelines/main/misc/embedding.yml index 31f85633479a4..087ca0f68eb3d 100644 --- a/.buildkite/misc/embedding.yml +++ b/.buildkite/pipelines/main/misc/embedding.yml @@ -1,6 +1,3 @@ -# These steps should only run on `sandbox.jl` machines, not `docker`-isolated ones -# since we need nestable sandboxing. The rootfs images being used here are built from -# the `.buildkite/rootfs_images/llvm-passes.jl` file. agents: queue: "julia" # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing @@ -14,8 +11,8 @@ steps: - JuliaCI/julia#v1: version: 1.6 - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.0/package_linux.x86_64.tar.gz - rootfs_treehash: "d5722d586b93eb307bb6340d275afdbf7578a756" + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.1/package_linux.x86_64.tar.gz + rootfs_treehash: "8c33c341a864852629b8aac01a6eb6a79b73570e" uid: 1000 gid: 1000 workspaces: diff --git a/.buildkite/misc/llvmpasses.yml b/.buildkite/pipelines/main/misc/llvmpasses.yml similarity index 83% rename from .buildkite/misc/llvmpasses.yml rename to .buildkite/pipelines/main/misc/llvmpasses.yml index 7660b3978e967..eed2f957855bb 100644 --- a/.buildkite/misc/llvmpasses.yml +++ b/.buildkite/pipelines/main/misc/llvmpasses.yml @@ -1,6 +1,3 @@ -# These steps should only run on `sandbox.jl` machines, not `docker`-isolated ones -# since we need nestable sandboxing. The rootfs images being used here are built from -# the `.buildkite/rootfs_images/llvm-passes.jl` file. agents: queue: "julia" # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing @@ -36,8 +33,8 @@ steps: - JuliaCI/julia#v1: version: 1.6 - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.0/package_linux.x86_64.tar.gz - rootfs_treehash: "d5722d586b93eb307bb6340d275afdbf7578a756" + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.1/package_linux.x86_64.tar.gz + rootfs_treehash: "8c33c341a864852629b8aac01a6eb6a79b73570e" uid: 1000 gid: 1000 workspaces: diff --git a/.buildkite/signed_pipeline_test.yml b/.buildkite/pipelines/main/misc/signed_pipeline_test.yml similarity index 100% rename from .buildkite/signed_pipeline_test.yml rename to .buildkite/pipelines/main/misc/signed_pipeline_test.yml diff --git a/.buildkite/misc/whitespace.yml b/.buildkite/pipelines/main/misc/whitespace.yml similarity index 63% rename from .buildkite/misc/whitespace.yml rename to .buildkite/pipelines/main/misc/whitespace.yml index 01b4c79e7822b..3f9bf13421d8e 100644 --- a/.buildkite/misc/whitespace.yml +++ b/.buildkite/pipelines/main/misc/whitespace.yml @@ -1,6 +1,3 @@ -# These steps should only run on `sandbox.jl` machines, not `docker`-isolated ones -# since we need nestable sandboxing. The rootfs images being used here are built from -# the `.buildkite/rootfs_images/llvm-passes.jl` file. agents: queue: "julia" # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing @@ -14,8 +11,8 @@ steps: - JuliaCI/julia#v1: version: 1.6 - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.0/package_linux.x86_64.tar.gz - rootfs_treehash: "d5722d586b93eb307bb6340d275afdbf7578a756" + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.1/package_linux.x86_64.tar.gz + rootfs_treehash: "8c33c341a864852629b8aac01a6eb6a79b73570e" workspaces: - "/cache/repos:/cache/repos" commands: | diff --git a/.buildkite/platforms/linux64.yml b/.buildkite/pipelines/main/platforms/linux64.yml similarity index 79% rename from .buildkite/platforms/linux64.yml rename to .buildkite/pipelines/main/platforms/linux64.yml index 69befe1064df7..7780c574951e8 100644 --- a/.buildkite/platforms/linux64.yml +++ b/.buildkite/pipelines/main/platforms/linux64.yml @@ -1,6 +1,3 @@ -# These steps should only run on `sandbox.jl` machines, not `docker`-isolated ones -# since we need nestable sandboxing. The rootfs images being used here are built from -# the `.buildkite/rootfs_images/llvm-passes.jl` file. agents: queue: "julia" # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing @@ -14,8 +11,8 @@ steps: - JuliaCI/julia#v1: version: 1.6 - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.0/package_linux.x86_64.tar.gz - rootfs_treehash: "d5722d586b93eb307bb6340d275afdbf7578a756" + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.1/package_linux.x86_64.tar.gz + rootfs_treehash: "8c33c341a864852629b8aac01a6eb6a79b73570e" uid: 1000 gid: 1000 workspaces: @@ -42,6 +39,7 @@ steps: - github_commit_status: context: "package_linux64" + # TODO: uncomment the following lines in order to enable the `tester_linux64` builder # - label: "tester_linux64" # key: tester_linux64 # depends_on: package_linux64 @@ -50,8 +48,8 @@ steps: # version: 1.6 # - staticfloat/sandbox#v1: # # TODO: use a separate `tester_linux` image, instead of using the `package_linux` image. - # rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.0/package_linux.x86_64.tar.gz - # rootfs_treehash: "d5722d586b93eb307bb6340d275afdbf7578a756" + # rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.1/package_linux.x86_64.tar.gz + # rootfs_treehash: "8c33c341a864852629b8aac01a6eb6a79b73570e" # uid: 1000 # gid: 1000 # workspaces: @@ -73,7 +71,7 @@ steps: # # echo "--- Run the Julia test suite" # unset JULIA_DEPOT_PATH - # julia-artifact/bin/julia .buildkite/rr_capture.jl julia-artifact/bin/julia -e 'Base.runtests(["all"]; ncores = Sys.CPU_THREADS)' + # julia-artifact/bin/julia .buildkite/utilities/rr/rr_capture.jl julia-artifact/bin/julia -e 'Base.runtests(["all"]; ncores = Sys.CPU_THREADS)' # timeout_in_minutes: 120 # notify: # - github_commit_status: diff --git a/.buildkite/pipelines/scheduled/0_webui.yml b/.buildkite/pipelines/scheduled/0_webui.yml new file mode 100644 index 0000000000000..8aaf812376b5c --- /dev/null +++ b/.buildkite/pipelines/scheduled/0_webui.yml @@ -0,0 +1,24 @@ +# This file represents what is put into the webUI. +# It is purely for keeping track of the changes we make to the webUI configuration; modifying this file has no effect. +# We use the `cryptic` buildkite plugin to provide secrets management, which requires some integration into the WebUI's steps. +agents: + queue: "julia" + sandbox.jl: "true" + +steps: + - label: ":unlock: Unlock secrets, launch pipelines" + plugins: + - staticfloat/cryptic: + # Our list of pipelines that should be launched (but don't require a signature) + # These pipelines can be modified by any contributor and CI will still run. + # Build secrets will not be available in these pipelines (or their children) + # but some of our signed pipelines can wait upon the completion of these unsigned + # pipelines. + # unsigned_pipelines: + # - .buildkite/pipelines/scheduled/launch_unsigned_builders.yml + + # Our signed pipelines must have a `signature` or `signature_file` parameter that + # verifies the treehash of the pipeline itself and the inputs listed in `inputs` + signed_pipelines: + - pipeline: .buildkite/pipelines/scheduled/coverage/coverage_linux64.yml + signature: "U2FsdGVkX1+lpFo/nKzx3c6xCZPKYTAuunXpOsZG4+s4+iU5LfEpMvtNvpKQjDugRoxQxCItMqB6vr4KZN3KtKhjkLbr8ExAyaPil/N/uFhrLlpwNem9dxHbPrU2l7qo" diff --git a/.buildkite/pipelines/scheduled/README.md b/.buildkite/pipelines/scheduled/README.md new file mode 100644 index 0000000000000..ca071dceb2a44 --- /dev/null +++ b/.buildkite/pipelines/scheduled/README.md @@ -0,0 +1,5 @@ +## Scheduled pipeline (`master` branch only) + +This is the [`julia-master->scheduled`](https://buildkite.com/julialang/julia-master-scheduled) pipeline. + +We use this pipeline for scheduled builds. The builders in this pipeline run on a schedule once per day. They are not triggered by GitHub webhooks. diff --git a/.buildkite/pipelines/scheduled/coverage/coverage_linux64.yml b/.buildkite/pipelines/scheduled/coverage/coverage_linux64.yml new file mode 100644 index 0000000000000..ce7a3aca4227d --- /dev/null +++ b/.buildkite/pipelines/scheduled/coverage/coverage_linux64.yml @@ -0,0 +1,40 @@ +agents: + queue: "julia" + # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing + sandbox.jl: "true" + os: "linux" + +steps: + - label: ":unlock: :coverage: Run coverage test" + plugins: + - staticfloat/cryptic: + variables: + - CODECOV_TOKEN="U2FsdGVkX19l0fhdBabbuiEdysyEabkJLRHfxm7CNRkuGbnwPV365sxxC7Czs/CVcws0N1oB4pVwALRRMe36oA==" + - COVERALLS_TOKEN="U2FsdGVkX19zopI0hMNzzi2UUOvNVFD8Y0iisFnO/ryVxU7Tit8ZEaeN+gxodRx4CosUUh192F1+q3dTMWRIvw==" + - JuliaCI/julia#v1: + version: 1.6 + - staticfloat/sandbox#v1: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.1/package_linux.x86_64.tar.gz + rootfs_treehash: "8c33c341a864852629b8aac01a6eb6a79b73570e" + uid: 1000 + gid: 1000 + commands: | + echo "--- Build Julia from source" + make -j 6 + + echo "--- Print Julia version info" + ./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()' + ./julia -e '@info "" Sys.CPU_THREADS' + # this is necessary to make sure that the LibGit2 tests passes + git config --global init.defaultBranch master + + echo "--- Run Julia tests in parallel with code coverage enabled" + ./julia --code-coverage=all --sysimage-native-code=no .buildkite/pipelines/scheduled/coverage/run_tests_parallel.jl + + echo "--- Process and upload coverage information" + ./julia .buildkite/pipelines/scheduled/coverage/upload_coverage.jl + timeout_in_minutes: 240 # 240 minutes = 4 hours + +# We must accept the signed job id secret in order to propagate secrets +env: + BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} diff --git a/.buildkite/pipelines/scheduled/coverage/run_tests_parallel.jl b/.buildkite/pipelines/scheduled/coverage/run_tests_parallel.jl new file mode 100644 index 0000000000000..6da608b5e8be9 --- /dev/null +++ b/.buildkite/pipelines/scheduled/coverage/run_tests_parallel.jl @@ -0,0 +1,25 @@ +# Important note: even if one or more tests fail, we will still exit with status code 0. + +# The reason for this is that we always want to upload code coverage, even if some of the +# tests fail. Therefore, even if the `coverage_linux64` builder passes, you should not +# assume that all of the tests passed. If you want to know if all of the tests are passing, +# please look at the status of the `tester_*` builders (e.g. `tester_linux64`). + +# When running this file, make sure to set all of the following command-line flags: +# 1. `--code-coverage=all` +# 2. `--sysimage-native-code=no` + +empty!(Base.DEPOT_PATH) +push!(Base.DEPOT_PATH, mktempdir(; cleanup = true)) + +const tests = "all" +const ncores = Sys.CPU_THREADS + +@info "" Sys.CPU_THREADS +@info "" tests ncores + +try + Base.runtests(tests; ncores) +catch ex + @error "" exception=(ex, catch_backtrace()) +end diff --git a/.buildkite/coverage-linux64/upload_coverage.jl b/.buildkite/pipelines/scheduled/coverage/upload_coverage.jl similarity index 100% rename from .buildkite/coverage-linux64/upload_coverage.jl rename to .buildkite/pipelines/scheduled/coverage/upload_coverage.jl diff --git a/.buildkite/pipelines/scheduled/launch_unsigned_builders.yml b/.buildkite/pipelines/scheduled/launch_unsigned_builders.yml new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/.buildkite/rr_capture.jl b/.buildkite/utilities/rr/rr_capture.jl similarity index 100% rename from .buildkite/rr_capture.jl rename to .buildkite/utilities/rr/rr_capture.jl From 6a5c98614918ba9c13c264a1eb7ba55c85990c00 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Sun, 25 Jul 2021 21:42:39 -0400 Subject: [PATCH 31/73] CI (Buildkite): include the short commit hash in the artifact filename (#41700) (cherry picked from commit 343c2b19bc83dfa58cbb701babba4a0721e18903) --- .../pipelines/main/platforms/linux64.yml | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/.buildkite/pipelines/main/platforms/linux64.yml b/.buildkite/pipelines/main/platforms/linux64.yml index 7780c574951e8..1e88db1144b4c 100644 --- a/.buildkite/pipelines/main/platforms/linux64.yml +++ b/.buildkite/pipelines/main/platforms/linux64.yml @@ -19,6 +19,17 @@ steps: # Include `/cache/repos` so that our `git` version introspection works. - "/cache/repos:/cache/repos" commands: | + echo "--- Print the short and long commit hashes" + SHORT_COMMIT_LENGTH=10 + SHORT_COMMIT=`echo $$BUILDKITE_COMMIT | cut -c1-$$SHORT_COMMIT_LENGTH` + JULIA_DIRECTORY_NAME="julia-$$SHORT_COMMIT" + JULIA_BINARYDIST_FILENAME=`make print-JULIA_BINARYDIST_FILENAME | cut -c27-` + ARTIFACT_FILE_EXTENSION="tar.gz" + ARTIFACT_FILENAME="$$JULIA_BINARYDIST_FILENAME.$$ARTIFACT_FILE_EXTENSION" + echo "The full commit is $$BUILDKITE_COMMIT" + echo "The Julia directory name will be $$JULIA_DIRECTORY_NAME" + echo "The artifact filename will be $$ARTIFACT_FILENAME" + echo "--- Build Julia from source" make -j 6 make release @@ -28,12 +39,13 @@ steps: ./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()' echo "--- Compress build artifacts" - mv julia-* julia-artifact - rm -rf julia-linux64.tar.gz - tar czf julia-linux64.tar.gz julia-artifact/ + ls -ld $$JULIA_DIRECTORY_NAME/ + rm -rf $$ARTIFACT_FILENAME + tar czf $$ARTIFACT_FILENAME $$JULIA_DIRECTORY_NAME/ + ls -l $$ARTIFACT_FILENAME echo "--- Upload build artifacts" - buildkite-agent artifact upload julia-linux64.tar.gz + buildkite-agent artifact upload $$ARTIFACT_FILENAME timeout_in_minutes: 60 notify: - github_commit_status: From 5a62f7b4ed5b2272305e79f8519ee547d724aa81 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Sun, 25 Jul 2021 22:20:55 -0400 Subject: [PATCH 32/73] CI (Buildkite): for the signed pipeline test, use `signature_file` instead of `signature` (#41706) * CI (Buildkite): for the signed pipeline test, use `signature_file` instead of `signature` * Base64-decode the signature before writing it to the signature file (cherry picked from commit f7f46af8ff39a1b4c7000651c680058e9c0639f5) --- .buildkite/pipelines/main/0_webui.yml | 2 +- .../pipelines/main/misc/signed_pipeline_test.yml.signature | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 .buildkite/pipelines/main/misc/signed_pipeline_test.yml.signature diff --git a/.buildkite/pipelines/main/0_webui.yml b/.buildkite/pipelines/main/0_webui.yml index 96dc81cef36fe..8e7b9c58ea423 100644 --- a/.buildkite/pipelines/main/0_webui.yml +++ b/.buildkite/pipelines/main/0_webui.yml @@ -21,4 +21,4 @@ steps: # verifies the treehash of the pipeline itself and the inputs listed in `inputs` signed_pipelines: - pipeline: .buildkite/pipelines/main/misc/signed_pipeline_test.yml - signature: "U2FsdGVkX18ZrMlORSIi0TvW99eZ3JyUEhntMyXjrOSJ9jAtUohgHc8+dMnWUc6qWjYt8k63wfPrth33SGWDiF814Bm1W5Zk3+R6HgVu6UCrQEI5sLm+SJPbrvET+Jkn" + signature_file: .buildkite/pipelines/main/misc/signed_pipeline_test.yml.signature diff --git a/.buildkite/pipelines/main/misc/signed_pipeline_test.yml.signature b/.buildkite/pipelines/main/misc/signed_pipeline_test.yml.signature new file mode 100644 index 0000000000000..10220c758086a --- /dev/null +++ b/.buildkite/pipelines/main/misc/signed_pipeline_test.yml.signature @@ -0,0 +1 @@ +Salted__¬ÉNE""Ñ;Ö÷יܜ”í3%ã¬ä‰ö0-Rˆ`Ï>tÉÖQΪZ6-òN·Áóë¶÷Heƒˆ_5àµ[–dßäzné@«@B9°¹¾H“Û®ñø™' \ No newline at end of file From 6bb23a31a9cccb28efc17e042604ef98efdfaa9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mos=C3=A8=20Giordano?= Date: Mon, 21 Jun 2021 18:04:58 +0100 Subject: [PATCH 33/73] [CompilerSupportLibraries_jll] Update to v0.5.0 (#41292) This includes libraries from GCC 11. (cherry picked from commit d68e570f19fac4ea3841b97cfb87096f796ec518) --- deps/checksums/compilersupportlibraries | 184 +++++++++--------- .../CompilerSupportLibraries_jll/Project.toml | 2 +- 2 files changed, 93 insertions(+), 93 deletions(-) diff --git a/deps/checksums/compilersupportlibraries b/deps/checksums/compilersupportlibraries index f181febda4642..e351d100cb481 100644 --- a/deps/checksums/compilersupportlibraries +++ b/deps/checksums/compilersupportlibraries @@ -1,92 +1,92 @@ -CompilerSupportLibraries.v0.3.6+1.aarch64-apple-darwin-libgfortran5.tar.gz/md5/e2617b4caaf70a6cfc22f789c856a447 -CompilerSupportLibraries.v0.3.6+1.aarch64-apple-darwin-libgfortran5.tar.gz/sha512/60a195de3affdd6e5d82d8471f3537b859aa3f4b24b6e4cf20c318f2a21d7cd71f8be6b4e56fa8e49361a6b3b25620d99d68d4116cfd4a6f37946fe9ab653973 -CompilerSupportLibraries.v0.3.6+1.aarch64-linux-gnu-libgfortran3.tar.gz/md5/7a793318d44c87141b495d715bd75cfd -CompilerSupportLibraries.v0.3.6+1.aarch64-linux-gnu-libgfortran3.tar.gz/sha512/061a7791ffde78d536239222ba2a8a20451b5a9b2b587c6a158a3434f072e41ba0b9da37d9afdbadb48c2768dbf03f0621989db09243d9d91dc06a7be480b8b5 -CompilerSupportLibraries.v0.3.6+1.aarch64-linux-gnu-libgfortran4.tar.gz/md5/b962679d33709701ac5640945ea7ef23 -CompilerSupportLibraries.v0.3.6+1.aarch64-linux-gnu-libgfortran4.tar.gz/sha512/7a494b749e0dfa26898088212205d8b26d20cf67a4c8a228290c51f90bd1da3e2fdb7aec508e861d515d166380de56acda4236059c06c2cc9699120273dd9ff2 -CompilerSupportLibraries.v0.3.6+1.aarch64-linux-gnu-libgfortran5.tar.gz/md5/55da155faf3d8a00727a5243e4bb4cee -CompilerSupportLibraries.v0.3.6+1.aarch64-linux-gnu-libgfortran5.tar.gz/sha512/69ceab960374780cc20fa83fd8a55a5e3ead9ad39ae95d23db66ee40d7892e33d94ace9d5d66ca4c578d8e5eaf6b25b76e1c9326efdbeb72c2758e193d8cbddc -CompilerSupportLibraries.v0.3.6+1.aarch64-linux-musl-libgfortran3.tar.gz/md5/d1d01472d321bca9e49c5d4fd7a67efe -CompilerSupportLibraries.v0.3.6+1.aarch64-linux-musl-libgfortran3.tar.gz/sha512/7d228a66c575cd962d71739112cef619082ace2114c64e3d7b71f25cfba65215a2e1e48a387273e9474495d6f3c8b40619174fbba1424b92e3af188a4e4b93da -CompilerSupportLibraries.v0.3.6+1.aarch64-linux-musl-libgfortran4.tar.gz/md5/91cc6f967a3f539981a57a5c2d9bd655 -CompilerSupportLibraries.v0.3.6+1.aarch64-linux-musl-libgfortran4.tar.gz/sha512/747954cbbd6135231f297e543987e70c2df2764eba6317d7b35ea727e107a4013f0c04096675d03ec4c4b309903ef17d3ec2cecdce95e7b0ddf392cddeb831c1 -CompilerSupportLibraries.v0.3.6+1.aarch64-linux-musl-libgfortran5.tar.gz/md5/49e3d15b927dbedcdce25649a0a1aa5d -CompilerSupportLibraries.v0.3.6+1.aarch64-linux-musl-libgfortran5.tar.gz/sha512/0cfc57bbf30d106f5e6debae5fed60ce49e03fae4b1606dcd138aebfae6ab0f06f256291cdc5723d100e947ac33fdda43f3d5d1eaaacf1c40759e65872517319 -CompilerSupportLibraries.v0.3.6+1.armv6l-linux-gnueabihf-libgfortran3.tar.gz/md5/faef5a6105813b6386860e51fafb8b8e -CompilerSupportLibraries.v0.3.6+1.armv6l-linux-gnueabihf-libgfortran3.tar.gz/sha512/fb3401631864425efffb1d41050a2166a4a1bb7ef0a4834f88116ada268426e595341ba48d7759c16958d2dbb397e17c49fd0fc360c1ae3097ddd0301d3006ef -CompilerSupportLibraries.v0.3.6+1.armv6l-linux-gnueabihf-libgfortran4.tar.gz/md5/4e2b35488b72576ef6fc6408bba4a43e -CompilerSupportLibraries.v0.3.6+1.armv6l-linux-gnueabihf-libgfortran4.tar.gz/sha512/53dc1be5781cc26718fe7d250c6a92609224bdf792e6e93ec4c29c2c2c8f2aef98bed09f32cdd274a9c13d46ade61745e7ce28b077693c0fff5334c09e03cff4 -CompilerSupportLibraries.v0.3.6+1.armv6l-linux-gnueabihf-libgfortran5.tar.gz/md5/feaa3b7611af9eace55a01607f4b3c2f -CompilerSupportLibraries.v0.3.6+1.armv6l-linux-gnueabihf-libgfortran5.tar.gz/sha512/94c980068fceecc5f64939f686543335c323cb98a46de6a6c361972b8033f4845a9cfe3b01cc218b9ef67408177506f4c1c99bcd8f596581fde29205edb47f54 -CompilerSupportLibraries.v0.3.6+1.armv6l-linux-musleabihf-libgfortran3.tar.gz/md5/b3c9636596534a24154de59f1ab3eef2 -CompilerSupportLibraries.v0.3.6+1.armv6l-linux-musleabihf-libgfortran3.tar.gz/sha512/f109152758f9a3c2bd7a91e10455be963c69aba102598565462c3e48900f241fadc9cc12ecf20febd19afe28c77b698b11618635d90a2bf6f3c5f105de59ddf5 -CompilerSupportLibraries.v0.3.6+1.armv6l-linux-musleabihf-libgfortran4.tar.gz/md5/aeefaa508f55c8695d38759a26045153 -CompilerSupportLibraries.v0.3.6+1.armv6l-linux-musleabihf-libgfortran4.tar.gz/sha512/6999d33b9dc575bbd079e3bf16abea25e278203b4b2d740cba089a187acce935ddac9d7226f736875f6e0c744baea4ae46a6af742e885ad65270e3a0c26777f5 -CompilerSupportLibraries.v0.3.6+1.armv6l-linux-musleabihf-libgfortran5.tar.gz/md5/eab52efe430621e5149dfdcb12dc2f6b -CompilerSupportLibraries.v0.3.6+1.armv6l-linux-musleabihf-libgfortran5.tar.gz/sha512/b45a922639fe2981c7e49b4b8e61fa56a9bcc6ec49bc5d9b693f202e4e712e6661b9bb91d1cbfe7fc12e79c630a879181a9618abbcbc8b172086b8dc620feb0c -CompilerSupportLibraries.v0.3.6+1.armv7l-linux-gnueabihf-libgfortran3.tar.gz/md5/3ce0215c2fb1aac133ddc2cfc849b568 -CompilerSupportLibraries.v0.3.6+1.armv7l-linux-gnueabihf-libgfortran3.tar.gz/sha512/91be3e075c6d98c901bab767538fa98ec9bf7bda45c747522964543317b89c6914013d319d04049daa66903d5459933707d47e584be77c6a03af526e6156f9b1 -CompilerSupportLibraries.v0.3.6+1.armv7l-linux-gnueabihf-libgfortran4.tar.gz/md5/edb5f07f117c933519ce09b79b79c002 -CompilerSupportLibraries.v0.3.6+1.armv7l-linux-gnueabihf-libgfortran4.tar.gz/sha512/d2dccd7d2b839caef2c1f76b759c701db81de9fc034f1975cf3921370198459c0df8b8f62d96adf6e98a237192cd9bcc50ab9eaeb5ed865ea4eb7183fcc16b2c -CompilerSupportLibraries.v0.3.6+1.armv7l-linux-gnueabihf-libgfortran5.tar.gz/md5/68d1d97ede12403b9512ed43eca14066 -CompilerSupportLibraries.v0.3.6+1.armv7l-linux-gnueabihf-libgfortran5.tar.gz/sha512/b343156bb2c6c9c578ad98abe3b807a8fd5a46d9805fc7758af35629d502179a1e4959f1f0571c33b21548a91403fb2aeb7487264e70554a83e9fc68cce042df -CompilerSupportLibraries.v0.3.6+1.armv7l-linux-musleabihf-libgfortran3.tar.gz/md5/cb461ec3bfd7947cb0000671a4ed9951 -CompilerSupportLibraries.v0.3.6+1.armv7l-linux-musleabihf-libgfortran3.tar.gz/sha512/bf0e33de68b40f540fea2017319e126876dd6d95f8dc96bfe43d35f8b39bd94b524a037401e43803e3042a9e5a0b35dd4a18a451b7261e89d41b695cc936fef1 -CompilerSupportLibraries.v0.3.6+1.armv7l-linux-musleabihf-libgfortran4.tar.gz/md5/7a14ff05027b4bf14307f2a8bd5757ee -CompilerSupportLibraries.v0.3.6+1.armv7l-linux-musleabihf-libgfortran4.tar.gz/sha512/e5997228417300c9db59bc5206d742c7b7913fdbfcf68d9e3de82ff75be0be598c7c01c75e41d707fd9b79f40072efc94bda1e9347bce851dd2130b07ccb6f03 -CompilerSupportLibraries.v0.3.6+1.armv7l-linux-musleabihf-libgfortran5.tar.gz/md5/c427644efcf406e8d4d4a4d59007b773 -CompilerSupportLibraries.v0.3.6+1.armv7l-linux-musleabihf-libgfortran5.tar.gz/sha512/b19d4f5370d65fea5cc7ef1bd31355389992059f07f782c77f7137c657e453265385ff8309ee27d15eea969cf4719fa394d8f969190353b6adba80fab2ff966c -CompilerSupportLibraries.v0.3.6+1.i686-linux-gnu-libgfortran3.tar.gz/md5/5e2169dd2444c39c8475dc9ce6824bbb -CompilerSupportLibraries.v0.3.6+1.i686-linux-gnu-libgfortran3.tar.gz/sha512/89d4c2320c2a6f1c3c897c4cb2e61227d94c921772e71a4c9486fcaceb0ee8bceb055067d0ffd185a799a26b5d2de91e306f39631d531db4c82049e5d33582c7 -CompilerSupportLibraries.v0.3.6+1.i686-linux-gnu-libgfortran4.tar.gz/md5/8d65d12bd9967b9d07406663e0321a28 -CompilerSupportLibraries.v0.3.6+1.i686-linux-gnu-libgfortran4.tar.gz/sha512/b17c49209dee3a075ea750e5381500789a90c8c8a8bed35c36a65daf4a4ba35d231df3ed048a69fda226ce843047c38fbfcb7fc130317a67f4336b4a7b5e77c9 -CompilerSupportLibraries.v0.3.6+1.i686-linux-gnu-libgfortran5.tar.gz/md5/4c1c3aaeb4911d962c6581b9d4a49627 -CompilerSupportLibraries.v0.3.6+1.i686-linux-gnu-libgfortran5.tar.gz/sha512/ff7868c80666616f3ed150c4fd31e954cc6359447fa309be824feef1db2037b500720ff8e1d173d4660e44691323e52d157b02224421a9717848cfd8a231b301 -CompilerSupportLibraries.v0.3.6+1.i686-linux-musl-libgfortran3.tar.gz/md5/cf531cf590d019c425a0a61e0bd13e2d -CompilerSupportLibraries.v0.3.6+1.i686-linux-musl-libgfortran3.tar.gz/sha512/1e2aae084ea8f7d40b8ec83898b1c6c467b807a99693afc1f71dfdc90818ffde921b9d6924e52faf31f199977a76b7f09809c6a2e72401f2f887b7c1234d5f71 -CompilerSupportLibraries.v0.3.6+1.i686-linux-musl-libgfortran4.tar.gz/md5/6b57a31396e45ab4e5e568d55823349c -CompilerSupportLibraries.v0.3.6+1.i686-linux-musl-libgfortran4.tar.gz/sha512/5c629e1d5fa59f450915a7c6d46bc9358e37c18d30dd376d3724981f27291b1b1be196d2544673630b0248078efb8267164e078abb3c506b4fe43a052738da47 -CompilerSupportLibraries.v0.3.6+1.i686-linux-musl-libgfortran5.tar.gz/md5/3a4c0fc120c8fd938ae5ca1d06c5e271 -CompilerSupportLibraries.v0.3.6+1.i686-linux-musl-libgfortran5.tar.gz/sha512/bf86cb58113c54c76e6beb0f072baf6acbb8a83b2062f33373303de52dcd1bae68a8e2427cdd5b526a1ce3cace4fca66e1b0ba58f02f03be15ca1c8308fe2091 -CompilerSupportLibraries.v0.3.6+1.i686-w64-mingw32-libgfortran3.tar.gz/md5/4fe7195e1d78f96b78b70de510b97d49 -CompilerSupportLibraries.v0.3.6+1.i686-w64-mingw32-libgfortran3.tar.gz/sha512/91756b7116340ddf92eb33d0354717ba0f2c53a397e8896a65777ce17b881e8f50c8fee0aed334825e246c567e1dcdac89040528ee5c5457049f05fca32a572d -CompilerSupportLibraries.v0.3.6+1.i686-w64-mingw32-libgfortran4.tar.gz/md5/6e1627bf331afd0ce66a7c3893d6c998 -CompilerSupportLibraries.v0.3.6+1.i686-w64-mingw32-libgfortran4.tar.gz/sha512/216c2c2f7d11184a5e7528712dbaa8d79eb09b49136c47052f133a39fc1343fb03c2be07433cd363cdeed0535c44efafb853a1e15ef882c9cca383e21f5c7a83 -CompilerSupportLibraries.v0.3.6+1.i686-w64-mingw32-libgfortran5.tar.gz/md5/40549153660a73e01c076b507708ae1d -CompilerSupportLibraries.v0.3.6+1.i686-w64-mingw32-libgfortran5.tar.gz/sha512/ba8051a3d73a3abee0045f39d7bf7eb26ac5aca23060020c690e7eb8acbe66fa719b1ee5156af38dbbabaf24580dc6ed6778f7cd688542cf745aebdacfab4a65 -CompilerSupportLibraries.v0.3.6+1.powerpc64le-linux-gnu-libgfortran3.tar.gz/md5/822767633d333501c1a3ca794e978049 -CompilerSupportLibraries.v0.3.6+1.powerpc64le-linux-gnu-libgfortran3.tar.gz/sha512/58bb8c2b12e044adf07f8f256a4b2a49f5a89d617875fcf29d2e255ab1987c307e6a557027daa449661a33507c660f7f5602a2e3e15427537d7b9e7fbd9c6f1e -CompilerSupportLibraries.v0.3.6+1.powerpc64le-linux-gnu-libgfortran4.tar.gz/md5/beb6a4e6610af2846d1c2e9fa39fdbd9 -CompilerSupportLibraries.v0.3.6+1.powerpc64le-linux-gnu-libgfortran4.tar.gz/sha512/b56459be8ad26830886f47fa784dd94bad4ebfb20aaf238df6d6ede90bd164b1136abc02e90c799403c0f5c96c566f136af7e7fed0e6fb67d82046302875284f -CompilerSupportLibraries.v0.3.6+1.powerpc64le-linux-gnu-libgfortran5.tar.gz/md5/bdb0c130d9ee8406cb1436ceba1c5e9b -CompilerSupportLibraries.v0.3.6+1.powerpc64le-linux-gnu-libgfortran5.tar.gz/sha512/8a50449b06caa2fbf5e12b7eb55cfbb9fe44a722cf5ae15d5a3032b3ab90f6e4a12831074a380871ee97d0e95c4da8475b61708020732a20574c841d068cfe8e -CompilerSupportLibraries.v0.3.6+1.x86_64-apple-darwin-libgfortran3.tar.gz/md5/3861d39bd40e2f19b4b388c8768f65bd -CompilerSupportLibraries.v0.3.6+1.x86_64-apple-darwin-libgfortran3.tar.gz/sha512/a3e7e3b1dc9c3d379ea29c91b9ca52722ad4c842c7f37f45ef291879856becdbe86ba965d7e5ced5d1909c4d86560fc0c49d6ee98094133fd6f306ddd2321275 -CompilerSupportLibraries.v0.3.6+1.x86_64-apple-darwin-libgfortran4.tar.gz/md5/81d22b79197d2e8efc7760c8118b3d8b -CompilerSupportLibraries.v0.3.6+1.x86_64-apple-darwin-libgfortran4.tar.gz/sha512/85cbcb21974b1e228bf64f1195bebf7180461c34080d7f065918c3e3d15b6bbe17e2661a8c6002f84a7bf5292d376c727c442bb1c5fed5c0d21fbc235c46e535 -CompilerSupportLibraries.v0.3.6+1.x86_64-apple-darwin-libgfortran5.tar.gz/md5/e8c88a9fc2665d0a533e5ee4d5975f69 -CompilerSupportLibraries.v0.3.6+1.x86_64-apple-darwin-libgfortran5.tar.gz/sha512/590fff684b1d8f13c98fb887b300ec1008ded65942dae420036f1f0e95dc92219f68b45ed937914fa8714fb680cff52c10ca64ed00205e1303d60325ed0f34d4 -CompilerSupportLibraries.v0.3.6+1.x86_64-linux-gnu-libgfortran3.tar.gz/md5/ff14a7f01d946e0fe9c44a78eea2dfd7 -CompilerSupportLibraries.v0.3.6+1.x86_64-linux-gnu-libgfortran3.tar.gz/sha512/ebb883a300ddc00bffb8395c032dc90f0b6bf5ed69732bfb0e1979833c620782668bc817b7f9ecc1a9dc6ce07b4ab1dc68a6c5b5531d59a0392bd66fb21ac496 -CompilerSupportLibraries.v0.3.6+1.x86_64-linux-gnu-libgfortran4.tar.gz/md5/e9ae17296ab6489c0b9ec0a122d0532e -CompilerSupportLibraries.v0.3.6+1.x86_64-linux-gnu-libgfortran4.tar.gz/sha512/fa3cdf5fb23a881257b9a781dec90a0899a9a7d784a7ed470c8d2771e44e989cbf6cbae5b0e09cf440265165d974fd02b9b6d0546c7695fad96de6aace280127 -CompilerSupportLibraries.v0.3.6+1.x86_64-linux-gnu-libgfortran5.tar.gz/md5/b96d49f7b28ade84f1d2afcb52365e4e -CompilerSupportLibraries.v0.3.6+1.x86_64-linux-gnu-libgfortran5.tar.gz/sha512/43af76d6e832780b16697acdc1e250cc5d8302df63c228c2e8b4d0e8b5f50126110134beb1d5c4b18520445574411f26daaad1aff35205479d3bdecfa91daa94 -CompilerSupportLibraries.v0.3.6+1.x86_64-linux-musl-libgfortran3.tar.gz/md5/9429f875e5fafe3f202d06652f860dcc -CompilerSupportLibraries.v0.3.6+1.x86_64-linux-musl-libgfortran3.tar.gz/sha512/1fa658c41d5d9c629fafb55989f49e584618d4b58f60743d2b5d793219d5ef73a2e915d390ba274c94b61ef4954076115e8d00882a5e6a7bed8ae09376bd0236 -CompilerSupportLibraries.v0.3.6+1.x86_64-linux-musl-libgfortran4.tar.gz/md5/1ab1db32747768c0b7729219b1d93c41 -CompilerSupportLibraries.v0.3.6+1.x86_64-linux-musl-libgfortran4.tar.gz/sha512/73431efc5e629804571fb1915094a7fda78cd7a0c8ff221d1101f613fb1cfcf8f9c4e774a56d4cb5b032b3f5ef43db1176726aa74a5cbf3e01c8d05049bf1b1f -CompilerSupportLibraries.v0.3.6+1.x86_64-linux-musl-libgfortran5.tar.gz/md5/2f31cbd608f126358efa062748572262 -CompilerSupportLibraries.v0.3.6+1.x86_64-linux-musl-libgfortran5.tar.gz/sha512/c67c3594dd39fc602a9d4cd890f64a135017aaa2cbd119b23dcf5a4dfc5d38dbfd79bb16952153a53ddc87a2c3ece32f0ea8524070567e1b81bf7e12187da7ac -CompilerSupportLibraries.v0.3.6+1.x86_64-unknown-freebsd-libgfortran3.tar.gz/md5/243b1ca96c14601eaae76c551e6951f7 -CompilerSupportLibraries.v0.3.6+1.x86_64-unknown-freebsd-libgfortran3.tar.gz/sha512/f2835403f96cbe8877b910f05b75e81bb396c9f32f7cd0d2e6007fde83de8be263a5fbafd659ac96fa39204dbb9703153eeef3945efe2910798daae124591542 -CompilerSupportLibraries.v0.3.6+1.x86_64-unknown-freebsd-libgfortran4.tar.gz/md5/5c33ea039c39e4a225a437c4f2a50306 -CompilerSupportLibraries.v0.3.6+1.x86_64-unknown-freebsd-libgfortran4.tar.gz/sha512/c3621099ffd21d0c0cc4e371cf5618a7355c3a6f28aa150dc5826aafb879b9cc56e21f793306c50c45d8d37781d19cb6f376c460cb36f27395bba052f69f4918 -CompilerSupportLibraries.v0.3.6+1.x86_64-unknown-freebsd-libgfortran5.tar.gz/md5/9998ba283644e521f560bcc1ae93d921 -CompilerSupportLibraries.v0.3.6+1.x86_64-unknown-freebsd-libgfortran5.tar.gz/sha512/3549fc7f4fdf3298d44b33e81620984089d0b9606af27bbce5ad3a766d1f35372756f100c7434f98962148a839b3c4fcdb445180738729fd50365d1421dfeb80 -CompilerSupportLibraries.v0.3.6+1.x86_64-w64-mingw32-libgfortran3.tar.gz/md5/c3dc43bd0f25b65d89c8f3207152a8c7 -CompilerSupportLibraries.v0.3.6+1.x86_64-w64-mingw32-libgfortran3.tar.gz/sha512/a94baa8b6fa123a476c08ca9fea8b6bcc7643c293210a865145637d39bc0cfd9b7d4c023e0eee8a482b569f0c9df34222e4d35ea7134a7e91af1410b595cb77d -CompilerSupportLibraries.v0.3.6+1.x86_64-w64-mingw32-libgfortran4.tar.gz/md5/8e5d9cf3b90773f875c16cfd96169625 -CompilerSupportLibraries.v0.3.6+1.x86_64-w64-mingw32-libgfortran4.tar.gz/sha512/798e866fcbf629400deee8b2a6dfb07897fa79f493ee15ff7a11739a83a1a599cfcfbdc7a5c74fc287fa528881c72da25dc544bd939bccfb864cc60d9baf930c -CompilerSupportLibraries.v0.3.6+1.x86_64-w64-mingw32-libgfortran5.tar.gz/md5/c61c1cdd1963aea7035c1ff9d0e85230 -CompilerSupportLibraries.v0.3.6+1.x86_64-w64-mingw32-libgfortran5.tar.gz/sha512/11fbb579f38ff85aa7032021c087ddaed2016b545dbe4373d37a447621aad7510406d2174e53dd223db010c0e6283940c35c6c93e2f1ac9e0bf08d561af52c6f +CompilerSupportLibraries.v0.5.0+0.aarch64-apple-darwin-libgfortran5.tar.gz/md5/307711def378e337a999c182aa7e07d8 +CompilerSupportLibraries.v0.5.0+0.aarch64-apple-darwin-libgfortran5.tar.gz/sha512/0dcad5e315e045397320f667b27fc378da898ebfea9b55a2837e68b29434fe2c2ddc9652cc75a4551062ce70a2bfaffa8223c77398aa41fe1a73ccb44952cd8f +CompilerSupportLibraries.v0.5.0+0.aarch64-linux-gnu-libgfortran3.tar.gz/md5/177f2665038919c3f8ed968226ff3b56 +CompilerSupportLibraries.v0.5.0+0.aarch64-linux-gnu-libgfortran3.tar.gz/sha512/ea67c3b9986106aee12e5f22ab3d3c5d71a58759a7d20a7724bbb198e5c71f42fa2034e46f3147006a2d2277b3881f0546030d1040cb9393e58eeae87eb82c4d +CompilerSupportLibraries.v0.5.0+0.aarch64-linux-gnu-libgfortran4.tar.gz/md5/f16db35be9018a5c61eaafaaf7226d10 +CompilerSupportLibraries.v0.5.0+0.aarch64-linux-gnu-libgfortran4.tar.gz/sha512/051b5a0dd2235eaa90557e487c83499b3d7e0b9e921f7b2f14e77c81152c338acd5bac8040bdf6679db656cd8039093db43565f843dede253717425e464e61b0 +CompilerSupportLibraries.v0.5.0+0.aarch64-linux-gnu-libgfortran5.tar.gz/md5/e6082f3e46b627fdaef09f1ef81c1d7b +CompilerSupportLibraries.v0.5.0+0.aarch64-linux-gnu-libgfortran5.tar.gz/sha512/13d0ab1c0e84a65db729ea6bd45a868d9d65e1a0ec95412448846d1044e2bbf11b11d96cfa576dccf3d7eccc4bed4eb9ae4bac0989e9b1b97adad5e404dfe4a4 +CompilerSupportLibraries.v0.5.0+0.aarch64-linux-musl-libgfortran3.tar.gz/md5/00703177897f8c46a577c2b0518432bc +CompilerSupportLibraries.v0.5.0+0.aarch64-linux-musl-libgfortran3.tar.gz/sha512/af14ad1303f3918dd691e0b509ea0fd52ac7c9f0c285e8dbb741bd34ce0b1927f89f219fcf8d260315c503b18bf98b3df117810328066a9964917cc34968ce98 +CompilerSupportLibraries.v0.5.0+0.aarch64-linux-musl-libgfortran4.tar.gz/md5/f823b692319cd370ca59189ad2ba4a3d +CompilerSupportLibraries.v0.5.0+0.aarch64-linux-musl-libgfortran4.tar.gz/sha512/b0c4131bf4d15c482bbed83fcc570da2f7bb8ef99d507e0e13eb0c8f5519ec73ff234c58d505294be3f8d39b6dd1c7022578db02005ae111c7873243e8ddc8ef +CompilerSupportLibraries.v0.5.0+0.aarch64-linux-musl-libgfortran5.tar.gz/md5/a9ef1a68518058fe6c945e8b00f8400f +CompilerSupportLibraries.v0.5.0+0.aarch64-linux-musl-libgfortran5.tar.gz/sha512/6aa53edf48a17ec8515cad5c79a15ab0e40cc44c9ffb188fd57fc560dde7a99d6487ead6e4caafaa9912c6590c6a391f914016fd4342589da09d56c657ad2c07 +CompilerSupportLibraries.v0.5.0+0.armv6l-linux-gnueabihf-libgfortran3.tar.gz/md5/d3aaf50955ad671917e941e0dcf3803f +CompilerSupportLibraries.v0.5.0+0.armv6l-linux-gnueabihf-libgfortran3.tar.gz/sha512/72983b2272300c2332cfe6864b5dd5249bbbb181bd65b10bf6bfb3a37e5e582bb9c159db0b63a077066a325899a2864717f28c60c85027be3b637bb80f994e52 +CompilerSupportLibraries.v0.5.0+0.armv6l-linux-gnueabihf-libgfortran4.tar.gz/md5/e221d51df9b18b2562a0f3e8dc8012cd +CompilerSupportLibraries.v0.5.0+0.armv6l-linux-gnueabihf-libgfortran4.tar.gz/sha512/758b07b4a559dda747574649926333a70355e2d80acb2ea37bb39777c0b1cecf8f308a5f8062110c378db2230ec8baf23385ae313d1c58de8bfc651573c64c1f +CompilerSupportLibraries.v0.5.0+0.armv6l-linux-gnueabihf-libgfortran5.tar.gz/md5/96f7feef9b1dd7944130de2e9cda68b8 +CompilerSupportLibraries.v0.5.0+0.armv6l-linux-gnueabihf-libgfortran5.tar.gz/sha512/8b4aaff1388cd506bef7f3a9edd42ed8ee1db468a18d34cd5d58d7da305853dbf48d4665e99c06c6fb0115e421d19dba5c36e947cb06defe7f479a05b547f112 +CompilerSupportLibraries.v0.5.0+0.armv6l-linux-musleabihf-libgfortran3.tar.gz/md5/a1e3642a7ce2b7834aa2f1b695a9977c +CompilerSupportLibraries.v0.5.0+0.armv6l-linux-musleabihf-libgfortran3.tar.gz/sha512/9d22b1fa8fa8eaaa5316cb494eb223e0fe73660aa5ca7518180e40d296d6d07a9863938501e5d5350bf79e79d975d7d66dca12768a0a69527d2c17baf7aaf345 +CompilerSupportLibraries.v0.5.0+0.armv6l-linux-musleabihf-libgfortran4.tar.gz/md5/d897098fd98928c2d644ed5ee26c3faa +CompilerSupportLibraries.v0.5.0+0.armv6l-linux-musleabihf-libgfortran4.tar.gz/sha512/4aad051f4f1e3d744825c650363a49f39e04cbd44dad25197ddee1890339e9441aa872f893478a2d8ff556c9a70a89c2885cd779ba3efd3c0f7193c386b820b7 +CompilerSupportLibraries.v0.5.0+0.armv6l-linux-musleabihf-libgfortran5.tar.gz/md5/c36bfd4c5b90d55c55bc18feaf51b134 +CompilerSupportLibraries.v0.5.0+0.armv6l-linux-musleabihf-libgfortran5.tar.gz/sha512/ab16c638780a0118b930ac587df81fa74d2731bf1af402266106e1ecb791df353c1f368a8e7fc9147d390825ff8624e600aae45f1f6ccfc0015ce131368452d7 +CompilerSupportLibraries.v0.5.0+0.armv7l-linux-gnueabihf-libgfortran3.tar.gz/md5/feb76551e6f7407de3006a3d363cee7a +CompilerSupportLibraries.v0.5.0+0.armv7l-linux-gnueabihf-libgfortran3.tar.gz/sha512/976f8e34e72231b013ea0418feff9c3c9efa7b9c34688aca115a03f2bade8760ca9f259f8f502ef5012fbb389f4bf365fd7639b066daca16fb7ec1d32b5cd789 +CompilerSupportLibraries.v0.5.0+0.armv7l-linux-gnueabihf-libgfortran4.tar.gz/md5/560ca43fa6dbd3f2e9052401477df165 +CompilerSupportLibraries.v0.5.0+0.armv7l-linux-gnueabihf-libgfortran4.tar.gz/sha512/333c7f4fbc172e7fd3d99e2673dbed1d9c699a5bb29a20095a255fadc89ded05abda755fc167aa8a16a4e93f524390c9c817df7b67fccdca88754d0301259977 +CompilerSupportLibraries.v0.5.0+0.armv7l-linux-gnueabihf-libgfortran5.tar.gz/md5/d3ac5f871599ab225a1128c302486345 +CompilerSupportLibraries.v0.5.0+0.armv7l-linux-gnueabihf-libgfortran5.tar.gz/sha512/adb706882e923978b6e18c7134578bc86ed4e031a7a0120222018cd1b8efcf530854e426b6442dbd80b8c77c3677f1906aedb12c0ddeb33efcdd3bcd2c4a109a +CompilerSupportLibraries.v0.5.0+0.armv7l-linux-musleabihf-libgfortran3.tar.gz/md5/58774aa398a63479af3f4c69678d0191 +CompilerSupportLibraries.v0.5.0+0.armv7l-linux-musleabihf-libgfortran3.tar.gz/sha512/fe9307e6fb0b54522495fc9cc48756a60fc79af27d9e73bfb3ee49cbb366dddec1beedca03614f15761b308bc28014205f174f673fa258e76d5947446b87b039 +CompilerSupportLibraries.v0.5.0+0.armv7l-linux-musleabihf-libgfortran4.tar.gz/md5/af1a8ce693ba307e61184f4023d73d67 +CompilerSupportLibraries.v0.5.0+0.armv7l-linux-musleabihf-libgfortran4.tar.gz/sha512/2ea581bb44408fc789ac306734736f6eb6cf0a15b234f43a6f50ae8f10014b5689f5aa8356112c2b54a86b9a7734ace3479c4e4aba1e5df636dda3dcd09b7e28 +CompilerSupportLibraries.v0.5.0+0.armv7l-linux-musleabihf-libgfortran5.tar.gz/md5/20d62064f495877f12b7e87e684ad43a +CompilerSupportLibraries.v0.5.0+0.armv7l-linux-musleabihf-libgfortran5.tar.gz/sha512/31b1c7c9fe3378e8bb788c897bbac0505a5ae70f500f3b1457325dbbb149c14224a88d17fbcf453465d8a572f33157766bb0e815cce7c8a2aa8a44422d34a365 +CompilerSupportLibraries.v0.5.0+0.i686-linux-gnu-libgfortran3.tar.gz/md5/fd4035aef1c83be0b865d70aa35e770b +CompilerSupportLibraries.v0.5.0+0.i686-linux-gnu-libgfortran3.tar.gz/sha512/a72047e7071838899d75896b4dcbdc102bca884507f4758b4e0dd62f50c9ce584f2b2b86d8b67dfc4fce9864faf9723056820e464bbab1a6173be47ad941d6da +CompilerSupportLibraries.v0.5.0+0.i686-linux-gnu-libgfortran4.tar.gz/md5/89715bfa0e69528d4d294ed449ef0e09 +CompilerSupportLibraries.v0.5.0+0.i686-linux-gnu-libgfortran4.tar.gz/sha512/6eb7947c72ec32d189221de42d5a76423a1fb5745db0812d88afe7f961d8f42669c7cf487235c1dcc81fbe73106b785c906bd6741e98f60e9931f4083be0e9ce +CompilerSupportLibraries.v0.5.0+0.i686-linux-gnu-libgfortran5.tar.gz/md5/5c1c73dc72029781847f74bcb1189c4b +CompilerSupportLibraries.v0.5.0+0.i686-linux-gnu-libgfortran5.tar.gz/sha512/642d35ed41a65c7a2d7f4f127f936d3cb1665c207aa5feef25cce09cc11e733d7ec129673fea873403567c35cf16122ed1635c303ba13bb3349be44585f3ca82 +CompilerSupportLibraries.v0.5.0+0.i686-linux-musl-libgfortran3.tar.gz/md5/f91c962e7bc3ffb825c7e5fb1e099ba6 +CompilerSupportLibraries.v0.5.0+0.i686-linux-musl-libgfortran3.tar.gz/sha512/f89df221ff80bcbb1e6edc2f9cc28dc138d7d6ae99ac018a3cdc9a09ba637f1a9938b1f0876086f4f822fb911853286dd4f1776d603a403190bee052431ae572 +CompilerSupportLibraries.v0.5.0+0.i686-linux-musl-libgfortran4.tar.gz/md5/d2a81da3371a638f76087629ae0a6507 +CompilerSupportLibraries.v0.5.0+0.i686-linux-musl-libgfortran4.tar.gz/sha512/67941af15a0f032a853cdea180e4f87249bed2dfd09ade6fca9760f5a44b26fc94a0d6932803edbd27b75aa8d26e64c377af2d64ddcba3206562be1427a64c80 +CompilerSupportLibraries.v0.5.0+0.i686-linux-musl-libgfortran5.tar.gz/md5/cec9f3b9d4924a49a34c632efd167752 +CompilerSupportLibraries.v0.5.0+0.i686-linux-musl-libgfortran5.tar.gz/sha512/9320eee2b6dbadd4e0ed3f8763d58854eb179b1d1661c8f1dba75c22af2330812040507944b0ab20b7a7cb233c9953a1d3a4b27937e7b7a858aed2255ad0fbbc +CompilerSupportLibraries.v0.5.0+0.i686-w64-mingw32-libgfortran3.tar.gz/md5/c36411b24c8bec4805230bd4fe0f2391 +CompilerSupportLibraries.v0.5.0+0.i686-w64-mingw32-libgfortran3.tar.gz/sha512/839b447efa46caffa699258ec8ae5e0a55d7f98a7fc037b48e6a6c29193e3d8bf48397575cc518716f41e2e9344daa670693df605a1b9d4a23d3f454ec5ab399 +CompilerSupportLibraries.v0.5.0+0.i686-w64-mingw32-libgfortran4.tar.gz/md5/d2e392edff3525afff6734fdf47c9ab1 +CompilerSupportLibraries.v0.5.0+0.i686-w64-mingw32-libgfortran4.tar.gz/sha512/1816c7ed409acc1435c7fcfd550b7664a08b31ecf433a906d8903a60ed458dab0fa712bd0d1590a0dc8506763a617446ba402efc78a2c010562c45e8eca66a88 +CompilerSupportLibraries.v0.5.0+0.i686-w64-mingw32-libgfortran5.tar.gz/md5/2cfeb5cd0a7e2400c9be3e846a1875d2 +CompilerSupportLibraries.v0.5.0+0.i686-w64-mingw32-libgfortran5.tar.gz/sha512/ca620dd8542ffe9a177b0f95712e77e59b0fc1044e0186dd7468a86aba4d2b92931a1d6f980e75cceb26c6c5f9dab427f4ce32e0f77998b9a827b3ce9151041c +CompilerSupportLibraries.v0.5.0+0.powerpc64le-linux-gnu-libgfortran3.tar.gz/md5/8ba0e4070358839909934d8a1bc9e0bf +CompilerSupportLibraries.v0.5.0+0.powerpc64le-linux-gnu-libgfortran3.tar.gz/sha512/8750769ca321f863fbb354f6e4e76b1241f7e24e5f4ea14ea511486dc5bc4fe8274740f1500149c5ac85a8214a0193c9a09332f35eb47e6222bef9070eecc6c8 +CompilerSupportLibraries.v0.5.0+0.powerpc64le-linux-gnu-libgfortran4.tar.gz/md5/50554a092af3a4a651b53e3ce3cf8a2d +CompilerSupportLibraries.v0.5.0+0.powerpc64le-linux-gnu-libgfortran4.tar.gz/sha512/53ec765d4de3b0bae9727b3b2a27437b184f2072aecda5d0b22d648a95fbba777bb89da823bc851d7242cd3f8c212e3fdaea8e5af11db21c578c2e12db51991d +CompilerSupportLibraries.v0.5.0+0.powerpc64le-linux-gnu-libgfortran5.tar.gz/md5/b09a5913b537b26aa7f8996b1877c748 +CompilerSupportLibraries.v0.5.0+0.powerpc64le-linux-gnu-libgfortran5.tar.gz/sha512/b68020c1b1acf4a1c51822bccc1eb67574ceffae3c133e7efe22ec0cc3a674a7c056c01be02c1c681f469fe1443d76baf4b0e305bec8181e57c3ce5a446a5c22 +CompilerSupportLibraries.v0.5.0+0.x86_64-apple-darwin-libgfortran3.tar.gz/md5/1e4c5d2084f76eacb4419214668c6594 +CompilerSupportLibraries.v0.5.0+0.x86_64-apple-darwin-libgfortran3.tar.gz/sha512/696155b560bfaf592bf7024ba0e6f084382dd269cdd25416fa8840387c101132901e94709c8d0534f038666a6f6849c3d55e8bed4223b5be499e099b49610e77 +CompilerSupportLibraries.v0.5.0+0.x86_64-apple-darwin-libgfortran4.tar.gz/md5/63b386e59f3732d03459c59000fc1382 +CompilerSupportLibraries.v0.5.0+0.x86_64-apple-darwin-libgfortran4.tar.gz/sha512/f6c7e0611df7fd86cc9ca63b380e112561d10b489bc8fbfe911c441ef5e87776761d3c161ff5f6aade479f7e96456084c6939d7eff175ced4f42b3b9ee29426a +CompilerSupportLibraries.v0.5.0+0.x86_64-apple-darwin-libgfortran5.tar.gz/md5/07e22a4b58aaaf145e52b36602c5b08d +CompilerSupportLibraries.v0.5.0+0.x86_64-apple-darwin-libgfortran5.tar.gz/sha512/8a047b0098e8504e2dde0113170416686bc70f9d685fcb19bf3eb76afe30dc16a3b0d2023eb704c25025bbef87e99603dbd2a2708b1a3df908747b06cbfc92ee +CompilerSupportLibraries.v0.5.0+0.x86_64-linux-gnu-libgfortran3.tar.gz/md5/23048b3be33f184ffc9be42ca914aa3a +CompilerSupportLibraries.v0.5.0+0.x86_64-linux-gnu-libgfortran3.tar.gz/sha512/4573b21e34f4d8127a86c18f95065039da92eeb9ade4058bd8459034bb4a003ceefe29e865089126fdc36cffd95a9c12bcb72ed74bff5987a9d1f4b300ecfe45 +CompilerSupportLibraries.v0.5.0+0.x86_64-linux-gnu-libgfortran4.tar.gz/md5/3314ec0668abf069c900558de0690b65 +CompilerSupportLibraries.v0.5.0+0.x86_64-linux-gnu-libgfortran4.tar.gz/sha512/d012c4674401773000f0de831cb8b4b6c454d0ab68d51fbbe970504e76c693211086a24a7df34de2390eaeb438ab23f63c68b480a408ab2136f442aba5094bd7 +CompilerSupportLibraries.v0.5.0+0.x86_64-linux-gnu-libgfortran5.tar.gz/md5/e7768c00909613b8f29f6a5860ff4247 +CompilerSupportLibraries.v0.5.0+0.x86_64-linux-gnu-libgfortran5.tar.gz/sha512/43c29456a0fc74c4fda42d088903651c6bbac6b842f2aa600e3019b391b04158ee97f884e6962bd9e7a9cf337dbb1cdb2151d103e1dee5214ba798b167b1ed32 +CompilerSupportLibraries.v0.5.0+0.x86_64-linux-musl-libgfortran3.tar.gz/md5/b2a30e92ba8e40ef070e3ec7c16b97f0 +CompilerSupportLibraries.v0.5.0+0.x86_64-linux-musl-libgfortran3.tar.gz/sha512/64a4029dd1e84922728b2c93a455d7d6b262c979dddf59301ff96e9c28980fbd9c1db57e81afaece96ccb51b9751e5a0180b84e412427430487280c56d8da266 +CompilerSupportLibraries.v0.5.0+0.x86_64-linux-musl-libgfortran4.tar.gz/md5/b0610d32a80b3f87baebf0250b0f92d6 +CompilerSupportLibraries.v0.5.0+0.x86_64-linux-musl-libgfortran4.tar.gz/sha512/3b7098fbb82e4a7a903b82f942303b248e0e35be13a47e4839a036085c4a33925f1f78fe941b852331cc52de80f32bcdb9a64ccff0386e1070a6ca4600c08eb8 +CompilerSupportLibraries.v0.5.0+0.x86_64-linux-musl-libgfortran5.tar.gz/md5/3f905dd4e8b3cfd2cc3f8efcaa50a407 +CompilerSupportLibraries.v0.5.0+0.x86_64-linux-musl-libgfortran5.tar.gz/sha512/22af14d245e3c062131dd274afa6d9c7cde9a11ee2455e27ae2f7725a025fc2cd6cdb3a1a3c899988c6c3412a714c1f0763f4e08924726212405938c3cf66da5 +CompilerSupportLibraries.v0.5.0+0.x86_64-unknown-freebsd-libgfortran3.tar.gz/md5/2c56a22c935dda76831f36c713cca099 +CompilerSupportLibraries.v0.5.0+0.x86_64-unknown-freebsd-libgfortran3.tar.gz/sha512/6bd9bd6ec8b6b18013b3c6de344de134835c9281d39bc5e6e31928970c60b584fa625df18efbce3ea571dee53011dec73e9aae9159e812f219692fbb4dd86a2d +CompilerSupportLibraries.v0.5.0+0.x86_64-unknown-freebsd-libgfortran4.tar.gz/md5/e483c3e85b4d4b2685ee4e8f09951ac1 +CompilerSupportLibraries.v0.5.0+0.x86_64-unknown-freebsd-libgfortran4.tar.gz/sha512/47c2f305237ccd55ed2ba445cbcd599c23f9c1392388017506f9d61a4dc8fec4ba4136be81a0e82de4f161f6788c4a62acc9d71efe6cf90b766e5339950ed337 +CompilerSupportLibraries.v0.5.0+0.x86_64-unknown-freebsd-libgfortran5.tar.gz/md5/41c25d9cf7545721b8d4dd2386e95ead +CompilerSupportLibraries.v0.5.0+0.x86_64-unknown-freebsd-libgfortran5.tar.gz/sha512/173570bbf4eb60d678472058ec2c18732cd27ad2911457c83f47a1d97c1c0028d91005cf56539e51d4a04178544ac0bba47ea27e74b6b4e8d3310551ad3167fe +CompilerSupportLibraries.v0.5.0+0.x86_64-w64-mingw32-libgfortran3.tar.gz/md5/f124c93580a038ce806f479568b46597 +CompilerSupportLibraries.v0.5.0+0.x86_64-w64-mingw32-libgfortran3.tar.gz/sha512/c313390dbcffaea6cb5202645b5304134a1ce6aac5a3835696f45316c8170b237c04f13166694eee0f31903ac1e5c3cd73ad8974ba19b44289da3504d3436f8c +CompilerSupportLibraries.v0.5.0+0.x86_64-w64-mingw32-libgfortran4.tar.gz/md5/050fe7a6bdf980c198f4c201629d15e0 +CompilerSupportLibraries.v0.5.0+0.x86_64-w64-mingw32-libgfortran4.tar.gz/sha512/211e435f5e2b7209aedaf4a81b5e0d5e615b9144de248c06e43dc61b31890dbde80d718e74454b489bd1f77476d34bd01d3f9a25355bc50fca0dc07df0264cad +CompilerSupportLibraries.v0.5.0+0.x86_64-w64-mingw32-libgfortran5.tar.gz/md5/3566d0f714c1503b92160b486a4eaa4a +CompilerSupportLibraries.v0.5.0+0.x86_64-w64-mingw32-libgfortran5.tar.gz/sha512/b2f29c1c6dc35e1002021f8f15a20a72a57c346b33a6d045ff7a261e88767738a4da1dd88aa71a20514bdf6376099979c9d938173fa3ae28641c40372c94db60 diff --git a/stdlib/CompilerSupportLibraries_jll/Project.toml b/stdlib/CompilerSupportLibraries_jll/Project.toml index b4b4fddfac9d0..ac78369b12602 100644 --- a/stdlib/CompilerSupportLibraries_jll/Project.toml +++ b/stdlib/CompilerSupportLibraries_jll/Project.toml @@ -1,6 +1,6 @@ name = "CompilerSupportLibraries_jll" uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" -version = "0.3.6+1" +version = "0.5.0+0" [deps] Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" From 062bd3d435b5db9f273a00d2a39ac1ee0ffb3db8 Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Wed, 21 Jul 2021 20:27:25 -0700 Subject: [PATCH 34/73] [build] Automatically set `USE_BINARYBUILDER_CSL=0` when local CSL is new (#41645) We ship our own compiler support libraries to ensure a minimum level of support for BB-built libraries, however certain distros provide very bleeding-edge compilers. This can be a problem if we download an _older_ `libstdc++.so` and forcibly link against that when launching Julia, as when Julia itself is built with the local `g++`, it may use symbols that don't exist in the BB-sourced `libstdc++.so`. To address this, we default to not using BB-sourced CSLs if the `libstdc++.so` that the native compiler would use contains a symbol that our BB-sourced CSLs do not have. We use the monotonically-climbing `GLIBCXX_3.4.XX` version symbols for this purpose; encoding the "next" version number within `deps/csl.mk`, and triggering the BB-avoidance if that version exists within the system-provided `libstdc++.so`. (cherry picked from commit 27c0291f7fbecd811a3f9eb62b1159cff6b61e2a) --- Make.inc | 5 ++- deps/csl.mk | 40 ++++++++++++++++++- .../CompilerSupportLibraries_jll/Project.toml | 4 ++ 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/Make.inc b/Make.inc index 438ab7fa02815..4719a3d49b039 100644 --- a/Make.inc +++ b/Make.inc @@ -1196,7 +1196,10 @@ BB_TRIPLET := $(subst $(SPACE),-,$(filter-out cxx%,$(filter-out libgfortran%,$(s LIBGFORTRAN_VERSION := $(subst libgfortran,,$(filter libgfortran%,$(subst -,$(SPACE),$(BB_TRIPLET_LIBGFORTRAN)))) # This is the set of projects that BinaryBuilder dependencies are hooked up for. -BB_PROJECTS := OPENBLAS LLVM SUITESPARSE OPENLIBM GMP MBEDTLS LIBSSH2 NGHTTP2 MPFR CURL LIBGIT2 PCRE LIBUV LIBUNWIND DSFMT OBJCONV ZLIB P7ZIP CSL +# Note: we explicitly _do not_ define `CSL` here, since it requires some more +# advanced techniques to decide whether it should be installed from a BB source +# or not. See `deps/csl.mk` for more detail. +BB_PROJECTS := OPENBLAS LLVM SUITESPARSE OPENLIBM GMP MBEDTLS LIBSSH2 NGHTTP2 MPFR CURL LIBGIT2 PCRE LIBUV LIBUNWIND DSFMT OBJCONV ZLIB P7ZIP define SET_BB_DEFAULT # First, check to see if BB is disabled on a global setting ifeq ($$(USE_BINARYBUILDER),0) diff --git a/deps/csl.mk b/deps/csl.mk index 75e970f6882d3..4dc497b80cbea 100644 --- a/deps/csl.mk +++ b/deps/csl.mk @@ -1,5 +1,3 @@ -ifeq ($(USE_BINARYBUILDER_CSL),0) - # Interrogate the fortran compiler (which is always GCC based) on where it is keeping its libraries STD_LIB_PATH := $(shell LANG=C $(FC) -print-search-dirs | grep '^programs: =' | sed -e "s/^programs: =//") STD_LIB_PATH += :$(shell LANG=C $(FC) -print-search-dirs | grep '^libraries: =' | sed -e "s/^libraries: =//") @@ -12,6 +10,44 @@ define pathsearch $(firstword $(wildcard $(addsuffix /$(1),$(subst :, ,$(2))))) endef +# CSL bundles lots of system compiler libraries, and while it is quite bleeding-edge +# as compared to what most distros ship, if someone tries to build an older branch, +# the version of CSL that ships with that branch may become relatively old. This is +# not a problem for code that is built in BB, but when we build Julia with the system +# compiler, that compiler uses the version of `libstdc++` that it is bundled with, +# and we can get linker errors when trying to run that `julia` executable with the +# `libstdc++` that comes from the (now old) BB-built CSL. +# +# To fix this, we take note when the system `libstdc++.so` is newer than whatever we +# would get from CSL (by searching for a `GLIBCXX_3.4.X` symbol that does not exist +# in our CSL, but would in a newer one), and default to `USE_BINARYBUILDER_CSL=0` in +# this case. +CSL_NEXT_GLIBCXX_VERSION=GLIBCXX_3\.4\.30|GLIBCXX_3\.5\.|GLIBCXX_4\. + +# First, check to see if BB is disabled on a global setting +ifeq ($(USE_BINARYBUILDER),0) +USE_BINARYBUILDER_CSL ?= 0 +else +# If it's not, check to see if it's disabled by a USE_SYSTEM_xxx flag +ifeq ($(USE_SYSTEM_CSL),1) +USE_BINARYBUILDER_CSL ?= 0 +else +# If it's not, see if we should disable it due to `libstdc++` being newer: +LIBSTDCXX_PATH := $(eval $(call pathsearch,libstdc++,$(STD_LIB_PATH))) +ifneq (,$(and $(LIBSTDCXX_PATH),$(shell objdump -p $(LIBSTDCXX_PATH) | grep $(CSL_NEXT_GLIBCXX_VERSION)))) +# Found `libstdc++`, grepped it for strings and found a `GLIBCXX` symbol +# that is newer that whatever we have in CSL. Default to not using BB. +USE_BINARYBUILDER_CSL ?= 0 +else +# Either we didn't find `libstdc++` (e.g. we're using `clang`), or we +# found it and couldn't find the new symbol in it (it's older than what +# BB provides, so let's use BB instead) +USE_BINARYBUILDER_CSL ?= 1 +endif +endif +endif + +ifeq ($(USE_BINARYBUILDER_CSL),0) define copy_csl install-csl: | $$(build_shlibdir) $$(build_shlibdir)/$(1) $$(build_shlibdir)/$(1): | $$(build_shlibdir) diff --git a/stdlib/CompilerSupportLibraries_jll/Project.toml b/stdlib/CompilerSupportLibraries_jll/Project.toml index ac78369b12602..b76517c828d0e 100644 --- a/stdlib/CompilerSupportLibraries_jll/Project.toml +++ b/stdlib/CompilerSupportLibraries_jll/Project.toml @@ -1,5 +1,9 @@ name = "CompilerSupportLibraries_jll" uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" + +# NOTE: When updating this, also make sure to update the value +# `CSL_NEXT_GLIBCXX_VERSION` in `deps/csl.mk`, to properly disable +# automatic usage of BB-built CSLs on extremely up-to-date systems! version = "0.5.0+0" [deps] From 07a5c4d7bc1e07f300c43705eb3b0c48ec0ff505 Mon Sep 17 00:00:00 2001 From: Sukera Date: Tue, 20 Jul 2021 09:49:43 +0200 Subject: [PATCH 35/73] Add isinf/isnan check for %g/%G formatting (cherry picked from commit 860f99433210b2cc4b7ac8e9c4f4bfa1f9cdcf6d) --- stdlib/Printf/src/Printf.jl | 42 ++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/stdlib/Printf/src/Printf.jl b/stdlib/Printf/src/Printf.jl index b40ce3406f212..89a25d03f477f 100644 --- a/stdlib/Printf/src/Printf.jl +++ b/stdlib/Printf/src/Printf.jl @@ -421,27 +421,31 @@ const __BIG_FLOAT_MAX__ = 8192 elseif T == Val{'f'} || T == Val{'F'} newpos = Ryu.writefixed(buf, pos, x, prec, plus, space, hash, UInt8('.')) elseif T == Val{'g'} || T == Val{'G'} - # C11-compliant general format - prec = prec == 0 ? 1 : prec - # format the value in scientific notation and parse the exponent part - exp = let p = Ryu.writeexp(buf, pos, x, prec) - b1, b2, b3, b4 = buf[p-4], buf[p-3], buf[p-2], buf[p-1] - Z = UInt8('0') - if b1 == UInt8('e') - # two-digit exponent - sign = b2 == UInt8('+') ? 1 : -1 - exp = 10 * (b3 - Z) + (b4 - Z) + if isinf(x) || isnan(x) + newpos = Ryu.writeshortest(buf, pos, x, plus, space) + else + # C11-compliant general format + prec = prec == 0 ? 1 : prec + # format the value in scientific notation and parse the exponent part + exp = let p = Ryu.writeexp(buf, pos, x, prec) + b1, b2, b3, b4 = buf[p-4], buf[p-3], buf[p-2], buf[p-1] + Z = UInt8('0') + if b1 == UInt8('e') + # two-digit exponent + sign = b2 == UInt8('+') ? 1 : -1 + exp = 10 * (b3 - Z) + (b4 - Z) + else + # three-digit exponent + sign = b1 == UInt8('+') ? 1 : -1 + exp = 100 * (b2 - Z) + 10 * (b3 - Z) + (b4 - Z) + end + flipsign(exp, sign) + end + if -4 ≤ exp < prec + newpos = Ryu.writefixed(buf, pos, x, prec - (exp + 1), plus, space, hash, UInt8('.'), !hash) else - # three-digit exponent - sign = b1 == UInt8('+') ? 1 : -1 - exp = 100 * (b2 - Z) + 10 * (b3 - Z) + (b4 - Z) + newpos = Ryu.writeexp(buf, pos, x, prec - 1, plus, space, hash, T == Val{'g'} ? UInt8('e') : UInt8('E'), UInt8('.'), !hash) end - flipsign(exp, sign) - end - if -4 ≤ exp < prec - newpos = Ryu.writefixed(buf, pos, x, prec - (exp + 1), plus, space, hash, UInt8('.'), !hash) - else - newpos = Ryu.writeexp(buf, pos, x, prec - 1, plus, space, hash, T == Val{'g'} ? UInt8('e') : UInt8('E'), UInt8('.'), !hash) end elseif T == Val{'a'} || T == Val{'A'} x, neg = x < 0 || x === -Base.zero(x) ? (-x, true) : (x, false) From ff2ce6b63c02c817883b3f1ca9d53a791a62dbad Mon Sep 17 00:00:00 2001 From: Sukera Date: Tue, 20 Jul 2021 09:54:39 +0200 Subject: [PATCH 36/73] Add test for Inf/NaN handling in %g/%G Printf (cherry picked from commit 6a7c78b8a5ed0423fde478268bda42b4ff62f3ff) --- stdlib/Printf/test/runtests.jl | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/stdlib/Printf/test/runtests.jl b/stdlib/Printf/test/runtests.jl index fc20276e3479d..ca6a333eeec24 100644 --- a/stdlib/Printf/test/runtests.jl +++ b/stdlib/Printf/test/runtests.jl @@ -94,6 +94,15 @@ end @test Printf.@sprintf("%g", 123456.7) == "123457" @test Printf.@sprintf("%g", 1234567.8) == "1.23457e+06" + # %g regression gh #41631 + for (val, res) in ((Inf, "Inf"), + (-Inf, "-Inf"), + (NaN, "NaN"), + (-NaN, "NaN")) + @test Printf.@sprintf("%g", val) == res + @test Printf.@sprintf("%G", val) == res + end + # zeros @test Printf.@sprintf("%.15g", 0) == "0" @test Printf.@sprintf("%#.15g", 0) == "0.00000000000000" From 6ccee0cc86c2117a5f82b0c6ab6b56b053c838d7 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Thu, 5 Aug 2021 23:11:54 -0400 Subject: [PATCH 37/73] CI (Buildkite): on the `platform_linux` jobs, after building Julia, make sure that the working directory is clean (#41796) (cherry picked from commit cb30aa7a089f4d5f1c9f834d96c3788f2e93ebcc) --- .buildkite/pipelines/main/platforms/linux64.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.buildkite/pipelines/main/platforms/linux64.yml b/.buildkite/pipelines/main/platforms/linux64.yml index 1e88db1144b4c..44b0ebe29f557 100644 --- a/.buildkite/pipelines/main/platforms/linux64.yml +++ b/.buildkite/pipelines/main/platforms/linux64.yml @@ -35,6 +35,9 @@ steps: make release make install + echo "--- Make sure that the working directory is clean" + if [ -z "$(git status --short)" ]; then echo "INFO: The working directory is clean."; else echo "ERROR: The working directory is dirty."; echo "Output of git status:"; git status; exit 1; fi + echo "--- Print Julia version info" ./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()' From a264ee765f01dadbc20ed6054b6d63df1c3d8004 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Fri, 6 Aug 2021 22:11:27 -0400 Subject: [PATCH 38/73] CI: Disable Codecov commit statuses (#41812) (cherry picked from commit 131b75b327557db49059aa20fc6404bc61b32148) --- .codecov.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .codecov.yml diff --git a/.codecov.yml b/.codecov.yml new file mode 100644 index 0000000000000..35cde5cd5e854 --- /dev/null +++ b/.codecov.yml @@ -0,0 +1,4 @@ +coverage: + status: + project: off + patch: off From 389338bca8be2630a7b31e58b3ab00f140f06718 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 10 Aug 2021 17:31:22 -0400 Subject: [PATCH 39/73] win: fix lock function call for backtrace collection (#41849) Avoids an assert when this is triggered (cherry picked from commit 2d8174254805b96ce936f233ce75264ad18051cf) --- src/signals-win.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/signals-win.c b/src/signals-win.c index c871c59aa1316..da03281b8a237 100644 --- a/src/signals-win.c +++ b/src/signals-win.c @@ -137,7 +137,7 @@ void jl_throw_in_ctx(jl_value_t *excpt, PCONTEXT ctxThread) ptls->bt_size = rec_backtrace_ctx(ptls->bt_data, JL_MAX_BT_SIZE, ctxThread, ptls->pgcstack); } else if (have_backtrace_fiber) { - JL_LOCK(&backtrace_lock); + JL_LOCK_NOGC(&backtrace_lock); stkerror_ctx = ctxThread; stkerror_ptls = ptls; jl_swapcontext(&error_return_fiber, &collect_backtrace_fiber); From ec28bcab932edc586db3212af78283c80b868f2f Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Thu, 12 Aug 2021 13:29:12 -0400 Subject: [PATCH 40/73] CI (Buildkite): `llvmpasses`: use the latest rootfs image (#41864) (cherry picked from commit 01d439e7e1d304903ccf2e37b47acea3d791238f) --- .buildkite/pipelines/main/misc/llvmpasses.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipelines/main/misc/llvmpasses.yml b/.buildkite/pipelines/main/misc/llvmpasses.yml index eed2f957855bb..a012ace41acff 100644 --- a/.buildkite/pipelines/main/misc/llvmpasses.yml +++ b/.buildkite/pipelines/main/misc/llvmpasses.yml @@ -33,8 +33,8 @@ steps: - JuliaCI/julia#v1: version: 1.6 - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.1/package_linux.x86_64.tar.gz - rootfs_treehash: "8c33c341a864852629b8aac01a6eb6a79b73570e" + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.8/package_linux.x86_64.tar.gz + rootfs_treehash: "84a323ae8fcc724f8ea5aca5901bbbf4bda3e519" uid: 1000 gid: 1000 workspaces: From 8e0f454ccccbedbda89bdfc32f6f62567b49e16a Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Tue, 17 Aug 2021 10:43:36 +0200 Subject: [PATCH 41/73] ensure that the non-REPL precompile code actually get precompiled (#41898) (cherry picked from commit e8faf9d17b78cdc61146893f71b7902d9dd80d81) --- contrib/generate_precompile.jl | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl index 41e7c16e49e89..06166a9726930 100644 --- a/contrib/generate_precompile.jl +++ b/contrib/generate_precompile.jl @@ -202,16 +202,20 @@ function generate_precompile_statements() module $pkgname end """) - tmp = tempname() + tmp_prec = tempname() + tmp_proc = tempname() s = """ pushfirst!(DEPOT_PATH, $(repr(prec_path))); - Base.PRECOMPILE_TRACE_COMPILE[] = $(repr(tmp)); + Base.PRECOMPILE_TRACE_COMPILE[] = $(repr(tmp_prec)); Base.compilecache(Base.PkgId($(repr(pkgname))), $(repr(path))) $precompile_script """ - run(`$(julia_exepath()) -O0 --sysimage $sysimg --startup-file=no -Cnative -e $s`) - for statement in split(read(tmp, String), '\n') - push!(statements, statement) + run(`$(julia_exepath()) -O0 --sysimage $sysimg --trace-compile=$tmp_proc --startup-file=no -Cnative -e $s`) + for f in (tmp_prec, tmp_proc) + for statement in split(read(f, String), '\n') + occursin("Main.", statement) && continue + push!(statements, statement) + end end end From 1b13f82026d520ee3fb37b806d61187ab28ae437 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Sun, 22 Aug 2021 22:42:24 -0400 Subject: [PATCH 42/73] README: add a link to the `base-buildkite-docs` repository (#41956) (cherry picked from commit 310bf160e026f3867119bef7d14724279e4eec60) --- .buildkite/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.buildkite/README.md b/.buildkite/README.md index 12887536b90c1..b3f74f2b23137 100644 --- a/.buildkite/README.md +++ b/.buildkite/README.md @@ -3,3 +3,5 @@ This directory contains the Buildkite configuration files for Base Julia CI. The rootfs image definitions are located in the [rootfs-images](https://github.com/JuliaCI/rootfs-images) repository. + +The documentation for the Base Julia CI setup is located in the [base-buildkite-docs](https://github.com/JuliaCI/base-buildkite-docs) repository. From 73c2e0bb66c5099d2f77076b13df701b1504332a Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 18 Mar 2021 00:35:43 -0400 Subject: [PATCH 43/73] Try to avoid julia becoming unkillable after fatal errors (#40056) - don't smash the alt-stack when already using it - handle jl_critical_error on the original stack, leaving our signal handling thread free to handle more signals (and helping lock corruption detection in some cases) - unblock signals when handling signals: some libc apparently like to block all signals, which can cause mild havoc, since we'd really like the user or bad data to be able to still kill the process (and not just be ignored or cause it to hang) - reset signals to SIG_DFL earlier (so we recurse less) - destroy some state from the Task we co-opted to run the exit handlers, so that it can't accidentally jump back into the running program after we've started tearing down the process, from an untimely ^C (previously ^C might cancel the exit) or a jlbacktrace call. - mark functions as leaf with CFI instead of (potentially) smashing the stack, and add a bit of red-zone if we are recursing (to keep pgcstack sensible) - support safe_restore for the mach catch_exception_raise (while we're trying to generate the backtrace) (cherry picked from commit 107901d09a847eb43c0cde923861b01f969b428d) (cherry picked from commit f02a7908c14724903cf0f803547db3ccb0c1a3eb) --- src/gf.c | 2 +- src/julia_internal.h | 32 +++++++- src/signal-handling.c | 41 ++++++++-- src/signals-mach.c | 122 ++++++++++++++++++++---------- src/signals-unix.c | 169 +++++++++++++++++++++++------------------- src/signals-win.c | 9 ++- src/stackwalk.c | 5 +- src/task.c | 12 +-- 8 files changed, 252 insertions(+), 140 deletions(-) diff --git a/src/gf.c b/src/gf.c index 33ec6003c95f9..43d4cf5112b95 100644 --- a/src/gf.c +++ b/src/gf.c @@ -1817,7 +1817,7 @@ static void JL_NORETURN jl_method_error_bare(jl_function_t *f, jl_value_t *args, jl_static_show((JL_STREAM*)STDERR_FILENO,args); jl_printf((JL_STREAM*)STDERR_FILENO,"\n"); jl_ptls_t ptls = jl_get_ptls_states(); ptls->bt_size = rec_backtrace(ptls->bt_data, JL_MAX_BT_SIZE, 0); - jl_critical_error(0, NULL, ptls->bt_data, &ptls->bt_size); + jl_critical_error(0, NULL); abort(); } // not reached diff --git a/src/julia_internal.h b/src/julia_internal.h index d1c040d8ee71a..0b766967918e2 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -69,6 +69,36 @@ void __tsan_switch_to_fiber(void *fiber, unsigned flags); # define JL_USE_IFUNC 0 #endif +// If we've smashed the stack, (and not just normal NORETURN) +// this will smash stack-unwind too +#ifdef _OS_WINDOWS_ +#if defined(_CPU_X86_64_) + // install the unhandled exception handler at the top of our stack + // to call directly into our personality handler +#define CFI_NORETURN \ + asm volatile ("\t.seh_handler __julia_personality, @except\n\t.text"); +#else +#define CFI_NORETURN +#endif +#else +// wipe out the call-stack unwind capability beyond this function +// (we are noreturn, so it is not a total lie) +#if defined(_CPU_X86_64_) +// per nongnu libunwind: "x86_64 ABI specifies that end of call-chain is marked with a NULL RBP or undefined return address" +// so we do all 3, to be extra certain of it +#define CFI_NORETURN \ + asm volatile ("\t.cfi_undefined rip"); \ + asm volatile ("\t.cfi_undefined rbp"); \ + asm volatile ("\t.cfi_return_column rbp"); +#else + // per nongnu libunwind: "DWARF spec says undefined return address location means end of stack" + // we use whatever happens to be register 1 on this platform for this +#define CFI_NORETURN \ + asm volatile ("\t.cfi_undefined 1"); \ + asm volatile ("\t.cfi_return_column 1"); +#endif +#endif + // If this is detected in a backtrace of segfault, it means the functions // that use this value must be reworked into their async form with cb arg // provided and with JL_UV_LOCK used around the calls @@ -904,7 +934,7 @@ size_t rec_backtrace_ctx(jl_bt_element_t *bt_data, size_t maxsize, bt_context_t size_t rec_backtrace_ctx_dwarf(jl_bt_element_t *bt_data, size_t maxsize, bt_context_t *ctx, jl_gcframe_t *pgcstack) JL_NOTSAFEPOINT; #endif JL_DLLEXPORT jl_value_t *jl_get_backtrace(void); -void jl_critical_error(int sig, bt_context_t *context, jl_bt_element_t *bt_data, size_t *bt_size); +void jl_critical_error(int sig, bt_context_t *context); JL_DLLEXPORT void jl_raise_debugger(void); int jl_getFunctionInfo(jl_frame_t **frames, uintptr_t pointer, int skipC, int noInline) JL_NOTSAFEPOINT; JL_DLLEXPORT void jl_gdblookup(void* ip) JL_NOTSAFEPOINT; diff --git a/src/signal-handling.c b/src/signal-handling.c index 80dfdb3b2fc21..aa642eeedf2a2 100644 --- a/src/signal-handling.c +++ b/src/signal-handling.c @@ -231,15 +231,44 @@ void jl_show_sigill(void *_ctx) #endif } -// what to do on a critical error -void jl_critical_error(int sig, bt_context_t *context, jl_bt_element_t *bt_data, size_t *bt_size) +// what to do on a critical error on a thread +void jl_critical_error(int sig, bt_context_t *context) { - // This function is not allowed to reference any TLS variables. - // We need to explicitly pass in the TLS buffer pointer when - // we make `jl_filename` and `jl_lineno` thread local. + + jl_ptls_t ptls = jl_get_ptls_states(); + jl_bt_element_t *bt_data = ptls->bt_data; + size_t *bt_size = &ptls->bt_size; size_t i, n = *bt_size; - if (sig) + if (sig) { + // kill this task, so that we cannot get back to it accidentally (via an untimely ^C or jlbacktrace in jl_exit) + ptls->pgcstack = NULL; + ptls->safe_restore = NULL; + if (ptls->current_task) { + ptls->current_task->eh = NULL; + ptls->current_task->excstack = NULL; + } +#ifndef _OS_WINDOWS_ + sigset_t sset; + sigemptyset(&sset); + // n.b. In `abort()`, Apple's libSystem "helpfully" blocks all signals + // on all threads but SIGABRT. But we also don't know what the thread + // was doing, so unblock all critical signals so that they will crash + // hard, and not just get stuck. + sigaddset(&sset, SIGSEGV); + sigaddset(&sset, SIGBUS); + sigaddset(&sset, SIGILL); + // also unblock fatal signals now, so we won't get back here twice + sigaddset(&sset, SIGTERM); + sigaddset(&sset, SIGABRT); + sigaddset(&sset, SIGQUIT); + // and the original signal is now fatal too, in case it wasn't + // something already listed (?) + if (sig != SIGINT) + sigaddset(&sset, sig); + pthread_sigmask(SIG_UNBLOCK, &sset, NULL); +#endif jl_safe_printf("\nsignal (%d): %s\n", sig, strsignal(sig)); + } jl_safe_printf("in expression starting at %s:%d\n", jl_filename, jl_lineno); if (context) { // Must avoid extended backtrace frames here unless we're sure bt_data diff --git a/src/signals-mach.c b/src/signals-mach.c index 3737bab1002cd..0d97d3b0dce56 100644 --- a/src/signals-mach.c +++ b/src/signals-mach.c @@ -84,6 +84,7 @@ extern boolean_t exc_server(mach_msg_header_t *, mach_msg_header_t *); void *mach_segv_listener(void *arg) { (void)arg; + (void)jl_get_ptls_states(); while (1) { int ret = mach_msg_server(exc_server, 2048, segv_port, MACH_MSG_TIMEOUT_NONE); jl_safe_printf("mach_msg_server: %s\n", mach_error_string(ret)); @@ -91,7 +92,8 @@ void *mach_segv_listener(void *arg) } } -static void allocate_segv_handler() + +static void allocate_mach_handler() { // ensure KEYMGR_GCC3_DW2_OBJ_LIST is initialized, as this requires malloc // and thus can deadlock when used without first initializing it. @@ -122,7 +124,7 @@ static void allocate_segv_handler() jl_error("pthread_create failed"); } pthread_attr_destroy(&attr); - for (int16_t tid = 0;tid < jl_n_threads;tid++) { + for (int16_t tid = 0; tid < jl_n_threads; tid++) { attach_exception_port(pthread_mach_thread_np(jl_all_tls_states[tid]->system_id), 0); } } @@ -164,19 +166,31 @@ typedef arm_exception_state64_t host_exception_state_t; static void jl_call_in_state(jl_ptls_t ptls2, host_thread_state_t *state, void (*fptr)(void)) { - uint64_t rsp = (uint64_t)ptls2->signal_stack + sig_stack_size; +#ifdef _CPU_X86_64_ + uintptr_t rsp = state->__rsp; +#elif defined(_CPU_AARCH64_) + uintptr_t rsp = state->__sp; +#else +#error "julia: throw-in-context not supported on this platform" +#endif + if (ptls2->signal_stack == NULL || is_addr_on_sigstack(ptls2, (void*)rsp)) { + rsp = (rsp - 256) & ~(uintptr_t)15; // redzone and re-alignment + } + else { + rsp = (uintptr_t)ptls2->signal_stack + sig_stack_size; + } assert(rsp % 16 == 0); - // push (null) $RIP onto the stack - rsp -= sizeof(void*); - *(void**)rsp = NULL; - #ifdef _CPU_X86_64_ + rsp -= sizeof(void*); state->__rsp = rsp; // set stack pointer state->__rip = (uint64_t)fptr; // "call" the function -#else +#elif defined(_CPU_AARCH64_) state->__sp = rsp; state->__pc = (uint64_t)fptr; + state->__lr = 0; +#else +#error "julia: throw-in-context not supported on this platform" #endif } @@ -194,11 +208,22 @@ static void jl_throw_in_thread(int tid, mach_port_t thread, jl_value_t *exceptio ptls2->sig_exception = exception; } jl_call_in_state(ptls2, &state, &jl_sig_throw); - ret = thread_set_state(thread, THREAD_STATE, - (thread_state_t)&state, count); + ret = thread_set_state(thread, THREAD_STATE, (thread_state_t)&state, count); HANDLE_MACH_ERROR("thread_set_state", ret); } +static void segv_handler(int sig, siginfo_t *info, void *context) +{ + jl_ptls_t ptls = jl_get_ptls_states(); + assert(sig == SIGSEGV || sig == SIGBUS); + if (ptls->safe_restore) { // restarting jl_ or jl_unwind_stepn + jl_call_in_state(ptls, (host_thread_state_t*)jl_to_bt_context(context), &jl_sig_throw); + } + else { + sigdie_handler(sig, info, context); + } +} + //exc_server uses dlsym to find symbol JL_DLLEXPORT kern_return_t catch_exception_raise(mach_port_t exception_port, @@ -208,18 +233,16 @@ kern_return_t catch_exception_raise(mach_port_t exception_port, exception_data_t code, mach_msg_type_number_t code_count) { - unsigned int count = THREAD_STATE_COUNT; unsigned int exc_count = HOST_EXCEPTION_STATE_COUNT; host_exception_state_t exc_state; - host_thread_state_t state; -#ifdef LIBOSXUNWIND +#ifdef LLVMLIBUNWIND if (thread == mach_profiler_thread) { return profiler_segv_handler(exception_port, thread, task, exception, code, code_count); } #endif int16_t tid; jl_ptls_t ptls2 = NULL; - for (tid = 0;tid < jl_n_threads;tid++) { + for (tid = 0; tid < jl_n_threads; tid++) { jl_ptls_t _ptls2 = jl_all_tls_states[tid]; if (pthread_mach_thread_np(_ptls2->system_id) == thread) { ptls2 = _ptls2; @@ -288,11 +311,8 @@ kern_return_t catch_exception_raise(mach_port_t exception_port, return KERN_SUCCESS; } else { - kern_return_t ret = thread_get_state(thread, THREAD_STATE, (thread_state_t)&state, &count); - HANDLE_MACH_ERROR("thread_get_state", ret); - jl_critical_error(SIGSEGV, (unw_context_t*)&state, - ptls2->bt_data, &ptls2->bt_size); - return KERN_INVALID_ARGUMENT; + jl_exit_thread0(128 + SIGSEGV, NULL, 0); + return KERN_SUCCESS; } } @@ -307,24 +327,27 @@ static void attach_exception_port(thread_port_t thread, int segv_only) HANDLE_MACH_ERROR("thread_set_exception_ports", ret); } -static void jl_thread_suspend_and_get_state(int tid, unw_context_t **ctx) +static void jl_thread_suspend_and_get_state2(int tid, host_thread_state_t *ctx) { jl_ptls_t ptls2 = jl_all_tls_states[tid]; - mach_port_t tid_port = pthread_mach_thread_np(ptls2->system_id); + mach_port_t thread = pthread_mach_thread_np(ptls2->system_id); - kern_return_t ret = thread_suspend(tid_port); + kern_return_t ret = thread_suspend(thread); HANDLE_MACH_ERROR("thread_suspend", ret); // Do the actual sampling unsigned int count = THREAD_STATE_COUNT; - static unw_context_t state; - memset(&state, 0, sizeof(unw_context_t)); + memset(ctx, 0, sizeof(*ctx)); // Get the state of the suspended thread - ret = thread_get_state(tid_port, THREAD_STATE, (thread_state_t)&state, &count); + ret = thread_get_state(thread, THREAD_STATE, (thread_state_t)ctx, &count); +} - // Initialize the unwind context with the suspend thread's state - *ctx = &state; +static void jl_thread_suspend_and_get_state(int tid, unw_context_t **ctx) +{ + static host_thread_state_t state; + jl_thread_suspend_and_get_state2(tid, &state); + *ctx = (unw_context_t*)&state; } static void jl_thread_resume(int tid, int sig) @@ -366,29 +389,46 @@ static void jl_try_deliver_sigint(void) HANDLE_MACH_ERROR("thread_resume", ret); } -static void jl_exit_thread0(int exitstate) +static void JL_NORETURN jl_exit_thread0_cb(int exitstate) +{ +CFI_NORETURN + jl_critical_error(exitstate - 128, NULL); + jl_exit(exitstate); +} + +static void jl_exit_thread0(int exitstate, jl_bt_element_t *bt_data, size_t bt_size) { jl_ptls_t ptls2 = jl_all_tls_states[0]; mach_port_t thread = pthread_mach_thread_np(ptls2->system_id); - kern_return_t ret = thread_suspend(thread); - HANDLE_MACH_ERROR("thread_suspend", ret); + + host_thread_state_t state; + jl_thread_suspend_and_get_state2(0, &state); + unw_context_t *uc = (unw_context_t*)&state; // This aborts `sleep` and other syscalls. - ret = thread_abort(thread); + kern_return_t ret = thread_abort(thread); HANDLE_MACH_ERROR("thread_abort", ret); - unsigned int count = THREAD_STATE_COUNT; - host_thread_state_t state; - ret = thread_get_state(thread, THREAD_STATE, - (thread_state_t)&state, &count); + if (bt_data == NULL) { + // Must avoid extended backtrace frames here unless we're sure bt_data + // is properly rooted. + ptls2->bt_size = rec_backtrace_ctx(ptls2->bt_data, JL_MAX_BT_SIZE, uc, NULL); + } + else { + ptls2->bt_size = bt_size; // <= JL_MAX_BT_SIZE + memcpy(ptls2->bt_data, bt_data, ptls2->bt_size * sizeof(bt_data[0])); + } void (*exit_func)(int) = &_exit; if (thread0_exit_count <= 1) { - exit_func = &jl_exit; + exit_func = &jl_exit_thread0_cb; } else if (thread0_exit_count == 2) { exit_func = &exit; } + else { + exit_func = &_exit; + } #ifdef _CPU_X86_64_ // First integer argument. Not portable but good enough =) @@ -399,8 +439,8 @@ static void jl_exit_thread0(int exitstate) #error Fill in first integer argument here #endif jl_call_in_state(ptls2, &state, (void (*)(void))exit_func); - ret = thread_set_state(thread, THREAD_STATE, - (thread_state_t)&state, count); + unsigned int count = THREAD_STATE_COUNT; + ret = thread_set_state(thread, THREAD_STATE, (thread_state_t)&state, count); HANDLE_MACH_ERROR("thread_set_state", ret); ret = thread_resume(thread); @@ -498,8 +538,10 @@ void *mach_profile_listener(void *arg) break; } - unw_context_t *uc; - jl_thread_suspend_and_get_state(i, &uc); + host_thread_state_t state; + jl_thread_suspend_and_get_state2(i, &state); + unw_context_t *uc = (unw_context_t*)&state; + if (running) { #ifdef LIBOSXUNWIND /* diff --git a/src/signals-unix.c b/src/signals-unix.c index 57ce2439fcb90..de3b5e13c98df 100644 --- a/src/signals-unix.c +++ b/src/signals-unix.c @@ -58,7 +58,9 @@ static bt_context_t *jl_to_bt_context(void *sigctx) #endif } + static int thread0_exit_count = 0; +static void jl_exit_thread0(int exitstate, jl_bt_element_t *bt_data, size_t bt_size); static inline __attribute__((unused)) uintptr_t jl_get_rsp_from_ctx(const void *_ctx) { @@ -86,8 +88,17 @@ static inline __attribute__((unused)) uintptr_t jl_get_rsp_from_ctx(const void * #endif } +static int is_addr_on_sigstack(jl_ptls_t ptls, void *ptr) +{ + // One guard page for signal_stack. + return !((char*)ptr < (char*)ptls->signal_stack - jl_page_size || + (char*)ptr > (char*)ptls->signal_stack + sig_stack_size); +} + // Modify signal context `_ctx` so that `fptr` will execute when the signal // returns. `fptr` will execute on the signal stack, and must not return. +// jl_call_in_ctx is also currently executing on that signal stack, +// so be careful not to smash it static void jl_call_in_ctx(jl_ptls_t ptls, void (*fptr)(void), int sig, void *_ctx) { // Modifying the ucontext should work but there is concern that @@ -105,30 +116,32 @@ static void jl_call_in_ctx(jl_ptls_t ptls, void (*fptr)(void), int sig, void *_c fptr(); return; } - uintptr_t rsp = (uintptr_t)ptls->signal_stack + sig_stack_size; + uintptr_t rsp = jl_get_rsp_from_ctx(_ctx); + if (is_addr_on_sigstack(ptls, (void*)rsp)) { + rsp = (rsp - 256) & ~(uintptr_t)15; // redzone and re-alignment + } + else { + rsp = (uintptr_t)ptls->signal_stack + sig_stack_size; + } assert(rsp % 16 == 0); #if defined(_OS_LINUX_) && defined(_CPU_X86_64_) ucontext_t *ctx = (ucontext_t*)_ctx; rsp -= sizeof(void*); - *(void**)rsp = NULL; ctx->uc_mcontext.gregs[REG_RSP] = rsp; ctx->uc_mcontext.gregs[REG_RIP] = (uintptr_t)fptr; #elif defined(_OS_FREEBSD_) && defined(_CPU_X86_64_) ucontext_t *ctx = (ucontext_t*)_ctx; rsp -= sizeof(void*); - *(void**)rsp = NULL; ctx->uc_mcontext.mc_rsp = rsp; ctx->uc_mcontext.mc_rip = (uintptr_t)fptr; #elif defined(_OS_LINUX_) && defined(_CPU_X86_) ucontext_t *ctx = (ucontext_t*)_ctx; rsp -= sizeof(void*); - *(void**)rsp = NULL; ctx->uc_mcontext.gregs[REG_ESP] = rsp; ctx->uc_mcontext.gregs[REG_EIP] = (uintptr_t)fptr; #elif defined(_OS_FREEBSD_) && defined(_CPU_X86_) ucontext_t *ctx = (ucontext_t*)_ctx; rsp -= sizeof(void*); - *(void**)rsp = NULL; ctx->uc_mcontext.mc_esp = rsp; ctx->uc_mcontext.mc_eip = (uintptr_t)fptr; #elif defined(_OS_LINUX_) && defined(_CPU_AARCH64_) @@ -162,14 +175,14 @@ static void jl_call_in_ctx(jl_ptls_t ptls, void (*fptr)(void), int sig, void *_c // `catch_exception_raise`. It works fine when a signal is received // due to `kill`/`raise` though. ucontext64_t *ctx = (ucontext64_t*)_ctx; - rsp -= sizeof(void*); - *(void**)rsp = NULL; #if defined(_CPU_X86_64_) + rsp -= sizeof(void*); ctx->uc_mcontext64->__ss.__rsp = rsp; ctx->uc_mcontext64->__ss.__rip = (uintptr_t)fptr; #else ctx->uc_mcontext64->__ss.__sp = rsp; ctx->uc_mcontext64->__ss.__pc = (uintptr_t)fptr; + ctx->uc_mcontext64->__ss.__lr = 0; #endif #else #warning "julia: throw-in-context not supported on this platform" @@ -206,16 +219,11 @@ static int is_addr_on_stack(jl_ptls_t ptls, void *addr) static void sigdie_handler(int sig, siginfo_t *info, void *context) { - jl_ptls_t ptls = jl_get_ptls_states(); - sigset_t sset; + signal(sig, SIG_DFL); uv_tty_reset_mode(); if (sig == SIGILL) jl_show_sigill(context); - jl_critical_error(sig, jl_to_bt_context(context), - ptls->bt_data, &ptls->bt_size); - sigfillset(&sset); - sigprocmask(SIG_UNBLOCK, &sset, NULL); - signal(sig, SIG_DFL); + jl_critical_error(sig, jl_to_bt_context(context)); if (sig != SIGSEGV && sig != SIGBUS && sig != SIGILL) { @@ -228,12 +236,6 @@ static void sigdie_handler(int sig, siginfo_t *info, void *context) #include "signals-mach.c" #else -static int is_addr_on_sigstack(jl_ptls_t ptls, void *ptr) -{ - // One guard page for signal_stack. - return !((char*)ptr < (char*)ptls->signal_stack - jl_page_size || - (char*)ptr > (char*)ptls->signal_stack + sig_stack_size); -} static int jl_is_on_sigstack(jl_ptls_t ptls, void *ptr, void *context) { @@ -245,7 +247,6 @@ static void segv_handler(int sig, siginfo_t *info, void *context) { jl_ptls_t ptls = jl_get_ptls_states(); assert(sig == SIGSEGV || sig == SIGBUS); - if (jl_addr_is_safepoint((uintptr_t)info->si_addr)) { jl_set_gc_and_wait(); // Do not raise sigint on worker thread @@ -284,22 +285,6 @@ static void segv_handler(int sig, siginfo_t *info, void *context) } } -static void allocate_segv_handler(void) -{ - struct sigaction act; - memset(&act, 0, sizeof(struct sigaction)); - sigemptyset(&act.sa_mask); - act.sa_sigaction = segv_handler; - act.sa_flags = SA_ONSTACK | SA_SIGINFO; - if (sigaction(SIGSEGV, &act, NULL) < 0) { - jl_errorf("fatal error: sigaction: %s", strerror(errno)); - } - // On AArch64, stack overflow triggers a SIGBUS - if (sigaction(SIGBUS, &act, NULL) < 0) { - jl_errorf("fatal error: sigaction: %s", strerror(errno)); - } -} - #if !defined(JL_DISABLE_LIBUNWIND) static unw_context_t *volatile signal_context; static pthread_mutex_t in_signal_lock; @@ -319,9 +304,8 @@ static void jl_thread_suspend_and_get_state(int tid, unw_context_t **ctx) static void jl_thread_resume(int tid, int sig) { - (void)sig; jl_ptls_t ptls2 = jl_all_tls_states[tid]; - jl_atomic_store_release(&ptls2->signal_request, 1); + jl_atomic_store_release(&ptls2->signal_request, sig == -1 ? 3 : 1); pthread_cond_broadcast(&exit_signal_cond); pthread_cond_wait(&signal_caught_cond, &in_signal_lock); // wait for thread to acknowledge assert(jl_atomic_load_acquire(&ptls2->signal_request) == 0); @@ -344,12 +328,14 @@ static void jl_try_deliver_sigint(void) // Write only by signal handling thread, read only by main thread // no sync necessary. static int thread0_exit_state = 0; -static void jl_exit_thread0_cb(void) +static void JL_NORETURN jl_exit_thread0_cb(void) { +CFI_NORETURN // This can get stuck if it happens at an unfortunate spot // (unavoidable due to its async nature). // Try harder to exit each time if we get multiple exit requests. if (thread0_exit_count <= 1) { + jl_critical_error(thread0_exit_state - 128, NULL); jl_exit(thread0_exit_state); } else if (thread0_exit_count == 2) { @@ -360,12 +346,23 @@ static void jl_exit_thread0_cb(void) } } -static void jl_exit_thread0(int state) +static void jl_exit_thread0(int state, jl_bt_element_t *bt_data, size_t bt_size) { jl_ptls_t ptls2 = jl_all_tls_states[0]; - thread0_exit_state = state; - jl_atomic_store_release(&ptls2->signal_request, 3); - pthread_kill(ptls2->system_id, SIGUSR2); + if (thread0_exit_count <= 1) { + unw_context_t *signal_context; + jl_thread_suspend_and_get_state(0, &signal_context); + thread0_exit_state = state; + ptls2->bt_size = bt_size; // <= JL_MAX_BT_SIZE + memcpy(ptls2->bt_data, bt_data, ptls2->bt_size * sizeof(bt_data[0])); + jl_thread_resume(0, -1); + } + else { + thread0_exit_state = state; + jl_atomic_store_release(&ptls2->signal_request, 3); + // This also makes sure `sleep` is aborted. + pthread_kill(ptls2->system_id, SIGUSR2); + } } // request: @@ -387,12 +384,10 @@ void usr2_handler(int sig, siginfo_t *info, void *ctx) pthread_cond_broadcast(&signal_caught_cond); pthread_cond_wait(&exit_signal_cond, &in_signal_lock); request = jl_atomic_exchange(&ptls->signal_request, 0); - assert(request == 1); - (void)request; + assert(request == 1 || request == 3); pthread_cond_broadcast(&signal_caught_cond); pthread_mutex_unlock(&in_signal_lock); } - else #endif if (request == 2) { int force = jl_check_force_sigint(); @@ -483,43 +478,42 @@ JL_DLLEXPORT void jl_profile_stop_timer(void) #endif #endif // HAVE_MACH -static void *alloc_sigstack(size_t size) +static void allocate_segv_handler(void) { - size_t pagesz = jl_getpagesize(); - // Add one guard page to catch stack overflow in the signal handler - size = LLT_ALIGN(size, pagesz) + pagesz; - void *stackbuff = mmap(0, size, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (stackbuff == MAP_FAILED) - jl_errorf("fatal error allocating signal stack: mmap: %s", - strerror(errno)); - mprotect(stackbuff, pagesz, PROT_NONE); - return (void*)((char*)stackbuff + pagesz); + struct sigaction act; + memset(&act, 0, sizeof(struct sigaction)); + sigemptyset(&act.sa_mask); + act.sa_sigaction = segv_handler; + act.sa_flags = SA_ONSTACK | SA_SIGINFO; + if (sigaction(SIGSEGV, &act, NULL) < 0) { + jl_errorf("fatal error: sigaction: %s", strerror(errno)); + } + // On AArch64, stack overflow triggers a SIGBUS + if (sigaction(SIGBUS, &act, NULL) < 0) { + jl_errorf("fatal error: sigaction: %s", strerror(errno)); + } +} + +static void *alloc_sigstack(size_t *ssize) +{ + void *stk = jl_malloc_stack(ssize, NULL); + if (stk == MAP_FAILED) + jl_errorf("fatal error allocating signal stack: mmap: %s", strerror(errno)); + return stk; } void jl_install_thread_signal_handler(jl_ptls_t ptls) { - void *signal_stack = alloc_sigstack(sig_stack_size); + size_t ssize = sig_stack_size; + void *signal_stack = alloc_sigstack(&ssize); + ptls->signal_stack = signal_stack; stack_t ss; ss.ss_flags = 0; - ss.ss_size = sig_stack_size - 16; + ss.ss_size = ssize - 16; ss.ss_sp = signal_stack; if (sigaltstack(&ss, NULL) < 0) { jl_errorf("fatal error: sigaltstack: %s", strerror(errno)); } - -#if !defined(HAVE_MACH) - struct sigaction act; - memset(&act, 0, sizeof(struct sigaction)); - sigemptyset(&act.sa_mask); - act.sa_sigaction = usr2_handler; - act.sa_flags = SA_ONSTACK | SA_SIGINFO | SA_RESTART; - if (sigaction(SIGUSR2, &act, NULL) < 0) { - jl_errorf("fatal error: sigaction: %s", strerror(errno)); - } -#endif - - ptls->signal_stack = signal_stack; } static void jl_sigsetset(sigset_t *sset) @@ -737,10 +731,16 @@ static void *signal_listener(void *arg) // this part is async with the running of the rest of the program // and must be thread-safe, but not necessarily signal-handler safe if (critical) { - jl_critical_error(sig, NULL, bt_data, &bt_size); if (doexit) { thread0_exit_count++; - jl_exit_thread0(128 + sig); + jl_exit_thread0(128 + sig, bt_data, bt_size); + } + else { + jl_safe_printf("\nsignal (%d): %s\n", sig, strsignal(sig)); + size_t i; + for (i = 0; i < bt_size; i += jl_bt_entry_size(bt_data + i)) { + jl_print_bt_entry_codeloc(bt_data + i); + } } } } @@ -787,7 +787,7 @@ void jl_install_default_signal_handlers(void) memset(&actf, 0, sizeof(struct sigaction)); sigemptyset(&actf.sa_mask); actf.sa_sigaction = fpe_handler; - actf.sa_flags = SA_SIGINFO; + actf.sa_flags = SA_ONSTACK | SA_SIGINFO; if (sigaction(SIGFPE, &actf, NULL) < 0) { jl_errorf("fatal error: sigaction: %s", strerror(errno)); } @@ -806,13 +806,26 @@ void jl_install_default_signal_handlers(void) jl_error("fatal error: Couldn't set SIGTRAP"); } +#if defined(HAVE_MACH) + allocate_mach_handler(); +#else + struct sigaction act; + memset(&act, 0, sizeof(struct sigaction)); + sigemptyset(&act.sa_mask); + act.sa_sigaction = usr2_handler; + act.sa_flags = SA_ONSTACK | SA_SIGINFO | SA_RESTART; + if (sigaction(SIGUSR2, &act, NULL) < 0) { + jl_errorf("fatal error: sigaction: %s", strerror(errno)); + } +#endif + allocate_segv_handler(); struct sigaction act_die; memset(&act_die, 0, sizeof(struct sigaction)); sigemptyset(&act_die.sa_mask); act_die.sa_sigaction = sigdie_handler; - act_die.sa_flags = SA_SIGINFO; + act_die.sa_flags = SA_SIGINFO | SA_RESETHAND; if (sigaction(SIGILL, &act_die, NULL) < 0) { jl_errorf("fatal error: sigaction: %s", strerror(errno)); } @@ -823,7 +836,7 @@ void jl_install_default_signal_handlers(void) jl_errorf("fatal error: sigaction: %s", strerror(errno)); } // need to ensure the following signals are not SIG_IGN, even though they will be blocked - act_die.sa_flags = SA_SIGINFO | SA_RESTART; + act_die.sa_flags = SA_SIGINFO | SA_RESTART | SA_RESETHAND; #if defined(HAVE_ITIMER) if (sigaction(SIGPROF, &act_die, NULL) < 0) { jl_errorf("fatal error: sigaction: %s", strerror(errno)); diff --git a/src/signals-win.c b/src/signals-win.c index da03281b8a237..fc3041d0dbd32 100644 --- a/src/signals-win.c +++ b/src/signals-win.c @@ -92,7 +92,7 @@ void __cdecl crt_sig_handler(int sig, int num) RtlCaptureContext(&Context); if (sig == SIGILL) jl_show_sigill(&Context); - jl_critical_error(sig, &Context, ptls->bt_data, &ptls->bt_size); + jl_critical_error(sig, &Context); raise(sig); } } @@ -309,8 +309,7 @@ LONG WINAPI jl_exception_handler(struct _EXCEPTION_POINTERS *ExceptionInfo) jl_safe_printf(" at 0x%Ix -- ", (size_t)ExceptionInfo->ExceptionRecord->ExceptionAddress); jl_print_native_codeloc((uintptr_t)ExceptionInfo->ExceptionRecord->ExceptionAddress); - jl_critical_error(0, ExceptionInfo->ContextRecord, - ptls->bt_data, &ptls->bt_size); + jl_critical_error(0, ExceptionInfo->ContextRecord); static int recursion = 0; if (recursion++) exit(1); @@ -384,10 +383,12 @@ JL_DLLEXPORT int jl_profile_start_timer(void) { if (hBtThread == NULL) { - if (MMSYSERR_NOERROR != timeGetDevCaps(&timecaps, sizeof(timecaps))) { + TIMECAPS _timecaps; + if (MMSYSERR_NOERROR != timeGetDevCaps(&_timecaps, sizeof(_timecaps))) { fputs("failed to get timer resolution", stderr); return -2; } + timecaps = _timecaps; hBtThread = CreateThread( NULL, // default security attributes diff --git a/src/stackwalk.c b/src/stackwalk.c index 9150d48b29765..52d95e8fe8de7 100644 --- a/src/stackwalk.c +++ b/src/stackwalk.c @@ -698,7 +698,10 @@ JL_DLLEXPORT void jl_gdblookup(void* ip) // Print backtrace for current exception in catch block JL_DLLEXPORT void jlbacktrace(void) JL_NOTSAFEPOINT { - jl_excstack_t *s = jl_get_ptls_states()->current_task->excstack; + jl_ptls_t ptls = jl_get_ptls_states(); + if (ptls->current_task == NULL) + return; + jl_excstack_t *s = ptls->current_task->excstack; if (!s) return; size_t bt_size = jl_excstack_bt_size(s, s->top); diff --git a/src/task.c b/src/task.c index 4d94d90fc62b3..d65e7412bb7ab 100644 --- a/src/task.c +++ b/src/task.c @@ -647,8 +647,9 @@ JL_DLLEXPORT void jl_rethrow(void) // Special case throw for errors detected inside signal handlers. This is not // (cannot be) called directly in the signal handler itself, but is returned to // after the signal handler exits. -JL_DLLEXPORT void jl_sig_throw(void) +JL_DLLEXPORT void JL_NORETURN jl_sig_throw(void) { +CFI_NORETURN jl_ptls_t ptls = jl_get_ptls_states(); jl_value_t *e = ptls->sig_exception; ptls->sig_exception = NULL; @@ -800,14 +801,7 @@ void jl_init_tasks(void) JL_GC_DISABLED STATIC_OR_JS void NOINLINE JL_NORETURN start_task(void) { -#ifdef _OS_WINDOWS_ -#if defined(_CPU_X86_64_) - // install the unhandled exception hanlder at the top of our stack - // to call directly into our personality handler - asm volatile ("\t.seh_handler __julia_personality, @except\n\t.text"); -#endif -#endif - +CFI_NORETURN // this runs the first time we switch to a task sanitizer_finish_switch_fiber(); jl_ptls_t ptls = jl_get_ptls_states(); From cc664066b063e85a9c9c5b56d5d2611101e77b02 Mon Sep 17 00:00:00 2001 From: Kenta Sato Date: Mon, 2 Aug 2021 17:30:24 +0900 Subject: [PATCH 44/73] guard `isempty (close #41656) (#41752) (cherry picked from commit 4c3829d413a12c2b4a87952e0f6d1ecbbff63239) --- contrib/generate_precompile.jl | 2 +- test/misc.jl | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl index 06166a9726930..28be1a992efda 100644 --- a/contrib/generate_precompile.jl +++ b/contrib/generate_precompile.jl @@ -1,6 +1,6 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -if isempty(Base.ARGS) || Base.ARGS[1] !== "0" +if Base.isempty(Base.ARGS) || Base.ARGS[1] !== "0" Sys.__init_build() # Prevent this from being put into the Main namespace @eval Module() begin diff --git a/test/misc.jl b/test/misc.jl index e069267ae3bed..30369830cf04b 100644 --- a/test/misc.jl +++ b/test/misc.jl @@ -912,3 +912,6 @@ end @testset "issue #28188" begin @test `$(@__FILE__)` == let file = @__FILE__; `$file` end end + +# issue #41656 +@test success(`$(Base.julia_cmd()) -e 'isempty(x) = true'`) From c750018e359d0530bffaa4134156b9e76d6b2841 Mon Sep 17 00:00:00 2001 From: Philipp Gabler Date: Wed, 11 Aug 2021 22:01:06 +0200 Subject: [PATCH 45/73] Fix firstindex in replace_ref_begin_end (fixes #41630) (#41695) (cherry picked from commit 2ebbb2b3a0b54eae66549bd058ae334cb3642e50) --- base/views.jl | 2 +- test/offsetarray.jl | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/base/views.jl b/base/views.jl index ccf24d4cdea3a..153b5d57004d8 100644 --- a/base/views.jl +++ b/base/views.jl @@ -42,7 +42,7 @@ function replace_ref_begin_end_!(ex, withex) n = 1 J = lastindex(ex.args) for j = 2:J - exj, used = replace_ref_begin_end_!(ex.args[j], (:($firstindex($S)),:($lastindex($S,$n)))) + exj, used = replace_ref_begin_end_!(ex.args[j], (:($firstindex($S,$n)),:($lastindex($S,$n)))) used_S |= used ex.args[j] = exj if isa(exj,Expr) && exj.head === :... diff --git a/test/offsetarray.jl b/test/offsetarray.jl index 5deb442f36222..362aada64fe70 100644 --- a/test/offsetarray.jl +++ b/test/offsetarray.jl @@ -776,3 +776,24 @@ end strY = String(take!(io)) @test strX == strY end + +@testset "issue #41630: replace_ref_begin_end!/@view on offset-like arrays" begin + x = OffsetArray([1 2; 3 4], -10:-9, 9:10) # 2×2 OffsetArray{...} with indices -10:-9×9:10 + + # begin/end with offset indices + @test (@view x[begin, 9])[] == 1 + @test (@view x[-10, end])[] == 2 + @test (@view x[-9, begin])[] == 3 + @test (@view x[end, 10])[] == 4 + @test (@view x[begin, begin])[] == 1 + @test (@view x[begin, end])[] == 2 + @test (@view x[end, begin])[] == 3 + @test (@view x[end, end])[] == 4 + + # nested usages of begin/end + y = OffsetArray([-10, -9], (5,)) + @test (@view x[begin, -y[end]])[] == 1 + @test (@view x[y[begin], end])[] == 2 + @test (@view x[end, -y[end]])[] == 3 + @test (@view x[y[end], end])[] == 4 +end From cbbefc1d6342c98c2234a69856c0e529845eed80 Mon Sep 17 00:00:00 2001 From: Jakob Nybo Nissen Date: Wed, 25 Aug 2021 00:42:41 +0200 Subject: [PATCH 46/73] Fix collect on stateful generator (#41919) Previously this code would drop 1 from the length of some generators. Fixes #35530 (cherry picked from commit 8364a4ccd8885fa8d8c78094c7653c58e33d9f0d) --- base/array.jl | 10 +++++++--- test/iterators.jl | 9 +++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/base/array.jl b/base/array.jl index e7b87cf9739ce..cdc7ad7fedbbc 100644 --- a/base/array.jl +++ b/base/array.jl @@ -666,8 +666,10 @@ else end end -_array_for(::Type{T}, itr, ::HasLength) where {T} = Vector{T}(undef, Int(length(itr)::Integer)) -_array_for(::Type{T}, itr, ::HasShape{N}) where {T,N} = similar(Array{T,N}, axes(itr)) +_array_for(::Type{T}, itr, isz::HasLength) where {T} = _array_for(T, itr, isz, length(itr)) +_array_for(::Type{T}, itr, isz::HasShape{N}) where {T,N} = _array_for(T, itr, isz, axes(itr)) +_array_for(::Type{T}, itr, ::HasLength, len) where {T} = Vector{T}(undef, len) +_array_for(::Type{T}, itr, ::HasShape{N}, axs) where {T,N} = similar(Array{T,N}, axs) function collect(itr::Generator) isz = IteratorSize(itr.iter) @@ -675,12 +677,14 @@ function collect(itr::Generator) if isa(isz, SizeUnknown) return grow_to!(Vector{et}(), itr) else + shape = isz isa HasLength ? length(itr) : axes(itr) y = iterate(itr) if y === nothing return _array_for(et, itr.iter, isz) end v1, st = y - collect_to_with_first!(_array_for(typeof(v1), itr.iter, isz), v1, itr, st) + arr = _array_for(typeof(v1), itr.iter, isz, shape) + return collect_to_with_first!(arr, v1, itr, st) end end diff --git a/test/iterators.jl b/test/iterators.jl index b9bec84bf9a58..6007a59464f5a 100644 --- a/test/iterators.jl +++ b/test/iterators.jl @@ -291,6 +291,15 @@ let (a, b) = (1:3, [4 6; end end +# collect stateful iterator +let + itr = (i+1 for i in Base.Stateful([1,2,3])) + @test collect(itr) == [2, 3, 4] + A = zeros(Int, 0, 0) + itr = (i-1 for i in Base.Stateful(A)) + @test collect(itr) == Int[] # Stateful do not preserve shape +end + # with 1D inputs let a = 1:2, b = 1.0:10.0, From 94c1a3a788f9419cc7147aab0d9acb27aed2fb1a Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Wed, 9 Dec 2020 11:06:46 -0500 Subject: [PATCH 47/73] Set the `@JuliaLang/github-actions` team as the code owners for GitHub Actions workflow files (#38783) (cherry picked from commit ebf54b4b69e4fb3267881a093d18a955c3f1378c) --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000000000..865abeebd82af --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +/.github/workflows/ @JuliaLang/github-actions From 7f9928e40a97e7a9173008cabef4ec5af9ed48ec Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Sun, 23 May 2021 09:02:40 -0400 Subject: [PATCH 48/73] Set up a few code owners (#40925) Co-authored-by: Viral B. Shah (cherry picked from commit 3bbe22e589ecaa6062c370c4a9673a9b277a3a70) --- .github/CODEOWNERS | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 865abeebd82af..2e7d0f4692e73 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1,7 @@ -/.github/workflows/ @JuliaLang/github-actions +/.github/ @JuliaLang/github-actions +/.buildkite/ @JuliaLang/github-actions +/src/ @JuliaLang/compiler +/stdlib/LinearAlgebra/ @JuliaLang/linearalgebra +/stdlib/libblastrampoline_jll/ @staticfloat +/test/ @JuliaLang/test + From 49e2197104ddb80fd43cf42e1592ca11dab25b97 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Sun, 23 May 2021 09:29:21 -0400 Subject: [PATCH 49/73] Pare down the CODEOWNERS [skip ci] [ci skip] (#40929) (cherry picked from commit 4ecbc16df572f54d3f05d872f13f31f9e6abc7ef) --- .github/CODEOWNERS | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 2e7d0f4692e73..6cab5b68b11e9 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,7 +1,3 @@ +CODEOWNERS @JuliaLang/github-actions /.github/ @JuliaLang/github-actions /.buildkite/ @JuliaLang/github-actions -/src/ @JuliaLang/compiler -/stdlib/LinearAlgebra/ @JuliaLang/linearalgebra -/stdlib/libblastrampoline_jll/ @staticfloat -/test/ @JuliaLang/test - From 9b6bf884ac511aa37fb097a287b45eb5c4502657 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Fri, 6 Aug 2021 22:46:50 -0400 Subject: [PATCH 50/73] CI (Buildbot): As soon as a PR is created or updated, create pending (yellow) commit statuses for all Buildbot jobs (#41811) (cherry picked from commit 6fce8d50da57631e33ffc316c57a99bf5597d765) --- .github/CODEOWNERS | 6 ++- .github/workflows/statuses.yml | 97 ++++++++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/statuses.yml diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 6cab5b68b11e9..5fc00a73b47d2 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,3 +1,5 @@ -CODEOWNERS @JuliaLang/github-actions -/.github/ @JuliaLang/github-actions +CODEOWNERS @JuliaLang/github-actions +/.github/ @JuliaLang/github-actions /.buildkite/ @JuliaLang/github-actions + +/.github/workflows/statuses.yml @DilumAluthge diff --git a/.github/workflows/statuses.yml b/.github/workflows/statuses.yml new file mode 100644 index 0000000000000..27422b4e69345 --- /dev/null +++ b/.github/workflows/statuses.yml @@ -0,0 +1,97 @@ +# Please ping @DilumAluthge when making any changes to this file. + +# This is just a short-term solution until we have migrated all of CI to Buildkite. +# +# 1. TODO: delete this file once we have migrated all of CI to Buildkite. +# +# 2. TODO: disable GitHub Actions on the `JuliaLang/julia` repository once we have migrated all +# of CI to Buildkite. + +# Here are some steps that we take in this workflow file for security reasons: +# 1. We do not checkout any code. +# 2. We do not run any external actions. + +name: Statuses + +on: + push: + branches: + - 'master' + - 'release-*' + # When using the `pull_request_target` event, all PRs will get a `GITHUB_TOKEN` that has + # write permissions, even if the PR is from a fork. + # Therefore, for security reasons, we do not checkout any code in this workflow. + pull_request: + branches: + - 'master' + - 'release-*' + +# These are the permissions for the `GITHUB_TOKEN` token. +# We should only give the token the minimum necessary set of permissions. +permissions: + statuses: write + actions: none + checks: none + contents: none + deployments: none + issues: none + discussions: none + packages: none + pull-requests: none + repository-projects: none + security-events: none + +jobs: + statuses: + name: statuses + runs-on: ubuntu-latest + if: github.repository == 'JuliaLang/julia' + strategy: + fail-fast: false + steps: + - run: echo "SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV + if: github.event_name == 'pull_request' + + - run: echo "SHA=${{ env.GITHUB_SHA }}" >> $GITHUB_ENV + if: github.event_name != 'pull_request' + + - run: echo "The SHA is ${{ env.SHA }}" + + # As we incrementally migrate individual jobs from Buildbot to Buildkite, we should + # remove them from the `context_list`. + - run: | + declare -a CONTEXT_LIST=( + "buildbot/package_freebsd64" + "buildbot/package_linux32" + "buildbot/package_linuxaarch64" + "buildbot/package_linuxarmv7l" + "buildbot/package_linuxppc64le" + "buildbot/package_macos64" + "buildbot/package_macosaarch64" + "buildbot/package_musl64" + "buildbot/package_win32" + "buildbot/package_win64" + "buildbot/tester_freebsd64" + "buildbot/tester_linux32" + "buildbot/tester_linux64" + "buildbot/tester_linuxaarch64" + "buildbot/tester_linuxarmv7l" + "buildbot/tester_linuxppc64le" + "buildbot/tester_macos64" + "buildbot/tester_macosaarch64" + "buildbot/tester_musl64" + "buildbot/tester_win32" + "buildbot/tester_win64" + ) + for CONTEXT in "${CONTEXT_LIST[@]}" + do + curl \ + -X POST \ + -H "Authorization: token $GITHUB_TOKEN" \ + -H "Accept: application/vnd.github.v3+json" \ + -d "{\"context\": \"$CONTEXT\", \"state\": \"$STATE\"}" \ + https://api.github.com/repos/JuliaLang/julia/statuses/${{ env.SHA }} + done + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + STATE: "pending" From 1346ba071086cf1970ff327ff2cdcabc7bbbfca7 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Fri, 6 Aug 2021 23:10:19 -0400 Subject: [PATCH 51/73] CI (Buildbot): for the "Create Pending Statuses" GitHub Actions workflow, use the `pull_request_target` event instead of the `pull_request` event (#41814) (cherry picked from commit d62e2060584cd8005fbd04661952dff0618a6920) --- .github/workflows/statuses.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/statuses.yml b/.github/workflows/statuses.yml index 27422b4e69345..1c29288d4fe79 100644 --- a/.github/workflows/statuses.yml +++ b/.github/workflows/statuses.yml @@ -21,7 +21,7 @@ on: # When using the `pull_request_target` event, all PRs will get a `GITHUB_TOKEN` that has # write permissions, even if the PR is from a fork. # Therefore, for security reasons, we do not checkout any code in this workflow. - pull_request: + pull_request_target: branches: - 'master' - 'release-*' From 81941f2c97b0aaf90c3e1b8addb414846cf49515 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Fri, 6 Aug 2021 23:41:42 -0400 Subject: [PATCH 52/73] CI (Buildbot): Fix a bug in the "Create Pending Statuses" GitHub Actions workflow (#41815) (cherry picked from commit ed866b5f0f36a245317d257d062bd000226bd18c) --- .github/workflows/statuses.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/statuses.yml b/.github/workflows/statuses.yml index 1c29288d4fe79..fa448e9d9fbf0 100644 --- a/.github/workflows/statuses.yml +++ b/.github/workflows/statuses.yml @@ -50,10 +50,10 @@ jobs: fail-fast: false steps: - run: echo "SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV - if: github.event_name == 'pull_request' + if: github.event_name == 'pull_request_target' - run: echo "SHA=${{ env.GITHUB_SHA }}" >> $GITHUB_ENV - if: github.event_name != 'pull_request' + if: github.event_name != 'pull_request_target' - run: echo "The SHA is ${{ env.SHA }}" From cab69667b8f0f9cd44f680a53a3f6b403819903f Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Sat, 7 Aug 2021 00:39:27 -0400 Subject: [PATCH 53/73] CI (Buildbot): a simpler way of getting the SHA for the "Statuses" action (#41817) (cherry picked from commit 0e8bb9596e29703e65d11214284e2fcac1b19bda) --- .github/workflows/statuses.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/statuses.yml b/.github/workflows/statuses.yml index fa448e9d9fbf0..26ec245d1ad18 100644 --- a/.github/workflows/statuses.yml +++ b/.github/workflows/statuses.yml @@ -52,7 +52,7 @@ jobs: - run: echo "SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV if: github.event_name == 'pull_request_target' - - run: echo "SHA=${{ env.GITHUB_SHA }}" >> $GITHUB_ENV + - run: echo "SHA=${{ github.sha }}" >> $GITHUB_ENV if: github.event_name != 'pull_request_target' - run: echo "The SHA is ${{ env.SHA }}" From b05e7fbdaaaee01eff474e8bf3d7e0c7a1f8bbb7 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Sat, 7 Aug 2021 20:55:55 -0400 Subject: [PATCH 54/73] CI (Buildbot, GHA): in the "Statuses" workflow, we only need to create pending (yellow) statuses for the `tester_` jobs (#41822) (cherry picked from commit 08fae511b9a616f40179f85f0d429b88fc5c815e) --- .github/workflows/statuses.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/workflows/statuses.yml b/.github/workflows/statuses.yml index 26ec245d1ad18..ca358e6f79487 100644 --- a/.github/workflows/statuses.yml +++ b/.github/workflows/statuses.yml @@ -61,16 +61,6 @@ jobs: # remove them from the `context_list`. - run: | declare -a CONTEXT_LIST=( - "buildbot/package_freebsd64" - "buildbot/package_linux32" - "buildbot/package_linuxaarch64" - "buildbot/package_linuxarmv7l" - "buildbot/package_linuxppc64le" - "buildbot/package_macos64" - "buildbot/package_macosaarch64" - "buildbot/package_musl64" - "buildbot/package_win32" - "buildbot/package_win64" "buildbot/tester_freebsd64" "buildbot/tester_linux32" "buildbot/tester_linux64" From ec3d314c23050bc400a8f5d1d9b76e8ef7c410fd Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Sat, 7 Aug 2021 23:02:29 -0400 Subject: [PATCH 55/73] CI (Buildbot, GHA): in the "Statuses" workflow, remove the `macosaarch64` and `musl64` statuses (#41824) (cherry picked from commit bdd745722716693dc141545f7a83e59d1f13769c) --- .github/workflows/statuses.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/statuses.yml b/.github/workflows/statuses.yml index ca358e6f79487..a5e8161dd3b99 100644 --- a/.github/workflows/statuses.yml +++ b/.github/workflows/statuses.yml @@ -68,8 +68,6 @@ jobs: "buildbot/tester_linuxarmv7l" "buildbot/tester_linuxppc64le" "buildbot/tester_macos64" - "buildbot/tester_macosaarch64" - "buildbot/tester_musl64" "buildbot/tester_win32" "buildbot/tester_win64" ) From 95a0461f8b3d87eedce174e9e09a121c9bb04e49 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Mon, 9 Aug 2021 00:35:47 -0400 Subject: [PATCH 56/73] CI (Buildbot, GHA): in the "Statuses" workflow, remove the `tester_linuxppc64le` status (#41831) (cherry picked from commit 0d1e548d0a1fe9e0fcbe8c36caac2e344fc9a5fe) --- .github/workflows/statuses.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/statuses.yml b/.github/workflows/statuses.yml index a5e8161dd3b99..431b9fdfd89bf 100644 --- a/.github/workflows/statuses.yml +++ b/.github/workflows/statuses.yml @@ -66,7 +66,6 @@ jobs: "buildbot/tester_linux64" "buildbot/tester_linuxaarch64" "buildbot/tester_linuxarmv7l" - "buildbot/tester_linuxppc64le" "buildbot/tester_macos64" "buildbot/tester_win32" "buildbot/tester_win64" From 718b19bf0091b15df79283d2ab887c79a3cbd0f0 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Mon, 9 Aug 2021 01:20:13 -0400 Subject: [PATCH 57/73] CI (Buildbot, GHA): In the "Statuses" workflow, remove the `tester_linuxarmv7l` status (#41832) (cherry picked from commit 9665d2a9248d7c7cb228a9ee0d9282041c337989) --- .github/workflows/statuses.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/statuses.yml b/.github/workflows/statuses.yml index 431b9fdfd89bf..0fcb8b37586e4 100644 --- a/.github/workflows/statuses.yml +++ b/.github/workflows/statuses.yml @@ -65,7 +65,6 @@ jobs: "buildbot/tester_linux32" "buildbot/tester_linux64" "buildbot/tester_linuxaarch64" - "buildbot/tester_linuxarmv7l" "buildbot/tester_macos64" "buildbot/tester_win32" "buildbot/tester_win64" From 038b342a2320337be28f7f3dd1a8cae71f684d40 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Mon, 9 Aug 2021 03:57:21 -0400 Subject: [PATCH 58/73] CI (Buildbot, GHA): add a note about token permissions (#41825) (cherry picked from commit 7005b7d68be7e6dddb6fba70de855df0abdf3c22) --- .github/workflows/statuses.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/statuses.yml b/.github/workflows/statuses.yml index 0fcb8b37586e4..df86caa3acee3 100644 --- a/.github/workflows/statuses.yml +++ b/.github/workflows/statuses.yml @@ -10,6 +10,7 @@ # Here are some steps that we take in this workflow file for security reasons: # 1. We do not checkout any code. # 2. We do not run any external actions. +# 3. We only give `GITHUB_TOKEN` the minimum necessary set of permissions. name: Statuses From 3f77bb4814ba33a2c8b325f0586fd0940ac4bdcc Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Tue, 10 Aug 2021 13:33:04 -0400 Subject: [PATCH 59/73] CI (Buildbot, GHA): Simplify the `permissions` key in the workflow file for the "Statuses" workflow (#41851) (cherry picked from commit 34dc0449ce5d6ffa1a936b38a64e3d1590ffff80) --- .github/workflows/statuses.yml | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/.github/workflows/statuses.yml b/.github/workflows/statuses.yml index df86caa3acee3..97ec290abe013 100644 --- a/.github/workflows/statuses.yml +++ b/.github/workflows/statuses.yml @@ -30,17 +30,7 @@ on: # These are the permissions for the `GITHUB_TOKEN` token. # We should only give the token the minimum necessary set of permissions. permissions: - statuses: write - actions: none - checks: none - contents: none - deployments: none - issues: none - discussions: none - packages: none - pull-requests: none - repository-projects: none - security-events: none + statuses: write jobs: statuses: From c16ac527ccb1e023d4d835beb4591ee97eef8747 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Thu, 26 Aug 2021 01:40:22 -0400 Subject: [PATCH 60/73] Fix #41975 - Dropped typecheck in GotoIfNot (#42010) Recall the reproducer from the issue: ``` julia> f() = (if nothing; end; unsafe_load(Ptr{Int}(0))) f (generic function with 1 method) julia> f() Unreachable reached at 0x7fb33bb50090 signal (4): Illegal instruction in expression starting at REPL[13]:1 unsafe_load at ./pointer.jl:105 [inlined] unsafe_load at ./pointer.jl:105 [inlined] ``` There were actually two places where we were dropping the GotoIfNot, one in type annotation after inference, one in SSA conversion. The one in SSA conversion was structural: When both branches target the same jump destination, the GotoIfNot would be dropped. This was fine in general, except that as shown above, GotoIfNot can actually itself have a side effect, namely throwing a type error if the condition is not a boolean. Thus in order to actually drop the node we need to prove that the error check does not fire. The reason we want to drop the GotoIfNot node here is that IRCode has an invariant that every basic block is in the predecessor list only once (otherwise PhiNodes would have to carry extra state regarding which branch they refer to). To fix this, insert an `Expr(:call, typecheck, _, Bool)` when dropping the GotoIfNot. We do lose the ability to distinguish the GotoIfNot from literal typechecks as a result, but at the moment they generate identical errors. If we ever wanted to dinstinguish them, we could create another typecheck intrinsic that throws a different error or use an approach like #41994. (cherry picked from commit 24450009edcd819238cc633ec3270be768716a13) --- base/compiler/ssair/slot2ssa.jl | 2 +- base/compiler/typeinfer.jl | 2 +- test/compiler/ssair.jl | 4 ++++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/base/compiler/ssair/slot2ssa.jl b/base/compiler/ssair/slot2ssa.jl index 601019113a15e..b278262cd3884 100644 --- a/base/compiler/ssair/slot2ssa.jl +++ b/base/compiler/ssair/slot2ssa.jl @@ -817,7 +817,7 @@ function construct_ssa!(ci::CodeInfo, ir::IRCode, domtree::DomTree, defuse, narg new_dest = block_for_inst(cfg, stmt.dest) if new_dest == bb+1 # Drop this node - it's a noop - new_code[idx] = stmt.cond + new_code[idx] = Expr(:call, GlobalRef(Core, :typeassert), stmt.cond, GlobalRef(Core, :Bool)) else new_code[idx] = GotoIfNot(stmt.cond, new_dest) end diff --git a/base/compiler/typeinfer.jl b/base/compiler/typeinfer.jl index 510eec52ed623..74555629f0995 100644 --- a/base/compiler/typeinfer.jl +++ b/base/compiler/typeinfer.jl @@ -626,7 +626,7 @@ function type_annotate!(sv::InferenceState, run_optimizer::Bool) expr = body[i] if isa(expr, GotoIfNot) if !isa(states[expr.dest], VarTable) - body[i] = expr.cond + body[i] = Expr(:call, GlobalRef(Core, :typeassert), expr.cond, GlobalRef(Core, :Bool)) end end end diff --git a/test/compiler/ssair.jl b/test/compiler/ssair.jl index f90bb71e291d0..17a0753eddc64 100644 --- a/test/compiler/ssair.jl +++ b/test/compiler/ssair.jl @@ -310,3 +310,7 @@ let cfg = CFG(BasicBlock[ Compiler.domtree_insert_edge!(domtree, cfg.blocks, 1, 3) @test domtree.idoms_bb == Compiler.naive_idoms(cfg.blocks) == [0, 1, 1, 3, 1, 4] end + +# Issue #41975 - SSA conversion drops type check +f_if_typecheck() = (if nothing; end; unsafe_load(Ptr{Int}(0))) +@test_throws TypeError f_if_typecheck() From a89e7c346d64692e0b2ddcea8cc1d6fb7a53d6e1 Mon Sep 17 00:00:00 2001 From: Troels Nielsen Date: Thu, 26 Aug 2021 17:44:27 +0200 Subject: [PATCH 61/73] codegen: parameter attributes on CFunction closures sticks (#41827) When CFunction closures are created an extra argument is added to the function signature for holding the closure. Make sure that the parameter attributes on already existing parameters are not shifted when adding that parameter. (cherry picked from commit 08f342230e056c118dedd9889c558ebcbbb26603) --- src/codegen.cpp | 41 ++++++++++++++++++++++++++++++++++++++++- test/ccall.jl | 20 ++++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 60a5fb98da25f..59f5d9e8b7d84 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -4826,9 +4826,48 @@ static Function* gen_cfun_wrapper( // add nest parameter (pointer to jl_value_t* data array) after sret arg assert(closure_types); std::vector fargt_sig(sig.fargt_sig); + fargt_sig.insert(fargt_sig.begin() + sig.sret, T_pprjlvalue); + + // Shift LLVM attributes for parameters one to the right, as + // we are adding the extra nest parameter after sret arg. + std::vector> newAttributes; + newAttributes.reserve(attributes.getNumAttrSets() + 1); + auto it = attributes.index_begin(); + + // Skip past FunctionIndex + if (it == AttributeList::AttrIndex::FunctionIndex) { + ++it; + } + + // Move past ReturnValue and parameter return value + for (;it < AttributeList::AttrIndex::FirstArgIndex + sig.sret; ++it) { + if (attributes.hasAttributes(it)) { + newAttributes.emplace_back(it, attributes.getAttributes(it)); + } + } + + // Add the new nest attribute + AttrBuilder attrBuilder; + attrBuilder.addAttribute(Attribute::Nest); + newAttributes.emplace_back(it, AttributeSet::get(jl_LLVMContext, attrBuilder)); + + // Shift forward the rest of the attributes + for(;it < attributes.index_end(); ++it) { + if (attributes.hasAttributes(it)) { + newAttributes.emplace_back(it + 1, attributes.getAttributes(it)); + } + } + + // Remember to add back FunctionIndex + if (attributes.hasAttributes(AttributeList::AttrIndex::FunctionIndex)) { + newAttributes.emplace_back(AttributeList::AttrIndex::FunctionIndex, + attributes.getAttributes(AttributeList::AttrIndex::FunctionIndex)); + } + + // Create the new AttributeList + attributes = AttributeList::get(jl_LLVMContext, newAttributes); functype = FunctionType::get(sig.sret ? T_void : sig.prt, fargt_sig, /*isVa*/false); - attributes = attributes.addAttribute(jl_LLVMContext, 1 + sig.sret, Attribute::Nest); } else { functype = sig.functype(); diff --git a/test/ccall.jl b/test/ccall.jl index 5f5272c372a11..1da7e2a145ee2 100644 --- a/test/ccall.jl +++ b/test/ccall.jl @@ -979,6 +979,26 @@ for (t, v) in ((Complex{Int32}, :ci32), (Complex{Int64}, :ci64), end end + +#issue 40164 +@testset "llvm parameter attributes on cfunction closures" begin + struct Struct40164 + x::Cdouble + y::Cdouble + z::Cdouble + end + + function test_40164() + ret = Struct40164[] + f = x::Struct40164 -> (push!(ret, x); nothing) + f_c = @cfunction($f, Cvoid, (Struct40164,)) + ccall(f_c.ptr, Ptr{Cvoid}, (Struct40164,), Struct40164(0, 1, 2)) + ret + end + + @test test_40164() == [Struct40164(0, 1, 2)] +end + else @test_broken "cfunction: no support for closures on this platform" From 1dfff1e91d1e359389b50ed64e8c2556ebe7ea45 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Wed, 7 Jul 2021 16:54:52 -0400 Subject: [PATCH 62/73] fix #40773, bug in `summarysize` on arrays of inlined structs with pointers (#41492) (cherry picked from commit c49153dce6c29e9e04cd1b7e4c71ac89c4a64737) --- base/summarysize.jl | 5 +++-- test/misc.jl | 5 +++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/base/summarysize.jl b/base/summarysize.jl index 916214a69f88f..62458dcdc5b97 100644 --- a/base/summarysize.jl +++ b/base/summarysize.jl @@ -120,12 +120,13 @@ function (ss::SummarySize)(obj::Array) if !haskey(ss.seen, datakey) ss.seen[datakey] = true dsize = Core.sizeof(obj) - if isbitsunion(eltype(obj)) + T = eltype(obj) + if isbitsunion(T) # add 1 union selector byte for each element dsize += length(obj) end size += dsize - if !isempty(obj) && !Base.allocatedinline(eltype(obj)) + if !isempty(obj) && (!Base.allocatedinline(T) || (T isa DataType && !Base.datatype_pointerfree(T))) push!(ss.frontier_x, obj) push!(ss.frontier_i, 1) end diff --git a/test/misc.jl b/test/misc.jl index 30369830cf04b..bc518963b38f6 100644 --- a/test/misc.jl +++ b/test/misc.jl @@ -291,6 +291,11 @@ let vec = vcat(missing, ones(100000)) @test length(unique(summarysize(vec) for i = 1:20)) == 1 end +# issue #40773 +let s = Set(1:100) + @test summarysize([s]) > summarysize(s) +end + # issue #13021 let ex = try Main.x13021 = 0 From 2f74e1ef8119bc37f5b77dca84c1769347c3fb3b Mon Sep 17 00:00:00 2001 From: Martin Holters Date: Mon, 30 Aug 2021 22:44:44 +0200 Subject: [PATCH 63/73] Fix a precision issue in `abstract_iteration` (#41839) If the first loop exits in the first iteration, the `statetype` is still `Bottom`. In that case, the new `stateordonet` needs to be determined with the two-arg version of `iterate` again. Explicitly test that inference produces a sound (and reasonably precise) result when splatting an iterator (in this case a long range) that allows constant-propagation up to the `MAX_TUPLE_SPLAT` limit. Fixes #41022 Co-authored-by: Jameson Nash (cherry picked from commit 92337b560a3a5b71181ed420bea1890d196517a6) --- base/compiler/abstractinterpretation.jl | 35 ++++++++++++++++++------- test/compiler/inference.jl | 17 +++++++++++- 2 files changed, 41 insertions(+), 11 deletions(-) diff --git a/base/compiler/abstractinterpretation.jl b/base/compiler/abstractinterpretation.jl index 368dc602b98a7..c976c17e22767 100644 --- a/base/compiler/abstractinterpretation.jl +++ b/base/compiler/abstractinterpretation.jl @@ -615,9 +615,11 @@ function abstract_iteration(interp::AbstractInterpreter, @nospecialize(itft), @n return ret, AbstractIterationInfo(calls) end if Nothing <: stateordonet_widened || length(ret) >= InferenceParams(interp).MAX_TUPLE_SPLAT + stateordonet = stateordonet_widened break end if !isa(stateordonet_widened, DataType) || !(stateordonet_widened <: Tuple) || isvatuple(stateordonet_widened) || length(stateordonet_widened.parameters) != 2 + stateordonet = stateordonet_widened break end nstatetype = getfield_tfunc(stateordonet, Const(2)) @@ -635,27 +637,40 @@ function abstract_iteration(interp::AbstractInterpreter, @nospecialize(itft), @n end # From here on, we start asking for results on the widened types, rather than # the precise (potentially const) state type - statetype = widenconst(statetype) - valtype = widenconst(valtype) + # statetype and valtype are reinitialized in the first iteration below from the + # (widened) stateordonet, which has not yet been fully analyzed in the loop above + statetype = Bottom + valtype = Bottom + may_have_terminated = Nothing <: stateordonet while valtype !== Any - stateordonet = abstract_call_known(interp, iteratef, nothing, Any[Const(iteratef), itertype, statetype], sv).rt - stateordonet = widenconst(stateordonet) - nounion = typesubtract(stateordonet, Nothing, 0) - if !isa(nounion, DataType) || !(nounion <: Tuple) || isvatuple(nounion) || length(nounion.parameters) != 2 + nounion = typeintersect(stateordonet, Tuple{Any,Any}) + if nounion !== Union{} && !isa(nounion, DataType) + # nounion is of a type we cannot handle valtype = Any break end - if nounion.parameters[1] <: valtype && nounion.parameters[2] <: statetype + if nounion === Union{} || (nounion.parameters[1] <: valtype && nounion.parameters[2] <: statetype) + # reached a fixpoint or iterator failed/gave invalid answer if typeintersect(stateordonet, Nothing) === Union{} - # Reached a fixpoint, but Nothing is not possible => iterator is infinite or failing - return Any[Bottom], nothing + # ... but cannot terminate + if !may_have_terminated + # ... and cannot have terminated prior to this loop + return Any[Bottom], nothing + else + # iterator may have terminated prior to this loop, but not during it + valtype = Bottom + end end break end valtype = tmerge(valtype, nounion.parameters[1]) statetype = tmerge(statetype, nounion.parameters[2]) + stateordonet = abstract_call_known(interp, iteratef, nothing, Any[Const(iteratef), itertype, statetype], sv).rt + stateordonet = widenconst(stateordonet) + end + if valtype !== Union{} + push!(ret, Vararg{valtype}) end - push!(ret, Vararg{valtype}) return ret, nothing end diff --git a/test/compiler/inference.jl b/test/compiler/inference.jl index 1cbf36810f4d5..1e3b3cf5e20e3 100644 --- a/test/compiler/inference.jl +++ b/test/compiler/inference.jl @@ -2740,9 +2740,24 @@ partial_return_2(x) = Val{partial_return_1(x)[2]} @test Base.return_types(partial_return_2, (Int,)) == Any[Type{Val{1}}] -# Precision of abstract_iteration +# Soundness and precision of abstract_iteration +f41839() = (1:100...,) +@test NTuple{100,Int} <: only(Base.return_types(f41839, ())) <: Tuple{Vararg{Int}} f_splat(x) = (x...,) @test Base.return_types(f_splat, (Pair{Int,Int},)) == Any[Tuple{Int, Int}] +@test Base.return_types(f_splat, (UnitRange{Int},)) == Any[Tuple{Vararg{Int}}] +struct Itr41839_1 end # empty or infinite +Base.iterate(::Itr41839_1) = rand(Bool) ? (nothing, nothing) : nothing +Base.iterate(::Itr41839_1, ::Nothing) = (nothing, nothing) +@test Base.return_types(f_splat, (Itr41839_1,)) == Any[Tuple{}] +struct Itr41839_2 end # empty or failing +Base.iterate(::Itr41839_2) = rand(Bool) ? (nothing, nothing) : nothing +Base.iterate(::Itr41839_2, ::Nothing) = error() +@test Base.return_types(f_splat, (Itr41839_2,)) == Any[Tuple{}] +struct Itr41839_3 end +Base.iterate(::Itr41839_3 ) = rand(Bool) ? nothing : (nothing, 1) +Base.iterate(::Itr41839_3 , i) = i < 16 ? (i, i + 1) : nothing +@test only(Base.return_types(f_splat, (Itr41839_3,))) <: Tuple{Vararg{Union{Nothing, Int}}} # issue #32699 f32699(a) = (id = a[1],).id From 75597036cb6078afba8da782a061caa671d0a74e Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 1 Sep 2021 13:53:16 -0400 Subject: [PATCH 64/73] [AllocOpt] fix iterator invalidation (#42059) We might previously accidentally visit this use after deletion, if the orig_inst ended up back in the workqueue. Fixes #41916 (cherry picked from commit d8a8db23a5208480fe7cd56b413c9743ea58db83) --- src/llvm-alloc-opt.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/llvm-alloc-opt.cpp b/src/llvm-alloc-opt.cpp index 3bb11988b730e..f4a60cf739fc3 100644 --- a/src/llvm-alloc-opt.cpp +++ b/src/llvm-alloc-opt.cpp @@ -1156,6 +1156,7 @@ void Optimizer::optimizeTag(CallInst *orig_inst) { auto tag = orig_inst->getArgOperand(2); // `julia.typeof` is only legal on the original pointer, no need to scan recursively + size_t last_deleted = removed.size(); for (auto user: orig_inst->users()) { if (auto call = dyn_cast(user)) { auto callee = call->getCalledOperand(); @@ -1168,6 +1169,8 @@ void Optimizer::optimizeTag(CallInst *orig_inst) } } } + while (last_deleted < removed.size()) + removed[last_deleted++]->replaceUsesOfWith(orig_inst, UndefValue::get(orig_inst->getType())); } void Optimizer::splitOnStack(CallInst *orig_inst) From 7960f599a99b30bcb6a9557f373e1d2e470aecdf Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 1 Sep 2021 13:53:28 -0400 Subject: [PATCH 65/73] GMP: fix warning in init (#42062) (cherry picked from commit 4598966f8b3fabac1865a8fd1b5f69b4ac1e246a) --- base/gmp.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/base/gmp.jl b/base/gmp.jl index e0c11e5d34186..151d3efbb8cfb 100644 --- a/base/gmp.jl +++ b/base/gmp.jl @@ -94,10 +94,10 @@ const ALLOC_OVERFLOW_FUNCTION = Ref(false) function __init__() try if version().major != VERSION.major || bits_per_limb() != BITS_PER_LIMB - msg = bits_per_limb() != BITS_PER_LIMB ? error : warn - msg("The dynamically loaded GMP library (v\"$(version())\" with __gmp_bits_per_limb == $(bits_per_limb()))\n", - "does not correspond to the compile time version (v\"$VERSION\" with __gmp_bits_per_limb == $BITS_PER_LIMB).\n", - "Please rebuild Julia.") + msg = """The dynamically loaded GMP library (v\"$(version())\" with __gmp_bits_per_limb == $(bits_per_limb())) + does not correspond to the compile time version (v\"$VERSION\" with __gmp_bits_per_limb == $BITS_PER_LIMB). + Please rebuild Julia.""" + bits_per_limb() != BITS_PER_LIMB ? @error(msg) : @warn(msg) end ccall((:__gmp_set_memory_functions, :libgmp), Cvoid, From fc1fcefd8e0f26c8f74c63bf793fce95a669a020 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 3 Sep 2021 05:18:26 -0400 Subject: [PATCH 66/73] InteractiveUtils: recursive correctly in varinfo, et al. (#42061) * InteractiveUtils: recursive correctly in varinfo, et al. Fixes #42045 (cherry picked from commit a163e374a4b8c66978ac6ff7652742f1adee1a51) --- .../InteractiveUtils/src/InteractiveUtils.jl | 115 ++++++++---------- 1 file changed, 48 insertions(+), 67 deletions(-) diff --git a/stdlib/InteractiveUtils/src/InteractiveUtils.jl b/stdlib/InteractiveUtils/src/InteractiveUtils.jl index b50c3240c04ab..0130505bb13da 100644 --- a/stdlib/InteractiveUtils/src/InteractiveUtils.jl +++ b/stdlib/InteractiveUtils/src/InteractiveUtils.jl @@ -34,40 +34,40 @@ The memory consumption estimate is an approximate lower bound on the size of the - `sortby` : the column to sort results by. Options are `:name` (default), `:size`, and `:summary`. """ function varinfo(m::Module=Main, pattern::Regex=r""; all::Bool = false, imported::Bool = false, sortby::Symbol = :name, recursive::Bool = false) - @assert sortby in [:name, :size, :summary] "Unrecognized `sortby` value `:$sortby`. Possible options are `:name`, `:size`, and `:summary`" - function _populate_rows(m2::Module, allrows, include_self::Bool, prep::String) - newrows = Any[ - let - value = getfield(m2, v) - ssize_str, ssize = if value===Base || value===Main || value===Core + sortby in (:name, :size, :summary) || throw(ArgumentError("Unrecognized `sortby` value `:$sortby`. Possible options are `:name`, `:size`, and `:summary`")) + rows = Vector{Any}[] + workqueue = [(m, ""),] + while !isempty(workqueue) + m2, prep = popfirst!(workqueue) + for v in names(m2; all, imported) + if !isdefined(m2, v) || !occursin(pattern, string(v)) + continue + end + value = getfield(m2, v) + isbuiltin = value === Base || value === Main || value === Core + if recursive && !isbuiltin && isa(value, Module) && value !== m2 && nameof(value) === v && parentmodule(value) === m2 + push!(workqueue, (value, "$prep$v.")) + end + ssize_str, ssize = if isbuiltin ("", typemax(Int)) else ss = summarysize(value) (format_bytes(ss), ss) end - Any[string(prep, v), ssize_str, summary(value), ssize] - end - for v in names(m2; all, imported) - if (string(v) != split(string(m2), ".")[end] || include_self) && isdefined(m2, v) && occursin(pattern, string(v)) ] - append!(allrows, newrows) - if recursive - for row in newrows - if row[3] == "Module" && !in(split(row[1], ".")[end], [split(string(m2), ".")[end], "Base", "Main", "Core"]) - _populate_rows(getfield(m2, Symbol(split(row[1], ".")[end])), allrows, false, prep * "$(row[1]).") - end - end + push!(rows, Any[string(prep, v), ssize_str, summary(value), ssize]) end - return allrows end - rows = _populate_rows(m, Vector{Any}[], true, "") - if sortby == :name - col, reverse = 1, false - elseif sortby == :size - col, reverse = 4, true - elseif sortby == :summary - col, reverse = 3, false + let (col, rev) = if sortby == :name + 1, false + elseif sortby == :size + 4, true + elseif sortby == :summary + 3, false + else + @assert "unreachable" + end + sort!(rows; by=r->r[col], rev) end - rows = sort!(rows, by=r->r[col], rev=reverse) pushfirst!(rows, Any["name", "size", "summary"]) return Markdown.MD(Any[Markdown.Table(map(r->r[1:3], rows), Symbol[:l, :r, :l])]) @@ -206,54 +206,35 @@ function methodswith(t::Type; supertypes::Bool=false) end # subtypes -function _subtypes(m::Module, x::Type, sts=Base.IdSet{Any}(), visited=Base.IdSet{Module}()) - push!(visited, m) +function _subtypes_in!(mods::Array, x::Type) xt = unwrap_unionall(x) - if !isa(xt, DataType) - return sts + if !isabstracttype(x) || !isa(xt, DataType) + # Fast path + return Type[] end - xt = xt::DataType - for s in names(m, all = true) - if isdefined(m, s) && !isdeprecated(m, s) - t = getfield(m, s) - if isa(t, DataType) - t = t::DataType - if t.name.name === s && supertype(t).name == xt.name - ti = typeintersect(t, x) - ti != Bottom && push!(sts, ti) - end - elseif isa(t, UnionAll) - t = t::UnionAll - tt = unwrap_unionall(t) - isa(tt, DataType) || continue - tt = tt::DataType - if tt.name.name === s && supertype(tt).name == xt.name - ti = typeintersect(t, x) - ti != Bottom && push!(sts, ti) + sts = Vector{Any}() + while !isempty(mods) + m = pop!(mods) + xt = xt::DataType + for s in names(m, all = true) + if isdefined(m, s) && !isdeprecated(m, s) + t = getfield(m, s) + dt = isa(t, UnionAll) ? unwrap_unionall(t) : t + if isa(dt, DataType) + if dt.name.name === s && dt.name.module == m && supertype(dt).name == xt.name + ti = typeintersect(t, x) + ti != Bottom && push!(sts, ti) + end + elseif isa(t, Module) && nameof(t) === s && parentmodule(t) === m && t !== m + t === Base || push!(mods, t) # exclude Base, since it also parented by Main end - elseif isa(t, Module) - t = t::Module - in(t, visited) || _subtypes(t, x, sts, visited) end end end - return sts -end - -function _subtypes_in(mods::Array, x::Type) - if !isabstracttype(x) - # Fast path - return Type[] - end - sts = Base.IdSet{Any}() - visited = Base.IdSet{Module}() - for m in mods - _subtypes(m, x, sts, visited) - end - return sort!(collect(sts), by=string) + return permute!(sts, sortperm(map(string, sts))) end -subtypes(m::Module, x::Type) = _subtypes_in([m], x) +subtypes(m::Module, x::Type) = _subtypes_in!([m], x) """ subtypes(T::DataType) @@ -270,7 +251,7 @@ julia> subtypes(Integer) Unsigned ``` """ -subtypes(x::Type) = _subtypes_in(Base.loaded_modules_array(), x) +subtypes(x::Type) = _subtypes_in!(Base.loaded_modules_array(), x) """ supertypes(T::Type) From 51a0b8021d9fb9d4bd36f6f3ff2a706305244a8c Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 26 Aug 2021 11:42:14 -0400 Subject: [PATCH 67/73] fix ptrhash_remove (#42009) Same bug as 5e57c214f872083ccacafa0f753e794ec654a21a (#26833), same fix. (cherry picked from commit 82c4a2739f97c942508b29e7d946a0504f45fb23) --- src/Makefile | 4 +- src/support/htable.inc | 112 ++++++++++++++++++++++------------------- 2 files changed, 63 insertions(+), 53 deletions(-) diff --git a/src/Makefile b/src/Makefile index 0de23588bcfab..503366a86fabd 100644 --- a/src/Makefile +++ b/src/Makefile @@ -257,10 +257,10 @@ $(addprefix $(BUILDDIR)/,threading.o threading.dbg.obj gc.o gc.dbg.obj init.c in $(addprefix $(BUILDDIR)/,APInt-C.o APInt-C.dbg.obj runtime_intrinsics.o runtime_intrinsics.dbg.obj): $(SRCDIR)/APInt-C.h # archive library file rules -$(BUILDDIR)/support/libsupport.a: $(addprefix $(SRCDIR)/support/,*.h *.c *.S) $(SRCDIR)/support/*.c +$(BUILDDIR)/support/libsupport.a: $(addprefix $(SRCDIR)/support/,*.h *.c *.S *.inc) $(SRCDIR)/support/*.c $(MAKE) -C $(SRCDIR)/support BUILDDIR='$(abspath $(BUILDDIR)/support)' -$(BUILDDIR)/support/libsupport-debug.a: $(addprefix $(SRCDIR)/support/,*.h *.c *.S) $(SRCDIR)/support/*.c +$(BUILDDIR)/support/libsupport-debug.a: $(addprefix $(SRCDIR)/support/,*.h *.c *.S *.inc) $(SRCDIR)/support/*.c $(MAKE) -C $(SRCDIR)/support debug BUILDDIR='$(abspath $(BUILDDIR)/support)' $(FLISP_EXECUTABLE_release): $(BUILDDIR)/flisp/libflisp.a diff --git a/src/support/htable.inc b/src/support/htable.inc index fa59624a4998f..7a9be2514e2f0 100644 --- a/src/support/htable.inc +++ b/src/support/htable.inc @@ -13,67 +13,77 @@ static void **HTNAME##_lookup_bp_r(htable_t *h, void *key, void *ctx) \ { \ uint_t hv; \ - size_t i, orig, index, iter; \ + size_t i, orig, index, iter, empty_slot; \ size_t newsz, sz = hash_size(h); \ size_t maxprobe = max_probe(sz); \ void **tab = h->table; \ void **ol; \ \ hv = HFUNC((uintptr_t)key, ctx); \ - retry_bp: \ - iter = 0; \ - index = (size_t)(hv & (sz-1)) * 2; \ - sz *= 2; \ - orig = index; \ - \ - do { \ - if (tab[index+1] == HT_NOTFOUND) { \ - tab[index] = key; \ - return &tab[index+1]; \ + while (1) { \ + iter = 0; \ + index = (size_t)(hv & (sz-1)) * 2; \ + sz *= 2; \ + orig = index; \ + empty_slot = -1; \ + \ + do { \ + if (tab[index] == HT_NOTFOUND) { \ + if (empty_slot == -1) \ + empty_slot = index; \ + break; \ + } \ + if (tab[index+1] == HT_NOTFOUND) { \ + if (empty_slot == -1) \ + empty_slot = index; \ + } \ + \ + if (EQFUNC(key, tab[index], ctx)) \ + return &tab[index+1]; \ + \ + index = (index+2) & (sz-1); \ + iter++; \ + if (iter > maxprobe) \ + break; \ + } while (index != orig); \ + \ + if (empty_slot != -1) { \ + tab[empty_slot] = key; \ + return &tab[empty_slot+1]; \ } \ \ - if (EQFUNC(key, tab[index], ctx)) \ - return &tab[index+1]; \ - \ - index = (index+2) & (sz-1); \ - iter++; \ - if (iter > maxprobe) \ - break; \ - } while (index != orig); \ - \ - /* table full */ \ - /* quadruple size, rehash, retry the insert */ \ - /* it's important to grow the table really fast; otherwise we waste */ \ - /* lots of time rehashing all the keys over and over. */ \ - sz = h->size; \ - ol = h->table; \ - if (sz < HT_N_INLINE) \ - newsz = HT_N_INLINE; \ - else if (sz >= (1<<19) || (sz <= (1<<8))) \ - newsz = sz<<1; \ - else \ - newsz = sz<<2; \ - /*printf("trying to allocate %d words.\n", newsz); fflush(stdout);*/ \ - tab = (void**)LLT_ALLOC(newsz*sizeof(void*)); \ - if (tab == NULL) \ - return NULL; \ - for(i=0; i < newsz; i++) \ - tab[i] = HT_NOTFOUND; \ - h->table = tab; \ - h->size = newsz; \ - for(i=0; i < sz; i+=2) { \ - if (ol[i+1] != HT_NOTFOUND) { \ - (*HTNAME##_lookup_bp_r(h, ol[i], ctx)) = ol[i+1]; \ + /* table full */ \ + /* quadruple size, rehash, retry the insert */ \ + /* it's important to grow the table really fast; otherwise we waste */ \ + /* lots of time rehashing all the keys over and over. */ \ + sz = h->size; \ + ol = h->table; \ + if (sz < HT_N_INLINE) \ + newsz = HT_N_INLINE; \ + else if (sz >= (1<<19) || (sz <= (1<<8))) \ + newsz = sz<<1; \ + else \ + newsz = sz<<2; \ + /*printf("trying to allocate %d words.\n", newsz); fflush(stdout);*/ \ + tab = (void**)LLT_ALLOC(newsz*sizeof(void*)); \ + if (tab == NULL) \ + return NULL; \ + for (i = 0; i < newsz; i++) \ + tab[i] = HT_NOTFOUND; \ + h->table = tab; \ + h->size = newsz; \ + for (i = 0; i < sz; i += 2) { \ + if (ol[i+1] != HT_NOTFOUND) { \ + (*HTNAME##_lookup_bp_r(h, ol[i], ctx)) = ol[i+1]; \ + } \ } \ - } \ - if (ol != &h->_space[0]) \ - LLT_FREE(ol); \ + if (ol != &h->_space[0]) \ + LLT_FREE(ol); \ \ - sz = hash_size(h); \ - maxprobe = max_probe(sz); \ - tab = h->table; \ - \ - goto retry_bp; \ + sz = hash_size(h); \ + maxprobe = max_probe(sz); \ + tab = h->table; \ + } \ \ return NULL; \ } \ From 7d1e1c3737190eecfa2c950d6526efeeeb5998be Mon Sep 17 00:00:00 2001 From: SamuraiAku <61489439+SamuraiAku@users.noreply.github.com> Date: Tue, 29 Jun 2021 15:12:33 -0700 Subject: [PATCH 68/73] Refactor LICENSE.md to make it scanner friendly (#41095) Create THIRDPARTY.md to hold license information for all code not covered by the main MIT license. This format allows for automated scanning and categorization of Julia's license. The licenses were broken up this way because this is the format that many automated license scanners (including GitHub). (cherry picked from commit 161e38496dff86873c595428e4da12620e528c57) --- LICENSE.md | 96 +++++++++++---------------------------------------- THIRDPARTY.md | 56 ++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 76 deletions(-) create mode 100644 THIRDPARTY.md diff --git a/LICENSE.md b/LICENSE.md index d1438a5f68bfc..79127224d049b 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,82 +1,26 @@ -The Julia language is licensed under the MIT License. The "language" consists -of the compiler (the contents of src/), most of the standard library (base/), -and some utilities (most of the rest of the files in this repository). See below -for exceptions. +MIT License -> Copyright (c) 2009-2019: Jeff Bezanson, Stefan Karpinski, Viral B. Shah, -> and other contributors: -> -> https://github.com/JuliaLang/julia/contributors -> -> Permission is hereby granted, free of charge, to any person obtaining -> a copy of this software and associated documentation files (the -> "Software"), to deal in the Software without restriction, including -> without limitation the rights to use, copy, modify, merge, publish, -> distribute, sublicense, and/or sell copies of the Software, and to -> permit persons to whom the Software is furnished to do so, subject to -> the following conditions: -> -> The above copyright notice and this permission notice shall be -> included in all copies or substantial portions of the Software. -> -> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -> EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -> MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -> NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -> LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -> OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -> WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +Copyright (c) 2009-2021: Jeff Bezanson, Stefan Karpinski, Viral B. Shah, and other contributors: https://github.com/JuliaLang/julia/contributors -Julia includes code from the following projects, which have their own licenses: +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: -- [crc32c.c](https://stackoverflow.com/questions/17645167/implementing-sse-4-2s-crc32c-in-software) (CRC-32c checksum code by Mark Adler) [[ZLib](https://opensource.org/licenses/Zlib)]. -- [LDC](https://github.com/ldc-developers/ldc/blob/master/LICENSE) (for ccall/cfunction ABI definitions) [BSD-3]. The portion of code that Julia uses from LDC is [BSD-3] licensed. -- [LLVM](https://releases.llvm.org/3.9.0/LICENSE.TXT) (for parts of src/jitlayers.cpp and src/disasm.cpp) [BSD-3, effectively] -- [MUSL](https://git.musl-libc.org/cgit/musl/tree/COPYRIGHT) (for getopt implementation on Windows) [MIT] -- [MINGW](https://sourceforge.net/p/mingw/mingw-org-wsl/ci/legacy/tree/mingwrt/mingwex/dirname.c) (for dirname implementation on Windows) [MIT] -- [NetBSD](https://www.netbsd.org/about/redistribution.html) (for setjmp, longjmp, and strptime implementations on Windows) [BSD-3] -- [Python](https://docs.python.org/3/license.html) (for strtod and joinpath implementation on Windows) [BSD-3, effectively] -- [Google Benchmark](https://github.com/google/benchmark) (for cyclecount implementation) [Apache 2.0] +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. -The following components included in Julia `Base` have their own separate licenses: +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -- base/ryu/* [Boost] (see [ryu](https://github.com/ulfjack/ryu/blob/master/LICENSE-Boost)) -- base/grisu/* [BSD-3] (see [double-conversion](https://github.com/google/double-conversion/blob/master/LICENSE)) -- base/special/{exp,rem_pio2,hyperbolic}.jl [Freely distributable with preserved copyright notice] (see [FDLIBM](https://www.netlib.org/fdlibm)) +end of terms and conditions -The Julia language links to the following external libraries, which have their -own licenses: - -- [FEMTOLISP](https://github.com/JeffBezanson/femtolisp) [BSD-3] -- [LIBUNWIND](https://git.savannah.gnu.org/gitweb/?p=libunwind.git;a=blob_plain;f=LICENSE;hb=master) [MIT] -- [LIBUV](https://github.com/joyent/libuv/blob/master/LICENSE) [MIT] -- [LLVM](https://releases.llvm.org/6.0.0/LICENSE.TXT) [BSD-3, effectively] -- [UTF8PROC](https://github.com/JuliaStrings/utf8proc) [MIT] - -Julia's `stdlib` uses the following external libraries, which have their own licenses: - -- [DSFMT](http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/LICENSE.txt) [BSD-3] -- [OPENLIBM](https://github.com/JuliaMath/openlibm/blob/master/LICENSE.md) [MIT, BSD-2, ISC] -- [GMP](https://gmplib.org/manual/Copying.html#Copying) [LGPL3+ or GPL2+] -- [LIBGIT2](https://github.com/libgit2/libgit2/blob/development/COPYING) [GPL2+ with unlimited linking exception] -- [CURL](https://curl.haxx.se/docs/copyright.html) [MIT/X derivative] -- [LIBSSH2](https://github.com/libssh2/libssh2/blob/master/COPYING) [BSD-3] -- [MBEDTLS](https://tls.mbed.org/how-to-get) [either GPLv2 or Apache 2.0] -- [MPFR](https://www.mpfr.org/mpfr-current/mpfr.html#Copying) [LGPL3+] -- [OPENBLAS](https://raw.github.com/xianyi/OpenBLAS/master/LICENSE) [BSD-3] -- [LAPACK](https://netlib.org/lapack/LICENSE.txt) [BSD-3] -- [PCRE](https://www.pcre.org/licence.txt) [BSD-3] -- [SUITESPARSE](http://suitesparse.com) [mix of LGPL2+ and GPL2+; see individual module licenses] - -Julia's build process uses the following external tools: - -- [PATCHELF](https://nixos.org/patchelf.html) -- [OBJCONV](https://www.agner.org/optimize/#objconv) - -Julia bundles the following external programs and libraries: - -- [7-Zip](https://www.7-zip.org/license.txt) -- [ZLIB](https://zlib.net/zlib_license.html) - -On some platforms, distributions of Julia contain SSL certificate authority certificates, -released under the [Mozilla Public License](https://en.wikipedia.org/wiki/Mozilla_Public_License). +Please see THIRDPARTY.md for license information for other software used in this project. diff --git a/THIRDPARTY.md b/THIRDPARTY.md new file mode 100644 index 0000000000000..87304437183d6 --- /dev/null +++ b/THIRDPARTY.md @@ -0,0 +1,56 @@ +The Julia language is licensed under the MIT License (see `LICENSE.md`). The "language" consists +of the compiler (the contents of src/), most of the standard library (base/), +and some utilities (most of the rest of the files in this repository). See below +for exceptions. + +- [crc32c.c](https://stackoverflow.com/questions/17645167/implementing-sse-4-2s-crc32c-in-software) (CRC-32c checksum code by Mark Adler) [[ZLib](https://opensource.org/licenses/Zlib)]. +- [LDC](https://github.com/ldc-developers/ldc/blob/master/LICENSE) (for ccall/cfunction ABI definitions) [BSD-3]. The portion of code that Julia uses from LDC is [BSD-3] licensed. +- [LLVM](https://releases.llvm.org/3.9.0/LICENSE.TXT) (for parts of src/jitlayers.cpp and src/disasm.cpp) [BSD-3, effectively] +- [MUSL](https://git.musl-libc.org/cgit/musl/tree/COPYRIGHT) (for getopt implementation on Windows) [MIT] +- [MINGW](https://sourceforge.net/p/mingw/mingw-org-wsl/ci/legacy/tree/mingwrt/mingwex/dirname.c) (for dirname implementation on Windows) [MIT] +- [NetBSD](https://www.netbsd.org/about/redistribution.html) (for setjmp, longjmp, and strptime implementations on Windows) [BSD-3] +- [Python](https://docs.python.org/3/license.html) (for strtod and joinpath implementation on Windows) [BSD-3, effectively] +- [Google Benchmark](https://github.com/google/benchmark) (for cyclecount implementation) [Apache 2.0] + +The following components included in Julia `Base` have their own separate licenses: + +- base/ryu/* [Boost] (see [ryu](https://github.com/ulfjack/ryu/blob/master/LICENSE-Boost)) +- base/grisu/* [BSD-3] (see [double-conversion](https://github.com/google/double-conversion/blob/master/LICENSE)) +- base/special/{exp,rem_pio2,hyperbolic}.jl [Freely distributable with preserved copyright notice] (see [FDLIBM](https://www.netlib.org/fdlibm)) + +The Julia language links to the following external libraries, which have their +own licenses: + +- [FEMTOLISP](https://github.com/JeffBezanson/femtolisp) [BSD-3] +- [LIBUNWIND](https://git.savannah.gnu.org/gitweb/?p=libunwind.git;a=blob_plain;f=LICENSE;hb=master) [MIT] +- [LIBUV](https://github.com/joyent/libuv/blob/master/LICENSE) [MIT] +- [LLVM](https://releases.llvm.org/6.0.0/LICENSE.TXT) [BSD-3, effectively] +- [UTF8PROC](https://github.com/JuliaStrings/utf8proc) [MIT] + +Julia's `stdlib` uses the following external libraries, which have their own licenses: + +- [DSFMT](http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/LICENSE.txt) [BSD-3] +- [OPENLIBM](https://github.com/JuliaMath/openlibm/blob/master/LICENSE.md) [MIT, BSD-2, ISC] +- [GMP](https://gmplib.org/manual/Copying.html#Copying) [LGPL3+ or GPL2+] +- [LIBGIT2](https://github.com/libgit2/libgit2/blob/development/COPYING) [GPL2+ with unlimited linking exception] +- [CURL](https://curl.haxx.se/docs/copyright.html) [MIT/X derivative] +- [LIBSSH2](https://github.com/libssh2/libssh2/blob/master/COPYING) [BSD-3] +- [MBEDTLS](https://tls.mbed.org/how-to-get) [either GPLv2 or Apache 2.0] +- [MPFR](https://www.mpfr.org/mpfr-current/mpfr.html#Copying) [LGPL3+] +- [OPENBLAS](https://raw.github.com/xianyi/OpenBLAS/master/LICENSE) [BSD-3] +- [LAPACK](https://netlib.org/lapack/LICENSE.txt) [BSD-3] +- [PCRE](https://www.pcre.org/licence.txt) [BSD-3] +- [SUITESPARSE](http://suitesparse.com) [mix of LGPL2+ and GPL2+; see individual module licenses] + +Julia's build process uses the following external tools: + +- [PATCHELF](https://nixos.org/patchelf.html) +- [OBJCONV](https://www.agner.org/optimize/#objconv) + +Julia bundles the following external programs and libraries: + +- [7-Zip](https://www.7-zip.org/license.txt) +- [ZLIB](https://zlib.net/zlib_license.html) + +On some platforms, distributions of Julia contain SSL certificate authority certificates, +released under the [Mozilla Public License](https://en.wikipedia.org/wiki/Mozilla_Public_License). From c92b559a7db2149e5ee612c5f5481ff36a62e73e Mon Sep 17 00:00:00 2001 From: Martin Holters Date: Wed, 1 Sep 2021 19:43:37 +0200 Subject: [PATCH 69/73] Avoid impossible unionall normalization (#42003) If the unionall bounds are inconsistent with the wrapper's bound, avoid throwing due to an impossible type instantiation. (cherry picked from commit b5b0684ec0a66c1bd50e75c25c724c20526bb702) --- src/jltypes.c | 11 +++++++++-- test/core.jl | 3 +++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/jltypes.c b/src/jltypes.c index 935ddf749b70f..26288f83edd96 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -1200,8 +1200,15 @@ jl_value_t *normalize_unionalls(jl_value_t *t) u = (jl_unionall_t*)t; } - if (u->var->lb == u->var->ub || may_substitute_ub(body, u->var)) - t = jl_instantiate_unionall(u, u->var->ub); + if (u->var->lb == u->var->ub || may_substitute_ub(body, u->var)) { + JL_TRY { + t = jl_instantiate_unionall(u, u->var->ub); + } + JL_CATCH { + // just skip normalization + // (may happen for bounds inconsistent with the wrapper's bounds) + } + } } JL_GC_POP(); return t; diff --git a/test/core.jl b/test/core.jl index ffd11c8e783ef..69261e30504a8 100644 --- a/test/core.jl +++ b/test/core.jl @@ -7549,3 +7549,6 @@ const T35130 = Tuple{Vector{Int}, <:Any} end h35130(x) = A35130(Any[x][1]::Vector{T35130}) @test h35130(T35130[([1],1)]) isa A35130 + +# avoid impossible normalization (don't try to form Tuple{Complex{String}} here) +@test Tuple{Complex{T} where String<:T<:String} == Tuple{Complex{T} where String<:T<:String} From ef2874998a4818aae25dfee7de66f743ad64e8d6 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 2 Sep 2021 14:20:59 -0400 Subject: [PATCH 70/73] avoid loading duplicate libraries (#42058) We will not use the duplicate, so best to try to avoid loading it. (cherry picked from commit c53669f664c22d45cdd0dcc375f1b18cf7379ad2) --- cli/loader_lib.c | 17 ++++++++++++++++- src/jl_exported_funcs.inc | 2 +- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/cli/loader_lib.c b/cli/loader_lib.c index d0d5d7dca089e..414400c6d26dd 100644 --- a/cli/loader_lib.c +++ b/cli/loader_lib.c @@ -31,12 +31,27 @@ void jl_loader_print_stderr3(const char * msg1, const char * msg2, const char * /* Wrapper around dlopen(), with extra relative pathing thrown in*/ static void * load_library(const char * rel_path, const char * src_dir) { + void * handle = NULL; + + // See if a handle is already open to the basename + const char *basename = rel_path + strlen(rel_path); + while (basename-- > rel_path) + if (*basename == PATHSEPSTRING[0] || *basename == '/') + break; + basename++; +#if defined(_OS_WINDOWS_) + if ((handle = GetModuleHandleW(basename))) + return handle; +#else + if ((handle = dlopen(basename, RTLD_NOLOAD | RTLD_NOW | RTLD_GLOBAL))) + return handle; +#endif + char path[2*PATH_MAX + 1] = {0}; strncat(path, src_dir, sizeof(path) - 1); strncat(path, PATHSEPSTRING, sizeof(path) - 1); strncat(path, rel_path, sizeof(path) - 1); - void * handle = NULL; #if defined(_OS_WINDOWS_) wchar_t wpath[2*PATH_MAX + 1] = {0}; if (!utf8_to_wchar(path, wpath, 2*PATH_MAX)) { diff --git a/src/jl_exported_funcs.inc b/src/jl_exported_funcs.inc index ca49e08071aed..c80d4139f058e 100644 --- a/src/jl_exported_funcs.inc +++ b/src/jl_exported_funcs.inc @@ -756,4 +756,4 @@ XX(jl_wakeup_thread) \ XX(jl_xor_int) \ XX(jl_yield) \ - XX(jl_zext_int) + XX(jl_zext_int) \ From 9d13e1615aee8006c5a01a4d6710c12cae580dda Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Fri, 3 Sep 2021 06:41:29 +0900 Subject: [PATCH 71/73] inference: fix #42090, make sure not to wrap `Conditional` in `PartialStruct` (#42091) (cherry picked from commit 03e7b23033aa8d05233893ebdb73c3f081cf0ffe) --- base/compiler/abstractinterpretation.jl | 2 +- test/compiler/inference.jl | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/base/compiler/abstractinterpretation.jl b/base/compiler/abstractinterpretation.jl index c976c17e22767..8f81082699440 100644 --- a/base/compiler/abstractinterpretation.jl +++ b/base/compiler/abstractinterpretation.jl @@ -1190,7 +1190,7 @@ function abstract_eval_statement(interp::AbstractInterpreter, @nospecialize(e), anyconst = false allconst = true for i = 2:length(e.args) - at = abstract_eval_value(interp, e.args[i], vtypes, sv) + at = widenconditional(abstract_eval_value(interp, e.args[i], vtypes, sv)) if !anyconst anyconst = has_nontrivial_const_info(at) end diff --git a/test/compiler/inference.jl b/test/compiler/inference.jl index 1e3b3cf5e20e3..579a67d9a2833 100644 --- a/test/compiler/inference.jl +++ b/test/compiler/inference.jl @@ -1719,6 +1719,29 @@ for expr25261 in opt25261[i:end] end @test foundslot +# https://github.com/JuliaLang/julia/issues/42090#issuecomment-911824851 +# `PartialStruct` shoudln't wrap `Conditional` +let M = Module() + @eval M begin + struct BePartialStruct + val::Int + cond + end + end + + rt = @eval M begin + Base.return_types((Union{Nothing,Int},)) do a + cond = a === nothing + obj = $(Expr(:new, M.BePartialStruct, 42, :cond)) + r1 = getfield(obj, :cond) ? 0 : a # r1::Union{Nothing,Int}, not r1::Int (because PartialStruct doesn't wrap Conditional) + a = $(gensym(:anyvar))::Any + r2 = getfield(obj, :cond) ? a : nothing # r2::Any, not r2::Const(nothing) (we don't need to worry about constrait invalidation here) + return r1, r2 # ::Tuple{Union{Nothing,Int},Any} + end |> only + end + @test rt == Tuple{Union{Nothing,Int},Any} +end + function f25579(g) h = g[] t = (h === nothing) From 19e66b3caaba1b06fa83bc994ce4c8fbc176bb4c Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Sat, 4 Sep 2021 09:31:16 +0900 Subject: [PATCH 72/73] inference: propagate variable changes to all exception frames #42081 (#42110) cherry-picked from #42081 Co-Authored-By: Jameson Nash --- base/compiler/abstractinterpretation.jl | 45 +++++----- base/compiler/inferencestate.jl | 114 +++++++++++++++++++----- test/compiler/inference.jl | 45 ++++++++++ 3 files changed, 161 insertions(+), 43 deletions(-) diff --git a/base/compiler/abstractinterpretation.jl b/base/compiler/abstractinterpretation.jl index 8f81082699440..1091989537749 100644 --- a/base/compiler/abstractinterpretation.jl +++ b/base/compiler/abstractinterpretation.jl @@ -1346,19 +1346,18 @@ function typeinf_local(interp::AbstractInterpreter, frame::InferenceState) n = frame.nstmts while frame.pc´´ <= n # make progress on the active ip set - local pc::Int = frame.pc´´ # current program-counter + local pc::Int = frame.pc´´ while true # inner loop optimizes the common case where it can run straight from pc to pc + 1 #print(pc,": ",s[pc],"\n") local pc´::Int = pc + 1 # next program-counter (after executing instruction) if pc == frame.pc´´ - # need to update pc´´ to point at the new lowest instruction in W - min_pc = _bits_findnext(W.bits, pc + 1) - frame.pc´´ = min_pc == -1 ? n + 1 : min_pc + # want to update pc´´ to point at the new lowest instruction in W + frame.pc´´ = pc´ end delete!(W, pc) frame.currpc = pc - frame.cur_hand = frame.handler_at[pc] - frame.stmt_edges[pc] === nothing || empty!(frame.stmt_edges[pc]) + edges = frame.stmt_edges[pc] + edges === nothing || empty!(edges) frame.stmt_info[pc] = nothing stmt = frame.src.code[pc] changes = s[pc]::VarTable @@ -1392,7 +1391,6 @@ function typeinf_local(interp::AbstractInterpreter, frame::InferenceState) pc´ = l else # general case - frame.handler_at[l] = frame.cur_hand changes_else = changes if isa(condt, Conditional) if condt.elsetype !== Any && condt.elsetype !== changes[slot_id(condt.var)] @@ -1440,7 +1438,6 @@ function typeinf_local(interp::AbstractInterpreter, frame::InferenceState) end elseif hd === :enter l = stmt.args[1]::Int - frame.cur_hand = Pair{Any,Any}(l, frame.cur_hand) # propagate type info to exception handler old = s[l] newstate_catch = stupdate!(old, changes) @@ -1452,11 +1449,7 @@ function typeinf_local(interp::AbstractInterpreter, frame::InferenceState) s[l] = newstate_catch end typeassert(s[l], VarTable) - frame.handler_at[l] = frame.cur_hand elseif hd === :leave - for i = 1:((stmt.args[1])::Int) - frame.cur_hand = (frame.cur_hand::Pair{Any,Any}).second - end else if hd === :(=) t = abstract_eval_statement(interp, stmt.args[2], changes, frame) @@ -1482,16 +1475,22 @@ function typeinf_local(interp::AbstractInterpreter, frame::InferenceState) frame.src.ssavaluetypes[pc] = t end end - if frame.cur_hand !== nothing && isa(changes, StateUpdate) - # propagate new type info to exception handler - # the handling for Expr(:enter) propagates all changes from before the try/catch - # so this only needs to propagate any changes - l = frame.cur_hand.first::Int - if stupdate1!(s[l]::VarTable, changes::StateUpdate) !== false - if l < frame.pc´´ - frame.pc´´ = l + if isa(changes, StateUpdate) + let cur_hand = frame.handler_at[pc], l, enter + while cur_hand != 0 + enter = frame.src.code[cur_hand] + l = (enter::Expr).args[1]::Int + # propagate new type info to exception handler + # the handling for Expr(:enter) propagates all changes from before the try/catch + # so this only needs to propagate any changes + if stupdate1!(s[l]::VarTable, changes::StateUpdate) !== false + if l < frame.pc´´ + frame.pc´´ = l + end + push!(W, l) + end + cur_hand = frame.handler_at[cur_hand] end - push!(W, l) end end end @@ -1504,7 +1503,6 @@ function typeinf_local(interp::AbstractInterpreter, frame::InferenceState) end pc´ > n && break # can't proceed with the fast-path fall-through - frame.handler_at[pc´] = frame.cur_hand newstate = stupdate!(s[pc´], changes) if isa(stmt, GotoNode) && frame.pc´´ < pc´ # if we are processing a goto node anyways, @@ -1515,7 +1513,7 @@ function typeinf_local(interp::AbstractInterpreter, frame::InferenceState) s[pc´] = newstate end push!(W, pc´) - pc = frame.pc´´ + break elseif newstate !== nothing s[pc´] = newstate pc = pc´ @@ -1525,6 +1523,7 @@ function typeinf_local(interp::AbstractInterpreter, frame::InferenceState) break end end + frame.pc´´ = _bits_findnext(W.bits, frame.pc´´)::Int # next program-counter end frame.dont_work_on_me = false nothing diff --git a/base/compiler/inferencestate.jl b/base/compiler/inferencestate.jl index 3a25cf753ae82..31322e5c9b4da 100644 --- a/base/compiler/inferencestate.jl +++ b/base/compiler/inferencestate.jl @@ -28,9 +28,7 @@ mutable struct InferenceState pc´´::LineNum nstmts::Int # current exception handler info - cur_hand #::Union{Nothing, Pair{LineNum, prev_handler}} - handler_at::Vector{Any} - n_handlers::Int + handler_at::Vector{LineNum} # ssavalue sparsity and restart info ssavalue_uses::Vector{BitSet} throw_blocks::BitSet @@ -57,8 +55,9 @@ mutable struct InferenceState function InferenceState(result::InferenceResult, src::CodeInfo, cached::Bool, interp::AbstractInterpreter) linfo = result.linfo + def = linfo.def code = src.code::Array{Any,1} - toplevel = !isa(linfo.def, Method) + toplevel = !isa(def, Method) sp = sptypes_from_meth_instance(linfo::MethodInstance) @@ -87,30 +86,21 @@ mutable struct InferenceState throw_blocks = find_throw_blocks(code) # exception handlers - cur_hand = nothing - handler_at = Any[ nothing for i=1:n ] - n_handlers = 0 - - W = BitSet() - push!(W, 1) #initial pc to visit - - if !toplevel - meth = linfo.def - inmodule = meth.module - else - inmodule = linfo.def::Module - end + ip = BitSet() + handler_at = compute_trycatch(src.code, ip) + push!(ip, 1) + mod = isa(def, Method) ? def.module : def valid_worlds = WorldRange(src.min_world, src.max_world == typemax(UInt) ? get_world_counter() : src.max_world) + frame = new( InferenceParams(interp), result, linfo, - sp, slottypes, inmodule, 0, + sp, slottypes, mod, 0, IdSet{InferenceState}(), IdSet{InferenceState}(), src, get_world_counter(interp), valid_worlds, nargs, s_types, s_edges, stmt_info, - Union{}, W, 1, n, - cur_hand, handler_at, n_handlers, + Union{}, ip, 1, n, handler_at, ssavalue_uses, throw_blocks, Vector{Tuple{InferenceState,LineNum}}(), # cycle_backedges Vector{InferenceState}(), # callers_in_cycle @@ -124,6 +114,90 @@ mutable struct InferenceState end end +function compute_trycatch(code::Vector{Any}, ip::BitSet) + # The goal initially is to record the frame like this for the state at exit: + # 1: (enter 3) # == 0 + # 3: (expr) # == 1 + # 3: (leave 1) # == 1 + # 4: (expr) # == 0 + # then we can find all trys by walking backwards from :enter statements, + # and all catches by looking at the statement after the :enter + n = length(code) + empty!(ip) + ip.offset = 0 # for _bits_findnext + push!(ip, n + 1) + handler_at = fill(0, n) + + # start from all :enter statements and record the location of the try + for pc = 1:n + stmt = code[pc] + if isexpr(stmt, :enter) + l = stmt.args[1]::Int + handler_at[pc + 1] = pc + push!(ip, pc + 1) + handler_at[l] = pc + push!(ip, l) + end + end + + # now forward those marks to all :leave statements + pc´´ = 0 + while true + # make progress on the active ip set + pc = _bits_findnext(ip.bits, pc´´)::Int + pc > n && break + while true # inner loop optimizes the common case where it can run straight from pc to pc + 1 + pc´ = pc + 1 # next program-counter (after executing instruction) + if pc == pc´´ + pc´´ = pc´ + end + delete!(ip, pc) + cur_hand = handler_at[pc] + @assert cur_hand != 0 "unbalanced try/catch" + stmt = code[pc] + if isa(stmt, GotoNode) + pc´ = stmt.label + elseif isa(stmt, GotoIfNot) + l = stmt.dest::Int + if handler_at[l] != cur_hand + @assert handler_at[l] == 0 "unbalanced try/catch" + handler_at[l] = cur_hand + if l < pc´´ + pc´´ = l + end + push!(ip, l) + end + elseif isa(stmt, ReturnNode) + @assert !isdefined(stmt, :val) "unbalanced try/catch" + break + elseif isa(stmt, Expr) + head = stmt.head + if head === :enter + cur_hand = pc + elseif head === :leave + l = stmt.args[1]::Int + for i = 1:l + cur_hand = handler_at[cur_hand] + end + cur_hand == 0 && break + end + end + + pc´ > n && break # can't proceed with the fast-path fall-through + if handler_at[pc´] != cur_hand + @assert handler_at[pc´] == 0 "unbalanced try/catch" + handler_at[pc´] = cur_hand + elseif !in(pc´, ip) + break # already visited + end + pc = pc´ + end + end + + @assert first(ip) == n + 1 + return handler_at +end + method_table(interp::AbstractInterpreter, sv::InferenceState) = sv.method_table function InferenceState(result::InferenceResult, cached::Bool, interp::AbstractInterpreter) diff --git a/test/compiler/inference.jl b/test/compiler/inference.jl index 579a67d9a2833..9e461b9c39824 100644 --- a/test/compiler/inference.jl +++ b/test/compiler/inference.jl @@ -3040,3 +3040,48 @@ Base.return_types((Union{Int,Nothing},)) do x end x end == [Int] + +# issue #42022 +let x = Tuple{Int,Any}[ + #= 1=# (0, Expr(:(=), Core.SlotNumber(3), 1)) + #= 2=# (0, Expr(:enter, 18)) + #= 3=# (2, Expr(:(=), Core.SlotNumber(3), 2.0)) + #= 4=# (2, Expr(:enter, 12)) + #= 5=# (4, Expr(:(=), Core.SlotNumber(3), '3')) + #= 6=# (4, Core.GotoIfNot(Core.SlotNumber(2), 9)) + #= 7=# (4, Expr(:leave, 2)) + #= 8=# (0, Core.ReturnNode(1)) + #= 9=# (4, Expr(:call, GlobalRef(Main, :throw))) + #=10=# (4, Expr(:leave, 1)) + #=11=# (2, Core.GotoNode(16)) + #=12=# (4, Expr(:leave, 1)) + #=13=# (2, Expr(:(=), Core.SlotNumber(4), Expr(:the_exception))) + #=14=# (2, Expr(:call, GlobalRef(Main, :rethrow))) + #=15=# (2, Expr(:pop_exception, Core.SSAValue(4))) + #=16=# (2, Expr(:leave, 1)) + #=17=# (0, Core.GotoNode(22)) + #=18=# (2, Expr(:leave, 1)) + #=19=# (0, Expr(:(=), Core.SlotNumber(5), Expr(:the_exception))) + #=20=# (0, nothing) + #=21=# (0, Expr(:pop_exception, Core.SSAValue(2))) + #=22=# (0, Core.ReturnNode(Core.SlotNumber(3))) + ] + handler_at = Core.Compiler.compute_trycatch(last.(x), Core.Compiler.BitSet()) + @test handler_at == first.(x) +end + +@test only(Base.return_types((Bool,)) do y + x = 1 + try + x = 2.0 + try + x = '3' + y ? (return 1) : throw() + catch ex1 + rethrow() + end + catch ex2 + nothing + end + return x + end) === Union{Int, Float64, Char} From df3d35143be5f92d3898c84fad10e3d3d377ee14 Mon Sep 17 00:00:00 2001 From: Kristoffer Date: Tue, 7 Sep 2021 05:07:37 +0200 Subject: [PATCH 73/73] bump to latest Pkg release-1.6 --- .../Pkg-6aad670822489136cc5dd3e5ea8bfbe30df776ca.tar.gz/md5 | 1 + .../sha512 | 1 + .../Pkg-ab8f6c8e7fbfc86023e45d937b3298c7afdc872b.tar.gz/md5 | 1 - .../sha512 | 1 - stdlib/Pkg.version | 4 ++-- 5 files changed, 4 insertions(+), 4 deletions(-) create mode 100644 deps/checksums/Pkg-6aad670822489136cc5dd3e5ea8bfbe30df776ca.tar.gz/md5 create mode 100644 deps/checksums/Pkg-6aad670822489136cc5dd3e5ea8bfbe30df776ca.tar.gz/sha512 delete mode 100644 deps/checksums/Pkg-ab8f6c8e7fbfc86023e45d937b3298c7afdc872b.tar.gz/md5 delete mode 100644 deps/checksums/Pkg-ab8f6c8e7fbfc86023e45d937b3298c7afdc872b.tar.gz/sha512 diff --git a/deps/checksums/Pkg-6aad670822489136cc5dd3e5ea8bfbe30df776ca.tar.gz/md5 b/deps/checksums/Pkg-6aad670822489136cc5dd3e5ea8bfbe30df776ca.tar.gz/md5 new file mode 100644 index 0000000000000..077334d96943b --- /dev/null +++ b/deps/checksums/Pkg-6aad670822489136cc5dd3e5ea8bfbe30df776ca.tar.gz/md5 @@ -0,0 +1 @@ +c34463bbc0613bfd59d1742c03025e44 diff --git a/deps/checksums/Pkg-6aad670822489136cc5dd3e5ea8bfbe30df776ca.tar.gz/sha512 b/deps/checksums/Pkg-6aad670822489136cc5dd3e5ea8bfbe30df776ca.tar.gz/sha512 new file mode 100644 index 0000000000000..c52d844ada062 --- /dev/null +++ b/deps/checksums/Pkg-6aad670822489136cc5dd3e5ea8bfbe30df776ca.tar.gz/sha512 @@ -0,0 +1 @@ +fb2247b6396777506e65a8be3609bebeb38803c45d1afcd515fd1e6d9b9c56a8188d222bdeb35a6f10cb404c1cda54b7105d0d35b5902fa003fb5f5badee82bf diff --git a/deps/checksums/Pkg-ab8f6c8e7fbfc86023e45d937b3298c7afdc872b.tar.gz/md5 b/deps/checksums/Pkg-ab8f6c8e7fbfc86023e45d937b3298c7afdc872b.tar.gz/md5 deleted file mode 100644 index 8f8a0b8102d30..0000000000000 --- a/deps/checksums/Pkg-ab8f6c8e7fbfc86023e45d937b3298c7afdc872b.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -776cc56b4f3aa0be4b960923b91af2a8 diff --git a/deps/checksums/Pkg-ab8f6c8e7fbfc86023e45d937b3298c7afdc872b.tar.gz/sha512 b/deps/checksums/Pkg-ab8f6c8e7fbfc86023e45d937b3298c7afdc872b.tar.gz/sha512 deleted file mode 100644 index 46321d171c956..0000000000000 --- a/deps/checksums/Pkg-ab8f6c8e7fbfc86023e45d937b3298c7afdc872b.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -d2b76b9144468ce60e423021de0abfae855d15069c3186a2b4ffd079bc6ff2bf5fc2d79f60497c191922a1c58fed9245e609f14a66cc509295d1d4e877aa9483 diff --git a/stdlib/Pkg.version b/stdlib/Pkg.version index e30977e29ae9b..d2b531aa88809 100644 --- a/stdlib/Pkg.version +++ b/stdlib/Pkg.version @@ -1,2 +1,2 @@ -PKG_BRANCH = release-1.5 -PKG_SHA1 = ab8f6c8e7fbfc86023e45d937b3298c7afdc872b +PKG_BRANCH = release-1.6 +PKG_SHA1 = 6aad670822489136cc5dd3e5ea8bfbe30df776ca