Merge pull request #42217 from JuliaLang/backports-release-1.7

release-1.7: Backports for RC-1.7

Author: KristofferC

.buildkite/pipelines/scheduled/0_webui.yml

@@ -21,4 +21,4 @@ steps:
           # verifies the treehash of the pipeline itself and the inputs listed in `inputs`
           signed_pipelines:
             - pipeline: .buildkite/pipelines/scheduled/coverage/coverage_linux64.yml
-              signature: "U2FsdGVkX1+lpFo/nKzx3c6xCZPKYTAuunXpOsZG4+s4+iU5LfEpMvtNvpKQjDugRoxQxCItMqB6vr4KZN3KtKhjkLbr8ExAyaPil/N/uFhrLlpwNem9dxHbPrU2l7qo"
+              signature: U2FsdGVkX1+FtqbbxyzoI/j0InDefRQ3OR06BAM2EWRhDG3SiwiPcOREudCTJ+1Z+AEVwVz5KTgw9lBVO1yjcWts3XePIy/W+arN4V+t97Dfuf4wsAr9ubpQ10GaoFnK

.buildkite/pipelines/scheduled/coverage/coverage_linux64.yml

@@ -29,7 +29,10 @@ steps:
       git config --global init.defaultBranch master
 
       echo "--- Run Julia tests in parallel with code coverage enabled"
-      ./julia --code-coverage=all --sysimage-native-code=no .buildkite/pipelines/scheduled/coverage/run_tests_parallel.jl
+      export JULIA_NUM_THREADS=1
+      export JULIA_WORKER_TIMEOUT=1200 # 1200 seconds = 20 minutes
+      ./julia -e 'import Distributed; @info "" Distributed.worker_timeout()'
+      ./julia .buildkite/pipelines/scheduled/coverage/run_tests_parallel.jl
 
       echo "--- Process and upload coverage information"
       ./julia .buildkite/pipelines/scheduled/coverage/upload_coverage.jl

.buildkite/pipelines/scheduled/coverage/run_tests_parallel.jl

@@ -1,25 +1,29 @@
 # Important note: even if one or more tests fail, we will still exit with status code 0.
-
+#
 # The reason for this is that we always want to upload code coverage, even if some of the
 # tests fail. Therefore, even if the `coverage_linux64` builder passes, you should not
 # assume that all of the tests passed. If you want to know if all of the tests are passing,
 # please look at the status of the `tester_*` builders (e.g. `tester_linux64`).
 
-# When running this file, make sure to set all of the following command-line flags:
-# 1. `--code-coverage=all`
-# 2. `--sysimage-native-code=no`
+const ncores = Sys.CPU_THREADS
+@info "" Sys.CPU_THREADS
+@info "" ncores
 
-empty!(Base.DEPOT_PATH)
-push!(Base.DEPOT_PATH, mktempdir(; cleanup = true))
+script_native_yes = """
+    Base.runtests(["cmdlineargs"]; ncores = $(ncores))
+"""
+script_native_no = """
+    Base.runtests(["all", "--skip", "cmdlineargs"]; ncores = $(ncores))
+"""
 
-const tests = "all"
-const ncores = Sys.CPU_THREADS
+base_cmd       = `$(Base.julia_cmd()) --code-coverage=all`
+cmd_native_yes = `$(base_cmd) --sysimage-native-code=yes -e $(script_native_yes)`
+cmd_native_no  = `$(base_cmd) --sysimage-native-code=no  -e $(script_native_no)`
 
-@info "" Sys.CPU_THREADS
-@info "" tests ncores
+@info "Running command" cmd_native_yes
+p1 = run(pipeline(cmd_native_yes; stdin, stdout, stderr); wait = false)
+wait(p1)
 
-try
-    Base.runtests(tests; ncores)
-catch ex
-    @error "" exception=(ex, catch_backtrace())
-end
+@info "Running command" cmd_native_no
+p2 = run(pipeline(cmd_native_no; stdin, stdout, stderr); wait = false)
+wait(p2)

.buildkite/pipelines/scheduled/coverage/upload_coverage.jl

@@ -91,7 +91,7 @@ function print_coverage_summary(
         cov_pct = floor(Int, cov_lines/tot_lines * 100)
     end
     @info "$(description): $(cov_pct)% ($(cov_lines)/$(tot_lines))"
-    return nothing
+    return (; cov_pct)
 end
 
 function buildkite_env(name::String)
@@ -198,7 +198,7 @@ end;
 sort!(fcs; by = fc -> fc.filename);
 
 print_coverage_summary.(fcs);
-print_coverage_summary(fcs, "Total")
+const total_cov_pct = print_coverage_summary(fcs, "Total").cov_pct
 
 let
     git_info = coveralls_buildkite_query_git_info()
@@ -217,3 +217,12 @@ let
     # In order to upload to Codecov, you need to have the `CODECOV_TOKEN` environment variable defined.
     Coverage.Codecov.submit_generic(fcs, kwargs)
 end
+
+if total_cov_pct < 50
+    msg = string(
+        "The total coverage is less than 50%. This should never happen, ",
+        "so it means that something has probably gone wrong with the code coverage job.",
+    )
+    @error msg total_cov_pct
+    throw(ErrorException(msg))
+end

.github/CODEOWNERS

@@ -2,4 +2,5 @@ CODEOWNERS @JuliaLang/github-actions
 /.github/ @JuliaLang/github-actions
 /.buildkite/ @JuliaLang/github-actions
 
+/.github/workflows/retry.yml @DilumAluthge
 /.github/workflows/statuses.yml @DilumAluthge

.github/workflows/retry.yml

@@ -0,0 +1,59 @@
+# Please ping @DilumAluthge when making any changes to this file.
+
+# Here are some steps that we take in this workflow file for security reasons:
+# 1. We do not checkout any code.
+# 2. We only run actions that are defined in a repository in the `JuliaLang` GitHub organization.
+# 3. We do not give the `GITHUB_TOKEN` any permissions.
+# 4. We only give the Buildkite API token (`BUILDKITE_API_TOKEN_RETRY`) the minimum necessary
+#    set of permissions.
+
+# Important note to Buildkite maintainers:
+# In order to make this work, you need to tell Buildkite that it should NOT create a brand-new
+# build when someone closes and reopens a pull request. To do so:
+# 1. Go to the relevant pipeline (e.g. https://buildkite.com/julialang/julia-master).
+# 2. Click on the "Pipeline Settings" button.
+# 3. In the left sidebar, under "Pipeline Settings", click on "GitHub".
+# 4. In the "GitHub Settings", under "Build Pull Requests", make sure that the "Skip pull
+#    request builds for existing commits" checkbox is checked. This is the setting that tells
+#    Buildkite that it should NOT create a brand-new build when someone closes and reopens a
+#    pull request.
+# 5. At the bottom of the page, click the "Save GitHub Settings" button.
+
+name: Retry Failed Buildkite Jobs
+
+on:
+  # When using the `pull_request_target` event, all PRs will get access to secret environment
+  # variables (such as the `BUILDKITE_API_TOKEN_RETRY` secret environment variable), even if
+  # the PR is from a fork. Therefore, for security reasons, we do not checkout any code in
+  # this workflow.
+  pull_request_target:
+
+    # TODO: delete the following line (once we have completely transitioned from Buildbot to Buildkite)
+    types: [ reopened, labeled ]
+
+    # TODO: uncomment the following line (once we have completely transitioned from Buildbot to Buildkite)
+    # types: [ reopened ]
+
+# We do not give the `GITHUB_TOKEN` any permissions.
+permissions:
+  statuses: none
+
+jobs:
+  retry:
+    name: retry
+    runs-on: ubuntu-latest
+
+    # TODO: delete the following line (once we have completely transitioned from Buildbot to Buildkite)
+    if: github.repository == 'JuliaLang/julia' && (github.event.label.name == 'Buildkite - retry failed jobs' || github.event.action == 'reopened')
+
+    # TODO: uncomment the following line (once we have completely transitioned from Buildbot to Buildkite)
+    # if: github.repository == 'JuliaLang/julia'
+
+    steps:
+      # For security reasons, we do not checkout any code in this workflow.
+      - uses: JuliaLang/retry-buildkite@24e8341f74e0d6760717235eac936db639d7e9eb
+        with:
+          buildkite_api_token: ${{ secrets.BUILDKITE_API_TOKEN_RETRY }}
+          buildkite_organization_slug: 'julialang'
+          buildkite_pipeline_slug: 'julia-master'
+          pr_number: ${{github.event.number}}

.github/workflows/statuses.yml

@@ -3,14 +3,11 @@
 # This is just a short-term solution until we have migrated all of CI to Buildkite.
 #
 # 1. TODO: delete this file once we have migrated all of CI to Buildkite.
-#
-# 2. TODO: disable GitHub Actions on the `JuliaLang/julia` repository once we have migrated all
-# of CI to Buildkite.
 
 # Here are some steps that we take in this workflow file for security reasons:
 # 1. We do not checkout any code.
 # 2. We do not run any external actions.
-# 3. We only give `GITHUB_TOKEN` the minimum necessary set of permissions.
+# 3. We only give the `GITHUB_TOKEN` the minimum necessary set of permissions.
 
 name: Statuses
 
@@ -27,7 +24,7 @@ on:
       - 'master'
       - 'release-*'
 
-# These are the permissions for the `GITHUB_TOKEN` token.
+# These are the permissions for the `GITHUB_TOKEN`.
 # We should only give the token the minimum necessary set of permissions.
 permissions:
   statuses: write
@@ -37,15 +34,12 @@ jobs:
     name: statuses
     runs-on: ubuntu-latest
     if: github.repository == 'JuliaLang/julia'
-    strategy:
-      fail-fast: false
     steps:
+      # For security reasons, we do not checkout any code in this workflow.
       - run: echo "SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
         if: github.event_name == 'pull_request_target'
-
       - run: echo "SHA=${{ github.sha }}" >> $GITHUB_ENV
         if: github.event_name != 'pull_request_target'
-
       - run: echo "The SHA is ${{ env.SHA }}"
 
       # As we incrementally migrate individual jobs from Buildbot to Buildkite, we should

NEWS.md

@@ -38,7 +38,7 @@ Language changes
 * The default random number generator has changed, so all random numbers will be different (even with the
   same seed) unless an explicit RNG object is used.
   See the section on the `Random` standard library below ([#40546]).
-* `Iterators.peel(itr)` now returns `nothing` when `itr` is empty instead of throwing a `BoundsError` ([#39569]).
+* `Iterators.peel(itr)` now returns `nothing` when `itr` is empty instead of throwing a `BoundsError` ([#39607]).
 * Multiple successive semicolons in an array expresion were previously ignored (e.g., `[1 ;; 2] == [1 ; 2]`).
   This syntax is now used to separate dimensions (see **New language features**).
 
@@ -76,7 +76,7 @@ New library functions
 ---------------------
 
 * Two argument methods `findmax(f, domain)`, `argmax(f, domain)` and the corresponding
-  `min` versions ([#27613]).
+  `min` versions ([#35316]).
 * `isunordered(x)` returns true if `x` is a value that is normally unordered, such as
   `NaN` or `missing` ([#35316]).
 * New `keepat!(vector, inds)` function which is the inplace equivalent of `vector[inds]`
@@ -302,7 +302,6 @@ Tooling Improvements
 
 
 <!--- generated by NEWS-update.jl: -->
-[#27613]: https://github.com/JuliaLang/julia/issues/27613
 [#29901]: https://github.com/JuliaLang/julia/issues/29901
 [#30676]: https://github.com/JuliaLang/julia/issues/30676
 [#31829]: https://github.com/JuliaLang/julia/issues/31829
@@ -340,9 +339,9 @@ Tooling Improvements
 [#39436]: https://github.com/JuliaLang/julia/issues/39436
 [#39455]: https://github.com/JuliaLang/julia/issues/39455
 [#39463]: https://github.com/JuliaLang/julia/issues/39463
-[#39569]: https://github.com/JuliaLang/julia/issues/39569
 [#39588]: https://github.com/JuliaLang/julia/issues/39588
 [#39594]: https://github.com/JuliaLang/julia/issues/39594
+[#39607]: https://github.com/JuliaLang/julia/issues/39607
 [#39710]: https://github.com/JuliaLang/julia/issues/39710
 [#39758]: https://github.com/JuliaLang/julia/issues/39758
 [#39794]: https://github.com/JuliaLang/julia/issues/39794

base/multidimensional.jl

@@ -353,8 +353,14 @@ module IteratorsMD
     # AbstractArray implementation
     Base.axes(iter::CartesianIndices{N,R}) where {N,R} = map(Base.axes1, iter.indices)
     Base.IndexStyle(::Type{CartesianIndices{N,R}}) where {N,R} = IndexCartesian()
-    @propagate_inbounds function Base.getindex(iter::CartesianIndices{N,R}, I::Vararg{Int, N}) where {N,R}
-        CartesianIndex(getindex.(iter.indices, I))
+    @inline function Base.getindex(iter::CartesianIndices{N,R}, I::Vararg{Int, N}) where {N,R}
+        # Eagerly do boundscheck before calculating each item of the CartesianIndex so that
+        # we can pass `@inbounds` hint to inside the map and generates more efficient SIMD codes (#42115)
+        @boundscheck checkbounds(iter, I...)
+        index = map(iter.indices, I) do r, i
+            @inbounds getindex(r, i)
+        end
+        CartesianIndex(index)
     end
 
     ndims(R::CartesianIndices) = ndims(typeof(R))

base/util.jl

@@ -208,6 +208,9 @@ function julia_cmd(julia=joinpath(Sys.BINDIR::String, julia_exename()))
     if opts.startupfile == 2
         push!(addflags, "--startup-file=no")
     end
+    if opts.use_sysimage_native_code == 0
+        push!(addflags, "--sysimage-native-code=no")
+    end
     return `$julia -C$cpu_target -J$image_file $addflags`
 end