Merge branch 'master' into shuffle_ntuple

Author: nsajko

.devcontainer/Dockerfile

@@ -1,8 +1,12 @@
 {
-    "extensions": [
-      "julialang.language-julia",
-      "ms-vscode.cpptools"
-    ],
-  
-    "dockerFile": "Dockerfile"
+  "image": "docker.io/library/julia:latest",
+  "customizations": {
+      "vscode": {
+        "extensions": [
+          "julialang.language-julia",
+          "ms-vscode.cpptools"
+        ]
+     }
+  },
+  "onCreateCommand": "apt-get update && apt-get install -y build-essential libatomic1 python3 gfortran perl wget m4 cmake pkg-config git"
 }

.devcontainer/devcontainer.json

@@ -11,6 +11,8 @@
 /julia-*
 /source-dist.tmp
 /source-dist.tmp1
+/test/results_*.json
+/test/results_*.dat
 
 *.expmap
 *.exe

.gitignore

@@ -4,37 +4,40 @@ Hi! If you are new to the Julia community: welcome, and thanks for trying Julia.
 
 If you are already familiar with Julia itself, this blog post by Katharine Hyatt on [Making your first Julia pull request](https://kshyatt.github.io/post/firstjuliapr/) is a great way to get started.
 
-## Learning Julia
-
-[The learning page](https://julialang.org/learning) has a great list of resources for new and experienced users alike.
 
-## Before filing an issue
+# Table of Contents
 
-- Reporting a potential bug? Please read the "[How to file a bug report](https://github.com/JuliaLang/julia/blob/master/CONTRIBUTING.md#how-to-file-a-bug-report)" section to make sure that all necessary information is included.
+1. [Learning Julia](#learning-julia)
+2. [Filing an issue](#filing-an-issue)
+    - [Before filing an issue](#before-filing-an-issue)
+    - [How to file a bug report](#how-to-file-a-bug-report)
+3. [Submitting contributions](#submitting-contributions)
+    - [Contributor Checklist](#contributor-checklist)
+    - [Writing tests](#writing-tests)
+    - [Improving documentation](#improving-documentation)
+    - [Contributing to core functionality or base libraries](#contributing-to-core-functionality-or-base-libraries)
+    - [Contributing to the standard library](#contributing-to-the-standard-library)
+    - [Contributing to patch releases](#contributing-to-patch-releases)
+    - [Code Formatting Guidelines](#code-formatting-guidelines)
+    - [Git Recommendations For Pull Requests](#git-recommendations-for-pull-requests)
+4. [Resources](#resources)
 
-- Contributing code? Be sure to review the [contributor checklist](https://github.com/JuliaLang/julia/blob/master/CONTRIBUTING.md#contributor-checklist) for helpful tips on the tools we use to build Julia.
 
-- Library feature requests are generally not accepted on this issue tracker. New libraries should be developed as [packages](https://julialang.github.io/Pkg.jl/v1/creating-packages/). Discuss ideas for libraries at the [Julia Discourse forum](https://discourse.julialang.org). Doing so will often lead to pointers to existing projects and bring together collaborators with common interests.
-
-## Contributor Checklist
-
-* Create a [GitHub account](https://github.com/signup/free).
-
-* [Fork Julia](https://github.com/JuliaLang/julia/fork).
+## Learning Julia
 
-* Build the software and libraries (the first time takes a while, but it's fast after that). Detailed build instructions are in the [README](https://github.com/JuliaLang/julia/tree/master/README.md). Julia depends on several external packages; most are automatically downloaded and installed, but are less frequently updated than Julia itself.
+[The learning page](https://julialang.org/learning) has a great list of resources for new and experienced users alike.
 
-* Keep Julia current. Julia is a fast-moving target, and many details of the language are still settling out. Keep the repository up-to-date and rebase work-in-progress frequently to make merges simpler.
+## Filing an issue
 
-* Learn to use [git](https://git-scm.com), the version control system used by GitHub and the Julia project. Try a tutorial such as the one [provided by GitHub](https://try.GitHub.io/levels/1/challenges/1).
+### Before filing an issue
 
-* Review discussions on the [Julia Discourse forum](https://discourse.julialang.org).
+- Reporting a potential bug? Please read the "[How to file a bug report](https://github.com/JuliaLang/julia/blob/master/CONTRIBUTING.md#how-to-file-a-bug-report)" section to make sure that all necessary information is included.
 
-* For more detailed tips, read the [submission guide](https://github.com/JuliaLang/julia/blob/master/CONTRIBUTING.md#submitting-contributions) below.
+- Contributing code? Be sure to review the [contributor checklist](https://github.com/JuliaLang/julia/blob/master/CONTRIBUTING.md#contributor-checklist) for helpful tips on the tools we use to build Julia.
 
-* Relax and have fun!
+- Library feature requests are generally not accepted on this issue tracker. New libraries should be developed as [packages](https://julialang.github.io/Pkg.jl/v1/creating-packages/). Discuss ideas for libraries at the [Julia Discourse forum](https://discourse.julialang.org). Doing so will often lead to pointers to existing projects and bring together collaborators with common interests.
 
-## How to file a bug report
+### How to file a bug report
 
 A useful bug report filed as a GitHub issue provides information about how to reproduce the error.
 
@@ -56,6 +59,24 @@ A useful bug report filed as a GitHub issue provides information about how to re
 
 ## Submitting contributions
 
+### Contributor Checklist
+
+* Create a [GitHub account](https://github.com/signup/free).
+
+* [Fork Julia](https://github.com/JuliaLang/julia/fork).
+
+* Build the software and libraries (the first time takes a while, but it's fast after that). Detailed build instructions are in the [README](https://github.com/JuliaLang/julia/tree/master/README.md). Julia depends on several external packages; most are automatically downloaded and installed, but are less frequently updated than Julia itself.
+
+* Keep Julia current. Julia is a fast-moving target, and many details of the language are still settling out. Keep the repository up-to-date and rebase work-in-progress frequently to make merges simpler.
+
+* Learn to use [git](https://git-scm.com), the version control system used by GitHub and the Julia project. Try a tutorial such as the one [provided by GitHub](https://try.GitHub.io/levels/1/challenges/1).
+
+* Review discussions on the [Julia Discourse forum](https://discourse.julialang.org).
+
+* For more detailed tips, read the [submission guide](https://github.com/JuliaLang/julia/blob/master/CONTRIBUTING.md#submitting-contributions) below.
+
+* Relax and have fun!
+
 ### Writing tests
 
 There are never enough tests. Track [code coverage at Codecov](https://codecov.io/github/JuliaLang/julia), and help improve it.
@@ -190,7 +211,7 @@ The Julia community uses [GitHub issues](https://github.com/JuliaLang/julia/issu
 Issues and pull requests should have self explanatory titles such that they can be understood from the list of PRs and Issues.
 i.e. `Add {feature}` and `Fix {bug}` are good, `Fix #12345. Corrects the bug.` is bad.
 
-You can make pull requests for incomplete features to get code review. The convention is to open these a draft PRs and prefix
+You can make pull requests for incomplete features to get code review. The convention is to open these as draft PRs and prefix
 the pull request title with "WIP:" for Work In Progress, or "RFC:" for Request for Comments when work is completed and ready
 for merging. This will prevent accidental merging of work that is in progress.
 
@@ -209,6 +230,7 @@ Add new code to Julia's base libraries as follows (this is the "basic" approach;
 Build as usual, and do `make clean testall` to test your contribution. If your contribution includes changes to Makefiles or external dependencies, make sure you can build Julia from a clean tree using `git clean -fdx` or equivalent (be careful – this command will delete any files lying around that aren't checked into git).
 
 #### Running specific tests
+
 There are `make` targets for running specific tests:
 
     make test-bitarray
@@ -323,13 +345,11 @@ please remove the `backport-X.Y` tag from the originating pull request for the c
 
 #### General Formatting Guidelines for Julia code contributions
 
- - 4 spaces per indentation level, no tabs
+ - Follow the latest dev version of [Julia Style Guide](https://docs.julialang.org/en/v1/manual/style-guide/).
  - use whitespace to make the code more readable
  - no whitespace at the end of a line (trailing whitespace)
  - comments are good, especially when they explain the algorithm
  - try to adhere to a 92 character line length limit
- - use upper camel case convention for modules, type names
- - use lower case with underscores for method names
  - it is generally preferred to use ASCII operators and identifiers over
    Unicode equivalents whenever possible
  - in docstrings refer to the language as "Julia" and the executable as "`julia`"
@@ -346,8 +366,8 @@ please remove the `backport-X.Y` tag from the originating pull request for the c
 
 ### Git Recommendations For Pull Requests
 
- - Avoid working from the `master` branch of your fork, creating a new branch will make it easier if Julia's `master` changes and you need to update your pull request.
- - Try to [squash](https://gitready.com/advanced/2009/02/10/squashing-commits-with-rebase.html) together small commits that make repeated changes to the same section of code so your pull request is easier to review. A reasonable number of separate well-factored commits is fine, especially for larger changes.
+ - Avoid working from the `master` branch of your fork. Create a new branch as it will make it easier to update your pull request if Julia's `master` changes.
+ - Try to [squash](https://gitready.com/advanced/2009/02/10/squashing-commits-with-rebase.html) together small commits that make repeated changes to the same section of code, so your pull request is easier to review. A reasonable number of separate well-factored commits is fine, especially for larger changes.
  - If any conflicts arise due to changes in Julia's `master`, prefer updating your pull request branch with `git rebase` versus `git merge` or `git pull`, since the latter will introduce merge commits that clutter the git history with noise that makes your changes more difficult to review.
  - Descriptive commit messages are good.
  - Using `git add -p` or `git add -i` can be useful to avoid accidentally committing unrelated changes.

CONTRIBUTING.md

@@ -0,0 +1 @@
+Manifest.toml

Compiler/.gitignore

@@ -1,6 +1,6 @@
 name = "Compiler"
 uuid = "807dbc54-b67e-4c79-8afb-eafe4df6f2e1"
-version = "0.0.2"
+version = "0.0.0"
 
 [compat]
 julia = "1.10"

Compiler/Project.toml

@@ -0,0 +1,45 @@
+# The `Compiler` module
+
+This directory maintains the implementation of the Julia compiler.
+
+Through a bootstrapping process, it is bundled into the Julia runtime as `Base.Compiler`.
+
+You can also use this `Compiler` module as the `Compiler` standard library by following the steps below.
+
+## How to use
+
+To utilize this `Compiler.jl` standard library, you need to declare it as a dependency in
+your `Project.toml` as follows:
+> Project.toml
+```toml
+[deps]
+Compiler = "807dbc54-b67e-4c79-8afb-eafe4df6f2e1"
+
+[compat]
+Compiler = "0"
+```
+
+With the setup above, [the special placeholder version (v0.0.0)](https://github.com/JuliaLang/BaseCompiler.jl)
+will be installed by default.[^1]
+
+[^1]: Currently, only version v0.0.0 is registered in the [General](https://github.com/JuliaRegistries/General) registry.
+
+If needed, you can switch to a custom implementation of the `Compiler` module by running
+```julia-repl
+pkg> dev /path/to/Compiler.jl # to use a local implementation
+```
+or
+```julia-repl
+pkg> add https://url/of/Compiler/branch # to use a remote implementation
+```
+This feature is particularly useful for developing or experimenting with alternative compiler implementations.
+
+> [!note]
+> The Compiler.jl standard library is available starting from Julia v1.10.
+> However, switching to a custom compiler implementation is supported only from
+> Julia v1.12 onwards.
+
+> [!warning]
+> When using a custom, non-`Base` version of `Compiler` implementation, it may be necessary
+> to run `InteractiveUtils.@activate Compiler` to ensure proper functionality of certain
+> reflection utilities.

Compiler/README.md

@@ -1,44 +1,77 @@
 module CompilerDevTools
 
 using Compiler
+using Compiler: argextype, widenconst
 using Core.IR
+using Base: isexpr
+
+mutable struct SplitCacheOwner end
 
-struct SplitCacheOwner; end
 struct SplitCacheInterp <: Compiler.AbstractInterpreter
     world::UInt
+    owner::SplitCacheOwner
     inf_params::Compiler.InferenceParams
     opt_params::Compiler.OptimizationParams
     inf_cache::Vector{Compiler.InferenceResult}
+    codegen_cache::IdDict{CodeInstance,CodeInfo}
     function SplitCacheInterp(;
         world::UInt = Base.get_world_counter(),
+        owner::SplitCacheOwner = SplitCacheOwner(),
         inf_params::Compiler.InferenceParams = Compiler.InferenceParams(),
         opt_params::Compiler.OptimizationParams = Compiler.OptimizationParams(),
         inf_cache::Vector{Compiler.InferenceResult} = Compiler.InferenceResult[])
-        new(world, inf_params, opt_params, inf_cache)
+        new(world, owner, inf_params, opt_params, inf_cache, IdDict{CodeInstance,CodeInfo}())
     end
 end
 
 Compiler.InferenceParams(interp::SplitCacheInterp) = interp.inf_params
 Compiler.OptimizationParams(interp::SplitCacheInterp) = interp.opt_params
 Compiler.get_inference_world(interp::SplitCacheInterp) = interp.world
 Compiler.get_inference_cache(interp::SplitCacheInterp) = interp.inf_cache
-Compiler.cache_owner(::SplitCacheInterp) = SplitCacheOwner()
+Compiler.cache_owner(interp::SplitCacheInterp) = interp.owner
+Compiler.codegen_cache(interp::SplitCacheInterp) = interp.codegen_cache
 
 import Core.OptimizedGenerics.CompilerPlugins: typeinf, typeinf_edge
-@eval @noinline typeinf(::SplitCacheOwner, mi::MethodInstance, source_mode::UInt8) =
-    Base.invoke_in_world(which(typeinf, Tuple{SplitCacheOwner, MethodInstance, UInt8}).primary_world, Compiler.typeinf_ext, SplitCacheInterp(; world=Base.tls_world_age()), mi, source_mode)
+@eval @noinline typeinf(owner::SplitCacheOwner, mi::MethodInstance, source_mode::UInt8) =
+    Base.invoke_in_world(which(typeinf, Tuple{SplitCacheOwner, MethodInstance, UInt8}).primary_world, Compiler.typeinf_ext_toplevel, SplitCacheInterp(; world=Base.tls_world_age(), owner), mi, source_mode)
 
-@eval @noinline function typeinf_edge(::SplitCacheOwner, mi::MethodInstance, parent_frame::Compiler.InferenceState, world::UInt, source_mode::UInt8)
+@eval @noinline function typeinf_edge(owner::SplitCacheOwner, mi::MethodInstance, parent_frame::Compiler.InferenceState, world::UInt, source_mode::UInt8)
     # TODO: This isn't quite right, we're just sketching things for now
-    interp = SplitCacheInterp(; world)
+    interp = SplitCacheInterp(; world, owner)
     Compiler.typeinf_edge(interp, mi.def, mi.specTypes, Core.svec(), parent_frame, false, false)
 end
 
-function with_new_compiler(f, args...)
-    mi = @ccall jl_method_lookup(Any[f, args...]::Ptr{Any}, (1+length(args))::Csize_t, Base.tls_world_age()::Csize_t)::Ref{Core.MethodInstance}
-    world = Base.tls_world_age()
+function lookup_method_instance(f, args...)
+    @ccall jl_method_lookup(Any[f, args...]::Ptr{Any}, (1+length(args))::Csize_t, Base.tls_world_age()::Csize_t)::Ref{Core.MethodInstance}
+end
+
+function Compiler.transform_result_for_cache(interp::SplitCacheInterp, result::Compiler.InferenceResult, edges::Compiler.SimpleVector)
+    opt = result.src::Compiler.OptimizationState
+    ir = opt.optresult.ir::Compiler.IRCode
+    override = with_new_compiler
+    for inst in ir.stmts
+        stmt = inst[:stmt]
+        isexpr(stmt, :call) || continue
+        f = stmt.args[1]
+        f === override && continue
+        T = widenconst(argextype(f, ir))
+        T <: Core.Builtin && continue
+        insert!(stmt.args, 1, override)
+        insert!(stmt.args, 3, interp.owner)
+    end
+    @invoke Compiler.transform_result_for_cache(interp::Compiler.AbstractInterpreter, result::Compiler.InferenceResult, edges::Compiler.SimpleVector)
+end
+
+with_new_compiler(f, args...; owner::SplitCacheOwner = SplitCacheOwner()) = with_new_compiler(f, owner, args...)
+
+function with_new_compiler(f, owner::SplitCacheOwner, args...)
+    # We try to avoid introducing `with_new_compiler` in the first place,
+    # but if we can't see the type, it's still possible to end up with a
+    # builtin here - simply forward to the ordinary builtin call.
+    isa(f, Core.Builtin) && return f(args...)
+    mi = lookup_method_instance(f, args...)
     new_compiler_ci = Core.OptimizedGenerics.CompilerPlugins.typeinf(
-        SplitCacheOwner(), mi, Compiler.SOURCE_MODE_ABI
+        owner, mi, Compiler.SOURCE_MODE_ABI
     )
     invoke(f, new_compiler_ci, args...)
 end

Compiler/extras/CompilerDevTools/src/CompilerDevTools.jl

@@ -0,0 +1,24 @@
+using Test
+using Compiler: code_cache
+using Base: inferencebarrier
+using CompilerDevTools
+using CompilerDevTools: lookup_method_instance, SplitCacheInterp
+
+@testset "CompilerDevTools" begin
+  do_work(x, y) = x + y
+  f1() = do_work(inferencebarrier(1), inferencebarrier(2))
+  interp = SplitCacheInterp()
+  cache = code_cache(interp)
+  mi = lookup_method_instance(f1)
+  @test !haskey(cache, mi)
+  @test with_new_compiler(f1, interp.owner) === 3
+  @test haskey(cache, mi)
+  # Here `do_work` is compiled at runtime, and so must have
+  # required extra work to be cached under the same cache owner.
+  mi = lookup_method_instance(do_work, 1, 2)
+  @test haskey(cache, mi)
+
+  # Should not error with a builtin whose type we do not know
+  f_unknown_builtin() = Base.compilerbarrier(:type, isa)(1, Int)
+  with_new_compiler(f_unknown_builtin, interp.owner) === true
+end;

Compiler/extras/CompilerDevTools/test/runtests.jl

@@ -0,0 +1,6 @@
+using Pkg
+
+Pkg.activate(dirname(@__DIR__)) do
+  Pkg.instantiate()
+  include("runtests.jl")
+end