Proposal: Extension-like Subpackages

base/exports.jl

@@ -864,6 +864,7 @@ export
     evalfile,
     include_string,
     include_dependency,
+    @subpackage_using,
 
 # RTS internals
     GC,

base/loading.jl

@@ -772,6 +772,11 @@ function manifest_uuid_path(env::String, pkg::PkgId)::Union{Nothing,String,Missi
     return nothing
 end
 
+function find_sub_path(project_path::String, subname::String)
+    subfiledir = joinpath(project_path, "sub", subname, subname * ".jl")
+    isfile(subfiledir) && return subfiledir
+    return joinpath(project_path, "sub", subname * ".jl")
+end
 
 function find_ext_path(project_path::String, extname::String)
     extfiledir = joinpath(project_path, "ext", extname, extname * ".jl")
@@ -991,7 +996,8 @@ function explicit_manifest_deps_get(project_file::String, where::PkgId, name::St
                         end
                     end
                 end
-            else # Check for extensions
+            else
+                # Check for extensions
                 extensions = get(entry, "extensions", nothing)
                 if extensions !== nothing
                     if haskey(extensions, where.name) && where.uuid == uuid5(UUID(uuid), where.name)
@@ -1023,6 +1029,38 @@ function explicit_manifest_deps_get(project_file::String, where::PkgId, name::St
                         return identify_package(PkgId(UUID(uuid), dep_name), name)
                     end
                 end
+                # Check for subpackages
+                subpackages = get(entry, "subpackages", nothing)
+                if subpackages !== nothing
+                    if haskey(subpackages, where.name) && where.uuid == uuid5(UUID(uuid), where.name)
+                        found_where = true
+                        if name == dep_name
+                            return PkgId(UUID(uuid), name)
+                        end
+                        subs = subpackages[where.name]::Union{String, Vector{String}}
+                        weakdeps = get(entry, "weakdeps", nothing)::Union{Vector{String}, Dict{String, Any}, Nothing}
+                        if (subs isa String && name == subs) || (subs isa Vector{String} && name in subs)
+                            for deps′ in [weakdeps, deps]
+                                if deps′ !== nothing
+                                    if deps′ isa Vector{String}
+                                        found_name = name in deps′
+                                        found_name && @goto done
+                                    elseif deps′ isa Dict{String, Any}
+                                        deps′ = deps′::Dict{String, Any}
+                                        for (dep, uuid) in deps′
+                                            uuid::String
+                                            if dep === name
+                                                return PkgId(UUID(uuid), name)
+                                            end
+                                        end
+                                    end
+                                end
+                            end
+                        end
+                        # `name` is not a subpackage, do standard lookup as if this was the parent
+                        return identify_package(PkgId(UUID(uuid), dep_name), name)
+                    end
+                end
             end
         end
     end
@@ -1057,19 +1095,33 @@ function explicit_manifest_uuid_path(project_file::String, pkg::PkgId)::Union{No
             end
         end
     end
-    # Extensions
     for (name, entries) in d
         entries = entries::Vector{Any}
         for entry in entries
             uuid = get(entry, "uuid", nothing)::Union{Nothing, String}
-            extensions = get(entry, "extensions", nothing)::Union{Nothing, Dict{String, Any}}
-            if extensions !== nothing && haskey(extensions, pkg.name) && uuid !== nothing && uuid5(UUID(uuid), pkg.name) == pkg.uuid
-                parent_path = locate_package(PkgId(UUID(uuid), name))
-                if parent_path === nothing
-                    error("failed to find source of parent package: \"$name\"")
+            # Extensions
+            let extensions = get(entry, "extensions", nothing)::Union{Nothing, Dict{String, Any}}
+                if extensions !== nothing && haskey(extensions, pkg.name) && uuid !== nothing && uuid5(UUID(uuid), pkg.name) == pkg.uuid
+                    parent_path = locate_package(PkgId(UUID(uuid), name))
+                    if parent_path === nothing
+                        error("failed to find source of parent package: \"$name\"")
+                    end
+                    p = normpath(dirname(parent_path), "..")
+                    return find_ext_path(p, pkg.name)
+                end
+            end
+            # Subpackages
+            let subpackages = get(entry, "subpackages", nothing)::Union{Nothing, Dict{String, Any}}
+                if subpackages !== nothing
+                    if haskey(subpackages, pkg.name) && uuid !== nothing && uuid5(UUID(uuid), pkg.name) == pkg.uuid
+                        parent_path = locate_package(PkgId(UUID(uuid), name))
+                        if parent_path === nothing
+                            error("failed to find source of parent package: \"$name\"")
+                        end
+                        p = normpath(dirname(parent_path), "..")
+                        return find_sub_path(p, pkg.name)
+                    end
                 end
-                p = normpath(dirname(parent_path), "..")
-                return find_ext_path(p, pkg.name)
             end
         end
     end
@@ -1670,6 +1722,63 @@ end
 # End extensions
 
 
+###############
+# Subpackages #
+###############
+function load_subpackage(into::Module, parentname::Symbol, subname::Union{Symbol, String})
+    str_parentname = string(parentname)
+    str_subname = string(subname)
+    parent_uuid = @lock require_lock begin
+        uuidkey_env = identify_package_env(into, String(parentname))
+        if uuidkey_env === nothing
+            where = PkgId(into)
+            if where.uuid === nothing
+                throw(ArgumentError("Package $parentname not found in current path."))
+            else
+                manifest_warnings = collect_manifest_warnings()
+                throw(ArgumentError("""
+                Package $(where.name) does not have $parentname in its dependencies:
+                $manifest_warnings- You may have a partially installed environment. Try `Pkg.instantiate()`
+                  to ensure all packages in the environment are installed.
+                - Or, if you have $(where.name) checked out for development and have
+                  added $parentname as a dependency but haven't updated your primary
+                  environment's manifest file, try `Pkg.resolve()`.
+                - Otherwise you may need to report an issue with $(where.name)"""))
+            end
+        end
+        uuidkey, env = uuidkey_env
+        uuidkey.uuid
+    end
+    sub_id = PkgId(uuid5(parent_uuid, str_subname), str_subname)
+    __require(sub_id)
+end
+function load_subpackage(mod::Module, subname::Union{Symbol, String})
+    parent_id = PkgId(mod)
+    str_subname = string(subname)
+    sub_id = PkgId(uuid5(parent_id.uuid, str_subname), str_subname)
+    __require(sub_id)
+end
+
+"""
+    @subpackage_using Parent.Sub
+
+Load the `Sub` subpackage from the `Parent` package.
+"""
+macro subpackage_using(parent_sub)
+    if isexpr(parent_sub, :(.), 2)
+        parent, sub = parent_sub.args
+        @gensym mod
+        esc(quote
+                $mod = $load_subpackage($__module__, $(QuoteNode(parent)), $sub)
+                using .$mod
+            end)
+    else
+        throw(ArgumentError("Malformed expression, `@subpackage_using` only takes expressions of the form `Parent.Subpackage`"))
+    end
+end
+# End subpackages
+
+
 struct CacheFlags
     # OOICCDDP - see jl_cache_flags
     use_pkgimages::Bool

test/loading.jl

@@ -1264,6 +1264,50 @@ end
     end
 end
 
+@testset "Subpackages" begin
+    old_depot_path = copy(DEPOT_PATH)
+    try
+        tmp = mktempdir()
+        push!(empty!(DEPOT_PATH), joinpath(tmp, "depot"))
+        proj = joinpath(@__DIR__, "project", "Subpackages", "HasDepWithSubpackages")
+        for i in 1:2 # Once when requiring precomilation, once where it is already precompiled
+            cmd = `$(Base.julia_cmd()) --project=$proj --startup-file=no -e '
+                    begin
+                    using HasSubpackages
+                    HasSubpackages.sub1_loaded && error("sub1_loaded set")
+                    using HasDepWithSubpackages
+                    HasSubpackages.sub1_loaded || error("sub1_loaded not set")
+                    HasDepWithSubpackages.g(1) == 1 || error("g failed")
+                    end
+                '`
+            @test success(cmd)
+        end
+        let cmd = `$(Base.julia_cmd()) --project=$proj --startup-file=no -e '
+                    begin
+                    using HasSubpackages
+                    using ADep
+                    HasSubpackages.sub2_loaded && error("sub2_loaded set!")
+                    @submodule_using HasSubpackages.Subpackage2 # currently broken
+                    HasSubpackages.sub2_loaded && error("sub2_loaded set!")
+                    end
+                '`
+            @test_broken success(cmd)
+        end
+        let cmd = `$(Base.julia_cmd()) --project=$proj --startup-file=no -e '
+                        begin
+                        @subpackage_using HasSubpackages.Subpackage3
+                        Subpackage3.x == 1 || error("Something went wrong with HasSubpackages.Subpackage3")
+                        "HasSubpackages" ∉ string.(values(Base.loaded_modules)) || error("HasSubpackages was loaded when it was not needed!")
+                        end
+                   '`
+            @test success(cmd)
+        end
+    finally
+        copy!(DEPOT_PATH, old_depot_path)
+    end
+end
+
+
 pkgimage(val) = val == 1 ? `--pkgimages=yes` : `--pkgimages=no`
 opt_level(val) = `-O$val`
 debug_level(val) = `-g$val`

test/project/Subpackages/ADep/Project.toml

@@ -0,0 +1,4 @@
+name = "ADep"
+uuid = "81016d05-d0fa-4c22-8125-5c67d6cf9f2f"
+authors = ["Mason Protter <mason.protter@icloud.com>"]
+version = "0.1.0"

test/project/Subpackages/ADep/src/ADep.jl

@@ -0,0 +1,7 @@
+module ADep
+
+foo(x) = x^2
+
+export foo
+
+end # module ADep

test/project/Subpackages/HasDepWithSubpackages/Manifest.toml

@@ -0,0 +1,24 @@
+julia_version = "1.13.0-DEV"
+manifest_format = "2.0"
+project_hash = "cd51eae9e59902fd893210e4f8a4d4dc2ac81722"
+
+[[deps.HasDepWithSubpackages]]
+deps = ["HasSubpackages"]
+path = "."
+uuid = "0237f425-1828-4bb9-8eeb-a4f4cdb55107"
+version = "0.1.0"
+
+[[deps.HasSubpackages]]
+path = "../HasSubpackages"
+uuid = "0f595f7f-271a-49d7-b31d-0f5ff0a9189a"
+version = "0.1.0"
+
+    [deps.HasSubpackages.subpackages]
+    Subpackage1 = "HasSubpackages"
+    Subpackage2 = ["HasSubpackages", "ADep"]
+    Subpackage3 = ""
+
+[[deps.ADep]]
+path = "../ADep"
+uuid = "81016d05-d0fa-4c22-8125-5c67d6cf9f2f"
+version = "0.1.0"

test/project/Subpackages/HasDepWithSubpackages/Project.toml

@@ -0,0 +1,7 @@
+name = "HasDepWithSubpackages"
+uuid = "0237f425-1828-4bb9-8eeb-a4f4cdb55107"
+version = "0.1.0"
+
+[deps]
+HasSubpackages = "0f595f7f-271a-49d7-b31d-0f5ff0a9189a"
+ADep = "81016d05-d0fa-4c22-8125-5c67d6cf9f2f"

test/project/Subpackages/HasDepWithSubpackages/src/HasDepWithSubpackages.jl

@@ -0,0 +1,10 @@
+module HasDepWithSubpackages
+
+using HasSubpackages
+@subpackage_using HasSubpackages.Subpackage1
+
+g(x) = f(x) - 1
+
+export g
+
+end

test/project/Subpackages/HasSubpackages/Project.toml

@@ -0,0 +1,11 @@
+name = "HasSubpackages"
+uuid = "0f595f7f-271a-49d7-b31d-0f5ff0a9189a"
+version = "0.1.0"
+
+[weakdeps]
+ADep = "81016d05-d0fa-4c22-8125-5c67d6cf9f2f"
+
+[subpackages]
+Subpackage1 = "HasSubpackages"
+Subpackage2 = ["HasSubpackages", "ADep"]
+Subpackage3 = ""

test/project/Subpackages/HasSubpackages/src/HasSubpackages.jl

@@ -0,0 +1,6 @@
+module HasSubpackages
+
+sub1_loaded = false
+sub2_loaded = false
+
+end

test/project/Subpackages/HasSubpackages/sub/Subpackage1.jl

@@ -0,0 +1,12 @@
+module Subpackage1
+using HasSubpackages
+
+function __init__()
+    HasSubpackages.sub1_loaded = true
+end
+
+f(x) = x + 1
+
+export f
+
+end

test/project/Subpackages/HasSubpackages/sub/Subpackage2.jl

@@ -0,0 +1,14 @@
+module Subpackage2
+
+using ADep
+using HasSubpackages
+
+bar(x) = foo(x) + 1
+
+export bar
+
+function __init__()
+    HasSubpackages.sub2_loaded = true
+end
+
+end