wasm-linker: Implement linker tests (#12006)

* test/link: initial wasm support

This adds basic parsing and dumping of wasm section so they
can be tested using the new linker-test infrastructure.

* test/link: all wasm sections parsing and dumping

We now parse and dump all sections for the wasm binary format.
Currently, this only dumps the name of a custom section.
Later this should also dump symbol table, name, linking metadata and relocations.
All of those live within the custom sections.

* Add wasm linker test

This also fixes a parser mistake in reading the flags.

* test/link: implement linker tests wasm & fixes

Adds several test cases to test the wasm self-hosted linker.
This also introduces fixes that were caught during the implementation
of those tests.

* test-runner: obey omit_stage2 for standalone

When a standalone test requires stage2, but stage2 is omit
from the compiler, such test case will not be included as part
of the test suite that is being ran. This is to support CI's
where we omit stage2 to lower the memory usage.

Author: Luukdegram

build.zig

@@ -481,8 +481,8 @@ pub fn build(b: *Builder) !void {
     ));
 
     toolchain_step.dependOn(tests.addCompareOutputTests(b, test_filter, modes));
-    toolchain_step.dependOn(tests.addStandaloneTests(b, test_filter, modes, skip_non_native, enable_macos_sdk, target));
-    toolchain_step.dependOn(tests.addLinkTests(b, test_filter, modes, enable_macos_sdk));
+    toolchain_step.dependOn(tests.addStandaloneTests(b, test_filter, modes, skip_non_native, enable_macos_sdk, target, omit_stage2));
+    toolchain_step.dependOn(tests.addLinkTests(b, test_filter, modes, enable_macos_sdk, omit_stage2));
     toolchain_step.dependOn(tests.addStackTraceTests(b, test_filter, modes));
     toolchain_step.dependOn(tests.addCliTests(b, test_filter, modes));
     toolchain_step.dependOn(tests.addAssembleAndLinkTests(b, test_filter, modes));

ci/azure/macos_script

@@ -76,7 +76,7 @@ release/bin/zig build test-run-translated-c -Denable-macos-sdk
 release/bin/zig build docs                  -Denable-macos-sdk
 release/bin/zig build test-fmt              -Denable-macos-sdk
 release/bin/zig build test-cases            -Denable-macos-sdk -Dsingle-threaded
-release/bin/zig build test-link             -Denable-macos-sdk
+release/bin/zig build test-link             -Denable-macos-sdk -Domit-stage2
 
 if [ "${BUILD_REASON}" != "PullRequest" ]; then
   mv ../LICENSE release/

lib/std/build.zig

@@ -1618,6 +1618,7 @@ pub const LibExeObjStep = struct {
     want_lto: ?bool = null,
     use_stage1: ?bool = null,
     use_llvm: ?bool = null,
+    use_lld: ?bool = null,
     ofmt: ?std.Target.ObjectFormat = null,
 
     output_path_source: GeneratedFile,
@@ -2474,6 +2475,14 @@ pub const LibExeObjStep = struct {
             }
         }
 
+        if (self.use_lld) |use_lld| {
+            if (use_lld) {
+                try zig_args.append("-fLLD");
+            } else {
+                try zig_args.append("-fno-LLD");
+            }
+        }
+
         if (self.ofmt) |ofmt| {
             try zig_args.append(try std.fmt.allocPrint(builder.allocator, "-ofmt={s}", .{@tagName(ofmt)}));
         }

lib/std/build/CheckObjectStep.zig

@@ -265,7 +265,10 @@ fn make(step: *Step) !void {
         }),
         .elf => @panic("TODO elf parser"),
         .coff => @panic("TODO coff parser"),
-        .wasm => @panic("TODO wasm parser"),
+        .wasm => try WasmDumper.parseAndDump(contents, .{
+            .gpa = gpa,
+            .dump_symtab = self.dump_symtab,
+        }),
         else => unreachable,
     };
 
@@ -522,3 +525,295 @@ const MachODumper = struct {
         }
     }
 };
+
+const WasmDumper = struct {
+    const symtab_label = "symbols";
+
+    fn parseAndDump(bytes: []const u8, opts: Opts) ![]const u8 {
+        const gpa = opts.gpa orelse unreachable; // Wasm dumper requires an allocator
+        if (opts.dump_symtab) {
+            @panic("TODO: Implement symbol table parsing and dumping");
+        }
+
+        var fbs = std.io.fixedBufferStream(bytes);
+        const reader = fbs.reader();
+
+        const buf = try reader.readBytesNoEof(8);
+        if (!mem.eql(u8, buf[0..4], &std.wasm.magic)) {
+            return error.InvalidMagicByte;
+        }
+        if (!mem.eql(u8, buf[4..], &std.wasm.version)) {
+            return error.UnsupportedWasmVersion;
+        }
+
+        var output = std.ArrayList(u8).init(gpa);
+        errdefer output.deinit();
+        const writer = output.writer();
+
+        while (reader.readByte()) |current_byte| {
+            const section = std.meta.intToEnum(std.wasm.Section, current_byte) catch |err| {
+                std.debug.print("Found invalid section id '{d}'\n", .{current_byte});
+                return err;
+            };
+
+            const section_length = try std.leb.readULEB128(u32, reader);
+            try parseAndDumpSection(section, bytes[fbs.pos..][0..section_length], writer);
+            fbs.pos += section_length;
+        } else |_| {} // reached end of stream
+
+        return output.toOwnedSlice();
+    }
+
+    fn parseAndDumpSection(section: std.wasm.Section, data: []const u8, writer: anytype) !void {
+        var fbs = std.io.fixedBufferStream(data);
+        const reader = fbs.reader();
+
+        try writer.print(
+            \\Section {s}
+            \\size {d}
+        , .{ @tagName(section), data.len });
+
+        switch (section) {
+            .type,
+            .import,
+            .function,
+            .table,
+            .memory,
+            .global,
+            .@"export",
+            .element,
+            .code,
+            .data,
+            => {
+                const entries = try std.leb.readULEB128(u32, reader);
+                try writer.print("\nentries {d}\n", .{entries});
+                try dumpSection(section, data[fbs.pos..], entries, writer);
+            },
+            .custom => {
+                const name_length = try std.leb.readULEB128(u32, reader);
+                const name = data[fbs.pos..][0..name_length];
+                fbs.pos += name_length;
+                try writer.print("\nname {s}\n", .{name});
+
+                if (mem.eql(u8, name, "name")) {
+                    try parseDumpNames(reader, writer, data);
+                }
+                // TODO: Implement parsing and dumping other custom sections (such as relocations)
+            },
+            .start => {
+                const start = try std.leb.readULEB128(u32, reader);
+                try writer.print("\nstart {d}\n", .{start});
+            },
+            else => {}, // skip unknown sections
+        }
+    }
+
+    fn dumpSection(section: std.wasm.Section, data: []const u8, entries: u32, writer: anytype) !void {
+        var fbs = std.io.fixedBufferStream(data);
+        const reader = fbs.reader();
+
+        switch (section) {
+            .type => {
+                var i: u32 = 0;
+                while (i < entries) : (i += 1) {
+                    const func_type = try reader.readByte();
+                    if (func_type != std.wasm.function_type) {
+                        std.debug.print("Expected function type, found byte '{d}'\n", .{func_type});
+                        return error.UnexpectedByte;
+                    }
+                    const params = try std.leb.readULEB128(u32, reader);
+                    try writer.print("params {d}\n", .{params});
+                    var index: u32 = 0;
+                    while (index < params) : (index += 1) {
+                        try parseDumpType(std.wasm.Valtype, reader, writer);
+                    } else index = 0;
+                    const returns = try std.leb.readULEB128(u32, reader);
+                    try writer.print("returns {d}\n", .{returns});
+                    while (index < returns) : (index += 1) {
+                        try parseDumpType(std.wasm.Valtype, reader, writer);
+                    }
+                }
+            },
+            .import => {
+                var i: u32 = 0;
+                while (i < entries) : (i += 1) {
+                    const module_name_len = try std.leb.readULEB128(u32, reader);
+                    const module_name = data[fbs.pos..][0..module_name_len];
+                    fbs.pos += module_name_len;
+                    const name_len = try std.leb.readULEB128(u32, reader);
+                    const name = data[fbs.pos..][0..name_len];
+                    fbs.pos += name_len;
+
+                    const kind = std.meta.intToEnum(std.wasm.ExternalKind, try reader.readByte()) catch |err| {
+                        std.debug.print("Invalid import kind\n", .{});
+                        return err;
+                    };
+
+                    try writer.print(
+                        \\module {s}
+                        \\name {s}
+                        \\kind {s}
+                    , .{ module_name, name, @tagName(kind) });
+                    try writer.writeByte('\n');
+                    switch (kind) {
+                        .function => {
+                            try writer.print("index {d}\n", .{try std.leb.readULEB128(u32, reader)});
+                        },
+                        .memory => {
+                            try parseDumpLimits(reader, writer);
+                        },
+                        .global => {
+                            try parseDumpType(std.wasm.Valtype, reader, writer);
+                            try writer.print("mutable {}\n", .{0x01 == try std.leb.readULEB128(u32, reader)});
+                        },
+                        .table => {
+                            try parseDumpType(std.wasm.RefType, reader, writer);
+                            try parseDumpLimits(reader, writer);
+                        },
+                    }
+                }
+            },
+            .function => {
+                var i: u32 = 0;
+                while (i < entries) : (i += 1) {
+                    try writer.print("index {d}\n", .{try std.leb.readULEB128(u32, reader)});
+                }
+            },
+            .table => {
+                var i: u32 = 0;
+                while (i < entries) : (i += 1) {
+                    try parseDumpType(std.wasm.RefType, reader, writer);
+                    try parseDumpLimits(reader, writer);
+                }
+            },
+            .memory => {
+                var i: u32 = 0;
+                while (i < entries) : (i += 1) {
+                    try parseDumpLimits(reader, writer);
+                }
+            },
+            .global => {
+                var i: u32 = 0;
+                while (i < entries) : (i += 1) {
+                    try parseDumpType(std.wasm.Valtype, reader, writer);
+                    try writer.print("mutable {}\n", .{0x01 == try std.leb.readULEB128(u1, reader)});
+                    try parseDumpInit(reader, writer);
+                }
+            },
+            .@"export" => {
+                var i: u32 = 0;
+                while (i < entries) : (i += 1) {
+                    const name_len = try std.leb.readULEB128(u32, reader);
+                    const name = data[fbs.pos..][0..name_len];
+                    fbs.pos += name_len;
+                    const kind_byte = try std.leb.readULEB128(u8, reader);
+                    const kind = std.meta.intToEnum(std.wasm.ExternalKind, kind_byte) catch |err| {
+                        std.debug.print("invalid export kind value '{d}'\n", .{kind_byte});
+                        return err;
+                    };
+                    const index = try std.leb.readULEB128(u32, reader);
+                    try writer.print(
+                        \\name {s}
+                        \\kind {s}
+                        \\index {d}
+                    , .{ name, @tagName(kind), index });
+                    try writer.writeByte('\n');
+                }
+            },
+            .element => {
+                var i: u32 = 0;
+                while (i < entries) : (i += 1) {
+                    try writer.print("table index {d}\n", .{try std.leb.readULEB128(u32, reader)});
+                    try parseDumpInit(reader, writer);
+
+                    const function_indexes = try std.leb.readULEB128(u32, reader);
+                    var function_index: u32 = 0;
+                    try writer.print("indexes {d}\n", .{function_indexes});
+                    while (function_index < function_indexes) : (function_index += 1) {
+                        try writer.print("index {d}\n", .{try std.leb.readULEB128(u32, reader)});
+                    }
+                }
+            },
+            .code => {}, // code section is considered opaque to linker
+            .data => {
+                var i: u32 = 0;
+                while (i < entries) : (i += 1) {
+                    const index = try std.leb.readULEB128(u32, reader);
+                    try writer.print("memory index 0x{x}\n", .{index});
+                    try parseDumpInit(reader, writer);
+                    const size = try std.leb.readULEB128(u32, reader);
+                    try writer.print("size {d}\n", .{size});
+                    try reader.skipBytes(size, .{}); // we do not care about the content of the segments
+                }
+            },
+            else => unreachable,
+        }
+    }
+
+    fn parseDumpType(comptime WasmType: type, reader: anytype, writer: anytype) !void {
+        const type_byte = try reader.readByte();
+        const valtype = std.meta.intToEnum(WasmType, type_byte) catch |err| {
+            std.debug.print("Invalid wasm type value '{d}'\n", .{type_byte});
+            return err;
+        };
+        try writer.print("type {s}\n", .{@tagName(valtype)});
+    }
+
+    fn parseDumpLimits(reader: anytype, writer: anytype) !void {
+        const flags = try std.leb.readULEB128(u8, reader);
+        const min = try std.leb.readULEB128(u32, reader);
+
+        try writer.print("min {x}\n", .{min});
+        if (flags != 0) {
+            try writer.print("max {x}\n", .{try std.leb.readULEB128(u32, reader)});
+        }
+    }
+
+    fn parseDumpInit(reader: anytype, writer: anytype) !void {
+        const byte = try std.leb.readULEB128(u8, reader);
+        const opcode = std.meta.intToEnum(std.wasm.Opcode, byte) catch |err| {
+            std.debug.print("invalid wasm opcode '{d}'\n", .{byte});
+            return err;
+        };
+        switch (opcode) {
+            .i32_const => try writer.print("i32.const {x}\n", .{try std.leb.readILEB128(i32, reader)}),
+            .i64_const => try writer.print("i64.const {x}\n", .{try std.leb.readILEB128(i64, reader)}),
+            .f32_const => try writer.print("f32.const {x}\n", .{@bitCast(f32, try reader.readIntLittle(u32))}),
+            .f64_const => try writer.print("f64.const {x}\n", .{@bitCast(f64, try reader.readIntLittle(u64))}),
+            .global_get => try writer.print("global.get {x}\n", .{try std.leb.readULEB128(u32, reader)}),
+            else => unreachable,
+        }
+        const end_opcode = try std.leb.readULEB128(u8, reader);
+        if (end_opcode != std.wasm.opcode(.end)) {
+            std.debug.print("expected 'end' opcode in init expression\n", .{});
+            return error.MissingEndOpcode;
+        }
+    }
+
+    fn parseDumpNames(reader: anytype, writer: anytype, data: []const u8) !void {
+        while (reader.context.pos < data.len) {
+            try parseDumpType(std.wasm.NameSubsection, reader, writer);
+            const size = try std.leb.readULEB128(u32, reader);
+            const entries = try std.leb.readULEB128(u32, reader);
+            try writer.print(
+                \\size {d}
+                \\names {d}
+            , .{ size, entries });
+            try writer.writeByte('\n');
+            var i: u32 = 0;
+            while (i < entries) : (i += 1) {
+                const index = try std.leb.readULEB128(u32, reader);
+                const name_len = try std.leb.readULEB128(u32, reader);
+                const pos = reader.context.pos;
+                const name = data[pos..][0..name_len];
+                reader.context.pos += name_len;
+
+                try writer.print(
+                    \\index {d}
+                    \\name {s}
+                , .{ index, name });
+                try writer.writeByte('\n');
+            }
+        }
+    }
+};

src/link.zig

@@ -351,7 +351,7 @@ pub const File = struct {
 
     pub fn makeWritable(base: *File) !void {
         switch (base.tag) {
-            .coff, .elf, .macho, .plan9 => {
+            .coff, .elf, .macho, .plan9, .wasm => {
                 if (base.file != null) return;
                 const emit = base.options.emit orelse return;
                 base.file = try emit.directory.handle.createFile(emit.sub_path, .{
@@ -360,7 +360,7 @@ pub const File = struct {
                     .mode = determineMode(base.options),
                 });
             },
-            .c, .wasm, .spirv, .nvptx => {},
+            .c, .spirv, .nvptx => {},
         }
     }
 
@@ -394,7 +394,7 @@ pub const File = struct {
                     base.file = null;
                 }
             },
-            .coff, .elf, .plan9 => if (base.file) |f| {
+            .coff, .elf, .plan9, .wasm => if (base.file) |f| {
                 if (base.intermediary_basename != null) {
                     // The file we have open is not the final file that we want to
                     // make executable, so we don't have to close it.
@@ -403,7 +403,7 @@ pub const File = struct {
                 f.close();
                 base.file = null;
             },
-            .c, .wasm, .spirv, .nvptx => {},
+            .c, .spirv, .nvptx => {},
         }
     }