wasm-linker: Implement linker tests (#12006)

* test/link: initial wasm support

This adds basic parsing and dumping of wasm section so they
can be tested using the new linker-test infrastructure.

* test/link: all wasm sections parsing and dumping

We now parse and dump all sections for the wasm binary format.
Currently, this only dumps the name of a custom section.
Later this should also dump symbol table, name, linking metadata and relocations.
All of those live within the custom sections.

* Add wasm linker test

This also fixes a parser mistake in reading the flags.

* test/link: implement linker tests wasm & fixes

Adds several test cases to test the wasm self-hosted linker.
This also introduces fixes that were caught during the implementation
of those tests.

* test-runner: obey omit_stage2 for standalone

When a standalone test requires stage2, but stage2 is omit
from the compiler, such test case will not be included as part
of the test suite that is being ran. This is to support CI's
where we omit stage2 to lower the memory usage.
This commit is contained in:
Luuk de Gram 2022-07-12 14:36:33 +02:00 committed by GitHub
parent 7d2e142679
commit 8033767082
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 519 additions and 9 deletions

View File

@ -481,8 +481,8 @@ pub fn build(b: *Builder) !void {
));
toolchain_step.dependOn(tests.addCompareOutputTests(b, test_filter, modes));
toolchain_step.dependOn(tests.addStandaloneTests(b, test_filter, modes, skip_non_native, enable_macos_sdk, target));
toolchain_step.dependOn(tests.addLinkTests(b, test_filter, modes, enable_macos_sdk));
toolchain_step.dependOn(tests.addStandaloneTests(b, test_filter, modes, skip_non_native, enable_macos_sdk, target, omit_stage2));
toolchain_step.dependOn(tests.addLinkTests(b, test_filter, modes, enable_macos_sdk, omit_stage2));
toolchain_step.dependOn(tests.addStackTraceTests(b, test_filter, modes));
toolchain_step.dependOn(tests.addCliTests(b, test_filter, modes));
toolchain_step.dependOn(tests.addAssembleAndLinkTests(b, test_filter, modes));

View File

@ -76,7 +76,7 @@ release/bin/zig build test-run-translated-c -Denable-macos-sdk
release/bin/zig build docs -Denable-macos-sdk
release/bin/zig build test-fmt -Denable-macos-sdk
release/bin/zig build test-cases -Denable-macos-sdk -Dsingle-threaded
release/bin/zig build test-link -Denable-macos-sdk
release/bin/zig build test-link -Denable-macos-sdk -Domit-stage2
if [ "${BUILD_REASON}" != "PullRequest" ]; then
mv ../LICENSE release/

View File

@ -1618,6 +1618,7 @@ pub const LibExeObjStep = struct {
want_lto: ?bool = null,
use_stage1: ?bool = null,
use_llvm: ?bool = null,
use_lld: ?bool = null,
ofmt: ?std.Target.ObjectFormat = null,
output_path_source: GeneratedFile,
@ -2474,6 +2475,14 @@ pub const LibExeObjStep = struct {
}
}
if (self.use_lld) |use_lld| {
if (use_lld) {
try zig_args.append("-fLLD");
} else {
try zig_args.append("-fno-LLD");
}
}
if (self.ofmt) |ofmt| {
try zig_args.append(try std.fmt.allocPrint(builder.allocator, "-ofmt={s}", .{@tagName(ofmt)}));
}

View File

@ -265,7 +265,10 @@ fn make(step: *Step) !void {
}),
.elf => @panic("TODO elf parser"),
.coff => @panic("TODO coff parser"),
.wasm => @panic("TODO wasm parser"),
.wasm => try WasmDumper.parseAndDump(contents, .{
.gpa = gpa,
.dump_symtab = self.dump_symtab,
}),
else => unreachable,
};
@ -522,3 +525,295 @@ const MachODumper = struct {
}
}
};
const WasmDumper = struct {
const symtab_label = "symbols";
fn parseAndDump(bytes: []const u8, opts: Opts) ![]const u8 {
const gpa = opts.gpa orelse unreachable; // Wasm dumper requires an allocator
if (opts.dump_symtab) {
@panic("TODO: Implement symbol table parsing and dumping");
}
var fbs = std.io.fixedBufferStream(bytes);
const reader = fbs.reader();
const buf = try reader.readBytesNoEof(8);
if (!mem.eql(u8, buf[0..4], &std.wasm.magic)) {
return error.InvalidMagicByte;
}
if (!mem.eql(u8, buf[4..], &std.wasm.version)) {
return error.UnsupportedWasmVersion;
}
var output = std.ArrayList(u8).init(gpa);
errdefer output.deinit();
const writer = output.writer();
while (reader.readByte()) |current_byte| {
const section = std.meta.intToEnum(std.wasm.Section, current_byte) catch |err| {
std.debug.print("Found invalid section id '{d}'\n", .{current_byte});
return err;
};
const section_length = try std.leb.readULEB128(u32, reader);
try parseAndDumpSection(section, bytes[fbs.pos..][0..section_length], writer);
fbs.pos += section_length;
} else |_| {} // reached end of stream
return output.toOwnedSlice();
}
fn parseAndDumpSection(section: std.wasm.Section, data: []const u8, writer: anytype) !void {
var fbs = std.io.fixedBufferStream(data);
const reader = fbs.reader();
try writer.print(
\\Section {s}
\\size {d}
, .{ @tagName(section), data.len });
switch (section) {
.type,
.import,
.function,
.table,
.memory,
.global,
.@"export",
.element,
.code,
.data,
=> {
const entries = try std.leb.readULEB128(u32, reader);
try writer.print("\nentries {d}\n", .{entries});
try dumpSection(section, data[fbs.pos..], entries, writer);
},
.custom => {
const name_length = try std.leb.readULEB128(u32, reader);
const name = data[fbs.pos..][0..name_length];
fbs.pos += name_length;
try writer.print("\nname {s}\n", .{name});
if (mem.eql(u8, name, "name")) {
try parseDumpNames(reader, writer, data);
}
// TODO: Implement parsing and dumping other custom sections (such as relocations)
},
.start => {
const start = try std.leb.readULEB128(u32, reader);
try writer.print("\nstart {d}\n", .{start});
},
else => {}, // skip unknown sections
}
}
fn dumpSection(section: std.wasm.Section, data: []const u8, entries: u32, writer: anytype) !void {
var fbs = std.io.fixedBufferStream(data);
const reader = fbs.reader();
switch (section) {
.type => {
var i: u32 = 0;
while (i < entries) : (i += 1) {
const func_type = try reader.readByte();
if (func_type != std.wasm.function_type) {
std.debug.print("Expected function type, found byte '{d}'\n", .{func_type});
return error.UnexpectedByte;
}
const params = try std.leb.readULEB128(u32, reader);
try writer.print("params {d}\n", .{params});
var index: u32 = 0;
while (index < params) : (index += 1) {
try parseDumpType(std.wasm.Valtype, reader, writer);
} else index = 0;
const returns = try std.leb.readULEB128(u32, reader);
try writer.print("returns {d}\n", .{returns});
while (index < returns) : (index += 1) {
try parseDumpType(std.wasm.Valtype, reader, writer);
}
}
},
.import => {
var i: u32 = 0;
while (i < entries) : (i += 1) {
const module_name_len = try std.leb.readULEB128(u32, reader);
const module_name = data[fbs.pos..][0..module_name_len];
fbs.pos += module_name_len;
const name_len = try std.leb.readULEB128(u32, reader);
const name = data[fbs.pos..][0..name_len];
fbs.pos += name_len;
const kind = std.meta.intToEnum(std.wasm.ExternalKind, try reader.readByte()) catch |err| {
std.debug.print("Invalid import kind\n", .{});
return err;
};
try writer.print(
\\module {s}
\\name {s}
\\kind {s}
, .{ module_name, name, @tagName(kind) });
try writer.writeByte('\n');
switch (kind) {
.function => {
try writer.print("index {d}\n", .{try std.leb.readULEB128(u32, reader)});
},
.memory => {
try parseDumpLimits(reader, writer);
},
.global => {
try parseDumpType(std.wasm.Valtype, reader, writer);
try writer.print("mutable {}\n", .{0x01 == try std.leb.readULEB128(u32, reader)});
},
.table => {
try parseDumpType(std.wasm.RefType, reader, writer);
try parseDumpLimits(reader, writer);
},
}
}
},
.function => {
var i: u32 = 0;
while (i < entries) : (i += 1) {
try writer.print("index {d}\n", .{try std.leb.readULEB128(u32, reader)});
}
},
.table => {
var i: u32 = 0;
while (i < entries) : (i += 1) {
try parseDumpType(std.wasm.RefType, reader, writer);
try parseDumpLimits(reader, writer);
}
},
.memory => {
var i: u32 = 0;
while (i < entries) : (i += 1) {
try parseDumpLimits(reader, writer);
}
},
.global => {
var i: u32 = 0;
while (i < entries) : (i += 1) {
try parseDumpType(std.wasm.Valtype, reader, writer);
try writer.print("mutable {}\n", .{0x01 == try std.leb.readULEB128(u1, reader)});
try parseDumpInit(reader, writer);
}
},
.@"export" => {
var i: u32 = 0;
while (i < entries) : (i += 1) {
const name_len = try std.leb.readULEB128(u32, reader);
const name = data[fbs.pos..][0..name_len];
fbs.pos += name_len;
const kind_byte = try std.leb.readULEB128(u8, reader);
const kind = std.meta.intToEnum(std.wasm.ExternalKind, kind_byte) catch |err| {
std.debug.print("invalid export kind value '{d}'\n", .{kind_byte});
return err;
};
const index = try std.leb.readULEB128(u32, reader);
try writer.print(
\\name {s}
\\kind {s}
\\index {d}
, .{ name, @tagName(kind), index });
try writer.writeByte('\n');
}
},
.element => {
var i: u32 = 0;
while (i < entries) : (i += 1) {
try writer.print("table index {d}\n", .{try std.leb.readULEB128(u32, reader)});
try parseDumpInit(reader, writer);
const function_indexes = try std.leb.readULEB128(u32, reader);
var function_index: u32 = 0;
try writer.print("indexes {d}\n", .{function_indexes});
while (function_index < function_indexes) : (function_index += 1) {
try writer.print("index {d}\n", .{try std.leb.readULEB128(u32, reader)});
}
}
},
.code => {}, // code section is considered opaque to linker
.data => {
var i: u32 = 0;
while (i < entries) : (i += 1) {
const index = try std.leb.readULEB128(u32, reader);
try writer.print("memory index 0x{x}\n", .{index});
try parseDumpInit(reader, writer);
const size = try std.leb.readULEB128(u32, reader);
try writer.print("size {d}\n", .{size});
try reader.skipBytes(size, .{}); // we do not care about the content of the segments
}
},
else => unreachable,
}
}
fn parseDumpType(comptime WasmType: type, reader: anytype, writer: anytype) !void {
const type_byte = try reader.readByte();
const valtype = std.meta.intToEnum(WasmType, type_byte) catch |err| {
std.debug.print("Invalid wasm type value '{d}'\n", .{type_byte});
return err;
};
try writer.print("type {s}\n", .{@tagName(valtype)});
}
fn parseDumpLimits(reader: anytype, writer: anytype) !void {
const flags = try std.leb.readULEB128(u8, reader);
const min = try std.leb.readULEB128(u32, reader);
try writer.print("min {x}\n", .{min});
if (flags != 0) {
try writer.print("max {x}\n", .{try std.leb.readULEB128(u32, reader)});
}
}
fn parseDumpInit(reader: anytype, writer: anytype) !void {
const byte = try std.leb.readULEB128(u8, reader);
const opcode = std.meta.intToEnum(std.wasm.Opcode, byte) catch |err| {
std.debug.print("invalid wasm opcode '{d}'\n", .{byte});
return err;
};
switch (opcode) {
.i32_const => try writer.print("i32.const {x}\n", .{try std.leb.readILEB128(i32, reader)}),
.i64_const => try writer.print("i64.const {x}\n", .{try std.leb.readILEB128(i64, reader)}),
.f32_const => try writer.print("f32.const {x}\n", .{@bitCast(f32, try reader.readIntLittle(u32))}),
.f64_const => try writer.print("f64.const {x}\n", .{@bitCast(f64, try reader.readIntLittle(u64))}),
.global_get => try writer.print("global.get {x}\n", .{try std.leb.readULEB128(u32, reader)}),
else => unreachable,
}
const end_opcode = try std.leb.readULEB128(u8, reader);
if (end_opcode != std.wasm.opcode(.end)) {
std.debug.print("expected 'end' opcode in init expression\n", .{});
return error.MissingEndOpcode;
}
}
fn parseDumpNames(reader: anytype, writer: anytype, data: []const u8) !void {
while (reader.context.pos < data.len) {
try parseDumpType(std.wasm.NameSubsection, reader, writer);
const size = try std.leb.readULEB128(u32, reader);
const entries = try std.leb.readULEB128(u32, reader);
try writer.print(
\\size {d}
\\names {d}
, .{ size, entries });
try writer.writeByte('\n');
var i: u32 = 0;
while (i < entries) : (i += 1) {
const index = try std.leb.readULEB128(u32, reader);
const name_len = try std.leb.readULEB128(u32, reader);
const pos = reader.context.pos;
const name = data[pos..][0..name_len];
reader.context.pos += name_len;
try writer.print(
\\index {d}
\\name {s}
, .{ index, name });
try writer.writeByte('\n');
}
}
}
};

View File

@ -351,7 +351,7 @@ pub const File = struct {
pub fn makeWritable(base: *File) !void {
switch (base.tag) {
.coff, .elf, .macho, .plan9 => {
.coff, .elf, .macho, .plan9, .wasm => {
if (base.file != null) return;
const emit = base.options.emit orelse return;
base.file = try emit.directory.handle.createFile(emit.sub_path, .{
@ -360,7 +360,7 @@ pub const File = struct {
.mode = determineMode(base.options),
});
},
.c, .wasm, .spirv, .nvptx => {},
.c, .spirv, .nvptx => {},
}
}
@ -394,7 +394,7 @@ pub const File = struct {
base.file = null;
}
},
.coff, .elf, .plan9 => if (base.file) |f| {
.coff, .elf, .plan9, .wasm => if (base.file) |f| {
if (base.intermediary_basename != null) {
// The file we have open is not the final file that we want to
// make executable, so we don't have to close it.
@ -403,7 +403,7 @@ pub const File = struct {
f.close();
base.file = null;
},
.c, .wasm, .spirv, .nvptx => {},
.c, .spirv, .nvptx => {},
}
}

View File

@ -1206,6 +1206,14 @@ fn parseAtom(self: *Wasm, atom: *Atom, kind: Kind) !void {
};
symbol.tag = .data;
// when creating an object file, or importing memory and the data belongs in the .bss segment
// we set the entire region of it to zeroes.
// We do not have to do this when exporting the memory (the default) because the runtime
// will do it for us, and we do not emit the bss segment at all.
if ((self.base.options.output_mode == .Obj or self.base.options.import_memory) and kind.data == .uninitialized) {
std.mem.set(u8, atom.code.items, 0);
}
const should_merge = self.base.options.output_mode != .Obj;
const gop = try self.data_segments.getOrPut(self.base.allocator, segment_info.outputName(should_merge));
if (gop.found_existing) {
@ -2014,9 +2022,10 @@ pub fn flushModule(self: *Wasm, comp: *Compilation, prog_node: *std.Progress.Nod
}
if (import_memory) {
const mem_name = if (is_obj) "__linear_memory" else "memory";
const mem_imp: types.Import = .{
.module_name = try self.string_table.put(self.base.allocator, self.host_name),
.name = try self.string_table.put(self.base.allocator, "__linear_memory"),
.name = try self.string_table.put(self.base.allocator, mem_name),
.kind = .{ .memory = self.memories.limits },
};
try self.emitImport(writer, mem_imp);

View File

@ -27,6 +27,26 @@ pub fn addCases(cases: *tests.StandaloneContext) void {
.build_modes = true,
});
cases.addBuildFile("test/link/wasm/type/build.zig", .{
.build_modes = true,
.requires_stage2 = true,
});
cases.addBuildFile("test/link/wasm/segments/build.zig", .{
.build_modes = true,
.requires_stage2 = true,
});
cases.addBuildFile("test/link/wasm/stack_pointer/build.zig", .{
.build_modes = true,
.requires_stage2 = true,
});
cases.addBuildFile("test/link/wasm/bss/build.zig", .{
.build_modes = true,
.requires_stage2 = true,
});
if (builtin.os.tag == .macos) {
cases.addBuildFile("test/link/macho/entry/build.zig", .{
.build_modes = true,

View File

@ -0,0 +1,41 @@
const std = @import("std");
const Builder = std.build.Builder;
pub fn build(b: *Builder) void {
const mode = b.standardReleaseOptions();
const test_step = b.step("test", "Test");
test_step.dependOn(b.getInstallStep());
const lib = b.addSharedLibrary("lib", "lib.zig", .unversioned);
lib.setBuildMode(mode);
lib.setTarget(.{ .cpu_arch = .wasm32, .os_tag = .freestanding });
lib.use_llvm = false;
lib.use_stage1 = false;
lib.use_lld = false;
// to make sure the bss segment is emitted, we must import memory
lib.import_memory = true;
lib.install();
const check_lib = lib.checkObject(.wasm);
// since we import memory, make sure it exists with the correct naming
check_lib.checkStart("Section import");
check_lib.checkNext("entries 1");
check_lib.checkNext("module env"); // default module name is "env"
check_lib.checkNext("name memory"); // as per linker specification
// since we are importing memory, ensure it's not exported
check_lib.checkStart("Section export");
check_lib.checkNext("entries 1"); // we're exporting function 'foo' so only 1 entry
// validate the name of the stack pointer
check_lib.checkStart("Section custom");
check_lib.checkNext("type data_segment");
check_lib.checkNext("names 2");
check_lib.checkNext("index 0");
check_lib.checkNext("name .rodata");
check_lib.checkNext("index 1"); // bss section always last
check_lib.checkNext("name .bss");
test_step.dependOn(&check_lib.step);
}

View File

@ -0,0 +1,5 @@
pub var bss: u32 = undefined;
export fn foo() void {
_ = bss;
}

View File

@ -0,0 +1,31 @@
const std = @import("std");
const Builder = std.build.Builder;
pub fn build(b: *Builder) void {
const mode = b.standardReleaseOptions();
const test_step = b.step("test", "Test");
test_step.dependOn(b.getInstallStep());
const lib = b.addSharedLibrary("lib", "lib.zig", .unversioned);
lib.setBuildMode(mode);
lib.setTarget(.{ .cpu_arch = .wasm32, .os_tag = .freestanding });
lib.use_llvm = false;
lib.use_stage1 = false;
lib.use_lld = false;
lib.install();
const check_lib = lib.checkObject(.wasm);
check_lib.checkStart("Section data");
check_lib.checkNext("entries 2"); // rodata & data, no bss because we're exporting memory
check_lib.checkStart("Section custom");
check_lib.checkStart("name name"); // names custom section
check_lib.checkStart("type data_segment");
check_lib.checkNext("names 2");
check_lib.checkNext("index 0");
check_lib.checkNext("name .rodata");
check_lib.checkNext("index 1");
check_lib.checkNext("name .data");
test_step.dependOn(&check_lib.step);
}

View File

@ -0,0 +1,9 @@
pub const rodata: u32 = 5;
pub var data: u32 = 10;
pub var bss: u32 = undefined;
export fn foo() void {
_ = rodata;
_ = data;
_ = bss;
}

View File

@ -0,0 +1,41 @@
const std = @import("std");
const Builder = std.build.Builder;
pub fn build(b: *Builder) void {
const mode = b.standardReleaseOptions();
const test_step = b.step("test", "Test");
test_step.dependOn(b.getInstallStep());
const lib = b.addSharedLibrary("lib", "lib.zig", .unversioned);
lib.setBuildMode(mode);
lib.setTarget(.{ .cpu_arch = .wasm32, .os_tag = .freestanding });
lib.use_llvm = false;
lib.use_stage1 = false;
lib.use_lld = false;
lib.stack_size = std.wasm.page_size * 2; // set an explicit stack size
lib.install();
const check_lib = lib.checkObject(.wasm);
// ensure global exists and its initial value is equal to explitic stack size
check_lib.checkStart("Section global");
check_lib.checkNext("entries 1");
check_lib.checkNext("type i32"); // on wasm32 the stack pointer must be i32
check_lib.checkNext("mutable true"); // must be able to mutate the stack pointer
check_lib.checkNext("i32.const {stack_pointer}");
check_lib.checkComputeCompare("stack_pointer", .{ .op = .eq, .value = .{ .literal = lib.stack_size.? } });
// validate memory section starts after virtual stack
check_lib.checkNext("Section data");
check_lib.checkNext("i32.const {data_start}");
check_lib.checkComputeCompare("data_start", .{ .op = .eq, .value = .{ .variable = "stack_pointer" } });
// validate the name of the stack pointer
check_lib.checkStart("Section custom");
check_lib.checkNext("type global");
check_lib.checkNext("names 1");
check_lib.checkNext("index 0");
check_lib.checkNext("name __stack_pointer");
test_step.dependOn(&check_lib.step);
}

View File

@ -0,0 +1 @@
export fn foo() void {}

View File

@ -0,0 +1,32 @@
const std = @import("std");
const Builder = std.build.Builder;
pub fn build(b: *Builder) void {
const mode = b.standardReleaseOptions();
const test_step = b.step("test", "Test");
test_step.dependOn(b.getInstallStep());
const lib = b.addSharedLibrary("lib", "lib.zig", .unversioned);
lib.setBuildMode(mode);
lib.setTarget(.{ .cpu_arch = .wasm32, .os_tag = .freestanding });
lib.use_llvm = false;
lib.use_stage1 = false;
lib.use_lld = false;
lib.install();
const check_lib = lib.checkObject(.wasm);
check_lib.checkStart("Section type");
// only 2 entries, although we have 3 functions.
// This is to test functions with the same function signature
// have their types deduplicated.
check_lib.checkNext("entries 2");
check_lib.checkNext("params 1");
check_lib.checkNext("type i32");
check_lib.checkNext("returns 1");
check_lib.checkNext("type i64");
check_lib.checkNext("params 0");
check_lib.checkNext("returns 0");
test_step.dependOn(&check_lib.step);
}

View File

@ -0,0 +1,10 @@
export fn foo(x: u32) u64 {
return bar(x);
}
fn bar(x: u32) u64 {
y();
return x;
}
fn y() void {}

View File

@ -462,6 +462,7 @@ pub fn addStandaloneTests(
skip_non_native: bool,
enable_macos_sdk: bool,
target: std.zig.CrossTarget,
omit_stage2: bool,
) *build.Step {
const cases = b.allocator.create(StandaloneContext) catch unreachable;
cases.* = StandaloneContext{
@ -473,6 +474,7 @@ pub fn addStandaloneTests(
.skip_non_native = skip_non_native,
.enable_macos_sdk = enable_macos_sdk,
.target = target,
.omit_stage2 = omit_stage2,
};
standalone.addCases(cases);
@ -485,6 +487,7 @@ pub fn addLinkTests(
test_filter: ?[]const u8,
modes: []const Mode,
enable_macos_sdk: bool,
omit_stage2: bool,
) *build.Step {
const cases = b.allocator.create(StandaloneContext) catch unreachable;
cases.* = StandaloneContext{
@ -496,6 +499,7 @@ pub fn addLinkTests(
.skip_non_native = true,
.enable_macos_sdk = enable_macos_sdk,
.target = .{},
.omit_stage2 = omit_stage2,
};
link.addCases(cases);
return cases.step;
@ -957,6 +961,7 @@ pub const StandaloneContext = struct {
skip_non_native: bool,
enable_macos_sdk: bool,
target: std.zig.CrossTarget,
omit_stage2: bool,
pub fn addC(self: *StandaloneContext, root_src: []const u8) void {
self.addAllArgs(root_src, true);
@ -970,10 +975,12 @@ pub const StandaloneContext = struct {
build_modes: bool = false,
cross_targets: bool = false,
requires_macos_sdk: bool = false,
requires_stage2: bool = false,
}) void {
const b = self.b;
if (features.requires_macos_sdk and !self.enable_macos_sdk) return;
if (features.requires_stage2 and self.omit_stage2) return;
const annotated_case_name = b.fmt("build {s}", .{build_file});
if (self.test_filter) |filter| {