From d99dc21b9f00dffcf919188ef6751ee85a2c662e Mon Sep 17 00:00:00 2001 From: Michael Dusan Date: Sat, 10 Apr 2021 19:51:27 -0400 Subject: [PATCH] test: overhaul stack_trace testing - limit expected-output to main source file; ie. tolerate changes to start.zig - when mode != .Debug the function name is now symbolically represented; ie. tolerate changes in llvm optimizer effects on the callstack - cleanup how test cases are specified - add test case predicates for excluding by arch, os or custom fn --- test/stack_traces.zig | 617 +++++++++++------------------------------- test/tests.zig | 133 ++++++--- 2 files changed, 257 insertions(+), 493 deletions(-) diff --git a/test/stack_traces.zig b/test/stack_traces.zig index 190e216563..cb2cf94b70 100644 --- a/test/stack_traces.zig +++ b/test/stack_traces.zig @@ -2,18 +2,52 @@ const std = @import("std"); const os = std.os; const tests = @import("tests.zig"); -// zig fmt: off pub fn addCases(cases: *tests.StackTracesContext) void { - const source_return = - \\const std = @import("std"); - \\ + cases.addCase(.{ + .name = "return", + .source = \\pub fn main() !void { \\ return error.TheSkyIsFalling; \\} - ; - const source_try_return = - \\const std = @import("std"); - \\ + , + .Debug = .{ + .expect = + \\error: TheSkyIsFalling + \\source.zig:2:5: [address] in main (test) + \\ return error.TheSkyIsFalling; + \\ ^ + \\ + , + }, + .ReleaseSafe = .{ + .exclude_os = .{ + .windows, // segfault + }, + .expect = + \\error: TheSkyIsFalling + \\source.zig:2:5: [address] in [function] + \\ return error.TheSkyIsFalling; + \\ ^ + \\ + , + }, + .ReleaseFast = .{ + .expect = + \\error: TheSkyIsFalling + \\ + , + }, + .ReleaseSmall = .{ + .expect = + \\error: TheSkyIsFalling + \\ + , + }, + }); + + cases.addCase(.{ + .name = "try return", + .source = \\fn foo() !void { \\ return error.TheSkyIsFalling; \\} @@ -21,10 +55,51 @@ pub fn addCases(cases: *tests.StackTracesContext) void { \\pub fn main() !void { \\ try foo(); \\} - ; - const source_try_try_return_return = - \\const std = @import("std"); - \\ + , + .Debug = .{ + .expect = + \\error: TheSkyIsFalling + \\source.zig:2:5: [address] in foo (test) + \\ return error.TheSkyIsFalling; + \\ ^ + \\source.zig:6:5: [address] in main (test) + \\ try foo(); + \\ ^ + \\ + , + }, + .ReleaseSafe = .{ + .exclude_os = .{ + .windows, // segfault + }, + .expect = + \\error: TheSkyIsFalling + \\source.zig:2:5: [address] in [function] + \\ return error.TheSkyIsFalling; + \\ ^ + \\source.zig:6:5: [address] in [function] + \\ try foo(); + \\ ^ + \\ + , + }, + .ReleaseFast = .{ + .expect = + \\error: TheSkyIsFalling + \\ + , + }, + .ReleaseSmall = .{ + .expect = + \\error: TheSkyIsFalling + \\ + , + }, + }); + + cases.addCase(.{ + .name = "try try return return", + .source = \\fn foo() !void { \\ try bar(); \\} @@ -40,9 +115,66 @@ pub fn addCases(cases: *tests.StackTracesContext) void { \\pub fn main() !void { \\ try foo(); \\} - ; + , + .Debug = .{ + .expect = + \\error: TheSkyIsFalling + \\source.zig:10:5: [address] in make_error (test) + \\ return error.TheSkyIsFalling; + \\ ^ + \\source.zig:6:5: [address] in bar (test) + \\ return make_error(); + \\ ^ + \\source.zig:2:5: [address] in foo (test) + \\ try bar(); + \\ ^ + \\source.zig:14:5: [address] in main (test) + \\ try foo(); + \\ ^ + \\ + , + }, + .ReleaseSafe = .{ + .exclude_os = .{ + .windows, // segfault + }, + .expect = + \\error: TheSkyIsFalling + \\source.zig:10:5: [address] in [function] + \\ return error.TheSkyIsFalling; + \\ ^ + \\source.zig:6:5: [address] in [function] + \\ return make_error(); + \\ ^ + \\source.zig:2:5: [address] in [function] + \\ try bar(); + \\ ^ + \\source.zig:14:5: [address] in [function] + \\ try foo(); + \\ ^ + \\ + , + }, + .ReleaseFast = .{ + .expect = + \\error: TheSkyIsFalling + \\ + , + }, + .ReleaseSmall = .{ + .expect = + \\error: TheSkyIsFalling + \\ + , + }, + }); - const source_dumpCurrentStackTrace = + cases.addCase(.{ + .exclude_os = .{ + .windows, + }, + .name = "dumpCurrentStackTrace", + .source = \\const std = @import("std"); \\ \\fn bar() void { @@ -55,450 +187,17 @@ pub fn addCases(cases: *tests.StackTracesContext) void { \\ foo(); \\ return 1; \\} - ; - - switch (std.Target.current.os.tag) { - .freebsd => { - cases.addCase( - "return", - source_return, - [_][]const u8{ - // debug - \\error: TheSkyIsFalling - \\source.zig:4:5: [address] in main (test) - \\ return error.TheSkyIsFalling; - \\ ^ - \\ - , - // release-safe - \\error: TheSkyIsFalling - \\source.zig:4:5: [address] in std.start.main (test) - \\ return error.TheSkyIsFalling; - \\ ^ - \\ - , - // release-fast - \\error: TheSkyIsFalling - \\ - , - // release-small - \\error: TheSkyIsFalling - \\ - }, - ); - cases.addCase( - "try return", - source_try_return, - [_][]const u8{ - // debug - \\error: TheSkyIsFalling - \\source.zig:4:5: [address] in foo (test) - \\ return error.TheSkyIsFalling; - \\ ^ - \\source.zig:8:5: [address] in main (test) - \\ try foo(); - \\ ^ - \\ - , - // release-safe - \\error: TheSkyIsFalling - \\source.zig:4:5: [address] in std.start.main (test) - \\ return error.TheSkyIsFalling; - \\ ^ - \\source.zig:8:5: [address] in std.start.main (test) - \\ try foo(); - \\ ^ - \\ - , - // release-fast - \\error: TheSkyIsFalling - \\ - , - // release-small - \\error: TheSkyIsFalling - \\ - }, - ); - cases.addCase( - "try try return return", - source_try_try_return_return, - [_][]const u8{ - // debug - \\error: TheSkyIsFalling - \\source.zig:12:5: [address] in make_error (test) - \\ return error.TheSkyIsFalling; - \\ ^ - \\source.zig:8:5: [address] in bar (test) - \\ return make_error(); - \\ ^ - \\source.zig:4:5: [address] in foo (test) - \\ try bar(); - \\ ^ - \\source.zig:16:5: [address] in main (test) - \\ try foo(); - \\ ^ - \\ - , - // release-safe - \\error: TheSkyIsFalling - \\source.zig:12:5: [address] in std.start.main (test) - \\ return error.TheSkyIsFalling; - \\ ^ - \\source.zig:8:5: [address] in std.start.main (test) - \\ return make_error(); - \\ ^ - \\source.zig:4:5: [address] in std.start.main (test) - \\ try bar(); - \\ ^ - \\source.zig:16:5: [address] in std.start.main (test) - \\ try foo(); - \\ ^ - \\ - , - // release-fast - \\error: TheSkyIsFalling - \\ - , - // release-small - \\error: TheSkyIsFalling - \\ - }, - ); + , + .Debug = .{ + .expect = + \\source.zig:7:8: [address] in foo (test) + \\ bar(); + \\ ^ + \\source.zig:10:8: [address] in main (test) + \\ foo(); + \\ ^ + \\ + , }, - .linux => { - cases.addCase( - "return", - source_return, - [_][]const u8{ - // debug - \\error: TheSkyIsFalling - \\source.zig:4:5: [address] in main (test) - \\ return error.TheSkyIsFalling; - \\ ^ - \\ - , - // release-safe - \\error: TheSkyIsFalling - \\source.zig:4:5: [address] in std.start.posixCallMainAndExit (test) - \\ return error.TheSkyIsFalling; - \\ ^ - \\ - , - // release-fast - \\error: TheSkyIsFalling - \\ - , - // release-small - \\error: TheSkyIsFalling - \\ - }, - ); - cases.addCase( - "try return", - source_try_return, - [_][]const u8{ - // debug - \\error: TheSkyIsFalling - \\source.zig:4:5: [address] in foo (test) - \\ return error.TheSkyIsFalling; - \\ ^ - \\source.zig:8:5: [address] in main (test) - \\ try foo(); - \\ ^ - \\ - , - // release-safe - \\error: TheSkyIsFalling - \\source.zig:4:5: [address] in std.start.posixCallMainAndExit (test) - \\ return error.TheSkyIsFalling; - \\ ^ - \\source.zig:8:5: [address] in std.start.posixCallMainAndExit (test) - \\ try foo(); - \\ ^ - \\ - , - // release-fast - \\error: TheSkyIsFalling - \\ - , - // release-small - \\error: TheSkyIsFalling - \\ - }, - ); - cases.addCase( - "try try return return", - source_try_try_return_return, - [_][]const u8{ - // debug - \\error: TheSkyIsFalling - \\source.zig:12:5: [address] in make_error (test) - \\ return error.TheSkyIsFalling; - \\ ^ - \\source.zig:8:5: [address] in bar (test) - \\ return make_error(); - \\ ^ - \\source.zig:4:5: [address] in foo (test) - \\ try bar(); - \\ ^ - \\source.zig:16:5: [address] in main (test) - \\ try foo(); - \\ ^ - \\ - , - // release-safe - \\error: TheSkyIsFalling - \\source.zig:12:5: [address] in std.start.posixCallMainAndExit (test) - \\ return error.TheSkyIsFalling; - \\ ^ - \\source.zig:8:5: [address] in std.start.posixCallMainAndExit (test) - \\ return make_error(); - \\ ^ - \\source.zig:4:5: [address] in std.start.posixCallMainAndExit (test) - \\ try bar(); - \\ ^ - \\source.zig:16:5: [address] in std.start.posixCallMainAndExit (test) - \\ try foo(); - \\ ^ - \\ - , - // release-fast - \\error: TheSkyIsFalling - \\ - , - // release-small - \\error: TheSkyIsFalling - \\ - }, - ); - cases.addCase( - "dumpCurrentStackTrace", - source_dumpCurrentStackTrace, - [_][]const u8{ - // debug - \\source.zig:7:8: [address] in foo (test) - \\ bar(); - \\ ^ - \\source.zig:10:8: [address] in main (test) - \\ foo(); - \\ ^ - \\start.zig:404:29: [address] in std.start.posixCallMainAndExit (test) - \\ return root.main(); - \\ ^ - \\start.zig:225:5: [address] in std.start._start (test) - \\ @call(.{ .modifier = .never_inline }, posixCallMainAndExit, .{}); - \\ ^ - \\ - , - // release-safe - switch (std.Target.current.cpu.arch) { - .aarch64 => "", // TODO disabled; results in segfault - else => - \\start.zig:225:5: [address] in std.start._start (test) - \\ @call(.{ .modifier = .never_inline }, posixCallMainAndExit, .{}); - \\ ^ - \\ - , - }, - // release-fast - \\ - , - // release-small - \\ - }, - ); - }, - .macos => { - cases.addCase( - "return", - source_return, - [_][]const u8{ - // debug - \\error: TheSkyIsFalling - \\source.zig:4:5: [address] in main (test) - \\ return error.TheSkyIsFalling; - \\ ^ - \\ - , - // release-safe - \\error: TheSkyIsFalling - \\source.zig:4:5: [address] in std.start.main (test) - \\ return error.TheSkyIsFalling; - \\ ^ - \\ - , - // release-fast - \\error: TheSkyIsFalling - \\ - , - // release-small - \\error: TheSkyIsFalling - \\ - }, - ); - cases.addCase( - "try return", - source_try_return, - [_][]const u8{ - // debug - \\error: TheSkyIsFalling - \\source.zig:4:5: [address] in foo (test) - \\ return error.TheSkyIsFalling; - \\ ^ - \\source.zig:8:5: [address] in main (test) - \\ try foo(); - \\ ^ - \\ - , - // release-safe - \\error: TheSkyIsFalling - \\source.zig:4:5: [address] in std.start.main (test) - \\ return error.TheSkyIsFalling; - \\ ^ - \\source.zig:8:5: [address] in std.start.main (test) - \\ try foo(); - \\ ^ - \\ - , - // release-fast - \\error: TheSkyIsFalling - \\ - , - // release-small - \\error: TheSkyIsFalling - \\ - }, - ); - cases.addCase( - "try try return return", - source_try_try_return_return, - [_][]const u8{ - // debug - \\error: TheSkyIsFalling - \\source.zig:12:5: [address] in make_error (test) - \\ return error.TheSkyIsFalling; - \\ ^ - \\source.zig:8:5: [address] in bar (test) - \\ return make_error(); - \\ ^ - \\source.zig:4:5: [address] in foo (test) - \\ try bar(); - \\ ^ - \\source.zig:16:5: [address] in main (test) - \\ try foo(); - \\ ^ - \\ - , - // release-safe - \\error: TheSkyIsFalling - \\source.zig:12:5: [address] in std.start.main (test) - \\ return error.TheSkyIsFalling; - \\ ^ - \\source.zig:8:5: [address] in std.start.main (test) - \\ return make_error(); - \\ ^ - \\source.zig:4:5: [address] in std.start.main (test) - \\ try bar(); - \\ ^ - \\source.zig:16:5: [address] in std.start.main (test) - \\ try foo(); - \\ ^ - \\ - , - // release-fast - \\error: TheSkyIsFalling - \\ - , - // release-small - \\error: TheSkyIsFalling - \\ - }, - ); - }, - .windows => { - cases.addCase( - "return", - source_return, - [_][]const u8{ - // debug - \\error: TheSkyIsFalling - \\source.zig:4:5: [address] in main (test.obj) - \\ return error.TheSkyIsFalling; - \\ ^ - \\ - , - // release-safe - // --disabled-- results in segmenetation fault - "", - // release-fast - \\error: TheSkyIsFalling - \\ - , - // release-small - \\error: TheSkyIsFalling - \\ - }, - ); - cases.addCase( - "try return", - source_try_return, - [_][]const u8{ - // debug - \\error: TheSkyIsFalling - \\source.zig:4:5: [address] in foo (test.obj) - \\ return error.TheSkyIsFalling; - \\ ^ - \\source.zig:8:5: [address] in main (test.obj) - \\ try foo(); - \\ ^ - \\ - , - // release-safe - // --disabled-- results in segmenetation fault - "", - // release-fast - \\error: TheSkyIsFalling - \\ - , - // release-small - \\error: TheSkyIsFalling - \\ - }, - ); - cases.addCase( - "try try return return", - source_try_try_return_return, - [_][]const u8{ - // debug - \\error: TheSkyIsFalling - \\source.zig:12:5: [address] in make_error (test.obj) - \\ return error.TheSkyIsFalling; - \\ ^ - \\source.zig:8:5: [address] in bar (test.obj) - \\ return make_error(); - \\ ^ - \\source.zig:4:5: [address] in foo (test.obj) - \\ try bar(); - \\ ^ - \\source.zig:16:5: [address] in main (test.obj) - \\ try foo(); - \\ ^ - \\ - , - // release-safe - // --disabled-- results in segmenetation fault - "", - // release-fast - \\error: TheSkyIsFalling - \\ - , - // release-small - \\error: TheSkyIsFalling - \\ - }, - ); - }, - else => {}, - } + }); } -// zig fmt: off diff --git a/test/tests.zig b/test/tests.zig index a0a50d29a5..a7d7952aca 100644 --- a/test/tests.zig +++ b/test/tests.zig @@ -558,42 +558,87 @@ pub const StackTracesContext = struct { const Expect = [@typeInfo(Mode).Enum.fields.len][]const u8; - pub fn addCase( + pub fn addCase(self: *StackTracesContext, config: anytype) void { + if (@hasField(@TypeOf(config), "exclude")) { + if (config.exclude.exclude()) return; + } + if (@hasField(@TypeOf(config), "exclude_arch")) { + const exclude_arch: []const builtin.Cpu.Arch = &config.exclude_arch; + for (exclude_arch) |arch| if (arch == builtin.cpu.arch) return; + } + if (@hasField(@TypeOf(config), "exclude_os")) { + const exclude_os: []const builtin.Os.Tag = &config.exclude_os; + for (exclude_os) |os| if (os == builtin.os.tag) return; + } + for (self.modes) |mode| { + switch (mode) { + .Debug => { + if (@hasField(@TypeOf(config), "Debug")) { + self.addExpect(config.name, config.source, mode, config.Debug); + } + }, + .ReleaseSafe => { + if (@hasField(@TypeOf(config), "ReleaseSafe")) { + self.addExpect(config.name, config.source, mode, config.ReleaseSafe); + } + }, + .ReleaseFast => { + if (@hasField(@TypeOf(config), "ReleaseFast")) { + self.addExpect(config.name, config.source, mode, config.ReleaseFast); + } + }, + .ReleaseSmall => { + if (@hasField(@TypeOf(config), "ReleaseSmall")) { + self.addExpect(config.name, config.source, mode, config.ReleaseSmall); + } + }, + } + } + } + + fn addExpect( self: *StackTracesContext, name: []const u8, source: []const u8, - expect: Expect, + mode: Mode, + mode_config: anytype, ) void { - const b = self.b; - - for (self.modes) |mode| { - const expect_for_mode = expect[@enumToInt(mode)]; - if (expect_for_mode.len == 0) continue; - - const annotated_case_name = fmt.allocPrint(self.b.allocator, "{s} {s} ({s})", .{ - "stack-trace", - name, - @tagName(mode), - }) catch unreachable; - if (self.test_filter) |filter| { - if (mem.indexOf(u8, annotated_case_name, filter) == null) continue; - } - - const src_basename = "source.zig"; - const write_src = b.addWriteFile(src_basename, source); - const exe = b.addExecutableFromWriteFileStep("test", write_src, src_basename); - exe.setBuildMode(mode); - - const run_and_compare = RunAndCompareStep.create( - self, - exe, - annotated_case_name, - mode, - expect_for_mode, - ); - - self.step.dependOn(&run_and_compare.step); + if (@hasField(@TypeOf(mode_config), "exclude")) { + if (mode_config.exclude.exclude()) return; } + if (@hasField(@TypeOf(mode_config), "exclude_arch")) { + const exclude_arch: []const builtin.Cpu.Arch = &mode_config.exclude_arch; + for (exclude_arch) |arch| if (arch == builtin.cpu.arch) return; + } + if (@hasField(@TypeOf(mode_config), "exclude_os")) { + const exclude_os: []const builtin.Os.Tag = &mode_config.exclude_os; + for (exclude_os) |os| if (os == builtin.os.tag) return; + } + + const annotated_case_name = fmt.allocPrint(self.b.allocator, "{s} {s} ({s})", .{ + "stack-trace", + name, + @tagName(mode), + }) catch unreachable; + if (self.test_filter) |filter| { + if (mem.indexOf(u8, annotated_case_name, filter) == null) return; + } + + const b = self.b; + const src_basename = "source.zig"; + const write_src = b.addWriteFile(src_basename, source); + const exe = b.addExecutableFromWriteFileStep("test", write_src, src_basename); + exe.setBuildMode(mode); + + const run_and_compare = RunAndCompareStep.create( + self, + exe, + annotated_case_name, + mode, + mode_config.expect, + ); + + self.step.dependOn(&run_and_compare.step); } const RunAndCompareStep = struct { @@ -695,6 +740,7 @@ pub const StackTracesContext = struct { // process result // - keep only basename of source file path // - replace address with symbolic string + // - replace function name with symbolic string when mode != .Debug // - skip empty lines const got: []const u8 = got_result: { var buf = ArrayList(u8).init(b.allocator); @@ -703,26 +749,45 @@ pub const StackTracesContext = struct { var it = mem.split(stderr, "\n"); process_lines: while (it.next()) |line| { if (line.len == 0) continue; - const delims = [_][]const u8{ ":", ":", ":", " in " }; - var marks = [_]usize{0} ** 4; // offset search past `[drive]:` on windows var pos: usize = if (std.Target.current.os.tag == .windows) 2 else 0; + // locate delims/anchor + const delims = [_][]const u8{ ":", ":", ":", " in ", "(", ")" }; + var marks = [_]usize{0} ** delims.len; for (delims) |delim, i| { marks[i] = mem.indexOfPos(u8, line, pos, delim) orelse { + // unexpected pattern: emit raw line and cont try buf.appendSlice(line); try buf.appendSlice("\n"); continue :process_lines; }; pos = marks[i] + delim.len; } + // locate source basename pos = mem.lastIndexOfScalar(u8, line[0..marks[0]], fs.path.sep) orelse { + // unexpected pattern: emit raw line and cont try buf.appendSlice(line); try buf.appendSlice("\n"); continue :process_lines; }; + // end processing if source basename changes + if (!mem.eql(u8, "source.zig", line[pos + 1 .. marks[0]])) break; + // emit substituted line try buf.appendSlice(line[pos + 1 .. marks[2] + delims[2].len]); try buf.appendSlice(" [address]"); - try buf.appendSlice(line[marks[3]..]); + if (self.mode == .Debug) { + if (mem.lastIndexOfScalar(u8, line[marks[4]..marks[5]], '.')) |idot| { + // On certain platforms (windows) or possibly depending on how we choose to link main + // the object file extension may be present so we simply strip any extension. + try buf.appendSlice(line[marks[3] .. marks[4] + idot]); + try buf.appendSlice(line[marks[5]..]); + } else { + try buf.appendSlice(line[marks[3]..]); + } + } else { + try buf.appendSlice(line[marks[3] .. marks[3] + delims[3].len]); + try buf.appendSlice("[function]"); + } try buf.appendSlice("\n"); } break :got_result buf.toOwnedSlice();