test: overhaul stack_trace testing

- limit expected-output to main source file;
  ie. tolerate changes to start.zig
- when mode != .Debug the function name is now symbolically represented;
  ie. tolerate changes in llvm optimizer effects on the callstack
- cleanup how test cases are specified
- add test case predicates for excluding by arch, os or custom fn
This commit is contained in:
Michael Dusan 2021-04-10 19:51:27 -04:00
parent 75f86dab3f
commit d99dc21b9f
2 changed files with 257 additions and 493 deletions

View File

@ -2,18 +2,52 @@ const std = @import("std");
const os = std.os;
const tests = @import("tests.zig");
// zig fmt: off
pub fn addCases(cases: *tests.StackTracesContext) void {
const source_return =
\\const std = @import("std");
\\
cases.addCase(.{
.name = "return",
.source =
\\pub fn main() !void {
\\ return error.TheSkyIsFalling;
\\}
;
const source_try_return =
\\const std = @import("std");
\\
,
.Debug = .{
.expect =
\\error: TheSkyIsFalling
\\source.zig:2:5: [address] in main (test)
\\ return error.TheSkyIsFalling;
\\ ^
\\
,
},
.ReleaseSafe = .{
.exclude_os = .{
.windows, // segfault
},
.expect =
\\error: TheSkyIsFalling
\\source.zig:2:5: [address] in [function]
\\ return error.TheSkyIsFalling;
\\ ^
\\
,
},
.ReleaseFast = .{
.expect =
\\error: TheSkyIsFalling
\\
,
},
.ReleaseSmall = .{
.expect =
\\error: TheSkyIsFalling
\\
,
},
});
cases.addCase(.{
.name = "try return",
.source =
\\fn foo() !void {
\\ return error.TheSkyIsFalling;
\\}
@ -21,10 +55,51 @@ pub fn addCases(cases: *tests.StackTracesContext) void {
\\pub fn main() !void {
\\ try foo();
\\}
;
const source_try_try_return_return =
\\const std = @import("std");
\\
,
.Debug = .{
.expect =
\\error: TheSkyIsFalling
\\source.zig:2:5: [address] in foo (test)
\\ return error.TheSkyIsFalling;
\\ ^
\\source.zig:6:5: [address] in main (test)
\\ try foo();
\\ ^
\\
,
},
.ReleaseSafe = .{
.exclude_os = .{
.windows, // segfault
},
.expect =
\\error: TheSkyIsFalling
\\source.zig:2:5: [address] in [function]
\\ return error.TheSkyIsFalling;
\\ ^
\\source.zig:6:5: [address] in [function]
\\ try foo();
\\ ^
\\
,
},
.ReleaseFast = .{
.expect =
\\error: TheSkyIsFalling
\\
,
},
.ReleaseSmall = .{
.expect =
\\error: TheSkyIsFalling
\\
,
},
});
cases.addCase(.{
.name = "try try return return",
.source =
\\fn foo() !void {
\\ try bar();
\\}
@ -40,9 +115,66 @@ pub fn addCases(cases: *tests.StackTracesContext) void {
\\pub fn main() !void {
\\ try foo();
\\}
;
,
.Debug = .{
.expect =
\\error: TheSkyIsFalling
\\source.zig:10:5: [address] in make_error (test)
\\ return error.TheSkyIsFalling;
\\ ^
\\source.zig:6:5: [address] in bar (test)
\\ return make_error();
\\ ^
\\source.zig:2:5: [address] in foo (test)
\\ try bar();
\\ ^
\\source.zig:14:5: [address] in main (test)
\\ try foo();
\\ ^
\\
,
},
.ReleaseSafe = .{
.exclude_os = .{
.windows, // segfault
},
.expect =
\\error: TheSkyIsFalling
\\source.zig:10:5: [address] in [function]
\\ return error.TheSkyIsFalling;
\\ ^
\\source.zig:6:5: [address] in [function]
\\ return make_error();
\\ ^
\\source.zig:2:5: [address] in [function]
\\ try bar();
\\ ^
\\source.zig:14:5: [address] in [function]
\\ try foo();
\\ ^
\\
,
},
.ReleaseFast = .{
.expect =
\\error: TheSkyIsFalling
\\
,
},
.ReleaseSmall = .{
.expect =
\\error: TheSkyIsFalling
\\
,
},
});
const source_dumpCurrentStackTrace =
cases.addCase(.{
.exclude_os = .{
.windows,
},
.name = "dumpCurrentStackTrace",
.source =
\\const std = @import("std");
\\
\\fn bar() void {
@ -55,450 +187,17 @@ pub fn addCases(cases: *tests.StackTracesContext) void {
\\ foo();
\\ return 1;
\\}
;
switch (std.Target.current.os.tag) {
.freebsd => {
cases.addCase(
"return",
source_return,
[_][]const u8{
// debug
\\error: TheSkyIsFalling
\\source.zig:4:5: [address] in main (test)
\\ return error.TheSkyIsFalling;
\\ ^
\\
,
// release-safe
\\error: TheSkyIsFalling
\\source.zig:4:5: [address] in std.start.main (test)
\\ return error.TheSkyIsFalling;
\\ ^
\\
,
// release-fast
\\error: TheSkyIsFalling
\\
,
// release-small
\\error: TheSkyIsFalling
\\
},
);
cases.addCase(
"try return",
source_try_return,
[_][]const u8{
// debug
\\error: TheSkyIsFalling
\\source.zig:4:5: [address] in foo (test)
\\ return error.TheSkyIsFalling;
\\ ^
\\source.zig:8:5: [address] in main (test)
\\ try foo();
\\ ^
\\
,
// release-safe
\\error: TheSkyIsFalling
\\source.zig:4:5: [address] in std.start.main (test)
\\ return error.TheSkyIsFalling;
\\ ^
\\source.zig:8:5: [address] in std.start.main (test)
\\ try foo();
\\ ^
\\
,
// release-fast
\\error: TheSkyIsFalling
\\
,
// release-small
\\error: TheSkyIsFalling
\\
},
);
cases.addCase(
"try try return return",
source_try_try_return_return,
[_][]const u8{
// debug
\\error: TheSkyIsFalling
\\source.zig:12:5: [address] in make_error (test)
\\ return error.TheSkyIsFalling;
\\ ^
\\source.zig:8:5: [address] in bar (test)
\\ return make_error();
\\ ^
\\source.zig:4:5: [address] in foo (test)
\\ try bar();
\\ ^
\\source.zig:16:5: [address] in main (test)
\\ try foo();
\\ ^
\\
,
// release-safe
\\error: TheSkyIsFalling
\\source.zig:12:5: [address] in std.start.main (test)
\\ return error.TheSkyIsFalling;
\\ ^
\\source.zig:8:5: [address] in std.start.main (test)
\\ return make_error();
\\ ^
\\source.zig:4:5: [address] in std.start.main (test)
\\ try bar();
\\ ^
\\source.zig:16:5: [address] in std.start.main (test)
\\ try foo();
\\ ^
\\
,
// release-fast
\\error: TheSkyIsFalling
\\
,
// release-small
\\error: TheSkyIsFalling
\\
},
);
,
.Debug = .{
.expect =
\\source.zig:7:8: [address] in foo (test)
\\ bar();
\\ ^
\\source.zig:10:8: [address] in main (test)
\\ foo();
\\ ^
\\
,
},
.linux => {
cases.addCase(
"return",
source_return,
[_][]const u8{
// debug
\\error: TheSkyIsFalling
\\source.zig:4:5: [address] in main (test)
\\ return error.TheSkyIsFalling;
\\ ^
\\
,
// release-safe
\\error: TheSkyIsFalling
\\source.zig:4:5: [address] in std.start.posixCallMainAndExit (test)
\\ return error.TheSkyIsFalling;
\\ ^
\\
,
// release-fast
\\error: TheSkyIsFalling
\\
,
// release-small
\\error: TheSkyIsFalling
\\
},
);
cases.addCase(
"try return",
source_try_return,
[_][]const u8{
// debug
\\error: TheSkyIsFalling
\\source.zig:4:5: [address] in foo (test)
\\ return error.TheSkyIsFalling;
\\ ^
\\source.zig:8:5: [address] in main (test)
\\ try foo();
\\ ^
\\
,
// release-safe
\\error: TheSkyIsFalling
\\source.zig:4:5: [address] in std.start.posixCallMainAndExit (test)
\\ return error.TheSkyIsFalling;
\\ ^
\\source.zig:8:5: [address] in std.start.posixCallMainAndExit (test)
\\ try foo();
\\ ^
\\
,
// release-fast
\\error: TheSkyIsFalling
\\
,
// release-small
\\error: TheSkyIsFalling
\\
},
);
cases.addCase(
"try try return return",
source_try_try_return_return,
[_][]const u8{
// debug
\\error: TheSkyIsFalling
\\source.zig:12:5: [address] in make_error (test)
\\ return error.TheSkyIsFalling;
\\ ^
\\source.zig:8:5: [address] in bar (test)
\\ return make_error();
\\ ^
\\source.zig:4:5: [address] in foo (test)
\\ try bar();
\\ ^
\\source.zig:16:5: [address] in main (test)
\\ try foo();
\\ ^
\\
,
// release-safe
\\error: TheSkyIsFalling
\\source.zig:12:5: [address] in std.start.posixCallMainAndExit (test)
\\ return error.TheSkyIsFalling;
\\ ^
\\source.zig:8:5: [address] in std.start.posixCallMainAndExit (test)
\\ return make_error();
\\ ^
\\source.zig:4:5: [address] in std.start.posixCallMainAndExit (test)
\\ try bar();
\\ ^
\\source.zig:16:5: [address] in std.start.posixCallMainAndExit (test)
\\ try foo();
\\ ^
\\
,
// release-fast
\\error: TheSkyIsFalling
\\
,
// release-small
\\error: TheSkyIsFalling
\\
},
);
cases.addCase(
"dumpCurrentStackTrace",
source_dumpCurrentStackTrace,
[_][]const u8{
// debug
\\source.zig:7:8: [address] in foo (test)
\\ bar();
\\ ^
\\source.zig:10:8: [address] in main (test)
\\ foo();
\\ ^
\\start.zig:404:29: [address] in std.start.posixCallMainAndExit (test)
\\ return root.main();
\\ ^
\\start.zig:225:5: [address] in std.start._start (test)
\\ @call(.{ .modifier = .never_inline }, posixCallMainAndExit, .{});
\\ ^
\\
,
// release-safe
switch (std.Target.current.cpu.arch) {
.aarch64 => "", // TODO disabled; results in segfault
else =>
\\start.zig:225:5: [address] in std.start._start (test)
\\ @call(.{ .modifier = .never_inline }, posixCallMainAndExit, .{});
\\ ^
\\
,
},
// release-fast
\\
,
// release-small
\\
},
);
},
.macos => {
cases.addCase(
"return",
source_return,
[_][]const u8{
// debug
\\error: TheSkyIsFalling
\\source.zig:4:5: [address] in main (test)
\\ return error.TheSkyIsFalling;
\\ ^
\\
,
// release-safe
\\error: TheSkyIsFalling
\\source.zig:4:5: [address] in std.start.main (test)
\\ return error.TheSkyIsFalling;
\\ ^
\\
,
// release-fast
\\error: TheSkyIsFalling
\\
,
// release-small
\\error: TheSkyIsFalling
\\
},
);
cases.addCase(
"try return",
source_try_return,
[_][]const u8{
// debug
\\error: TheSkyIsFalling
\\source.zig:4:5: [address] in foo (test)
\\ return error.TheSkyIsFalling;
\\ ^
\\source.zig:8:5: [address] in main (test)
\\ try foo();
\\ ^
\\
,
// release-safe
\\error: TheSkyIsFalling
\\source.zig:4:5: [address] in std.start.main (test)
\\ return error.TheSkyIsFalling;
\\ ^
\\source.zig:8:5: [address] in std.start.main (test)
\\ try foo();
\\ ^
\\
,
// release-fast
\\error: TheSkyIsFalling
\\
,
// release-small
\\error: TheSkyIsFalling
\\
},
);
cases.addCase(
"try try return return",
source_try_try_return_return,
[_][]const u8{
// debug
\\error: TheSkyIsFalling
\\source.zig:12:5: [address] in make_error (test)
\\ return error.TheSkyIsFalling;
\\ ^
\\source.zig:8:5: [address] in bar (test)
\\ return make_error();
\\ ^
\\source.zig:4:5: [address] in foo (test)
\\ try bar();
\\ ^
\\source.zig:16:5: [address] in main (test)
\\ try foo();
\\ ^
\\
,
// release-safe
\\error: TheSkyIsFalling
\\source.zig:12:5: [address] in std.start.main (test)
\\ return error.TheSkyIsFalling;
\\ ^
\\source.zig:8:5: [address] in std.start.main (test)
\\ return make_error();
\\ ^
\\source.zig:4:5: [address] in std.start.main (test)
\\ try bar();
\\ ^
\\source.zig:16:5: [address] in std.start.main (test)
\\ try foo();
\\ ^
\\
,
// release-fast
\\error: TheSkyIsFalling
\\
,
// release-small
\\error: TheSkyIsFalling
\\
},
);
},
.windows => {
cases.addCase(
"return",
source_return,
[_][]const u8{
// debug
\\error: TheSkyIsFalling
\\source.zig:4:5: [address] in main (test.obj)
\\ return error.TheSkyIsFalling;
\\ ^
\\
,
// release-safe
// --disabled-- results in segmenetation fault
"",
// release-fast
\\error: TheSkyIsFalling
\\
,
// release-small
\\error: TheSkyIsFalling
\\
},
);
cases.addCase(
"try return",
source_try_return,
[_][]const u8{
// debug
\\error: TheSkyIsFalling
\\source.zig:4:5: [address] in foo (test.obj)
\\ return error.TheSkyIsFalling;
\\ ^
\\source.zig:8:5: [address] in main (test.obj)
\\ try foo();
\\ ^
\\
,
// release-safe
// --disabled-- results in segmenetation fault
"",
// release-fast
\\error: TheSkyIsFalling
\\
,
// release-small
\\error: TheSkyIsFalling
\\
},
);
cases.addCase(
"try try return return",
source_try_try_return_return,
[_][]const u8{
// debug
\\error: TheSkyIsFalling
\\source.zig:12:5: [address] in make_error (test.obj)
\\ return error.TheSkyIsFalling;
\\ ^
\\source.zig:8:5: [address] in bar (test.obj)
\\ return make_error();
\\ ^
\\source.zig:4:5: [address] in foo (test.obj)
\\ try bar();
\\ ^
\\source.zig:16:5: [address] in main (test.obj)
\\ try foo();
\\ ^
\\
,
// release-safe
// --disabled-- results in segmenetation fault
"",
// release-fast
\\error: TheSkyIsFalling
\\
,
// release-small
\\error: TheSkyIsFalling
\\
},
);
},
else => {},
}
});
}
// zig fmt: off

View File

@ -558,42 +558,87 @@ pub const StackTracesContext = struct {
const Expect = [@typeInfo(Mode).Enum.fields.len][]const u8;
pub fn addCase(
pub fn addCase(self: *StackTracesContext, config: anytype) void {
if (@hasField(@TypeOf(config), "exclude")) {
if (config.exclude.exclude()) return;
}
if (@hasField(@TypeOf(config), "exclude_arch")) {
const exclude_arch: []const builtin.Cpu.Arch = &config.exclude_arch;
for (exclude_arch) |arch| if (arch == builtin.cpu.arch) return;
}
if (@hasField(@TypeOf(config), "exclude_os")) {
const exclude_os: []const builtin.Os.Tag = &config.exclude_os;
for (exclude_os) |os| if (os == builtin.os.tag) return;
}
for (self.modes) |mode| {
switch (mode) {
.Debug => {
if (@hasField(@TypeOf(config), "Debug")) {
self.addExpect(config.name, config.source, mode, config.Debug);
}
},
.ReleaseSafe => {
if (@hasField(@TypeOf(config), "ReleaseSafe")) {
self.addExpect(config.name, config.source, mode, config.ReleaseSafe);
}
},
.ReleaseFast => {
if (@hasField(@TypeOf(config), "ReleaseFast")) {
self.addExpect(config.name, config.source, mode, config.ReleaseFast);
}
},
.ReleaseSmall => {
if (@hasField(@TypeOf(config), "ReleaseSmall")) {
self.addExpect(config.name, config.source, mode, config.ReleaseSmall);
}
},
}
}
}
fn addExpect(
self: *StackTracesContext,
name: []const u8,
source: []const u8,
expect: Expect,
mode: Mode,
mode_config: anytype,
) void {
const b = self.b;
for (self.modes) |mode| {
const expect_for_mode = expect[@enumToInt(mode)];
if (expect_for_mode.len == 0) continue;
const annotated_case_name = fmt.allocPrint(self.b.allocator, "{s} {s} ({s})", .{
"stack-trace",
name,
@tagName(mode),
}) catch unreachable;
if (self.test_filter) |filter| {
if (mem.indexOf(u8, annotated_case_name, filter) == null) continue;
}
const src_basename = "source.zig";
const write_src = b.addWriteFile(src_basename, source);
const exe = b.addExecutableFromWriteFileStep("test", write_src, src_basename);
exe.setBuildMode(mode);
const run_and_compare = RunAndCompareStep.create(
self,
exe,
annotated_case_name,
mode,
expect_for_mode,
);
self.step.dependOn(&run_and_compare.step);
if (@hasField(@TypeOf(mode_config), "exclude")) {
if (mode_config.exclude.exclude()) return;
}
if (@hasField(@TypeOf(mode_config), "exclude_arch")) {
const exclude_arch: []const builtin.Cpu.Arch = &mode_config.exclude_arch;
for (exclude_arch) |arch| if (arch == builtin.cpu.arch) return;
}
if (@hasField(@TypeOf(mode_config), "exclude_os")) {
const exclude_os: []const builtin.Os.Tag = &mode_config.exclude_os;
for (exclude_os) |os| if (os == builtin.os.tag) return;
}
const annotated_case_name = fmt.allocPrint(self.b.allocator, "{s} {s} ({s})", .{
"stack-trace",
name,
@tagName(mode),
}) catch unreachable;
if (self.test_filter) |filter| {
if (mem.indexOf(u8, annotated_case_name, filter) == null) return;
}
const b = self.b;
const src_basename = "source.zig";
const write_src = b.addWriteFile(src_basename, source);
const exe = b.addExecutableFromWriteFileStep("test", write_src, src_basename);
exe.setBuildMode(mode);
const run_and_compare = RunAndCompareStep.create(
self,
exe,
annotated_case_name,
mode,
mode_config.expect,
);
self.step.dependOn(&run_and_compare.step);
}
const RunAndCompareStep = struct {
@ -695,6 +740,7 @@ pub const StackTracesContext = struct {
// process result
// - keep only basename of source file path
// - replace address with symbolic string
// - replace function name with symbolic string when mode != .Debug
// - skip empty lines
const got: []const u8 = got_result: {
var buf = ArrayList(u8).init(b.allocator);
@ -703,26 +749,45 @@ pub const StackTracesContext = struct {
var it = mem.split(stderr, "\n");
process_lines: while (it.next()) |line| {
if (line.len == 0) continue;
const delims = [_][]const u8{ ":", ":", ":", " in " };
var marks = [_]usize{0} ** 4;
// offset search past `[drive]:` on windows
var pos: usize = if (std.Target.current.os.tag == .windows) 2 else 0;
// locate delims/anchor
const delims = [_][]const u8{ ":", ":", ":", " in ", "(", ")" };
var marks = [_]usize{0} ** delims.len;
for (delims) |delim, i| {
marks[i] = mem.indexOfPos(u8, line, pos, delim) orelse {
// unexpected pattern: emit raw line and cont
try buf.appendSlice(line);
try buf.appendSlice("\n");
continue :process_lines;
};
pos = marks[i] + delim.len;
}
// locate source basename
pos = mem.lastIndexOfScalar(u8, line[0..marks[0]], fs.path.sep) orelse {
// unexpected pattern: emit raw line and cont
try buf.appendSlice(line);
try buf.appendSlice("\n");
continue :process_lines;
};
// end processing if source basename changes
if (!mem.eql(u8, "source.zig", line[pos + 1 .. marks[0]])) break;
// emit substituted line
try buf.appendSlice(line[pos + 1 .. marks[2] + delims[2].len]);
try buf.appendSlice(" [address]");
try buf.appendSlice(line[marks[3]..]);
if (self.mode == .Debug) {
if (mem.lastIndexOfScalar(u8, line[marks[4]..marks[5]], '.')) |idot| {
// On certain platforms (windows) or possibly depending on how we choose to link main
// the object file extension may be present so we simply strip any extension.
try buf.appendSlice(line[marks[3] .. marks[4] + idot]);
try buf.appendSlice(line[marks[5]..]);
} else {
try buf.appendSlice(line[marks[3]..]);
}
} else {
try buf.appendSlice(line[marks[3] .. marks[3] + delims[3].len]);
try buf.appendSlice("[function]");
}
try buf.appendSlice("\n");
}
break :got_result buf.toOwnedSlice();