From e897637d8d25f5b6b118356d2355da7c9148d8cb Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 6 Mar 2023 19:19:48 -0700 Subject: [PATCH] re-enable compare-output test cases --- build.zig | 2 +- lib/build_runner.zig | 4 +- test/src/CompareOutput.zig | 176 ++++++++++++++++++++++++++++++++++++ test/src/compare_output.zig | 175 ----------------------------------- test/tests.zig | 2 +- 5 files changed, 180 insertions(+), 179 deletions(-) create mode 100644 test/src/CompareOutput.zig delete mode 100644 test/src/compare_output.zig diff --git a/build.zig b/build.zig index 1f1b119ed3..40dc823eb7 100644 --- a/build.zig +++ b/build.zig @@ -444,7 +444,7 @@ pub fn build(b: *std.Build) !void { _ = enable_symlinks_windows; _ = enable_macos_sdk; - //test_step.dependOn(tests.addCompareOutputTests(b, test_filter, optimization_modes)); + test_step.dependOn(tests.addCompareOutputTests(b, test_filter, optimization_modes)); //test_step.dependOn(tests.addStandaloneTests( // b, // test_filter, diff --git a/lib/build_runner.zig b/lib/build_runner.zig index c30fbaea87..bb04b3e132 100644 --- a/lib/build_runner.zig +++ b/lib/build_runner.zig @@ -887,6 +887,8 @@ fn usage(builder: *std.Build, already_ran_build: bool, out_stream: anytype) !voi \\ --verbose Print commands before executing them \\ --color [auto|off|on] Enable or disable colored error messages \\ --prominent-compile-errors Output compile errors formatted for a human to read + \\ -fsummary Print the build summary, even on success + \\ -fno-summary Omit the build summary, even on failure \\ -j Limit concurrent jobs (default is to use all CPU cores) \\ --maxrss Limit memory usage (default is to use available memory) \\ @@ -920,8 +922,6 @@ fn usage(builder: *std.Build, already_ran_build: bool, out_stream: anytype) !voi \\Advanced Options: \\ -freference-trace[=num] How many lines of reference trace should be shown per compile error \\ -fno-reference-trace Disable reference trace - \\ -fsummary Print the build summary, even on success - \\ -fno-summary Omit the build summary, even on failure \\ --build-file [file] Override path to build.zig \\ --cache-dir [path] Override path to local Zig cache directory \\ --global-cache-dir [path] Override path to global Zig cache directory diff --git a/test/src/CompareOutput.zig b/test/src/CompareOutput.zig new file mode 100644 index 0000000000..68cf942684 --- /dev/null +++ b/test/src/CompareOutput.zig @@ -0,0 +1,176 @@ +//! This is the implementation of the test harness. +//! For the actual test cases, see test/compare_output.zig. + +b: *std.Build, +step: *std.Build.Step, +test_index: usize, +test_filter: ?[]const u8, +optimize_modes: []const OptimizeMode, + +const Special = enum { + None, + Asm, + RuntimeSafety, +}; + +const TestCase = struct { + name: []const u8, + sources: ArrayList(SourceFile), + expected_output: []const u8, + link_libc: bool, + special: Special, + cli_args: []const []const u8, + + const SourceFile = struct { + filename: []const u8, + source: []const u8, + }; + + pub fn addSourceFile(self: *TestCase, filename: []const u8, source: []const u8) void { + self.sources.append(SourceFile{ + .filename = filename, + .source = source, + }) catch @panic("OOM"); + } + + pub fn setCommandLineArgs(self: *TestCase, args: []const []const u8) void { + self.cli_args = args; + } +}; + +pub fn createExtra(self: *CompareOutput, name: []const u8, source: []const u8, expected_output: []const u8, special: Special) TestCase { + var tc = TestCase{ + .name = name, + .sources = ArrayList(TestCase.SourceFile).init(self.b.allocator), + .expected_output = expected_output, + .link_libc = false, + .special = special, + .cli_args = &[_][]const u8{}, + }; + const root_src_name = if (special == Special.Asm) "source.s" else "source.zig"; + tc.addSourceFile(root_src_name, source); + return tc; +} + +pub fn create(self: *CompareOutput, name: []const u8, source: []const u8, expected_output: []const u8) TestCase { + return createExtra(self, name, source, expected_output, Special.None); +} + +pub fn addC(self: *CompareOutput, name: []const u8, source: []const u8, expected_output: []const u8) void { + var tc = self.create(name, source, expected_output); + tc.link_libc = true; + self.addCase(tc); +} + +pub fn add(self: *CompareOutput, name: []const u8, source: []const u8, expected_output: []const u8) void { + const tc = self.create(name, source, expected_output); + self.addCase(tc); +} + +pub fn addAsm(self: *CompareOutput, name: []const u8, source: []const u8, expected_output: []const u8) void { + const tc = self.createExtra(name, source, expected_output, Special.Asm); + self.addCase(tc); +} + +pub fn addRuntimeSafety(self: *CompareOutput, name: []const u8, source: []const u8) void { + const tc = self.createExtra(name, source, undefined, Special.RuntimeSafety); + self.addCase(tc); +} + +pub fn addCase(self: *CompareOutput, case: TestCase) void { + const b = self.b; + + const write_src = b.addWriteFiles(); + for (case.sources.items) |src_file| { + write_src.add(src_file.filename, src_file.source); + } + + switch (case.special) { + Special.Asm => { + const annotated_case_name = fmt.allocPrint(self.b.allocator, "run assemble-and-link {s}", .{ + case.name, + }) catch @panic("OOM"); + if (self.test_filter) |filter| { + if (mem.indexOf(u8, annotated_case_name, filter) == null) return; + } + + const exe = b.addExecutable(.{ + .name = "test", + .target = .{}, + .optimize = .Debug, + }); + exe.addAssemblyFileSource(write_src.getFileSource(case.sources.items[0].filename).?); + + const run = exe.run(); + run.setName(annotated_case_name); + run.addArgs(case.cli_args); + run.expectStdErrEqual(""); + run.expectStdOutEqual(case.expected_output); + + self.step.dependOn(&run.step); + }, + Special.None => { + for (self.optimize_modes) |optimize| { + const annotated_case_name = fmt.allocPrint(self.b.allocator, "run compare-output {s} ({s})", .{ + case.name, @tagName(optimize), + }) catch @panic("OOM"); + if (self.test_filter) |filter| { + if (mem.indexOf(u8, annotated_case_name, filter) == null) continue; + } + + const basename = case.sources.items[0].filename; + const exe = b.addExecutable(.{ + .name = "test", + .root_source_file = write_src.getFileSource(basename).?, + .optimize = optimize, + .target = .{}, + }); + if (case.link_libc) { + exe.linkSystemLibrary("c"); + } + + const run = exe.run(); + run.setName(annotated_case_name); + run.addArgs(case.cli_args); + run.expectStdErrEqual(""); + run.expectStdOutEqual(case.expected_output); + + self.step.dependOn(&run.step); + } + }, + Special.RuntimeSafety => { + // TODO iterate over self.optimize_modes and test this in both + // debug and release safe mode + const annotated_case_name = fmt.allocPrint(self.b.allocator, "run safety {s}", .{case.name}) catch @panic("OOM"); + if (self.test_filter) |filter| { + if (mem.indexOf(u8, annotated_case_name, filter) == null) return; + } + + const basename = case.sources.items[0].filename; + const exe = b.addExecutable(.{ + .name = "test", + .root_source_file = write_src.getFileSource(basename).?, + .target = .{}, + .optimize = .Debug, + }); + if (case.link_libc) { + exe.linkSystemLibrary("c"); + } + + const run = exe.run(); + run.setName(annotated_case_name); + run.addArgs(case.cli_args); + run.expectExitCode(126); + + self.step.dependOn(&run.step); + }, + } +} + +const CompareOutput = @This(); +const std = @import("std"); +const ArrayList = std.ArrayList; +const fmt = std.fmt; +const mem = std.mem; +const fs = std.fs; +const OptimizeMode = std.builtin.OptimizeMode; diff --git a/test/src/compare_output.zig b/test/src/compare_output.zig deleted file mode 100644 index 20bd62d8e2..0000000000 --- a/test/src/compare_output.zig +++ /dev/null @@ -1,175 +0,0 @@ -// This is the implementation of the test harness. -// For the actual test cases, see test/compare_output.zig. -const std = @import("std"); -const ArrayList = std.ArrayList; -const fmt = std.fmt; -const mem = std.mem; -const fs = std.fs; -const OptimizeMode = std.builtin.OptimizeMode; - -pub const CompareOutputContext = struct { - b: *std.Build, - step: *std.Build.Step, - test_index: usize, - test_filter: ?[]const u8, - optimize_modes: []const OptimizeMode, - - const Special = enum { - None, - Asm, - RuntimeSafety, - }; - - const TestCase = struct { - name: []const u8, - sources: ArrayList(SourceFile), - expected_output: []const u8, - link_libc: bool, - special: Special, - cli_args: []const []const u8, - - const SourceFile = struct { - filename: []const u8, - source: []const u8, - }; - - pub fn addSourceFile(self: *TestCase, filename: []const u8, source: []const u8) void { - self.sources.append(SourceFile{ - .filename = filename, - .source = source, - }) catch unreachable; - } - - pub fn setCommandLineArgs(self: *TestCase, args: []const []const u8) void { - self.cli_args = args; - } - }; - - pub fn createExtra(self: *CompareOutputContext, name: []const u8, source: []const u8, expected_output: []const u8, special: Special) TestCase { - var tc = TestCase{ - .name = name, - .sources = ArrayList(TestCase.SourceFile).init(self.b.allocator), - .expected_output = expected_output, - .link_libc = false, - .special = special, - .cli_args = &[_][]const u8{}, - }; - const root_src_name = if (special == Special.Asm) "source.s" else "source.zig"; - tc.addSourceFile(root_src_name, source); - return tc; - } - - pub fn create(self: *CompareOutputContext, name: []const u8, source: []const u8, expected_output: []const u8) TestCase { - return createExtra(self, name, source, expected_output, Special.None); - } - - pub fn addC(self: *CompareOutputContext, name: []const u8, source: []const u8, expected_output: []const u8) void { - var tc = self.create(name, source, expected_output); - tc.link_libc = true; - self.addCase(tc); - } - - pub fn add(self: *CompareOutputContext, name: []const u8, source: []const u8, expected_output: []const u8) void { - const tc = self.create(name, source, expected_output); - self.addCase(tc); - } - - pub fn addAsm(self: *CompareOutputContext, name: []const u8, source: []const u8, expected_output: []const u8) void { - const tc = self.createExtra(name, source, expected_output, Special.Asm); - self.addCase(tc); - } - - pub fn addRuntimeSafety(self: *CompareOutputContext, name: []const u8, source: []const u8) void { - const tc = self.createExtra(name, source, undefined, Special.RuntimeSafety); - self.addCase(tc); - } - - pub fn addCase(self: *CompareOutputContext, case: TestCase) void { - const b = self.b; - - const write_src = b.addWriteFiles(); - for (case.sources.items) |src_file| { - write_src.add(src_file.filename, src_file.source); - } - - switch (case.special) { - Special.Asm => { - const annotated_case_name = fmt.allocPrint(self.b.allocator, "assemble-and-link {s}", .{ - case.name, - }) catch unreachable; - if (self.test_filter) |filter| { - if (mem.indexOf(u8, annotated_case_name, filter) == null) return; - } - - const exe = b.addExecutable(.{ - .name = "test", - .target = .{}, - .optimize = .Debug, - }); - exe.addAssemblyFileSource(write_src.getFileSource(case.sources.items[0].filename).?); - - const run = exe.run(); - run.addArgs(case.cli_args); - run.expectStdErrEqual(""); - run.expectStdOutEqual(case.expected_output); - - self.step.dependOn(&run.step); - }, - Special.None => { - for (self.optimize_modes) |optimize| { - const annotated_case_name = fmt.allocPrint(self.b.allocator, "{s} {s} ({s})", .{ - "compare-output", - case.name, - @tagName(optimize), - }) catch unreachable; - if (self.test_filter) |filter| { - if (mem.indexOf(u8, annotated_case_name, filter) == null) continue; - } - - const basename = case.sources.items[0].filename; - const exe = b.addExecutable(.{ - .name = "test", - .root_source_file = write_src.getFileSource(basename).?, - .optimize = optimize, - .target = .{}, - }); - if (case.link_libc) { - exe.linkSystemLibrary("c"); - } - - const run = exe.run(); - run.addArgs(case.cli_args); - run.expectStdErrEqual(""); - run.expectStdOutEqual(case.expected_output); - - self.step.dependOn(&run.step); - } - }, - Special.RuntimeSafety => { - // TODO iterate over self.optimize_modes and test this in both - // debug and release safe mode - const annotated_case_name = fmt.allocPrint(self.b.allocator, "safety {s}", .{case.name}) catch unreachable; - if (self.test_filter) |filter| { - if (mem.indexOf(u8, annotated_case_name, filter) == null) return; - } - - const basename = case.sources.items[0].filename; - const exe = b.addExecutable(.{ - .name = "test", - .root_source_file = write_src.getFileSource(basename).?, - .target = .{}, - .optimize = .Debug, - }); - if (case.link_libc) { - exe.linkSystemLibrary("c"); - } - - const run = exe.run(); - run.addArgs(case.cli_args); - run.expectExitCode(126); - - self.step.dependOn(&run.step); - }, - } - } -}; diff --git a/test/tests.zig b/test/tests.zig index 1a80d7de8d..e21e652cba 100644 --- a/test/tests.zig +++ b/test/tests.zig @@ -25,7 +25,7 @@ const link = @import("link.zig"); // Implementations pub const TranslateCContext = @import("src/translate_c.zig").TranslateCContext; pub const RunTranslatedCContext = @import("src/run_translated_c.zig").RunTranslatedCContext; -pub const CompareOutputContext = @import("src/compare_output.zig").CompareOutputContext; +pub const CompareOutputContext = @import("src/CompareOutput.zig"); pub const StackTracesContext = @import("src/StackTrace.zig"); pub const StandaloneContext = @import("src/Standalone.zig");