-femit-docs: creating sources.tar

It's always a good day when you get to use File.writeFileAll 😎

lib/compiler/std-docs.zig

@@ -167,9 +167,8 @@ fn serveSourcesTar(request: *std.http.Server.Request, context: *Context) !void {
             const remainder = stat.size % 512;
             break :p if (remainder > 0) 512 - remainder else 0;
         };
-        comptime assert(@sizeOf(TarHeader) == 512);
 
-        var file_header = TarHeader.init();
+        var file_header = std.tar.output.Header.init();
         file_header.typeflag = .regular;
         try file_header.setPath("std", entry.path);
         try file_header.setSize(stat.size);
@@ -383,77 +382,3 @@ fn openBrowserTabThread(gpa: Allocator, url: []const u8) !void {
     try child.spawn();
     _ = try child.wait();
 }
-
-/// Forked from https://github.com/mattnite/tar/blob/main/src/main.zig which is
-/// MIT licensed.
-pub const TarHeader = extern struct {
-    name: [100]u8,
-    mode: [7:0]u8,
-    uid: [7:0]u8,
-    gid: [7:0]u8,
-    size: [11:0]u8,
-    mtime: [11:0]u8,
-    checksum: [7:0]u8,
-    typeflag: FileType,
-    linkname: [100]u8,
-    magic: [5:0]u8,
-    version: [2]u8,
-    uname: [31:0]u8,
-    gname: [31:0]u8,
-    devmajor: [7:0]u8,
-    devminor: [7:0]u8,
-    prefix: [155]u8,
-    pad: [12]u8,
-
-    const FileType = enum(u8) {
-        regular = '0',
-        hard_link = '1',
-        symbolic_link = '2',
-        character = '3',
-        block = '4',
-        directory = '5',
-        fifo = '6',
-        reserved = '7',
-        pax_global = 'g',
-        extended = 'x',
-        _,
-    };
-
-    fn init() TarHeader {
-        var ret = std.mem.zeroes(TarHeader);
-        ret.magic = [_:0]u8{ 'u', 's', 't', 'a', 'r' };
-        ret.version = [_:0]u8{ '0', '0' };
-        return ret;
-    }
-
-    fn setPath(self: *TarHeader, prefix: []const u8, path: []const u8) !void {
-        if (prefix.len + 1 + path.len > 100) {
-            var i: usize = 0;
-            while (i < path.len and path.len - i > 100) {
-                while (path[i] != '/') : (i += 1) {}
-            }
-
-            _ = try std.fmt.bufPrint(&self.prefix, "{s}/{s}", .{ prefix, path[0..i] });
-            _ = try std.fmt.bufPrint(&self.name, "{s}", .{path[i + 1 ..]});
-        } else {
-            _ = try std.fmt.bufPrint(&self.name, "{s}/{s}", .{ prefix, path });
-        }
-    }
-
-    fn setSize(self: *TarHeader, size: u64) !void {
-        _ = try std.fmt.bufPrint(&self.size, "{o:0>11}", .{size});
-    }
-
-    fn updateChecksum(self: *TarHeader) !void {
-        const offset = @offsetOf(TarHeader, "checksum");
-        var checksum: usize = 0;
-        for (std.mem.asBytes(self), 0..) |val, i| {
-            checksum += if (i >= offset and i < offset + @sizeOf(@TypeOf(self.checksum)))
-                ' '
-            else
-                val;
-        }
-
-        _ = try std.fmt.bufPrint(&self.checksum, "{o:0>7}", .{checksum});
-    }
-};

lib/std/tar.zig

@@ -1,23 +1,25 @@
-/// Tar archive is single ordinary file which can contain many files (or
-/// directories, symlinks, ...). It's build by series of blocks each size of 512
-/// bytes. First block of each entry is header which defines type, name, size
-/// permissions and other attributes. Header is followed by series of blocks of
-/// file content, if any that entry has content. Content is padded to the block
-/// size, so next header always starts at block boundary.
-///
-/// This simple format is extended by GNU and POSIX pax extensions to support
-/// file names longer than 256 bytes and additional attributes.
-///
-/// This is not comprehensive tar parser. Here we are only file types needed to
-/// support Zig package manager; normal file, directory, symbolic link. And
-/// subset of attributes: name, size, permissions.
-///
-/// GNU tar reference: https://www.gnu.org/software/tar/manual/html_node/Standard.html
-/// pax reference: https://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html#tag_20_92_13
-///
+//! Tar archive is single ordinary file which can contain many files (or
+//! directories, symlinks, ...). It's build by series of blocks each size of 512
+//! bytes. First block of each entry is header which defines type, name, size
+//! permissions and other attributes. Header is followed by series of blocks of
+//! file content, if any that entry has content. Content is padded to the block
+//! size, so next header always starts at block boundary.
+//!
+//! This simple format is extended by GNU and POSIX pax extensions to support
+//! file names longer than 256 bytes and additional attributes.
+//!
+//! This is not comprehensive tar parser. Here we are only file types needed to
+//! support Zig package manager; normal file, directory, symbolic link. And
+//! subset of attributes: name, size, permissions.
+//!
+//! GNU tar reference: https://www.gnu.org/software/tar/manual/html_node/Standard.html
+//! pax reference: https://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html#tag_20_92_13
+
 const std = @import("std.zig");
 const assert = std.debug.assert;
 
+pub const output = @import("tar/output.zig");
+
 pub const Options = struct {
     /// Number of directory levels to skip when extracting files.
     strip_components: u32 = 0,

lib/std/tar/output.zig

@@ -0,0 +1,85 @@
+/// A struct that is exactly 512 bytes and matches tar file format. This is
+/// intended to be used for outputting tar files; for parsing there is
+/// `std.tar.Header`.
+pub const Header = extern struct {
+    // This struct was originally copied from
+    // https://github.com/mattnite/tar/blob/main/src/main.zig which is MIT
+    // licensed.
+
+    name: [100]u8,
+    mode: [7:0]u8,
+    uid: [7:0]u8,
+    gid: [7:0]u8,
+    size: [11:0]u8,
+    mtime: [11:0]u8,
+    checksum: [7:0]u8,
+    typeflag: FileType,
+    linkname: [100]u8,
+    magic: [5:0]u8,
+    version: [2]u8,
+    uname: [31:0]u8,
+    gname: [31:0]u8,
+    devmajor: [7:0]u8,
+    devminor: [7:0]u8,
+    prefix: [155]u8,
+    pad: [12]u8,
+
+    pub const FileType = enum(u8) {
+        regular = '0',
+        hard_link = '1',
+        symbolic_link = '2',
+        character = '3',
+        block = '4',
+        directory = '5',
+        fifo = '6',
+        reserved = '7',
+        pax_global = 'g',
+        extended = 'x',
+        _,
+    };
+
+    pub fn init() Header {
+        var ret = std.mem.zeroes(Header);
+        ret.magic = [_:0]u8{ 'u', 's', 't', 'a', 'r' };
+        ret.version = [_:0]u8{ '0', '0' };
+        return ret;
+    }
+
+    pub fn setPath(self: *Header, prefix: []const u8, path: []const u8) !void {
+        if (prefix.len + 1 + path.len > 100) {
+            var i: usize = 0;
+            while (i < path.len and path.len - i > 100) {
+                while (path[i] != '/') : (i += 1) {}
+            }
+
+            _ = try std.fmt.bufPrint(&self.prefix, "{s}/{s}", .{ prefix, path[0..i] });
+            _ = try std.fmt.bufPrint(&self.name, "{s}", .{path[i + 1 ..]});
+        } else {
+            _ = try std.fmt.bufPrint(&self.name, "{s}/{s}", .{ prefix, path });
+        }
+    }
+
+    pub fn setSize(self: *Header, size: u64) !void {
+        _ = try std.fmt.bufPrint(&self.size, "{o:0>11}", .{size});
+    }
+
+    pub fn updateChecksum(self: *Header) !void {
+        const offset = @offsetOf(Header, "checksum");
+        var checksum: usize = 0;
+        for (std.mem.asBytes(self), 0..) |val, i| {
+            checksum += if (i >= offset and i < offset + @sizeOf(@TypeOf(self.checksum)))
+                ' '
+            else
+                val;
+        }
+
+        _ = try std.fmt.bufPrint(&self.checksum, "{o:0>7}", .{checksum});
+    }
+
+    comptime {
+        assert(@sizeOf(Header) == 512);
+    }
+};
+
+const std = @import("../std.zig");
+const assert = std.debug.assert;

src/Compilation.zig

@@ -733,6 +733,7 @@ pub const MiscTask = enum {
     compiler_rt,
     zig_libc,
     analyze_mod,
+    docs_copy,
 
     @"musl crti.o",
     @"musl crtn.o",
@@ -3765,23 +3766,107 @@ fn taskDocsWasm(comp: *Compilation, wg: *WaitGroup) !void {
 
 fn workerDocsCopy(comp: *Compilation, wg: *WaitGroup) void {
     defer wg.finish();
+    docsCopyFallible(comp) catch |err| {
+        return comp.lockAndSetMiscFailure(
+            .docs_copy,
+            "unable to copy autodocs artifacts: {s}",
+            .{@errorName(err)},
+        );
+    };
+}
 
+fn docsCopyFallible(comp: *Compilation) anyerror!void {
     const emit = comp.docs_emit.?;
     var out_dir = emit.directory.handle.makeOpenPath(emit.sub_path, .{}) catch |err| {
-        // TODO create an error to be reported instead of logging
-        log.err("unable to create output directory '{}{s}': {s}", .{
-            emit.directory, emit.sub_path, @errorName(err),
-        });
-        return;
+        return comp.lockAndSetMiscFailure(
+            .docs_copy,
+            "unable to create output directory '{}{s}': {s}",
+            .{ emit.directory, emit.sub_path, @errorName(err) },
+        );
     };
     defer out_dir.close();
 
     for (&[_][]const u8{ "docs/main.js", "docs/index.html" }) |sub_path| {
         const basename = std.fs.path.basename(sub_path);
         comp.zig_lib_directory.handle.copyFile(sub_path, out_dir, basename, .{}) catch |err| {
-            log.err("unable to copy {s}: {s}", .{ sub_path, @errorName(err) });
+            comp.lockAndSetMiscFailure(.docs_copy, "unable to copy {s}: {s}", .{
+                sub_path,
+                @errorName(err),
+            });
+            return;
         };
     }
+
+    var tar_file = out_dir.createFile("sources.tar", .{}) catch |err| {
+        return comp.lockAndSetMiscFailure(
+            .docs_copy,
+            "unable to create '{}{s}/sources.tar': {s}",
+            .{ emit.directory, emit.sub_path, @errorName(err) },
+        );
+    };
+    defer tar_file.close();
+
+    const root = comp.root_mod.root;
+    const root_mod_name = comp.root_mod.fully_qualified_name;
+    const sub_path = if (root.sub_path.len == 0) "." else root.sub_path;
+    var mod_dir = root.root_dir.handle.openDir(sub_path, .{ .iterate = true }) catch |err| {
+        return comp.lockAndSetMiscFailure(.docs_copy, "unable to open directory '{}': {s}", .{
+            root, @errorName(err),
+        });
+    };
+
+    var walker = try mod_dir.walk(comp.gpa);
+    defer walker.deinit();
+
+    const padding_buffer = [1]u8{0} ** 512;
+
+    while (try walker.next()) |entry| {
+        switch (entry.kind) {
+            .file => {
+                if (!std.mem.endsWith(u8, entry.basename, ".zig")) continue;
+                if (std.mem.eql(u8, entry.basename, "test.zig")) continue;
+                if (std.mem.endsWith(u8, entry.basename, "_test.zig")) continue;
+            },
+            else => continue,
+        }
+
+        var file = mod_dir.openFile(entry.path, .{}) catch |err| {
+            return comp.lockAndSetMiscFailure(.docs_copy, "unable to open '{}{s}': {s}", .{
+                root, entry.path, @errorName(err),
+            });
+        };
+        defer file.close();
+
+        const stat = file.stat() catch |err| {
+            return comp.lockAndSetMiscFailure(.docs_copy, "unable to stat '{}{s}': {s}", .{
+                root, entry.path, @errorName(err),
+            });
+        };
+
+        var file_header = std.tar.output.Header.init();
+        file_header.typeflag = .regular;
+        try file_header.setPath(root_mod_name, entry.path);
+        try file_header.setSize(stat.size);
+        try file_header.updateChecksum();
+
+        const header_bytes = std.mem.asBytes(&file_header);
+        const padding = p: {
+            const remainder = stat.size % 512;
+            const n = if (remainder > 0) 512 - remainder else 0;
+            break :p padding_buffer[0..n];
+        };
+
+        var header_and_trailer: [2]std.os.iovec_const = .{
+            .{ .iov_base = header_bytes.ptr, .iov_len = header_bytes.len },
+            .{ .iov_base = padding.ptr, .iov_len = padding.len },
+        };
+
+        try tar_file.writeFileAll(file, .{
+            .in_len = stat.size,
+            .headers_and_trailers = &header_and_trailer,
+            .header_count = 1,
+        });
+    }
 }
 
 fn workerDocsWasm(comp: *Compilation, wg: *WaitGroup) void {