From cfcf9771c1bde357ad64d81cda9d61ba72d80b15 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 10 Jan 2023 20:21:58 -0700 Subject: [PATCH] zig build: support dependencies The `zig build` command now makes `@import("@dependencies")` available to the build runner package. It contains all the dependencies in a generated file that looks something like this: ```zig pub const imports = struct { pub const foo = @import("foo"); pub const @"bar.baz" = @import("bar.baz"); }; pub const build_root = struct { pub const foo = ""; pub const @"bar.baz" = ""; }; ``` The build runner exports this import so that `std.build.Builder` can access it. `std.build.Builder` uses it to implement the new `dependency` function which can be used like so: ```zig const libz_dep = b.dependency("libz", .{}); const libmp3lame_dep = b.dependency("libmp3lame", .{}); // ... lib.linkLibrary(libz_dep.artifact("z")); lib.linkLibrary(libmp3lame_dep.artifact("mp3lame")); ``` The `dependency` function calls the build.zig file of the dependency as a child Builder, and then can be ransacked for its build steps via the `artifact` function. This commit also renames `dependency.id` to `dependency.name` in the `build.zig.ini` file. --- lib/build_runner.zig | 14 ++-- lib/std/build.zig | 154 ++++++++++++++++++++++++++++++++++++++++++- src/Package.zig | 133 +++++++++++++++++++++++++++---------- src/main.zig | 53 +++++++++++---- 4 files changed, 297 insertions(+), 57 deletions(-) diff --git a/lib/build_runner.zig b/lib/build_runner.zig index 735ddb9de1..f27542d0f5 100644 --- a/lib/build_runner.zig +++ b/lib/build_runner.zig @@ -9,6 +9,8 @@ const process = std.process; const ArrayList = std.ArrayList; const File = std.fs.File; +pub const dependencies = @import("@dependencies"); + pub fn main() !void { // Here we use an ArenaAllocator backed by a DirectAllocator because a build is a short-lived, // one shot program. We don't need to waste time freeing memory and finding places to squish @@ -207,7 +209,7 @@ pub fn main() !void { builder.debug_log_scopes = debug_log_scopes.items; builder.resolveInstallPrefix(install_prefix, dir_list); - try runBuild(builder); + try builder.runBuild(root); if (builder.validateUserInputDidItFail()) return usageAndErr(builder, true, stderr_stream); @@ -223,19 +225,11 @@ pub fn main() !void { }; } -fn runBuild(builder: *Builder) anyerror!void { - switch (@typeInfo(@typeInfo(@TypeOf(root.build)).Fn.return_type.?)) { - .Void => root.build(builder), - .ErrorUnion => try root.build(builder), - else => @compileError("expected return type of build to be 'void' or '!void'"), - } -} - fn usage(builder: *Builder, already_ran_build: bool, out_stream: anytype) !void { // run the build script to collect the options if (!already_ran_build) { builder.resolveInstallPrefix(null, .{}); - try runBuild(builder); + try builder.runBuild(root); } try out_stream.print( diff --git a/lib/std/build.zig b/lib/std/build.zig index 7ce8ae2d10..43ec1eea20 100644 --- a/lib/std/build.zig +++ b/lib/std/build.zig @@ -69,13 +69,15 @@ pub const Builder = struct { search_prefixes: ArrayList([]const u8), libc_file: ?[]const u8 = null, installed_files: ArrayList(InstalledFile), + /// Path to the directory containing build.zig. build_root: []const u8, cache_root: []const u8, global_cache_root: []const u8, release_mode: ?std.builtin.Mode, is_release: bool, + /// zig lib dir override_lib_dir: ?[]const u8, - vcpkg_root: VcpkgRoot, + vcpkg_root: VcpkgRoot = .unattempted, pkg_config_pkg_list: ?(PkgConfigError![]const PkgConfigPkg) = null, args: ?[][]const u8 = null, debug_log_scopes: []const []const u8 = &.{}, @@ -100,6 +102,8 @@ pub const Builder = struct { /// Information about the native target. Computed before build() is invoked. host: NativeTargetInfo, + dep_prefix: []const u8 = "", + pub const ExecError = error{ ReadFailure, ExitCodeFailure, @@ -223,7 +227,6 @@ pub const Builder = struct { .is_release = false, .override_lib_dir = null, .install_path = undefined, - .vcpkg_root = VcpkgRoot{ .unattempted = {} }, .args = null, .host = host, }; @@ -233,6 +236,89 @@ pub const Builder = struct { return self; } + fn createChild( + parent: *Builder, + dep_name: []const u8, + build_root: []const u8, + args: anytype, + ) !*Builder { + const child = try createChildOnly(parent, dep_name, build_root); + try applyArgs(child, args); + return child; + } + + fn createChildOnly(parent: *Builder, dep_name: []const u8, build_root: []const u8) !*Builder { + const allocator = parent.allocator; + const child = try allocator.create(Builder); + child.* = .{ + .allocator = allocator, + .install_tls = .{ + .step = Step.initNoOp(.top_level, "install", allocator), + .description = "Copy build artifacts to prefix path", + }, + .uninstall_tls = .{ + .step = Step.init(.top_level, "uninstall", allocator, makeUninstall), + .description = "Remove build artifacts from prefix path", + }, + .user_input_options = UserInputOptionsMap.init(allocator), + .available_options_map = AvailableOptionsMap.init(allocator), + .available_options_list = ArrayList(AvailableOption).init(allocator), + .verbose = parent.verbose, + .verbose_link = parent.verbose_link, + .verbose_cc = parent.verbose_cc, + .verbose_air = parent.verbose_air, + .verbose_llvm_ir = parent.verbose_llvm_ir, + .verbose_cimport = parent.verbose_cimport, + .verbose_llvm_cpu_features = parent.verbose_llvm_cpu_features, + .prominent_compile_errors = parent.prominent_compile_errors, + .color = parent.color, + .reference_trace = parent.reference_trace, + .invalid_user_input = false, + .zig_exe = parent.zig_exe, + .default_step = undefined, + .env_map = parent.env_map, + .top_level_steps = ArrayList(*TopLevelStep).init(allocator), + .install_prefix = undefined, + .dest_dir = parent.dest_dir, + .lib_dir = parent.lib_dir, + .exe_dir = parent.exe_dir, + .h_dir = parent.h_dir, + .install_path = parent.install_path, + .sysroot = parent.sysroot, + .search_prefixes = ArrayList([]const u8).init(allocator), + .libc_file = parent.libc_file, + .installed_files = ArrayList(InstalledFile).init(allocator), + .build_root = build_root, + .cache_root = parent.cache_root, + .global_cache_root = parent.global_cache_root, + .release_mode = parent.release_mode, + .is_release = parent.is_release, + .override_lib_dir = parent.override_lib_dir, + .debug_log_scopes = parent.debug_log_scopes, + .debug_compile_errors = parent.debug_compile_errors, + .enable_darling = parent.enable_darling, + .enable_qemu = parent.enable_qemu, + .enable_rosetta = parent.enable_rosetta, + .enable_wasmtime = parent.enable_wasmtime, + .enable_wine = parent.enable_wine, + .glibc_runtimes_dir = parent.glibc_runtimes_dir, + .host = parent.host, + .dep_prefix = parent.fmt("{s}{s}.", .{ parent.dep_prefix, dep_name }), + }; + try child.top_level_steps.append(&child.install_tls); + try child.top_level_steps.append(&child.uninstall_tls); + child.default_step = &child.install_tls.step; + return child; + } + + pub fn applyArgs(b: *Builder, args: anytype) !void { + // TODO this function is the way that a build.zig file communicates + // options to its dependencies. It is the programmatic way to give + // command line arguments to a build.zig script. + _ = b; + _ = args; + } + pub fn destroy(self: *Builder) void { self.env_map.deinit(); self.top_level_steps.deinit(); @@ -1300,6 +1386,70 @@ pub const Builder = struct { &[_][]const u8{ base_dir, dest_rel_path }, ) catch unreachable; } + + pub const Dependency = struct { + builder: *Builder, + + pub fn artifact(d: *Dependency, name: []const u8) *LibExeObjStep { + var found: ?*LibExeObjStep = null; + for (d.builder.install_tls.step.dependencies.items) |dep_step| { + const inst = dep_step.cast(InstallArtifactStep) orelse continue; + if (mem.eql(u8, inst.artifact.name, name)) { + if (found != null) panic("artifact name '{s}' is ambiguous", .{name}); + found = inst.artifact; + } + } + return found orelse { + for (d.builder.install_tls.step.dependencies.items) |dep_step| { + const inst = dep_step.cast(InstallArtifactStep) orelse continue; + log.info("available artifact: '{s}'", .{inst.artifact.name}); + } + panic("unable to find artifact '{s}'", .{name}); + }; + } + }; + + pub fn dependency(b: *Builder, name: []const u8, args: anytype) *Dependency { + const build_runner = @import("root"); + const deps = build_runner.dependencies; + + inline for (@typeInfo(deps.imports).Struct.decls) |decl| { + if (mem.startsWith(u8, decl.name, b.dep_prefix) and + mem.endsWith(u8, decl.name, name) and + decl.name.len == b.dep_prefix.len + name.len) + { + const build_zig = @field(deps.imports, decl.name); + const build_root = @field(deps.build_root, decl.name); + return dependencyInner(b, name, build_root, build_zig, args); + } + } + + const full_path = b.pathFromRoot("build.zig.ini"); + std.debug.print("no dependency named '{s}' in '{s}'\n", .{ name, full_path }); + std.process.exit(1); + } + + fn dependencyInner( + b: *Builder, + name: []const u8, + build_root: []const u8, + comptime build_zig: type, + args: anytype, + ) *Dependency { + const sub_builder = b.createChild(name, build_root, args) catch unreachable; + sub_builder.runBuild(build_zig) catch unreachable; + const dep = b.allocator.create(Dependency) catch unreachable; + dep.* = .{ .builder = sub_builder }; + return dep; + } + + pub fn runBuild(b: *Builder, build_zig: anytype) anyerror!void { + switch (@typeInfo(@typeInfo(@TypeOf(build_zig.build)).Fn.return_type.?)) { + .Void => build_zig.build(b), + .ErrorUnion => try build_zig.build(b), + else => @compileError("expected return type of build to be 'void' or '!void'"), + } + } }; test "builder.findProgram compiles" { diff --git a/src/Package.zig b/src/Package.zig index c3a7f645a6..02baba4ca7 100644 --- a/src/Package.zig +++ b/src/Package.zig @@ -12,6 +12,8 @@ const Compilation = @import("Compilation.zig"); const Module = @import("Module.zig"); const ThreadPool = @import("ThreadPool.zig"); const WaitGroup = @import("WaitGroup.zig"); +const Cache = @import("Cache.zig"); +const build_options = @import("build_options"); pub const Table = std.StringHashMapUnmanaged(*Package); @@ -139,6 +141,9 @@ pub fn fetchAndAddDependencies( directory: Compilation.Directory, global_cache_directory: Compilation.Directory, local_cache_directory: Compilation.Directory, + dependencies_source: *std.ArrayList(u8), + build_roots_source: *std.ArrayList(u8), + name_prefix: []const u8, ) !void { const max_bytes = 10 * 1024 * 1024; const gpa = thread_pool.allocator; @@ -156,15 +161,15 @@ pub fn fetchAndAddDependencies( var it = ini.iterateSection("\n[dependency]\n"); while (it.next()) |dep| { var line_it = mem.split(u8, dep, "\n"); - var opt_id: ?[]const u8 = null; + var opt_name: ?[]const u8 = null; var opt_url: ?[]const u8 = null; var expected_hash: ?[]const u8 = null; while (line_it.next()) |kv| { const eq_pos = mem.indexOfScalar(u8, kv, '=') orelse continue; const key = kv[0..eq_pos]; const value = kv[eq_pos + 1 ..]; - if (mem.eql(u8, key, "id")) { - opt_id = value; + if (mem.eql(u8, key, "name")) { + opt_name = value; } else if (mem.eql(u8, key, "url")) { opt_url = value; } else if (mem.eql(u8, key, "hash")) { @@ -181,9 +186,9 @@ pub fn fetchAndAddDependencies( } } - const id = opt_id orelse { + const name = opt_name orelse { const loc = std.zig.findLineColumn(ini.bytes, @ptrToInt(dep.ptr) - @ptrToInt(ini.bytes.ptr)); - std.log.err("{s}/{s}:{d}:{d} missing key: 'id'", .{ + std.log.err("{s}/{s}:{d}:{d} missing key: 'name'", .{ directory.path orelse ".", "build.zig.ini", loc.line, @@ -195,7 +200,7 @@ pub fn fetchAndAddDependencies( const url = opt_url orelse { const loc = std.zig.findLineColumn(ini.bytes, @ptrToInt(dep.ptr) - @ptrToInt(ini.bytes.ptr)); - std.log.err("{s}/{s}:{d}:{d} missing key: 'id'", .{ + std.log.err("{s}/{s}:{d}:{d} missing key: 'name'", .{ directory.path orelse ".", "build.zig.ini", loc.line, @@ -205,6 +210,10 @@ pub fn fetchAndAddDependencies( continue; }; + const sub_prefix = try std.fmt.allocPrint(gpa, "{s}{s}.", .{ name_prefix, name }); + defer gpa.free(sub_prefix); + const fqn = sub_prefix[0 .. sub_prefix.len - 1]; + const sub_pkg = try fetchAndUnpack( thread_pool, http_client, @@ -213,22 +222,56 @@ pub fn fetchAndAddDependencies( expected_hash, ini, directory, + build_roots_source, + fqn, ); - try sub_pkg.fetchAndAddDependencies( + try pkg.fetchAndAddDependencies( thread_pool, http_client, sub_pkg.root_src_directory, global_cache_directory, local_cache_directory, + dependencies_source, + build_roots_source, + sub_prefix, ); - try addAndAdopt(pkg, gpa, id, sub_pkg); + try addAndAdopt(pkg, gpa, fqn, sub_pkg); + + try dependencies_source.writer().print(" pub const {s} = @import(\"{}\");\n", .{ + std.zig.fmtId(fqn), std.zig.fmtEscapes(fqn), + }); } if (any_error) return error.InvalidBuildZigIniFile; } +pub fn createFilePkg( + gpa: Allocator, + global_cache_directory: Compilation.Directory, + basename: []const u8, + contents: []const u8, +) !*Package { + const rand_int = std.crypto.random.int(u64); + const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ hex64(rand_int); + { + var tmp_dir = try global_cache_directory.handle.makeOpenPath(tmp_dir_sub_path, .{}); + defer tmp_dir.close(); + try tmp_dir.writeFile(basename, contents); + } + + var hh: Cache.HashHelper = .{}; + hh.addBytes(build_options.version); + hh.addBytes(contents); + const hex_digest = hh.final(); + + const o_dir_sub_path = "o" ++ fs.path.sep_str ++ hex_digest; + try renameTmpIntoCache(global_cache_directory.handle, tmp_dir_sub_path, o_dir_sub_path); + + return createWithDir(gpa, global_cache_directory, o_dir_sub_path, basename); +} + fn fetchAndUnpack( thread_pool: *ThreadPool, http_client: *std.http.Client, @@ -237,6 +280,8 @@ fn fetchAndUnpack( expected_hash: ?[]const u8, ini: std.Ini, comp_directory: Compilation.Directory, + build_roots_source: *std.ArrayList(u8), + fqn: []const u8, ) !*Package { const gpa = http_client.allocator; const s = fs.path.sep_str; @@ -267,14 +312,22 @@ fn fetchAndUnpack( const owned_src_path = try gpa.dupe(u8, build_zig_basename); errdefer gpa.free(owned_src_path); + const build_root = try global_cache_directory.join(gpa, &.{pkg_dir_sub_path}); + errdefer gpa.free(build_root); + + try build_roots_source.writer().print(" pub const {s} = \"{}\";\n", .{ + std.zig.fmtId(fqn), std.zig.fmtEscapes(build_root), + }); + ptr.* = .{ .root_src_directory = .{ - .path = try global_cache_directory.join(gpa, &.{pkg_dir_sub_path}), + .path = build_root, .handle = pkg_dir, }, .root_src_directory_owned = true, .root_src_path = owned_src_path, }; + return ptr; } @@ -331,31 +384,7 @@ fn fetchAndUnpack( }; const pkg_dir_sub_path = "p" ++ s ++ hexDigest(actual_hash); - - { - // Rename the temporary directory into the global package cache. - var handled_missing_dir = false; - while (true) { - global_cache_directory.handle.rename(tmp_dir_sub_path, pkg_dir_sub_path) catch |err| switch (err) { - error.FileNotFound => { - if (handled_missing_dir) return err; - global_cache_directory.handle.makeDir("p") catch |mkd_err| switch (mkd_err) { - error.PathAlreadyExists => handled_missing_dir = true, - else => |e| return e, - }; - continue; - }, - error.PathAlreadyExists => { - // Package has been already downloaded and may already be in use on the system. - global_cache_directory.handle.deleteTree(tmp_dir_sub_path) catch |del_err| { - std.log.warn("unable to delete temp directory: {s}", .{@errorName(del_err)}); - }; - }, - else => |e| return e, - }; - break; - } - } + try renameTmpIntoCache(global_cache_directory.handle, tmp_dir_sub_path, pkg_dir_sub_path); if (expected_hash) |h| { const actual_hex = hexDigest(actual_hash); @@ -378,6 +407,13 @@ fn fetchAndUnpack( ); } + const build_root = try global_cache_directory.join(gpa, &.{pkg_dir_sub_path}); + defer gpa.free(build_root); + + try build_roots_source.writer().print(" pub const {s} = \"{}\";\n", .{ + std.zig.fmtId(fqn), std.zig.fmtEscapes(build_root), + }); + return createWithDir(gpa, global_cache_directory, pkg_dir_sub_path, build_zig_basename); } @@ -516,3 +552,32 @@ fn hexDigest(digest: [Hash.digest_length]u8) [Hash.digest_length * 2]u8 { } return result; } + +fn renameTmpIntoCache( + cache_dir: fs.Dir, + tmp_dir_sub_path: []const u8, + dest_dir_sub_path: []const u8, +) !void { + assert(dest_dir_sub_path[1] == '/'); + var handled_missing_dir = false; + while (true) { + cache_dir.rename(tmp_dir_sub_path, dest_dir_sub_path) catch |err| switch (err) { + error.FileNotFound => { + if (handled_missing_dir) return err; + cache_dir.makeDir(dest_dir_sub_path[0..1]) catch |mkd_err| switch (mkd_err) { + error.PathAlreadyExists => handled_missing_dir = true, + else => |e| return e, + }; + continue; + }, + error.PathAlreadyExists => { + // Package has been already downloaded and may already be in use on the system. + cache_dir.deleteTree(tmp_dir_sub_path) catch |del_err| { + std.log.warn("unable to delete temp directory: {s}", .{@errorName(del_err)}); + }; + }, + else => |e| return e, + }; + break; + } +} diff --git a/src/main.zig b/src/main.zig index 739f8093e1..976ea26064 100644 --- a/src/main.zig +++ b/src/main.zig @@ -3983,11 +3983,6 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi }; defer zig_lib_directory.handle.close(); - var main_pkg: Package = .{ - .root_src_directory = zig_lib_directory, - .root_src_path = "build_runner.zig", - }; - var cleanup_build_dir: ?fs.Dir = null; defer if (cleanup_build_dir) |*dir| dir.close(); @@ -4031,12 +4026,6 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi }; child_argv.items[argv_index_build_file] = build_directory.path orelse cwd_path; - var build_pkg: Package = .{ - .root_src_directory = build_directory, - .root_src_path = build_zig_basename, - }; - try main_pkg.addAndAdopt(arena, "@build", &build_pkg); - var global_cache_directory: Compilation.Directory = l: { const p = override_global_cache_dir orelse try introspect.resolveGlobalCacheDir(arena); break :l .{ @@ -4083,23 +4072,65 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi try thread_pool.init(gpa); defer thread_pool.deinit(); + var main_pkg: Package = .{ + .root_src_directory = zig_lib_directory, + .root_src_path = "build_runner.zig", + }; + if (!build_options.omit_pkg_fetching_code) { var http_client: std.http.Client = .{ .allocator = gpa }; defer http_client.deinit(); try http_client.rescanRootCertificates(); + // Here we provide an import to the build runner that allows using reflection to find + // all of the dependencies. Without this, there would be no way to use `@import` to + // access dependencies by name, since `@import` requires string literals. + var dependencies_source = std.ArrayList(u8).init(gpa); + defer dependencies_source.deinit(); + try dependencies_source.appendSlice("pub const imports = struct {\n"); + + // This will go into the same package. It contains the file system paths + // to all the build.zig files. + var build_roots_source = std.ArrayList(u8).init(gpa); + defer build_roots_source.deinit(); + + // Here we borrow main package's table and will replace it with a fresh + // one after this process completes. main_pkg.fetchAndAddDependencies( &thread_pool, &http_client, build_directory, global_cache_directory, local_cache_directory, + &dependencies_source, + &build_roots_source, + "", ) catch |err| switch (err) { error.PackageFetchFailed => process.exit(1), else => |e| return e, }; + + try dependencies_source.appendSlice("};\npub const build_root = struct {\n"); + try dependencies_source.appendSlice(build_roots_source.items); + try dependencies_source.appendSlice("};\n"); + + const deps_pkg = try Package.createFilePkg( + gpa, + global_cache_directory, + "dependencies.zig", + dependencies_source.items, + ); + + mem.swap(Package.Table, &main_pkg.table, &deps_pkg.table); + try main_pkg.addAndAdopt(gpa, "@dependencies", deps_pkg); } + var build_pkg: Package = .{ + .root_src_directory = build_directory, + .root_src_path = build_zig_basename, + }; + try main_pkg.addAndAdopt(gpa, "@build", &build_pkg); + const comp = Compilation.create(gpa, .{ .zig_lib_directory = zig_lib_directory, .local_cache_directory = local_cache_directory,