From 94529ffb621fa633437ac48d8f90003e26e8ce5b Mon Sep 17 00:00:00 2001 From: mlugg Date: Wed, 13 Sep 2023 10:46:25 +0100 Subject: [PATCH] package manager: write deps in a flat format, eliminating the FQN concept The new `@depedencies` module contains generated code like the following (where strings like "abc123" represent hashes): ```zig pub const root_deps = [_]struct { []const u8, []const u8 }{ .{ "foo", "abc123" }, }; pub const packages = struct { pub const abc123 = struct { pub const build_root = "/home/mlugg/.cache/zig/blah/abc123"; pub const build_zig = @import("abc123"); pub const deps = [_]struct { []const u8, []const u8 }{ .{ "bar", "abc123" }, .{ "name", "ghi789" }, }; }; }; ``` Each package contains a build root string, the build.zig import, and a mapping from dependency names to package hashes. There is also such a mapping for the root package dependencies. In theory, we could now remove the `dep_prefix` field from `std.Build`, since its main purpose is now handled differently. I believe this is a desirable goal, as it doesn't really make sense to assign a single FQN to any package (because it may appear in many different places in the package hierarchy). This commit does not remove that field, as it's used non-trivially in a few places in the build runner and compiler tests: this will be a future enhancement. Resolves: #16354 Resolves: #17135 --- lib/build_runner.zig | 1 + lib/std/Build.zig | 41 ++++++++----- lib/std/Build/Step/Options.zig | 1 + src/Package.zig | 107 ++++++++++++++++++++++++--------- src/main.zig | 22 +++---- 5 files changed, 115 insertions(+), 57 deletions(-) diff --git a/lib/build_runner.zig b/lib/build_runner.zig index 7d036fa5bf..9fd94e40b3 100644 --- a/lib/build_runner.zig +++ b/lib/build_runner.zig @@ -81,6 +81,7 @@ pub fn main() !void { global_cache_directory, host, &cache, + dependencies.root_deps, ); defer builder.destroy(); diff --git a/lib/std/Build.zig b/lib/std/Build.zig index 472ef7d740..1852f75fba 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -132,6 +132,10 @@ modules: std.StringArrayHashMap(*Module), /// A map from build root dirs to the corresponding `*Dependency`. This is shared with all child /// `Build`s. initialized_deps: *InitializedDepMap, +/// A mapping from dependency names to package hashes. +available_deps: AvailableDeps, + +const AvailableDeps = []const struct { []const u8, []const u8 }; const InitializedDepMap = std.HashMap(InitializedDepKey, *Dependency, InitializedDepContext, std.hash_map.default_max_load_percentage); const InitializedDepKey = struct { @@ -248,6 +252,7 @@ pub fn create( global_cache_root: Cache.Directory, host: NativeTargetInfo, cache: *Cache, + available_deps: AvailableDeps, ) !*Build { const env_map = try allocator.create(EnvMap); env_map.* = try process.getEnvMap(allocator); @@ -308,6 +313,7 @@ pub fn create( .host = host, .modules = std.StringArrayHashMap(*Module).init(allocator), .initialized_deps = initialized_deps, + .available_deps = available_deps, }; try self.top_level_steps.put(allocator, self.install_tls.step.name, &self.install_tls); try self.top_level_steps.put(allocator, self.uninstall_tls.step.name, &self.uninstall_tls); @@ -319,14 +325,15 @@ fn createChild( parent: *Build, dep_name: []const u8, build_root: Cache.Directory, + pkg_deps: AvailableDeps, user_input_options: UserInputOptionsMap, ) !*Build { - const child = try createChildOnly(parent, dep_name, build_root, user_input_options); + const child = try createChildOnly(parent, dep_name, build_root, pkg_deps, user_input_options); try determineAndApplyInstallPrefix(child); return child; } -fn createChildOnly(parent: *Build, dep_name: []const u8, build_root: Cache.Directory, user_input_options: UserInputOptionsMap) !*Build { +fn createChildOnly(parent: *Build, dep_name: []const u8, build_root: Cache.Directory, pkg_deps: AvailableDeps, user_input_options: UserInputOptionsMap) !*Build { const allocator = parent.allocator; const child = try allocator.create(Build); child.* = .{ @@ -393,6 +400,7 @@ fn createChildOnly(parent: *Build, dep_name: []const u8, build_root: Cache.Direc .dep_prefix = parent.fmt("{s}{s}.", .{ parent.dep_prefix, dep_name }), .modules = std.StringArrayHashMap(*Module).init(allocator), .initialized_deps = parent.initialized_deps, + .available_deps = pkg_deps, }; try child.top_level_steps.put(allocator, child.install_tls.step.name, &child.install_tls); try child.top_level_steps.put(allocator, child.uninstall_tls.step.name, &child.uninstall_tls); @@ -1705,20 +1713,22 @@ pub fn dependency(b: *Build, name: []const u8, args: anytype) *Dependency { const build_runner = @import("root"); const deps = build_runner.dependencies; - inline for (@typeInfo(deps.imports).Struct.decls) |decl| { - if (mem.startsWith(u8, decl.name, b.dep_prefix) and - mem.endsWith(u8, decl.name, name) and - decl.name.len == b.dep_prefix.len + name.len) - { - const build_zig = @field(deps.imports, decl.name); - const build_root = @field(deps.build_root, decl.name); - return dependencyInner(b, name, build_root, build_zig, args); + const pkg_hash = for (b.available_deps) |dep| { + if (mem.eql(u8, dep[0], name)) break dep[1]; + } else { + const full_path = b.pathFromRoot("build.zig.zon"); + std.debug.print("no dependency named '{s}' in '{s}'. All packages used in build.zig must be declared in this file.\n", .{ name, full_path }); + process.exit(1); + }; + + inline for (@typeInfo(deps.packages).Struct.decls) |decl| { + if (mem.eql(u8, decl.name, pkg_hash)) { + const pkg = @field(deps.packages, decl.name); + return dependencyInner(b, name, pkg.build_root, pkg.build_zig, pkg.deps, args); } } - const full_path = b.pathFromRoot("build.zig.zon"); - std.debug.print("no dependency named '{s}' in '{s}'. All packages used in build.zig must be declared in this file.\n", .{ name, full_path }); - process.exit(1); + unreachable; // Bad @dependencies source } pub fn anonymousDependency( @@ -1737,7 +1747,7 @@ pub fn anonymousDependency( '/', '\\' => byte.* = '.', else => continue, }; - return dependencyInner(b, name, build_root, build_zig, args); + return dependencyInner(b, name, build_root, build_zig, &.{}, args); } fn userValuesAreSame(lhs: UserValue, rhs: UserValue) bool { @@ -1792,6 +1802,7 @@ pub fn dependencyInner( name: []const u8, build_root_string: []const u8, comptime build_zig: type, + pkg_deps: AvailableDeps, args: anytype, ) *Dependency { const user_input_options = userInputOptionsFromArgs(b.allocator, args); @@ -1810,7 +1821,7 @@ pub fn dependencyInner( process.exit(1); }, }; - const sub_builder = b.createChild(name, build_root, user_input_options) catch @panic("unhandled error"); + const sub_builder = b.createChild(name, build_root, pkg_deps, user_input_options) catch @panic("unhandled error"); sub_builder.runBuild(build_zig) catch @panic("unhandled error"); if (sub_builder.validateUserInputDidItFail()) { diff --git a/lib/std/Build/Step/Options.zig b/lib/std/Build/Step/Options.zig index 575ee747d1..5c3ea6d30b 100644 --- a/lib/std/Build/Step/Options.zig +++ b/lib/std/Build/Step/Options.zig @@ -314,6 +314,7 @@ test Options { .{ .path = "test", .handle = std.fs.cwd() }, host, &cache, + &.{}, ); defer builder.destroy(); diff --git a/src/Package.zig b/src/Package.zig index 9f18360201..bb2ab837f7 100644 --- a/src/Package.zig +++ b/src/Package.zig @@ -214,6 +214,8 @@ pub fn getName(target: *const Package, gpa: Allocator, mod: Module) ![]const u8 pub const build_zig_basename = "build.zig"; +/// Fetches a package and all of its dependencies recursively. Writes the +/// corresponding datastructures for the build runner into `dependencies_source`. pub fn fetchAndAddDependencies( pkg: *Package, deps_pkg: *Package, @@ -224,11 +226,11 @@ pub fn fetchAndAddDependencies( global_cache_directory: Compilation.Directory, local_cache_directory: Compilation.Directory, dependencies_source: *std.ArrayList(u8), - build_roots_source: *std.ArrayList(u8), - name_prefix: []const u8, error_bundle: *std.zig.ErrorBundle.Wip, all_modules: *AllModules, root_prog_node: *std.Progress.Node, + /// null for the root package + this_hash: ?[]const u8, ) !void { const max_bytes = 10 * 1024 * 1024; const gpa = thread_pool.allocator; @@ -242,6 +244,28 @@ pub fn fetchAndAddDependencies( ) catch |err| switch (err) { error.FileNotFound => { // Handle the same as no dependencies. + if (this_hash) |hash| { + const pkg_dir_sub_path = "p" ++ fs.path.sep_str ++ hash[0..hex_multihash_len]; + const build_root = try global_cache_directory.join(arena, &.{pkg_dir_sub_path}); + try dependencies_source.writer().print( + \\ pub const {} = struct {{ + \\ pub const build_root = "{}"; + \\ pub const build_zig = @import("{}"); + \\ pub const deps: []const struct {{ []const u8, []const u8 }} = &.{{}}; + \\ }}; + \\ + , .{ + std.zig.fmtId(hash), + std.zig.fmtEscapes(build_root), + std.zig.fmtEscapes(hash), + }); + } else { + try dependencies_source.writer().writeAll( + \\pub const packages = struct {}; + \\pub const root_deps: []const struct { []const u8, []const u8 } = &.{}; + \\ + ); + } return; }, else => |e| return e, @@ -284,23 +308,23 @@ pub fn fetchAndAddDependencies( root_prog_node.setEstimatedTotalItems(all_modules.count()); + if (this_hash == null) { + try dependencies_source.writer().writeAll("pub const packages = struct {\n"); + } + const deps_list = manifest.dependencies.values(); for (manifest.dependencies.keys(), 0..) |name, i| { const dep = deps_list[i]; - const sub_prefix = try std.fmt.allocPrint(arena, "{s}{s}.", .{ name_prefix, name }); - const fqn = sub_prefix[0 .. sub_prefix.len - 1]; - const sub = try fetchAndUnpack( thread_pool, http_client, global_cache_directory, dep, report, - build_roots_source, - fqn, all_modules, root_prog_node, + name, ); if (!sub.found_existing) { @@ -313,11 +337,10 @@ pub fn fetchAndAddDependencies( global_cache_directory, local_cache_directory, dependencies_source, - build_roots_source, - sub_prefix, error_bundle, all_modules, root_prog_node, + dep.hash.?, ); } @@ -329,10 +352,47 @@ pub fn fetchAndAddDependencies( } else { try deps_pkg.add(gpa, dep.hash.?, sub.mod); } + } - try dependencies_source.writer().print(" pub const {s} = @import(\"{}\");\n", .{ - std.zig.fmtId(fqn), std.zig.fmtEscapes(dep.hash.?), + if (this_hash) |hash| { + const pkg_dir_sub_path = "p" ++ fs.path.sep_str ++ hash[0..hex_multihash_len]; + const build_root = try global_cache_directory.join(arena, &.{pkg_dir_sub_path}); + try dependencies_source.writer().print( + \\ pub const {} = struct {{ + \\ pub const build_root = "{}"; + \\ pub const build_zig = @import("{}"); + \\ pub const deps: []const struct {{ []const u8, []const u8 }} = &.{{ + \\ + , .{ + std.zig.fmtId(hash), + std.zig.fmtEscapes(build_root), + std.zig.fmtEscapes(hash), }); + for (manifest.dependencies.keys(), manifest.dependencies.values()) |name, dep| { + try dependencies_source.writer().print( + " .{{ \"{}\", \"{}\" }},\n", + .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(dep.hash.?) }, + ); + } + try dependencies_source.writer().writeAll( + \\ }; + \\ }; + \\ + ); + } else { + try dependencies_source.writer().writeAll( + \\}; + \\ + \\pub const root_deps: []const struct { []const u8, []const u8 } = &.{ + \\ + ); + for (manifest.dependencies.keys(), manifest.dependencies.values()) |name, dep| { + try dependencies_source.writer().print( + " .{{ \"{}\", \"{}\" }},\n", + .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(dep.hash.?) }, + ); + } + try dependencies_source.writer().writeAll("};\n"); } } @@ -470,10 +530,11 @@ fn fetchAndUnpack( global_cache_directory: Compilation.Directory, dep: Manifest.Dependency, report: Report, - build_roots_source: *std.ArrayList(u8), - fqn: []const u8, all_modules: *AllModules, root_prog_node: *std.Progress.Node, + /// This does not have to be any form of canonical or fully-qualified name: it + /// is only intended to be human-readable for progress reporting. + name_for_prog: []const u8, ) !struct { mod: *Package, found_existing: bool } { const gpa = http_client.allocator; const s = fs.path.sep_str; @@ -484,25 +545,17 @@ fn fetchAndUnpack( const hex_digest = h[0..hex_multihash_len]; const pkg_dir_sub_path = "p" ++ s ++ hex_digest; - const build_root = try global_cache_directory.join(gpa, &.{pkg_dir_sub_path}); - errdefer gpa.free(build_root); - var pkg_dir = global_cache_directory.handle.openDir(pkg_dir_sub_path, .{}) catch |err| switch (err) { error.FileNotFound => break :cached, else => |e| return e, }; errdefer pkg_dir.close(); - try build_roots_source.writer().print(" pub const {s} = \"{}\";\n", .{ - std.zig.fmtId(fqn), std.zig.fmtEscapes(build_root), - }); - // The compiler has a rule that a file must not be included in multiple modules, // so we must detect if a module has been created for this package and reuse it. const gop = try all_modules.getOrPut(gpa, hex_digest.*); if (gop.found_existing) { if (gop.value_ptr.*) |mod| { - gpa.free(build_root); return .{ .mod = mod, .found_existing = true, @@ -510,6 +563,9 @@ fn fetchAndUnpack( } } + const build_root = try global_cache_directory.join(gpa, &.{pkg_dir_sub_path}); + errdefer gpa.free(build_root); + root_prog_node.completeOne(); const ptr = try gpa.create(Package); @@ -534,7 +590,7 @@ fn fetchAndUnpack( }; } - var pkg_prog_node = root_prog_node.start(fqn, 0); + var pkg_prog_node = root_prog_node.start(name_for_prog, 0); defer pkg_prog_node.end(); pkg_prog_node.activate(); pkg_prog_node.context.refresh(); @@ -666,13 +722,6 @@ fn fetchAndUnpack( return error.PackageFetchFailed; } - const build_root = try global_cache_directory.join(gpa, &.{pkg_dir_sub_path}); - defer gpa.free(build_root); - - try build_roots_source.writer().print(" pub const {s} = \"{}\";\n", .{ - std.zig.fmtId(fqn), std.zig.fmtEscapes(build_root), - }); - const mod = try createWithDir(gpa, global_cache_directory, pkg_dir_sub_path, build_zig_basename); try all_modules.put(gpa, actual_hex, mod); return .{ diff --git a/src/main.zig b/src/main.zig index 6e7b330c40..6c28503f9b 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4708,7 +4708,14 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi .root_src_directory = build_directory, .root_src_path = build_zig_basename, }; - if (!build_options.only_core_functionality) { + if (build_options.only_core_functionality) { + const deps_pkg = try Package.createFilePkg(gpa, local_cache_directory, "dependencies.zig", + \\pub const packages = struct {}; + \\pub const root_deps: []const struct { []const u8, []const u8 } = &.{}; + \\ + ); + try main_pkg.add(gpa, "@dependencies", deps_pkg); + } else { var http_client: std.http.Client = .{ .allocator = gpa }; defer http_client.deinit(); @@ -4717,12 +4724,6 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi // access dependencies by name, since `@import` requires string literals. var dependencies_source = std.ArrayList(u8).init(gpa); defer dependencies_source.deinit(); - try dependencies_source.appendSlice("pub const imports = struct {\n"); - - // This will go into the same package. It contains the file system paths - // to all the build.zig files. - var build_roots_source = std.ArrayList(u8).init(gpa); - defer build_roots_source.deinit(); var all_modules: Package.AllModules = .{}; defer all_modules.deinit(gpa); @@ -4746,11 +4747,10 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi global_cache_directory, local_cache_directory, &dependencies_source, - &build_roots_source, - "", &wip_errors, &all_modules, root_prog_node, + null, ); if (wip_errors.root_list.items.len > 0) { var errors = try wip_errors.toOwnedBundle(""); @@ -4760,10 +4760,6 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi } try fetch_result; - try dependencies_source.appendSlice("};\npub const build_root = struct {\n"); - try dependencies_source.appendSlice(build_roots_source.items); - try dependencies_source.appendSlice("};\n"); - const deps_pkg = try Package.createFilePkg( gpa, local_cache_directory,