jimzah/zig
1
0
Fork 0
mirror of https://github.com/ziglang/zig.git synced 2024-11-26 23:22:44 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

package manager: write deps in a flat format, eliminating the FQN concept

Browse Source 
The new `@depedencies` module contains generated code like the
following (where strings like "abc123" represent hashes):

```zig
pub const root_deps = [_]struct { []const u8, []const u8 }{
    .{ "foo", "abc123" },
};

pub const packages = struct {
    pub const abc123 = struct {
        pub const build_root = "/home/mlugg/.cache/zig/blah/abc123";
        pub const build_zig = @import("abc123");
        pub const deps = [_]struct { []const u8, []const u8 }{
            .{ "bar", "abc123" },
            .{ "name", "ghi789" },
        };
    };
};
```

Each package contains a build root string, the build.zig import, and a
mapping from dependency names to package hashes. There is also such a
mapping for the root package dependencies.

In theory, we could now remove the `dep_prefix` field from `std.Build`,
since its main purpose is now handled differently. I believe this is a
desirable goal, as it doesn't really make sense to assign a single FQN
to any package (because it may appear in many different places in the
package hierarchy). This commit does not remove that field, as it's used
non-trivially in a few places in the build runner and compiler tests:
this will be a future enhancement.

Resolves: #16354
Resolves: #17135
This commit is contained in:
mlugg 2023-09-13 10:46:25 +01:00 committed by Andrew Kelley
parent 1a0e6bcdb1
commit 94529ffb62
5 changed files with 115 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
lib/build_runner.zig
View File

@ -81,6 +81,7 @@ pub fn main() !void {
global_cache_directory,
host,
&cache,
dependencies.root_deps,
);
defer builder.destroy();

41
lib/std/Build.zig
View File

@ -132,6 +132,10 @@ modules: std.StringArrayHashMap(*Module),
/// A map from build root dirs to the corresponding `*Dependency`. This is shared with all child
/// `Build`s.
initialized_deps: *InitializedDepMap,
/// A mapping from dependency names to package hashes.
available_deps: AvailableDeps,
const AvailableDeps = []const struct { []const u8, []const u8 };
const InitializedDepMap = std.HashMap(InitializedDepKey, *Dependency, InitializedDepContext, std.hash_map.default_max_load_percentage);
const InitializedDepKey = struct {
@ -248,6 +252,7 @@ pub fn create(
global_cache_root: Cache.Directory,
host: NativeTargetInfo,
cache: *Cache,
available_deps: AvailableDeps,
) !*Build {
const env_map = try allocator.create(EnvMap);
env_map.* = try process.getEnvMap(allocator);
@ -308,6 +313,7 @@ pub fn create(
.host = host,
.modules = std.StringArrayHashMap(*Module).init(allocator),
.initialized_deps = initialized_deps,
.available_deps = available_deps,
};
try self.top_level_steps.put(allocator, self.install_tls.step.name, &self.install_tls);
try self.top_level_steps.put(allocator, self.uninstall_tls.step.name, &self.uninstall_tls);
@ -319,14 +325,15 @@ fn createChild(
parent: *Build,
dep_name: []const u8,
build_root: Cache.Directory,
pkg_deps: AvailableDeps,
user_input_options: UserInputOptionsMap,
) !*Build {
const child = try createChildOnly(parent, dep_name, build_root, user_input_options);
const child = try createChildOnly(parent, dep_name, build_root, pkg_deps, user_input_options);
try determineAndApplyInstallPrefix(child);
return child;
}
fn createChildOnly(parent: *Build, dep_name: []const u8, build_root: Cache.Directory, user_input_options: UserInputOptionsMap) !*Build {
fn createChildOnly(parent: *Build, dep_name: []const u8, build_root: Cache.Directory, pkg_deps: AvailableDeps, user_input_options: UserInputOptionsMap) !*Build {
const allocator = parent.allocator;
const child = try allocator.create(Build);
child.* = .{
@ -393,6 +400,7 @@ fn createChildOnly(parent: *Build, dep_name: []const u8, build_root: Cache.Direc
.dep_prefix = parent.fmt("{s}{s}.", .{ parent.dep_prefix, dep_name }),
.modules = std.StringArrayHashMap(*Module).init(allocator),
.initialized_deps = parent.initialized_deps,
.available_deps = pkg_deps,
};
try child.top_level_steps.put(allocator, child.install_tls.step.name, &child.install_tls);
try child.top_level_steps.put(allocator, child.uninstall_tls.step.name, &child.uninstall_tls);
@ -1705,20 +1713,22 @@ pub fn dependency(b: *Build, name: []const u8, args: anytype) *Dependency {
const build_runner = @import("root");
const deps = build_runner.dependencies;
inline for (@typeInfo(deps.imports).Struct.decls) |decl| {
if (mem.startsWith(u8, decl.name, b.dep_prefix) and
mem.endsWith(u8, decl.name, name) and
decl.name.len == b.dep_prefix.len + name.len)
{
const build_zig = @field(deps.imports, decl.name);
const build_root = @field(deps.build_root, decl.name);
return dependencyInner(b, name, build_root, build_zig, args);
const pkg_hash = for (b.available_deps) |dep| {
if (mem.eql(u8, dep[0], name)) break dep[1];
} else {
const full_path = b.pathFromRoot("build.zig.zon");
std.debug.print("no dependency named '{s}' in '{s}'. All packages used in build.zig must be declared in this file.\n", .{ name, full_path });
process.exit(1);
};
inline for (@typeInfo(deps.packages).Struct.decls) |decl| {
if (mem.eql(u8, decl.name, pkg_hash)) {
const pkg = @field(deps.packages, decl.name);
return dependencyInner(b, name, pkg.build_root, pkg.build_zig, pkg.deps, args);
}
}
const full_path = b.pathFromRoot("build.zig.zon");
std.debug.print("no dependency named '{s}' in '{s}'. All packages used in build.zig must be declared in this file.\n", .{ name, full_path });
process.exit(1);
unreachable; // Bad @dependencies source
}
pub fn anonymousDependency(
@ -1737,7 +1747,7 @@ pub fn anonymousDependency(
'/', '\\' => byte.* = '.',
else => continue,
};
return dependencyInner(b, name, build_root, build_zig, args);
return dependencyInner(b, name, build_root, build_zig, &.{}, args);
}
fn userValuesAreSame(lhs: UserValue, rhs: UserValue) bool {
@ -1792,6 +1802,7 @@ pub fn dependencyInner(
name: []const u8,
build_root_string: []const u8,
comptime build_zig: type,
pkg_deps: AvailableDeps,
args: anytype,
) *Dependency {
const user_input_options = userInputOptionsFromArgs(b.allocator, args);
@ -1810,7 +1821,7 @@ pub fn dependencyInner(
process.exit(1);
},
};
const sub_builder = b.createChild(name, build_root, user_input_options) catch @panic("unhandled error");
const sub_builder = b.createChild(name, build_root, pkg_deps, user_input_options) catch @panic("unhandled error");
sub_builder.runBuild(build_zig) catch @panic("unhandled error");
if (sub_builder.validateUserInputDidItFail()) {

1
lib/std/Build/Step/Options.zig
View File

@ -314,6 +314,7 @@ test Options {
.{ .path = "test", .handle = std.fs.cwd() },
host,
&cache,
&.{},
);
defer builder.destroy();

107
src/Package.zig
View File

@ -214,6 +214,8 @@ pub fn getName(target: *const Package, gpa: Allocator, mod: Module) ![]const u8
pub const build_zig_basename = "build.zig";
/// Fetches a package and all of its dependencies recursively. Writes the
/// corresponding datastructures for the build runner into `dependencies_source`.
pub fn fetchAndAddDependencies(
pkg: *Package,
deps_pkg: *Package,
@ -224,11 +226,11 @@ pub fn fetchAndAddDependencies(
global_cache_directory: Compilation.Directory,
local_cache_directory: Compilation.Directory,
dependencies_source: *std.ArrayList(u8),
build_roots_source: *std.ArrayList(u8),
name_prefix: []const u8,
error_bundle: *std.zig.ErrorBundle.Wip,
all_modules: *AllModules,
root_prog_node: *std.Progress.Node,
/// null for the root package
this_hash: ?[]const u8,
) !void {
const max_bytes = 10 * 1024 * 1024;
const gpa = thread_pool.allocator;
@ -242,6 +244,28 @@ pub fn fetchAndAddDependencies(
) catch |err| switch (err) {
error.FileNotFound => {
// Handle the same as no dependencies.
if (this_hash) |hash| {
const pkg_dir_sub_path = "p" ++ fs.path.sep_str ++ hash[0..hex_multihash_len];
const build_root = try global_cache_directory.join(arena, &.{pkg_dir_sub_path});
try dependencies_source.writer().print(
\\ pub const {} = struct {{
\\ pub const build_root = "{}";
\\ pub const build_zig = @import("{}");
\\ pub const deps: []const struct {{ []const u8, []const u8 }} = &.{{}};
\\ }};
\\
, .{
std.zig.fmtId(hash),
std.zig.fmtEscapes(build_root),
std.zig.fmtEscapes(hash),
});
} else {
try dependencies_source.writer().writeAll(
\\pub const packages = struct {};
\\pub const root_deps: []const struct { []const u8, []const u8 } = &.{};
\\
);
}
return;
},
else => |e| return e,
@ -284,23 +308,23 @@ pub fn fetchAndAddDependencies(
root_prog_node.setEstimatedTotalItems(all_modules.count());
if (this_hash == null) {
try dependencies_source.writer().writeAll("pub const packages = struct {\n");
}
const deps_list = manifest.dependencies.values();
for (manifest.dependencies.keys(), 0..) |name, i| {
const dep = deps_list[i];
const sub_prefix = try std.fmt.allocPrint(arena, "{s}{s}.", .{ name_prefix, name });
const fqn = sub_prefix[0 .. sub_prefix.len - 1];
const sub = try fetchAndUnpack(
thread_pool,
http_client,
global_cache_directory,
dep,
report,
build_roots_source,
fqn,
all_modules,
root_prog_node,
name,
);
if (!sub.found_existing) {
@ -313,11 +337,10 @@ pub fn fetchAndAddDependencies(
global_cache_directory,
local_cache_directory,
dependencies_source,
build_roots_source,
sub_prefix,
error_bundle,
all_modules,
root_prog_node,
dep.hash.?,
);
}
@ -329,10 +352,47 @@ pub fn fetchAndAddDependencies(
} else {
try deps_pkg.add(gpa, dep.hash.?, sub.mod);
}
}
try dependencies_source.writer().print(" pub const {s} = @import(\"{}\");\n", .{
std.zig.fmtId(fqn), std.zig.fmtEscapes(dep.hash.?),
if (this_hash) |hash| {
const pkg_dir_sub_path = "p" ++ fs.path.sep_str ++ hash[0..hex_multihash_len];
const build_root = try global_cache_directory.join(arena, &.{pkg_dir_sub_path});
try dependencies_source.writer().print(
\\ pub const {} = struct {{
\\ pub const build_root = "{}";
\\ pub const build_zig = @import("{}");
\\ pub const deps: []const struct {{ []const u8, []const u8 }} = &.{{
\\
, .{
std.zig.fmtId(hash),
std.zig.fmtEscapes(build_root),
std.zig.fmtEscapes(hash),
});
for (manifest.dependencies.keys(), manifest.dependencies.values()) |name, dep| {
try dependencies_source.writer().print(
" .{{ \"{}\", \"{}\" }},\n",
.{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(dep.hash.?) },
);
}
try dependencies_source.writer().writeAll(
\\ };
\\ };
\\
);
} else {
try dependencies_source.writer().writeAll(
\\};
\\
\\pub const root_deps: []const struct { []const u8, []const u8 } = &.{
\\
);
for (manifest.dependencies.keys(), manifest.dependencies.values()) |name, dep| {
try dependencies_source.writer().print(
" .{{ \"{}\", \"{}\" }},\n",
.{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(dep.hash.?) },
);
}
try dependencies_source.writer().writeAll("};\n");
}
}
@ -470,10 +530,11 @@ fn fetchAndUnpack(
global_cache_directory: Compilation.Directory,
dep: Manifest.Dependency,
report: Report,
build_roots_source: *std.ArrayList(u8),
fqn: []const u8,
all_modules: *AllModules,
root_prog_node: *std.Progress.Node,
/// This does not have to be any form of canonical or fully-qualified name: it
/// is only intended to be human-readable for progress reporting.
name_for_prog: []const u8,
) !struct { mod: *Package, found_existing: bool } {
const gpa = http_client.allocator;
const s = fs.path.sep_str;
@ -484,25 +545,17 @@ fn fetchAndUnpack(
const hex_digest = h[0..hex_multihash_len];
const pkg_dir_sub_path = "p" ++ s ++ hex_digest;
const build_root = try global_cache_directory.join(gpa, &.{pkg_dir_sub_path});
errdefer gpa.free(build_root);
var pkg_dir = global_cache_directory.handle.openDir(pkg_dir_sub_path, .{}) catch |err| switch (err) {
error.FileNotFound => break :cached,
else => |e| return e,
};
errdefer pkg_dir.close();
try build_roots_source.writer().print(" pub const {s} = \"{}\";\n", .{
std.zig.fmtId(fqn), std.zig.fmtEscapes(build_root),
});
// The compiler has a rule that a file must not be included in multiple modules,
// so we must detect if a module has been created for this package and reuse it.
const gop = try all_modules.getOrPut(gpa, hex_digest.*);
if (gop.found_existing) {
if (gop.value_ptr.*) |mod| {
gpa.free(build_root);
return .{
.mod = mod,
.found_existing = true,
@ -510,6 +563,9 @@ fn fetchAndUnpack(
}
}
const build_root = try global_cache_directory.join(gpa, &.{pkg_dir_sub_path});
errdefer gpa.free(build_root);
root_prog_node.completeOne();
const ptr = try gpa.create(Package);
@ -534,7 +590,7 @@ fn fetchAndUnpack(
};
}
var pkg_prog_node = root_prog_node.start(fqn, 0);
var pkg_prog_node = root_prog_node.start(name_for_prog, 0);
defer pkg_prog_node.end();
pkg_prog_node.activate();
pkg_prog_node.context.refresh();
@ -666,13 +722,6 @@ fn fetchAndUnpack(
return error.PackageFetchFailed;
}
const build_root = try global_cache_directory.join(gpa, &.{pkg_dir_sub_path});
defer gpa.free(build_root);
try build_roots_source.writer().print(" pub const {s} = \"{}\";\n", .{
std.zig.fmtId(fqn), std.zig.fmtEscapes(build_root),
});
const mod = try createWithDir(gpa, global_cache_directory, pkg_dir_sub_path, build_zig_basename);
try all_modules.put(gpa, actual_hex, mod);
return .{

22
src/main.zig
View File

@ -4708,7 +4708,14 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
.root_src_directory = build_directory,
.root_src_path = build_zig_basename,
};
if (!build_options.only_core_functionality) {
if (build_options.only_core_functionality) {
const deps_pkg = try Package.createFilePkg(gpa, local_cache_directory, "dependencies.zig",
\\pub const packages = struct {};
\\pub const root_deps: []const struct { []const u8, []const u8 } = &.{};
\\
);
try main_pkg.add(gpa, "@dependencies", deps_pkg);
} else {
var http_client: std.http.Client = .{ .allocator = gpa };
defer http_client.deinit();
@ -4717,12 +4724,6 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
// access dependencies by name, since `@import` requires string literals.
var dependencies_source = std.ArrayList(u8).init(gpa);
defer dependencies_source.deinit();
try dependencies_source.appendSlice("pub const imports = struct {\n");
// This will go into the same package. It contains the file system paths
// to all the build.zig files.
var build_roots_source = std.ArrayList(u8).init(gpa);
defer build_roots_source.deinit();
var all_modules: Package.AllModules = .{};
defer all_modules.deinit(gpa);
@ -4746,11 +4747,10 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
global_cache_directory,
local_cache_directory,
&dependencies_source,
&build_roots_source,
"",
&wip_errors,
&all_modules,
root_prog_node,
null,
);
if (wip_errors.root_list.items.len > 0) {
var errors = try wip_errors.toOwnedBundle("");
@ -4760,10 +4760,6 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
}
try fetch_result;
try dependencies_source.appendSlice("};\npub const build_root = struct {\n");
try dependencies_source.appendSlice(build_roots_source.items);
try dependencies_source.appendSlice("};\n");
const deps_pkg = try Package.createFilePkg(
gpa,
local_cache_directory,