From d4af35b3fe42a7ad5b808012b15d479526140730 Mon Sep 17 00:00:00 2001 From: daurnimator Date: Sat, 27 Feb 2021 13:49:02 +1100 Subject: [PATCH 01/13] HashMap.put returns !void, not a !bool --- lib/std/buf_set.zig | 2 +- lib/std/build.zig | 4 ++-- lib/std/hash_map.zig | 3 +-- lib/std/json.zig | 12 ++++++------ lib/std/json/write_stream.zig | 4 ++-- lib/std/priority_queue.zig | 2 +- src/link/Elf.zig | 6 +++--- src/link/MachO/DebugSymbols.zig | 4 ++-- src/liveness.zig | 4 ++-- src/translate_c.zig | 20 ++++++++++---------- tools/update_glibc.zig | 4 ++-- 11 files changed, 32 insertions(+), 33 deletions(-) diff --git a/lib/std/buf_set.zig b/lib/std/buf_set.zig index 75c5ae742d..e59dc9841b 100644 --- a/lib/std/buf_set.zig +++ b/lib/std/buf_set.zig @@ -32,7 +32,7 @@ pub const BufSet = struct { if (self.hash_map.get(key) == null) { const key_copy = try self.copy(key); errdefer self.free(key_copy); - _ = try self.hash_map.put(key_copy, {}); + try self.hash_map.put(key_copy, {}); } } diff --git a/lib/std/build.zig b/lib/std/build.zig index e11402e493..efeea4adb7 100644 --- a/lib/std/build.zig +++ b/lib/std/build.zig @@ -790,7 +790,7 @@ pub const Builder = struct { var list = ArrayList([]const u8).init(self.allocator); list.append(s) catch unreachable; list.append(value) catch unreachable; - _ = self.user_input_options.put(name, UserInputOption{ + self.user_input_options.put(name, UserInputOption{ .name = name, .value = UserValue{ .List = list }, .used = false, @@ -799,7 +799,7 @@ pub const Builder = struct { UserValue.List => |*list| { // append to the list list.append(value) catch unreachable; - _ = self.user_input_options.put(name, UserInputOption{ + self.user_input_options.put(name, UserInputOption{ .name = name, .value = UserValue{ .List = list.* }, .used = false, diff --git a/lib/std/hash_map.zig b/lib/std/hash_map.zig index 679575875d..e2e0f056e1 100644 --- a/lib/std/hash_map.zig +++ b/lib/std/hash_map.zig @@ -563,7 +563,6 @@ pub fn HashMapUnmanaged( } /// Insert an entry if the associated key is not already present, otherwise update preexisting value. - /// Returns true if the key was already present. pub fn put(self: *Self, allocator: *Allocator, key: K, value: V) !void { const result = try self.getOrPut(allocator, key); result.entry.value = value; @@ -1116,7 +1115,7 @@ test "std.hash_map put" { var i: u32 = 0; while (i < 16) : (i += 1) { - _ = try map.put(i, i); + try map.put(i, i); } i = 0; diff --git a/lib/std/json.zig b/lib/std/json.zig index f9fc371049..96ad066db3 100644 --- a/lib/std/json.zig +++ b/lib/std/json.zig @@ -2077,27 +2077,27 @@ pub const Parser = struct { p.state = .ArrayValue; }, .String => |s| { - _ = try object.put(key, try p.parseString(allocator, s, input, i)); + try object.put(key, try p.parseString(allocator, s, input, i)); _ = p.stack.pop(); p.state = .ObjectKey; }, .Number => |n| { - _ = try object.put(key, try p.parseNumber(n, input, i)); + try object.put(key, try p.parseNumber(n, input, i)); _ = p.stack.pop(); p.state = .ObjectKey; }, .True => { - _ = try object.put(key, Value{ .Bool = true }); + try object.put(key, Value{ .Bool = true }); _ = p.stack.pop(); p.state = .ObjectKey; }, .False => { - _ = try object.put(key, Value{ .Bool = false }); + try object.put(key, Value{ .Bool = false }); _ = p.stack.pop(); p.state = .ObjectKey; }, .Null => { - _ = try object.put(key, Value.Null); + try object.put(key, Value.Null); _ = p.stack.pop(); p.state = .ObjectKey; }, @@ -2184,7 +2184,7 @@ pub const Parser = struct { _ = p.stack.pop(); var object = &p.stack.items[p.stack.items.len - 1].Object; - _ = try object.put(key, value.*); + try object.put(key, value.*); p.state = .ObjectKey; }, // Array Parent -> [ ..., , value ] diff --git a/lib/std/json/write_stream.zig b/lib/std/json/write_stream.zig index b4a8aed84c..1cff0ed2b7 100644 --- a/lib/std/json/write_stream.zig +++ b/lib/std/json/write_stream.zig @@ -293,7 +293,7 @@ test "json write stream" { fn getJsonObject(allocator: *std.mem.Allocator) !std.json.Value { var value = std.json.Value{ .Object = std.json.ObjectMap.init(allocator) }; - _ = try value.Object.put("one", std.json.Value{ .Integer = @intCast(i64, 1) }); - _ = try value.Object.put("two", std.json.Value{ .Float = 2.0 }); + try value.Object.put("one", std.json.Value{ .Integer = @intCast(i64, 1) }); + try value.Object.put("two", std.json.Value{ .Float = 2.0 }); return value; } diff --git a/lib/std/priority_queue.zig b/lib/std/priority_queue.zig index f5c01edff5..ff671c9ff7 100644 --- a/lib/std/priority_queue.zig +++ b/lib/std/priority_queue.zig @@ -410,7 +410,7 @@ test "std.PriorityQueue: iterator" { const items = [_]u32{ 54, 12, 7, 23, 25, 13 }; for (items) |e| { _ = try queue.add(e); - _ = try map.put(e, {}); + try map.put(e, {}); } var it = queue.iterator(); diff --git a/src/link/Elf.zig b/src/link/Elf.zig index e1a6a1dff1..1b6fbb0f0f 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -2165,7 +2165,7 @@ pub fn freeDecl(self: *Elf, decl: *Module.Decl) void { // is desired for both. _ = self.dbg_line_fn_free_list.remove(&decl.fn_link.elf); if (decl.fn_link.elf.prev) |prev| { - _ = self.dbg_line_fn_free_list.put(self.base.allocator, prev, {}) catch {}; + self.dbg_line_fn_free_list.put(self.base.allocator, prev, {}) catch {}; prev.next = decl.fn_link.elf.next; if (decl.fn_link.elf.next) |next| { next.prev = prev; @@ -2423,7 +2423,7 @@ pub fn updateDecl(self: *Elf, module: *Module, decl: *Module.Decl) !void { if (src_fn.off + src_fn.len + min_nop_size > next.off) { // It grew too big, so we move it to a new location. if (src_fn.prev) |prev| { - _ = self.dbg_line_fn_free_list.put(self.base.allocator, prev, {}) catch {}; + self.dbg_line_fn_free_list.put(self.base.allocator, prev, {}) catch {}; prev.next = src_fn.next; } assert(src_fn.prev != next); @@ -2579,7 +2579,7 @@ fn updateDeclDebugInfoAllocation(self: *Elf, text_block: *TextBlock, len: u32) ! if (text_block.dbg_info_off + text_block.dbg_info_len + min_nop_size > next.dbg_info_off) { // It grew too big, so we move it to a new location. if (text_block.dbg_info_prev) |prev| { - _ = self.dbg_info_decl_free_list.put(self.base.allocator, prev, {}) catch {}; + self.dbg_info_decl_free_list.put(self.base.allocator, prev, {}) catch {}; prev.dbg_info_next = text_block.dbg_info_next; } next.dbg_info_prev = text_block.dbg_info_prev; diff --git a/src/link/MachO/DebugSymbols.zig b/src/link/MachO/DebugSymbols.zig index 645e17068b..042c1a12cf 100644 --- a/src/link/MachO/DebugSymbols.zig +++ b/src/link/MachO/DebugSymbols.zig @@ -1096,7 +1096,7 @@ pub fn commitDeclDebugInfo( if (src_fn.off + src_fn.len + min_nop_size > next.off) { // It grew too big, so we move it to a new location. if (src_fn.prev) |prev| { - _ = self.dbg_line_fn_free_list.put(allocator, prev, {}) catch {}; + self.dbg_line_fn_free_list.put(allocator, prev, {}) catch {}; prev.next = src_fn.next; } next.prev = src_fn.prev; @@ -1256,7 +1256,7 @@ fn updateDeclDebugInfoAllocation( if (text_block.dbg_info_off + text_block.dbg_info_len + min_nop_size > next.dbg_info_off) { // It grew too big, so we move it to a new location. if (text_block.dbg_info_prev) |prev| { - _ = self.dbg_info_decl_free_list.put(allocator, prev, {}) catch {}; + self.dbg_info_decl_free_list.put(allocator, prev, {}) catch {}; prev.dbg_info_next = text_block.dbg_info_next; } next.dbg_info_prev = text_block.dbg_info_prev; diff --git a/src/liveness.zig b/src/liveness.zig index b0aafa28f1..d652e7e954 100644 --- a/src/liveness.zig +++ b/src/liveness.zig @@ -119,7 +119,7 @@ fn analyzeInst( if (!else_table.contains(then_death)) { try else_entry_deaths.append(then_death); } - _ = try table.put(then_death, {}); + try table.put(then_death, {}); } } // Now we have to correctly populate new_set. @@ -195,7 +195,7 @@ fn analyzeInst( } } // undo resetting the table - _ = try table.put(case_death, {}); + try table.put(case_death, {}); } } diff --git a/src/translate_c.zig b/src/translate_c.zig index e82583bd07..170e6ef2e1 100644 --- a/src/translate_c.zig +++ b/src/translate_c.zig @@ -377,7 +377,7 @@ fn prepopulateGlobalNameTable(ast_unit: *clang.ASTUnit, c: *Context) !void { const macro = @ptrCast(*clang.MacroDefinitionRecord, entity); const raw_name = macro.getName_getNameStart(); const name = try c.str(raw_name); - _ = try c.global_names.put(c.gpa, name, {}); + try c.global_names.put(c.gpa, name, {}); }, else => {}, } @@ -399,7 +399,7 @@ fn declVisitorC(context: ?*c_void, decl: *const clang.Decl) callconv(.C) bool { fn declVisitorNamesOnly(c: *Context, decl: *const clang.Decl) Error!void { if (decl.castToNamedDecl()) |named_decl| { const decl_name = try c.str(named_decl.getName_bytes_begin()); - _ = try c.global_names.put(c.gpa, decl_name, {}); + try c.global_names.put(c.gpa, decl_name, {}); } } @@ -788,7 +788,7 @@ fn transRecordDecl(c: *Context, scope: *Scope, record_decl: *const clang.RecordD const is_pub = toplevel and !is_unnamed; const init_node = blk: { const record_def = record_decl.getDefinition() orelse { - _ = try c.opaque_demotes.put(c.gpa, @ptrToInt(record_decl.getCanonicalDecl()), {}); + try c.opaque_demotes.put(c.gpa, @ptrToInt(record_decl.getCanonicalDecl()), {}); break :blk Tag.opaque_literal.init(); }; @@ -805,13 +805,13 @@ fn transRecordDecl(c: *Context, scope: *Scope, record_decl: *const clang.RecordD const field_qt = field_decl.getType(); if (field_decl.isBitField()) { - _ = try c.opaque_demotes.put(c.gpa, @ptrToInt(record_decl.getCanonicalDecl()), {}); + try c.opaque_demotes.put(c.gpa, @ptrToInt(record_decl.getCanonicalDecl()), {}); try warn(c, scope, field_loc, "{s} demoted to opaque type - has bitfield", .{container_kind_name}); break :blk Tag.opaque_literal.init(); } if (qualTypeCanon(field_qt).isIncompleteOrZeroLengthArrayType(c.clang_context)) { - _ = try c.opaque_demotes.put(c.gpa, @ptrToInt(record_decl.getCanonicalDecl()), {}); + try c.opaque_demotes.put(c.gpa, @ptrToInt(record_decl.getCanonicalDecl()), {}); try warn(c, scope, field_loc, "{s} demoted to opaque type - has variable length array", .{container_kind_name}); break :blk Tag.opaque_literal.init(); } @@ -826,7 +826,7 @@ fn transRecordDecl(c: *Context, scope: *Scope, record_decl: *const clang.RecordD } const field_type = transQualType(c, scope, field_qt, field_loc) catch |err| switch (err) { error.UnsupportedType => { - _ = try c.opaque_demotes.put(c.gpa, @ptrToInt(record_decl.getCanonicalDecl()), {}); + try c.opaque_demotes.put(c.gpa, @ptrToInt(record_decl.getCanonicalDecl()), {}); try warn(c, scope, record_loc, "{s} demoted to opaque type - unable to translate type of field {s}", .{ container_kind_name, field_name }); break :blk Tag.opaque_literal.init(); }, @@ -972,7 +972,7 @@ fn transEnumDecl(c: *Context, scope: *Scope, enum_decl: *const clang.EnumDecl) E .fields = try c.arena.dupe(ast.Payload.Enum.Field, fields.items), }); } else blk: { - _ = try c.opaque_demotes.put(c.gpa, @ptrToInt(enum_decl.getCanonicalDecl()), {}); + try c.opaque_demotes.put(c.gpa, @ptrToInt(enum_decl.getCanonicalDecl()), {}); break :blk Tag.opaque_literal.init(); }; @@ -3199,7 +3199,7 @@ fn maybeSuppressResult( } fn addTopLevelDecl(c: *Context, name: []const u8, decl_node: Node) !void { - _ = try c.global_scope.sym_table.put(name, decl_node); + try c.global_scope.sym_table.put(name, decl_node); try c.global_scope.nodes.append(decl_node); } @@ -4235,7 +4235,7 @@ fn transMacroDefine(c: *Context, m: *MacroCtx) ParseError!void { return m.fail(c, "unable to translate C expr: unexpected token .{s}", .{@tagName(last)}); const var_decl = try Tag.pub_var_simple.create(c.arena, .{ .name = m.name, .init = init_node }); - _ = try c.global_scope.macro_table.put(m.name, var_decl); + try c.global_scope.macro_table.put(m.name, var_decl); } fn transMacroFnDefine(c: *Context, m: *MacroCtx) ParseError!void { @@ -4294,7 +4294,7 @@ fn transMacroFnDefine(c: *Context, m: *MacroCtx) ParseError!void { .return_type = return_type, .body = try block_scope.complete(c), }); - _ = try c.global_scope.macro_table.put(m.name, fn_decl); + try c.global_scope.macro_table.put(m.name, fn_decl); } const ParseError = Error || error{ParseError}; diff --git a/tools/update_glibc.zig b/tools/update_glibc.zig index 77be81d6d5..e3652acf1c 100644 --- a/tools/update_glibc.zig +++ b/tools/update_glibc.zig @@ -200,7 +200,7 @@ pub fn main() !void { continue; } if (std.mem.startsWith(u8, ver, "GCC_")) continue; - _ = try global_ver_set.put(ver, undefined); + try global_ver_set.put(ver, undefined); const gop = try global_fn_set.getOrPut(name); if (gop.found_existing) { if (!std.mem.eql(u8, gop.entry.value.lib, "c")) { @@ -242,7 +242,7 @@ pub fn main() !void { var buffered = std.io.bufferedWriter(vers_txt_file.writer()); const vers_txt = buffered.writer(); for (global_ver_list) |name, i| { - _ = global_ver_set.put(name, i) catch unreachable; + global_ver_set.put(name, i) catch unreachable; try vers_txt.print("{s}\n", .{name}); } try buffered.flush(); From e65b6d99ac38686228bf5b5b9c1121382e05da4b Mon Sep 17 00:00:00 2001 From: Michael Dusan Date: Sat, 27 Feb 2021 13:51:28 -0500 Subject: [PATCH 02/13] std.zig.fmtEscapes: update impacted test --- test/translate_c.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/translate_c.zig b/test/translate_c.zig index 2f95bb2d1e..367f69745b 100644 --- a/test/translate_c.zig +++ b/test/translate_c.zig @@ -2227,8 +2227,8 @@ pub fn addCases(cases: *tests.TranslateCContext) void { \\ var h: u8 = '\t'; \\ var i: u8 = '\x0b'; \\ var j: u8 = '\x00'; - \\ var k: u8 = '\"'; - \\ return "\'\\\x07\x08\x0c\n\r\t\x0b\x00\""; + \\ var k: u8 = '"'; + \\ return "'\\\x07\x08\x0c\n\r\t\x0b\x00\""; \\} }); From 566adc2510859eaa30eb7c318260c98e712daccf Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Sun, 28 Feb 2021 10:01:55 +0100 Subject: [PATCH 03/13] std: Swap arguments in Thread.spawn Beside the new order being consistent with the ThreadPool API and making more sense, this shuffling allows to write the context argument type in terms of the startFn arguments, reducing the use of anytype (eg. less explicit casts when using comptime_int parameters, yay). Sorry for the breakage. Closes #8082 --- lib/std/Thread.zig | 26 ++++++++++++++++++++------ lib/std/Thread/AutoResetEvent.zig | 4 ++-- lib/std/Thread/Mutex.zig | 2 +- lib/std/Thread/ResetEvent.zig | 4 ++-- lib/std/Thread/StaticResetEvent.zig | 4 ++-- lib/std/atomic/queue.zig | 4 ++-- lib/std/atomic/stack.zig | 4 ++-- lib/std/event/loop.zig | 10 +++++----- lib/std/fs/test.zig | 2 +- lib/std/net/test.zig | 4 ++-- lib/std/once.zig | 4 ++-- lib/std/os/test.zig | 14 +++++++------- src/ThreadPool.zig | 2 +- 13 files changed, 49 insertions(+), 35 deletions(-) diff --git a/lib/std/Thread.zig b/lib/std/Thread.zig index 77277bd154..b9a69d151f 100644 --- a/lib/std/Thread.zig +++ b/lib/std/Thread.zig @@ -165,18 +165,32 @@ pub const SpawnError = error{ Unexpected, }; -/// caller must call wait on the returned thread -/// fn startFn(@TypeOf(context)) T -/// where T is u8, noreturn, void, or !void -/// caller must call wait on the returned thread -pub fn spawn(context: anytype, comptime startFn: anytype) SpawnError!*Thread { +// Given `T`, the type of the thread startFn, extract the expected type for the +// context parameter. +fn SpawnContextType(comptime T: type) type { + const TI = @typeInfo(T); + if (TI != .Fn) + @compileError("expected function type, found " ++ @typeName(T)); + + if (TI.Fn.args.len != 1) + @compileError("expected function with single argument, found " ++ @typeName(T)); + + return TI.Fn.args[0].arg_type orelse + @compileError("cannot use a generic function as thread startFn"); +} + +/// Spawns a new thread executing startFn, returning an handle for it. +/// Caller must call wait on the returned thread. +/// The `startFn` function must take a single argument of type T and return a +/// value of type u8, noreturn, void or !void. +/// The `context` parameter is of type T and is passed to the spawned thread. +pub fn spawn(comptime startFn: anytype, context: SpawnContextType(@TypeOf(startFn))) SpawnError!*Thread { if (builtin.single_threaded) @compileError("cannot spawn thread when building in single-threaded mode"); // TODO compile-time call graph analysis to determine stack upper bound // https://github.com/ziglang/zig/issues/157 const default_stack_size = 16 * 1024 * 1024; const Context = @TypeOf(context); - comptime assert(@typeInfo(@TypeOf(startFn)).Fn.args[0].arg_type.? == Context); if (std.Target.current.os.tag == .windows) { const WinThread = struct { diff --git a/lib/std/Thread/AutoResetEvent.zig b/lib/std/Thread/AutoResetEvent.zig index 8b8b5658bf..0726dc794a 100644 --- a/lib/std/Thread/AutoResetEvent.zig +++ b/lib/std/Thread/AutoResetEvent.zig @@ -220,8 +220,8 @@ test "basic usage" { }; var context = Context{}; - const send_thread = try std.Thread.spawn(&context, Context.sender); - const recv_thread = try std.Thread.spawn(&context, Context.receiver); + const send_thread = try std.Thread.spawn(Context.sender, &context); + const recv_thread = try std.Thread.spawn(Context.receiver, &context); send_thread.wait(); recv_thread.wait(); diff --git a/lib/std/Thread/Mutex.zig b/lib/std/Thread/Mutex.zig index 94711bcda0..e7d11954bf 100644 --- a/lib/std/Thread/Mutex.zig +++ b/lib/std/Thread/Mutex.zig @@ -299,7 +299,7 @@ test "basic usage" { const thread_count = 10; var threads: [thread_count]*std.Thread = undefined; for (threads) |*t| { - t.* = try std.Thread.spawn(&context, worker); + t.* = try std.Thread.spawn(worker, &context); } for (threads) |t| t.wait(); diff --git a/lib/std/Thread/ResetEvent.zig b/lib/std/Thread/ResetEvent.zig index 622f9be98e..439fb0db40 100644 --- a/lib/std/Thread/ResetEvent.zig +++ b/lib/std/Thread/ResetEvent.zig @@ -281,7 +281,7 @@ test "basic usage" { var context: Context = undefined; try context.init(); defer context.deinit(); - const receiver = try std.Thread.spawn(&context, Context.receiver); + const receiver = try std.Thread.spawn(Context.receiver, &context); defer receiver.wait(); context.sender(); @@ -290,7 +290,7 @@ test "basic usage" { // https://github.com/ziglang/zig/issues/7009 var timed = Context.init(); defer timed.deinit(); - const sleeper = try std.Thread.spawn(&timed, Context.sleeper); + const sleeper = try std.Thread.spawn(Context.sleeper, &timed); defer sleeper.wait(); try timed.timedWaiter(); } diff --git a/lib/std/Thread/StaticResetEvent.zig b/lib/std/Thread/StaticResetEvent.zig index 6d90d7cf9a..07a8e50c16 100644 --- a/lib/std/Thread/StaticResetEvent.zig +++ b/lib/std/Thread/StaticResetEvent.zig @@ -379,7 +379,7 @@ test "basic usage" { }; var context = Context{}; - const receiver = try std.Thread.spawn(&context, Context.receiver); + const receiver = try std.Thread.spawn(Context.receiver, &context); defer receiver.wait(); context.sender(); @@ -388,7 +388,7 @@ test "basic usage" { // https://github.com/ziglang/zig/issues/7009 var timed = Context.init(); defer timed.deinit(); - const sleeper = try std.Thread.spawn(&timed, Context.sleeper); + const sleeper = try std.Thread.spawn(Context.sleeper, &timed); defer sleeper.wait(); try timed.timedWaiter(); } diff --git a/lib/std/atomic/queue.zig b/lib/std/atomic/queue.zig index f5f63944ab..4e427a1669 100644 --- a/lib/std/atomic/queue.zig +++ b/lib/std/atomic/queue.zig @@ -216,11 +216,11 @@ test "std.atomic.Queue" { var putters: [put_thread_count]*std.Thread = undefined; for (putters) |*t| { - t.* = try std.Thread.spawn(&context, startPuts); + t.* = try std.Thread.spawn(startPuts, &context); } var getters: [put_thread_count]*std.Thread = undefined; for (getters) |*t| { - t.* = try std.Thread.spawn(&context, startGets); + t.* = try std.Thread.spawn(startGets, &context); } for (putters) |t| diff --git a/lib/std/atomic/stack.zig b/lib/std/atomic/stack.zig index d55a8f81a3..4096c27354 100644 --- a/lib/std/atomic/stack.zig +++ b/lib/std/atomic/stack.zig @@ -123,11 +123,11 @@ test "std.atomic.stack" { } else { var putters: [put_thread_count]*std.Thread = undefined; for (putters) |*t| { - t.* = try std.Thread.spawn(&context, startPuts); + t.* = try std.Thread.spawn(startPuts, &context); } var getters: [put_thread_count]*std.Thread = undefined; for (getters) |*t| { - t.* = try std.Thread.spawn(&context, startGets); + t.* = try std.Thread.spawn(startGets, &context); } for (putters) |t| diff --git a/lib/std/event/loop.zig b/lib/std/event/loop.zig index 912f99e961..878cea4aa6 100644 --- a/lib/std/event/loop.zig +++ b/lib/std/event/loop.zig @@ -185,7 +185,7 @@ pub const Loop = struct { errdefer self.deinitOsData(); if (!builtin.single_threaded) { - self.fs_thread = try Thread.spawn(self, posixFsRun); + self.fs_thread = try Thread.spawn(posixFsRun, self); } errdefer if (!builtin.single_threaded) { self.posixFsRequest(&self.fs_end_request); @@ -264,7 +264,7 @@ pub const Loop = struct { } } while (extra_thread_index < extra_thread_count) : (extra_thread_index += 1) { - self.extra_threads[extra_thread_index] = try Thread.spawn(self, workerRun); + self.extra_threads[extra_thread_index] = try Thread.spawn(workerRun, self); } }, .macos, .freebsd, .netbsd, .dragonfly, .openbsd => { @@ -329,7 +329,7 @@ pub const Loop = struct { } } while (extra_thread_index < extra_thread_count) : (extra_thread_index += 1) { - self.extra_threads[extra_thread_index] = try Thread.spawn(self, workerRun); + self.extra_threads[extra_thread_index] = try Thread.spawn(workerRun, self); } }, .windows => { @@ -378,7 +378,7 @@ pub const Loop = struct { } } while (extra_thread_index < extra_thread_count) : (extra_thread_index += 1) { - self.extra_threads[extra_thread_index] = try Thread.spawn(self, workerRun); + self.extra_threads[extra_thread_index] = try Thread.spawn(workerRun, self); } }, else => {}, @@ -798,7 +798,7 @@ pub const Loop = struct { .event = std.Thread.AutoResetEvent{}, .is_running = true, // Must be last so that it can read the other state, such as `is_running`. - .thread = try std.Thread.spawn(self, DelayQueue.run), + .thread = try std.Thread.spawn(DelayQueue.run, self), }; } diff --git a/lib/std/fs/test.zig b/lib/std/fs/test.zig index e9f28a0b60..0e2d296bb7 100644 --- a/lib/std/fs/test.zig +++ b/lib/std/fs/test.zig @@ -762,7 +762,7 @@ test "open file with exclusive lock twice, make sure it waits" { try evt.init(); defer evt.deinit(); - const t = try std.Thread.spawn(S.C{ .dir = &tmp.dir, .evt = &evt }, S.checkFn); + const t = try std.Thread.spawn(S.checkFn, S.C{ .dir = &tmp.dir, .evt = &evt }); defer t.wait(); const SLEEP_TIMEOUT_NS = 10 * std.time.ns_per_ms; diff --git a/lib/std/net/test.zig b/lib/std/net/test.zig index 10a9c4e18b..4470d17aeb 100644 --- a/lib/std/net/test.zig +++ b/lib/std/net/test.zig @@ -161,7 +161,7 @@ test "listen on a port, send bytes, receive bytes" { } }; - const t = try std.Thread.spawn(server.listen_address, S.clientFn); + const t = try std.Thread.spawn(S.clientFn, server.listen_address); defer t.wait(); var client = try server.accept(); @@ -285,7 +285,7 @@ test "listen on a unix socket, send bytes, receive bytes" { } }; - const t = try std.Thread.spawn({}, S.clientFn); + const t = try std.Thread.spawn(S.clientFn, {}); defer t.wait(); var client = try server.accept(); diff --git a/lib/std/once.zig b/lib/std/once.zig index efa99060d3..79d273b86a 100644 --- a/lib/std/once.zig +++ b/lib/std/once.zig @@ -59,11 +59,11 @@ test "Once executes its function just once" { defer for (threads) |handle| handle.wait(); for (threads) |*handle| { - handle.* = try std.Thread.spawn(@as(u8, 0), struct { + handle.* = try std.Thread.spawn(struct { fn thread_fn(x: u8) void { global_once.call(); } - }.thread_fn); + }.thread_fn, 0); } } diff --git a/lib/std/os/test.zig b/lib/std/os/test.zig index f08d4d58fa..c8cef38d5d 100644 --- a/lib/std/os/test.zig +++ b/lib/std/os/test.zig @@ -317,7 +317,7 @@ test "std.Thread.getCurrentId" { if (builtin.single_threaded) return error.SkipZigTest; var thread_current_id: Thread.Id = undefined; - const thread = try Thread.spawn(&thread_current_id, testThreadIdFn); + const thread = try Thread.spawn(testThreadIdFn, &thread_current_id); const thread_id = thread.handle(); thread.wait(); if (Thread.use_pthreads) { @@ -336,10 +336,10 @@ test "spawn threads" { var shared_ctx: i32 = 1; - const thread1 = try Thread.spawn({}, start1); - const thread2 = try Thread.spawn(&shared_ctx, start2); - const thread3 = try Thread.spawn(&shared_ctx, start2); - const thread4 = try Thread.spawn(&shared_ctx, start2); + const thread1 = try Thread.spawn(start1, {}); + const thread2 = try Thread.spawn(start2, &shared_ctx); + const thread3 = try Thread.spawn(start2, &shared_ctx); + const thread4 = try Thread.spawn(start2, &shared_ctx); thread1.wait(); thread2.wait(); @@ -367,8 +367,8 @@ test "cpu count" { test "thread local storage" { if (builtin.single_threaded) return error.SkipZigTest; - const thread1 = try Thread.spawn({}, testTls); - const thread2 = try Thread.spawn({}, testTls); + const thread1 = try Thread.spawn(testTls, {}); + const thread2 = try Thread.spawn(testTls, {}); testTls({}); thread1.wait(); thread2.wait(); diff --git a/src/ThreadPool.zig b/src/ThreadPool.zig index ec580210e9..e66742b49e 100644 --- a/src/ThreadPool.zig +++ b/src/ThreadPool.zig @@ -74,7 +74,7 @@ pub fn init(self: *ThreadPool, allocator: *std.mem.Allocator) !void { try worker.idle_node.data.init(); errdefer worker.idle_node.data.deinit(); - worker.thread = try std.Thread.spawn(worker, Worker.run); + worker.thread = try std.Thread.spawn(Worker.run, worker); } } From d3d3e55fae2299d1ff594c77da2f1f41f44ab525 Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Sun, 28 Feb 2021 17:01:30 +0100 Subject: [PATCH 04/13] langref: Update usage of Thread.spawn() --- doc/langref.html.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/langref.html.in b/doc/langref.html.in index e49609fdbf..a716336015 100644 --- a/doc/langref.html.in +++ b/doc/langref.html.in @@ -933,8 +933,8 @@ const assert = std.debug.assert; threadlocal var x: i32 = 1234; test "thread local storage" { - const thread1 = try std.Thread.spawn({}, testTls); - const thread2 = try std.Thread.spawn({}, testTls); + const thread1 = try std.Thread.spawn(testTls, {}); + const thread2 = try std.Thread.spawn(testTls, {}); testTls({}); thread1.wait(); thread2.wait(); From b959029f88fa033e54e72e42004bc261baada1f1 Mon Sep 17 00:00:00 2001 From: Frank Denis Date: Sun, 28 Feb 2021 12:53:27 +0100 Subject: [PATCH 05/13] std/crypto/benchmark: update format strings Use the s formatter to format strings. --- lib/std/crypto/benchmark.zig | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/std/crypto/benchmark.zig b/lib/std/crypto/benchmark.zig index 00336aef87..476ed86ede 100644 --- a/lib/std/crypto/benchmark.zig +++ b/lib/std/crypto/benchmark.zig @@ -356,63 +356,63 @@ pub fn main() !void { inline for (hashes) |H| { if (filter == null or std.mem.indexOf(u8, H.name, filter.?) != null) { const throughput = try benchmarkHash(H.ty, mode(128 * MiB)); - try stdout.print("{:>17}: {:10} MiB/s\n", .{ H.name, throughput / (1 * MiB) }); + try stdout.print("{s:>17}: {:10} MiB/s\n", .{ H.name, throughput / (1 * MiB) }); } } inline for (macs) |M| { if (filter == null or std.mem.indexOf(u8, M.name, filter.?) != null) { const throughput = try benchmarkMac(M.ty, mode(128 * MiB)); - try stdout.print("{:>17}: {:10} MiB/s\n", .{ M.name, throughput / (1 * MiB) }); + try stdout.print("{s:>17}: {:10} MiB/s\n", .{ M.name, throughput / (1 * MiB) }); } } inline for (exchanges) |E| { if (filter == null or std.mem.indexOf(u8, E.name, filter.?) != null) { const throughput = try benchmarkKeyExchange(E.ty, mode(1000)); - try stdout.print("{:>17}: {:10} exchanges/s\n", .{ E.name, throughput }); + try stdout.print("{s:>17}: {:10} exchanges/s\n", .{ E.name, throughput }); } } inline for (signatures) |E| { if (filter == null or std.mem.indexOf(u8, E.name, filter.?) != null) { const throughput = try benchmarkSignature(E.ty, mode(1000)); - try stdout.print("{:>17}: {:10} signatures/s\n", .{ E.name, throughput }); + try stdout.print("{s:>17}: {:10} signatures/s\n", .{ E.name, throughput }); } } inline for (signature_verifications) |E| { if (filter == null or std.mem.indexOf(u8, E.name, filter.?) != null) { const throughput = try benchmarkSignatureVerification(E.ty, mode(1000)); - try stdout.print("{:>17}: {:10} verifications/s\n", .{ E.name, throughput }); + try stdout.print("{s:>17}: {:10} verifications/s\n", .{ E.name, throughput }); } } inline for (batch_signature_verifications) |E| { if (filter == null or std.mem.indexOf(u8, E.name, filter.?) != null) { const throughput = try benchmarkBatchSignatureVerification(E.ty, mode(1000)); - try stdout.print("{:>17}: {:10} verifications/s (batch)\n", .{ E.name, throughput }); + try stdout.print("{s:>17}: {:10} verifications/s (batch)\n", .{ E.name, throughput }); } } inline for (aeads) |E| { if (filter == null or std.mem.indexOf(u8, E.name, filter.?) != null) { const throughput = try benchmarkAead(E.ty, mode(128 * MiB)); - try stdout.print("{:>17}: {:10} MiB/s\n", .{ E.name, throughput / (1 * MiB) }); + try stdout.print("{s:>17}: {:10} MiB/s\n", .{ E.name, throughput / (1 * MiB) }); } } inline for (aes) |E| { if (filter == null or std.mem.indexOf(u8, E.name, filter.?) != null) { const throughput = try benchmarkAes(E.ty, mode(100000000)); - try stdout.print("{:>17}: {:10} ops/s\n", .{ E.name, throughput }); + try stdout.print("{s:>17}: {:10} ops/s\n", .{ E.name, throughput }); } } inline for (aes8) |E| { if (filter == null or std.mem.indexOf(u8, E.name, filter.?) != null) { const throughput = try benchmarkAes8(E.ty, mode(10000000)); - try stdout.print("{:>17}: {:10} ops/s\n", .{ E.name, throughput }); + try stdout.print("{s:>17}: {:10} ops/s\n", .{ E.name, throughput }); } } } From 0423f0f7d8afc24e48926849515f60a8a3a1e470 Mon Sep 17 00:00:00 2001 From: Frank Denis Date: Sun, 28 Feb 2021 00:33:50 +0100 Subject: [PATCH 06/13] std/crypto/aes: fix AES {encrypt,decrypt}Wide These functions are not used by anything yet, but run the last round only once. --- lib/std/crypto/aes/aesni.zig | 10 ++-------- lib/std/crypto/aes/armcrypto.zig | 10 ++-------- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/lib/std/crypto/aes/aesni.zig b/lib/std/crypto/aes/aesni.zig index 13b3f8e527..1d719af9c7 100644 --- a/lib/std/crypto/aes/aesni.zig +++ b/lib/std/crypto/aes/aesni.zig @@ -313,10 +313,7 @@ pub fn AesEncryptCtx(comptime Aes: type) type { inline while (i < rounds) : (i += 1) { ts = Block.parallel.encryptWide(count, ts, round_keys[i]); } - i = 1; - inline while (i < count) : (i += 1) { - ts = Block.parallel.encryptLastWide(count, ts, round_keys[i]); - } + ts = Block.parallel.encryptLastWide(count, ts, round_keys[i]); j = 0; inline while (j < count) : (j += 1) { dst[16 * j .. 16 * j + 16].* = ts[j].toBytes(); @@ -392,10 +389,7 @@ pub fn AesDecryptCtx(comptime Aes: type) type { inline while (i < rounds) : (i += 1) { ts = Block.parallel.decryptWide(count, ts, inv_round_keys[i]); } - i = 1; - inline while (i < count) : (i += 1) { - ts = Block.parallel.decryptLastWide(count, ts, inv_round_keys[i]); - } + ts = Block.parallel.decryptLastWide(count, ts, inv_round_keys[i]); j = 0; inline while (j < count) : (j += 1) { dst[16 * j .. 16 * j + 16].* = ts[j].toBytes(); diff --git a/lib/std/crypto/aes/armcrypto.zig b/lib/std/crypto/aes/armcrypto.zig index d331783284..85578fcad9 100644 --- a/lib/std/crypto/aes/armcrypto.zig +++ b/lib/std/crypto/aes/armcrypto.zig @@ -364,10 +364,7 @@ pub fn AesEncryptCtx(comptime Aes: type) type { inline while (i < rounds) : (i += 1) { ts = Block.parallel.encryptWide(count, ts, round_keys[i]); } - i = 1; - inline while (i < count) : (i += 1) { - ts = Block.parallel.encryptLastWide(count, ts, round_keys[i]); - } + ts = Block.parallel.encryptLastWide(count, ts, round_keys[i]); j = 0; inline while (j < count) : (j += 1) { dst[16 * j .. 16 * j + 16].* = ts[j].toBytes(); @@ -443,10 +440,7 @@ pub fn AesDecryptCtx(comptime Aes: type) type { inline while (i < rounds) : (i += 1) { ts = Block.parallel.decryptWide(count, ts, inv_round_keys[i]); } - i = 1; - inline while (i < count) : (i += 1) { - ts = Block.parallel.decryptLastWide(count, ts, inv_round_keys[i]); - } + ts = Block.parallel.decryptLastWide(count, ts, inv_round_keys[i]); j = 0; inline while (j < count) : (j += 1) { dst[16 * j .. 16 * j + 16].* = ts[j].toBytes(); From 294ee1bbc93a00ead5182234f824b0a665c579d9 Mon Sep 17 00:00:00 2001 From: Evan Haas Date: Thu, 25 Feb 2021 16:43:39 -0800 Subject: [PATCH 07/13] translate-c: add limited OffsetOfExpr support Add support for OffsetOfExpr that contain exactly 1 component, when that component is a field. For example, given: ```c struct S { float f; double d; }; struct T { long l; int i; struct S s[10]; }; ``` Then: ```c offsetof(struct T, i) // supported offsetof(struct T, s[2].d) // not supported currently ``` --- src/clang.zig | 38 ++++++++++++++++++++++++++++ src/translate_c.zig | 52 +++++++++++++++++++++++++++++++++++++++ src/translate_c/ast.zig | 8 ++++++ src/zig_clang.cpp | 45 +++++++++++++++++++++++++++++++++ src/zig_clang.h | 18 ++++++++++++++ test/run_translated_c.zig | 33 +++++++++++++++++++++++++ 6 files changed, 194 insertions(+) diff --git a/src/clang.zig b/src/clang.zig index 5ae4f50711..854ee7dc7e 100644 --- a/src/clang.zig +++ b/src/clang.zig @@ -432,6 +432,9 @@ pub const FieldDecl = opaque { pub const getLocation = ZigClangFieldDecl_getLocation; extern fn ZigClangFieldDecl_getLocation(*const FieldDecl) SourceLocation; + + pub const getParent = ZigClangFieldDecl_getParent; + extern fn ZigClangFieldDecl_getParent(*const FieldDecl) ?*const RecordDecl; }; pub const FileID = opaque {}; @@ -593,6 +596,34 @@ pub const TypeOfExprType = opaque { extern fn ZigClangTypeOfExprType_getUnderlyingExpr(*const TypeOfExprType) *const Expr; }; +pub const OffsetOfNode = opaque { + pub const getKind = ZigClangOffsetOfNode_getKind; + extern fn ZigClangOffsetOfNode_getKind(*const OffsetOfNode) OffsetOfNode_Kind; + + pub const getArrayExprIndex = ZigClangOffsetOfNode_getArrayExprIndex; + extern fn ZigClangOffsetOfNode_getArrayExprIndex(*const OffsetOfNode) c_uint; + + pub const getField = ZigClangOffsetOfNode_getField; + extern fn ZigClangOffsetOfNode_getField(*const OffsetOfNode) *FieldDecl; +}; + +pub const OffsetOfExpr = opaque { + pub const getNumComponents = ZigClangOffsetOfExpr_getNumComponents; + extern fn ZigClangOffsetOfExpr_getNumComponents(*const OffsetOfExpr) c_uint; + + pub const getNumExpressions = ZigClangOffsetOfExpr_getNumExpressions; + extern fn ZigClangOffsetOfExpr_getNumExpressions(*const OffsetOfExpr) c_uint; + + pub const getIndexExpr = ZigClangOffsetOfExpr_getIndexExpr; + extern fn ZigClangOffsetOfExpr_getIndexExpr(*const OffsetOfExpr, idx: c_uint) *const Expr; + + pub const getComponent = ZigClangOffsetOfExpr_getComponent; + extern fn ZigClangOffsetOfExpr_getComponent(*const OffsetOfExpr, idx: c_uint) *const OffsetOfNode; + + pub const getBeginLoc = ZigClangOffsetOfExpr_getBeginLoc; + extern fn ZigClangOffsetOfExpr_getBeginLoc(*const OffsetOfExpr) SourceLocation; +}; + pub const MemberExpr = opaque { pub const getBase = ZigClangMemberExpr_getBase; extern fn ZigClangMemberExpr_getBase(*const MemberExpr) *const Expr; @@ -1655,6 +1686,13 @@ pub const UnaryExprOrTypeTrait_Kind = extern enum { PreferredAlignOf, }; +pub const OffsetOfNode_Kind = extern enum { + Array, + Field, + Identifier, + Base, +}; + pub const Stage2ErrorMsg = extern struct { filename_ptr: ?[*]const u8, filename_len: usize, diff --git a/src/translate_c.zig b/src/translate_c.zig index 170e6ef2e1..6dac4bf0ef 100644 --- a/src/translate_c.zig +++ b/src/translate_c.zig @@ -1069,12 +1069,64 @@ fn transStmt( const expr = try transExpr(c, scope, source_expr, .used); return maybeSuppressResult(c, scope, result_used, expr); }, + .OffsetOfExprClass => return transOffsetOfExpr(c, scope, @ptrCast(*const clang.OffsetOfExpr, stmt), result_used), else => { return fail(c, error.UnsupportedTranslation, stmt.getBeginLoc(), "TODO implement translation of stmt class {s}", .{@tagName(sc)}); }, } } +/// Translate a "simple" offsetof expression containing exactly one component, +/// when that component is of kind .Field - e.g. offsetof(mytype, myfield) +fn transSimpleOffsetOfExpr( + c: *Context, + scope: *Scope, + expr: *const clang.OffsetOfExpr, +) TransError!Node { + assert(expr.getNumComponents() == 1); + const component = expr.getComponent(0); + if (component.getKind() == .Field) { + const field_decl = component.getField(); + if (field_decl.getParent()) |record_decl| { + if (c.decl_table.get(@ptrToInt(record_decl.getCanonicalDecl()))) |type_name| { + const type_node = try Tag.type.create(c.arena, type_name); + + var raw_field_name = try c.str(@ptrCast(*const clang.NamedDecl, field_decl).getName_bytes_begin()); + const quoted_field_name = try std.fmt.allocPrint(c.arena, "\"{s}\"", .{raw_field_name}); + const field_name_node = try Tag.string_literal.create(c.arena, quoted_field_name); + + return Tag.byte_offset_of.create(c.arena, .{ + .lhs = type_node, + .rhs = field_name_node, + }); + } + } + } + return fail(c, error.UnsupportedTranslation, expr.getBeginLoc(), "Failed to translate simple OffsetOfExpr", .{}); +} + +fn transOffsetOfExpr( + c: *Context, + scope: *Scope, + expr: *const clang.OffsetOfExpr, + result_used: ResultUsed, +) TransError!Node { + if (expr.getNumComponents() == 1) { + const offsetof_expr = try transSimpleOffsetOfExpr(c, scope, expr); + return maybeSuppressResult(c, scope, result_used, offsetof_expr); + } + + // TODO implement OffsetOfExpr with more than 1 component + // OffsetOfExpr API: + // call expr.getComponent(idx) while idx < expr.getNumComponents() + // component.getKind() will be either .Array or .Field (other kinds are C++-only) + // if .Field, use component.getField() to retrieve *clang.FieldDecl + // if .Array, use component.getArrayExprIndex() to get a c_uint which + // can be passed to expr.getIndexExpr(expr_index) to get the *clang.Expr for the array index + + return fail(c, error.UnsupportedTranslation, expr.getBeginLoc(), "TODO: implement complex OffsetOfExpr translation", .{}); +} + fn transBinaryOperator( c: *Context, scope: *Scope, diff --git a/src/translate_c/ast.zig b/src/translate_c/ast.zig index c02b4048f7..e984274c75 100644 --- a/src/translate_c/ast.zig +++ b/src/translate_c/ast.zig @@ -148,6 +148,8 @@ pub const Node = extern union { ptr_cast, /// @divExact(lhs, rhs) div_exact, + /// @byteOffsetOf(lhs, rhs) + byte_offset_of, negate, negate_wrap, @@ -303,6 +305,7 @@ pub const Node = extern union { .std_mem_zeroinit, .ptr_cast, .div_exact, + .byte_offset_of, => Payload.BinOp, .integer_literal, @@ -1135,6 +1138,10 @@ fn renderNode(c: *Context, node: Node) Allocator.Error!NodeIndex { const payload = node.castTag(.div_exact).?.data; return renderBuiltinCall(c, "@divExact", &.{ payload.lhs, payload.rhs }); }, + .byte_offset_of => { + const payload = node.castTag(.byte_offset_of).?.data; + return renderBuiltinCall(c, "@byteOffsetOf", &.{ payload.lhs, payload.rhs }); + }, .sizeof => { const payload = node.castTag(.sizeof).?.data; return renderBuiltinCall(c, "@sizeOf", &.{payload}); @@ -2001,6 +2008,7 @@ fn renderNodeGrouped(c: *Context, node: Node) !NodeIndex { .array_type, .bool_to_int, .div_exact, + .byte_offset_of, => { // no grouping needed return renderNode(c, node); diff --git a/src/zig_clang.cpp b/src/zig_clang.cpp index 8387f6fa80..0ddb597371 100644 --- a/src/zig_clang.cpp +++ b/src/zig_clang.cpp @@ -2609,6 +2609,46 @@ const struct ZigClangExpr *ZigClangTypeOfExprType_getUnderlyingExpr(const struct return reinterpret_cast(casted->getUnderlyingExpr()); } +enum ZigClangOffsetOfNode_Kind ZigClangOffsetOfNode_getKind(const struct ZigClangOffsetOfNode *self) { + auto casted = reinterpret_cast(self); + return (ZigClangOffsetOfNode_Kind)casted->getKind(); +} + +unsigned ZigClangOffsetOfNode_getArrayExprIndex(const struct ZigClangOffsetOfNode *self) { + auto casted = reinterpret_cast(self); + return casted->getArrayExprIndex(); +} + +struct ZigClangFieldDecl *ZigClangOffsetOfNode_getField(const struct ZigClangOffsetOfNode *self) { + auto casted = reinterpret_cast(self); + return reinterpret_cast(casted->getField()); +} + +unsigned ZigClangOffsetOfExpr_getNumComponents(const struct ZigClangOffsetOfExpr *self) { + auto casted = reinterpret_cast(self); + return casted->getNumComponents(); +} + +unsigned ZigClangOffsetOfExpr_getNumExpressions(const struct ZigClangOffsetOfExpr *self) { + auto casted = reinterpret_cast(self); + return casted->getNumExpressions(); +} + +const struct ZigClangExpr *ZigClangOffsetOfExpr_getIndexExpr(const struct ZigClangOffsetOfExpr *self, unsigned idx) { + auto casted = reinterpret_cast(self); + return reinterpret_cast(casted->getIndexExpr(idx)); +} + +const struct ZigClangOffsetOfNode *ZigClangOffsetOfExpr_getComponent(const struct ZigClangOffsetOfExpr *self, unsigned idx) { + auto casted = reinterpret_cast(self); + return reinterpret_cast(&casted->getComponent(idx)); +} + +ZigClangSourceLocation ZigClangOffsetOfExpr_getBeginLoc(const ZigClangOffsetOfExpr *self) { + auto casted = reinterpret_cast(self); + return bitcast(casted->getBeginLoc()); +} + struct ZigClangQualType ZigClangElaboratedType_getNamedType(const struct ZigClangElaboratedType *self) { auto casted = reinterpret_cast(self); return bitcast(casted->getNamedType()); @@ -3008,6 +3048,11 @@ ZigClangSourceLocation ZigClangFieldDecl_getLocation(const struct ZigClangFieldD return bitcast(casted->getLocation()); } +const struct ZigClangRecordDecl *ZigClangFieldDecl_getParent(const struct ZigClangFieldDecl *self) { + auto casted = reinterpret_cast(self); + return reinterpret_cast(casted->getParent()); +} + ZigClangQualType ZigClangFieldDecl_getType(const struct ZigClangFieldDecl *self) { auto casted = reinterpret_cast(self); return bitcast(casted->getType()); diff --git a/src/zig_clang.h b/src/zig_clang.h index 3c3e97f4a3..b3717946ea 100644 --- a/src/zig_clang.h +++ b/src/zig_clang.h @@ -928,6 +928,13 @@ enum ZigClangUnaryExprOrTypeTrait_Kind { ZigClangUnaryExprOrTypeTrait_KindPreferredAlignOf, }; +enum ZigClangOffsetOfNode_Kind { + ZigClangOffsetOfNode_KindArray, + ZigClangOffsetOfNode_KindField, + ZigClangOffsetOfNode_KindIdentifier, + ZigClangOffsetOfNode_KindBase, +}; + ZIG_EXTERN_C struct ZigClangSourceLocation ZigClangSourceManager_getSpellingLoc(const struct ZigClangSourceManager *, struct ZigClangSourceLocation Loc); ZIG_EXTERN_C const char *ZigClangSourceManager_getFilename(const struct ZigClangSourceManager *, @@ -1161,6 +1168,16 @@ ZIG_EXTERN_C struct ZigClangQualType ZigClangTypeOfType_getUnderlyingType(const ZIG_EXTERN_C const struct ZigClangExpr *ZigClangTypeOfExprType_getUnderlyingExpr(const struct ZigClangTypeOfExprType *); +ZIG_EXTERN_C enum ZigClangOffsetOfNode_Kind ZigClangOffsetOfNode_getKind(const struct ZigClangOffsetOfNode *); +ZIG_EXTERN_C unsigned ZigClangOffsetOfNode_getArrayExprIndex(const struct ZigClangOffsetOfNode *); +ZIG_EXTERN_C struct ZigClangFieldDecl * ZigClangOffsetOfNode_getField(const struct ZigClangOffsetOfNode *); + +ZIG_EXTERN_C unsigned ZigClangOffsetOfExpr_getNumComponents(const struct ZigClangOffsetOfExpr *); +ZIG_EXTERN_C unsigned ZigClangOffsetOfExpr_getNumExpressions(const struct ZigClangOffsetOfExpr *); +ZIG_EXTERN_C const struct ZigClangExpr *ZigClangOffsetOfExpr_getIndexExpr(const struct ZigClangOffsetOfExpr *, unsigned idx); +ZIG_EXTERN_C const struct ZigClangOffsetOfNode *ZigClangOffsetOfExpr_getComponent(const struct ZigClangOffsetOfExpr *, unsigned idx); +ZIG_EXTERN_C struct ZigClangSourceLocation ZigClangOffsetOfExpr_getBeginLoc(const struct ZigClangOffsetOfExpr *); + ZIG_EXTERN_C struct ZigClangQualType ZigClangElaboratedType_getNamedType(const struct ZigClangElaboratedType *); ZIG_EXTERN_C enum ZigClangElaboratedTypeKeyword ZigClangElaboratedType_getKeyword(const struct ZigClangElaboratedType *); @@ -1261,6 +1278,7 @@ ZIG_EXTERN_C bool ZigClangFieldDecl_isBitField(const struct ZigClangFieldDecl *) ZIG_EXTERN_C bool ZigClangFieldDecl_isAnonymousStructOrUnion(const ZigClangFieldDecl *); ZIG_EXTERN_C struct ZigClangQualType ZigClangFieldDecl_getType(const struct ZigClangFieldDecl *); ZIG_EXTERN_C struct ZigClangSourceLocation ZigClangFieldDecl_getLocation(const struct ZigClangFieldDecl *); +ZIG_EXTERN_C const struct ZigClangRecordDecl *ZigClangFieldDecl_getParent(const struct ZigClangFieldDecl *); ZIG_EXTERN_C const struct ZigClangExpr *ZigClangEnumConstantDecl_getInitExpr(const struct ZigClangEnumConstantDecl *); ZIG_EXTERN_C const struct ZigClangAPSInt *ZigClangEnumConstantDecl_getInitVal(const struct ZigClangEnumConstantDecl *); diff --git a/test/run_translated_c.zig b/test/run_translated_c.zig index 70dc1c835d..23b0a70395 100644 --- a/test/run_translated_c.zig +++ b/test/run_translated_c.zig @@ -1073,4 +1073,37 @@ pub fn addCases(cases: *tests.RunTranslatedCContext) void { \\ return 0; \\} , ""); + + cases.add("offsetof", + \\#include + \\#include + \\#define container_of(ptr, type, member) ({ \ + \\ const typeof( ((type *)0)->member ) *__mptr = (ptr); \ + \\ (type *)( (char *)__mptr - offsetof(type,member) );}) + \\typedef struct { + \\ int i; + \\ struct { int x; char y; int z; } s; + \\ float f; + \\} container; + \\int main(void) { + \\ if (offsetof(container, i) != 0) abort(); + \\ if (offsetof(container, s) <= offsetof(container, i)) abort(); + \\ if (offsetof(container, f) <= offsetof(container, s)) abort(); + \\ + \\ container my_container; + \\ typeof(my_container.s) *inner_member_pointer = &my_container.s; + \\ float *float_member_pointer = &my_container.f; + \\ int *anon_member_pointer = &my_container.s.z; + \\ container *my_container_p; + \\ + \\ my_container_p = container_of(inner_member_pointer, container, s); + \\ if (my_container_p != &my_container) abort(); + \\ + \\ my_container_p = container_of(float_member_pointer, container, f); + \\ if (my_container_p != &my_container) abort(); + \\ + \\ if (container_of(anon_member_pointer, typeof(my_container.s), z) != inner_member_pointer) abort(); + \\ return 0; + \\} + , ""); } From 1f17221bc4e17bcd7116fe12ab3f939346179799 Mon Sep 17 00:00:00 2001 From: daurnimator Date: Sat, 27 Feb 2021 14:19:38 +1100 Subject: [PATCH 08/13] std: add sendmsg --- lib/std/os.zig | 75 +++++++++++++++++++++++++++++++- lib/std/os/bits/linux/arm64.zig | 8 ++-- lib/std/os/bits/linux/x86_64.zig | 8 ++-- lib/std/os/linux.zig | 2 +- lib/std/os/windows.zig | 13 ++++++ 5 files changed, 96 insertions(+), 10 deletions(-) diff --git a/lib/std/os.zig b/lib/std/os.zig index 24d9041639..a2e62ed0ae 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -4840,7 +4840,7 @@ pub const SendError = error{ NetworkSubsystemFailed, } || UnexpectedError; -pub const SendToError = SendError || error{ +pub const SendMsgError = SendError || error{ /// The passed address didn't have the correct address family in its sa_family field. AddressFamilyNotSupported, @@ -4859,6 +4859,79 @@ pub const SendToError = SendError || error{ AddressNotAvailable, }; +pub fn sendmsg( + /// The file descriptor of the sending socket. + sockfd: socket_t, + /// Message header and iovecs + msg: msghdr_const, + flags: u32, +) SendMsgError!usize { + while (true) { + const rc = system.sendmsg(sockfd, &msg, flags); + if (builtin.os.tag == .windows) { + if (rc == windows.ws2_32.SOCKET_ERROR) { + switch (windows.ws2_32.WSAGetLastError()) { + .WSAEACCES => return error.AccessDenied, + .WSAEADDRNOTAVAIL => return error.AddressNotAvailable, + .WSAECONNRESET => return error.ConnectionResetByPeer, + .WSAEMSGSIZE => return error.MessageTooBig, + .WSAENOBUFS => return error.SystemResources, + .WSAENOTSOCK => return error.FileDescriptorNotASocket, + .WSAEAFNOSUPPORT => return error.AddressFamilyNotSupported, + .WSAEDESTADDRREQ => unreachable, // A destination address is required. + .WSAEFAULT => unreachable, // The lpBuffers, lpTo, lpOverlapped, lpNumberOfBytesSent, or lpCompletionRoutine parameters are not part of the user address space, or the lpTo parameter is too small. + .WSAEHOSTUNREACH => return error.NetworkUnreachable, + // TODO: WSAEINPROGRESS, WSAEINTR + .WSAEINVAL => unreachable, + .WSAENETDOWN => return error.NetworkSubsystemFailed, + .WSAENETRESET => return error.ConnectionResetByPeer, + .WSAENETUNREACH => return error.NetworkUnreachable, + .WSAENOTCONN => return error.SocketNotConnected, + .WSAESHUTDOWN => unreachable, // The socket has been shut down; it is not possible to WSASendTo on a socket after shutdown has been invoked with how set to SD_SEND or SD_BOTH. + .WSAEWOULDBLOCK => return error.WouldBlock, + .WSANOTINITIALISED => unreachable, // A successful WSAStartup call must occur before using this function. + else => |err| return windows.unexpectedWSAError(err), + } + } else { + return @intCast(usize, rc); + } + } else { + switch (errno(rc)) { + 0 => return @intCast(usize, rc), + + EACCES => return error.AccessDenied, + EAGAIN => return error.WouldBlock, + EALREADY => return error.FastOpenAlreadyInProgress, + EBADF => unreachable, // always a race condition + ECONNRESET => return error.ConnectionResetByPeer, + EDESTADDRREQ => unreachable, // The socket is not connection-mode, and no peer address is set. + EFAULT => unreachable, // An invalid user space address was specified for an argument. + EINTR => continue, + EINVAL => unreachable, // Invalid argument passed. + EISCONN => unreachable, // connection-mode socket was connected already but a recipient was specified + EMSGSIZE => return error.MessageTooBig, + ENOBUFS => return error.SystemResources, + ENOMEM => return error.SystemResources, + ENOTSOCK => unreachable, // The file descriptor sockfd does not refer to a socket. + EOPNOTSUPP => unreachable, // Some bit in the flags argument is inappropriate for the socket type. + EPIPE => return error.BrokenPipe, + EAFNOSUPPORT => return error.AddressFamilyNotSupported, + ELOOP => return error.SymLinkLoop, + ENAMETOOLONG => return error.NameTooLong, + ENOENT => return error.FileNotFound, + ENOTDIR => return error.NotDir, + EHOSTUNREACH => return error.NetworkUnreachable, + ENETUNREACH => return error.NetworkUnreachable, + ENOTCONN => return error.SocketNotConnected, + ENETDOWN => return error.NetworkSubsystemFailed, + else => |err| return unexpectedErrno(err), + } + } + } +} + +pub const SendToError = SendMsgError; + /// Transmit a message to another socket. /// /// The `sendto` call may be used only when the socket is in a connected state (so that the intended diff --git a/lib/std/os/bits/linux/arm64.zig b/lib/std/os/bits/linux/arm64.zig index a069b6adf1..e373d978e1 100644 --- a/lib/std/os/bits/linux/arm64.zig +++ b/lib/std/os/bits/linux/arm64.zig @@ -400,10 +400,10 @@ pub const msghdr = extern struct { msg_namelen: socklen_t, msg_iov: [*]iovec, msg_iovlen: i32, - __pad1: i32, + __pad1: i32 = 0, msg_control: ?*c_void, msg_controllen: socklen_t, - __pad2: socklen_t, + __pad2: socklen_t = 0, msg_flags: i32, }; @@ -412,10 +412,10 @@ pub const msghdr_const = extern struct { msg_namelen: socklen_t, msg_iov: [*]iovec_const, msg_iovlen: i32, - __pad1: i32, + __pad1: i32 = 0, msg_control: ?*c_void, msg_controllen: socklen_t, - __pad2: socklen_t, + __pad2: socklen_t = 0, msg_flags: i32, }; diff --git a/lib/std/os/bits/linux/x86_64.zig b/lib/std/os/bits/linux/x86_64.zig index 52fee679c5..88d277a0c5 100644 --- a/lib/std/os/bits/linux/x86_64.zig +++ b/lib/std/os/bits/linux/x86_64.zig @@ -495,10 +495,10 @@ pub const msghdr = extern struct { msg_namelen: socklen_t, msg_iov: [*]iovec, msg_iovlen: i32, - __pad1: i32, + __pad1: i32 = 0, msg_control: ?*c_void, msg_controllen: socklen_t, - __pad2: socklen_t, + __pad2: socklen_t = 0, msg_flags: i32, }; @@ -507,10 +507,10 @@ pub const msghdr_const = extern struct { msg_namelen: socklen_t, msg_iov: [*]iovec_const, msg_iovlen: i32, - __pad1: i32, + __pad1: i32 = 0, msg_control: ?*c_void, msg_controllen: socklen_t, - __pad2: socklen_t, + __pad2: socklen_t = 0, msg_flags: i32, }; diff --git a/lib/std/os/linux.zig b/lib/std/os/linux.zig index 035cdabe63..900e3f2871 100644 --- a/lib/std/os/linux.zig +++ b/lib/std/os/linux.zig @@ -977,7 +977,7 @@ pub fn getsockopt(fd: i32, level: u32, optname: u32, noalias optval: [*]u8, noal return syscall5(.getsockopt, @bitCast(usize, @as(isize, fd)), level, optname, @ptrToInt(optval), @ptrToInt(optlen)); } -pub fn sendmsg(fd: i32, msg: *msghdr_const, flags: u32) usize { +pub fn sendmsg(fd: i32, msg: *const msghdr_const, flags: u32) usize { if (builtin.arch == .i386) { return socketcall(SC_sendmsg, &[3]usize{ @bitCast(usize, @as(isize, fd)), @ptrToInt(msg), flags }); } diff --git a/lib/std/os/windows.zig b/lib/std/os/windows.zig index 6f67b65252..1791fec956 100644 --- a/lib/std/os/windows.zig +++ b/lib/std/os/windows.zig @@ -1291,6 +1291,19 @@ pub fn getsockname(s: ws2_32.SOCKET, name: *ws2_32.sockaddr, namelen: *ws2_32.so return ws2_32.getsockname(s, name, @ptrCast(*i32, namelen)); } +pub fn sendmsg( + s: ws2_32.SOCKET, + msg: *const ws2_32.WSAMSG, + flags: u32, +) i32 { + var bytes_send: DWORD = undefined; + if (ws2_32.WSASendMsg(s, msg, flags, &bytes_send, null, null) == ws2_32.SOCKET_ERROR) { + return ws2_32.SOCKET_ERROR; + } else { + return @as(i32, @intCast(u31, bytes_send)); + } +} + pub fn sendto(s: ws2_32.SOCKET, buf: [*]const u8, len: usize, flags: u32, to: ?*const ws2_32.sockaddr, to_len: ws2_32.socklen_t) i32 { var buffer = ws2_32.WSABUF{ .len = @truncate(u31, len), .buf = @intToPtr([*]u8, @ptrToInt(buf)) }; var bytes_send: DWORD = undefined; From 58b14d01aeb919989d1750a8894350a07fd4e844 Mon Sep 17 00:00:00 2001 From: jacob gw Date: Fri, 26 Feb 2021 09:56:16 -0500 Subject: [PATCH 09/13] stage2: remove value field from error This saves memory and from what I have heard allows threading to be easier. --- src/Module.zig | 1 - src/value.zig | 2 -- src/zir_sema.zig | 4 ++-- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/Module.zig b/src/Module.zig index 21dc953262..3a72f1272b 100644 --- a/src/Module.zig +++ b/src/Module.zig @@ -4101,7 +4101,6 @@ pub fn namedFieldPtr( scope.arena(), try Value.Tag.@"error".create(scope.arena(), .{ .name = entry.key, - .value = entry.value, }), ), }); diff --git a/src/value.zig b/src/value.zig index 0a6a48dbf7..ae94e4b424 100644 --- a/src/value.zig +++ b/src/value.zig @@ -1561,7 +1561,6 @@ pub const Value = extern union { .@"error" => { const payload = self.castTag(.@"error").?.data; hasher.update(payload.name); - std.hash.autoHash(&hasher, payload.value); }, .error_union => { const payload = self.castTag(.error_union).?.data; @@ -2157,7 +2156,6 @@ pub const Value = extern union { /// duration of the compilation. /// TODO revisit this when we have the concept of the error tag type name: []const u8, - value: u16, }, }; diff --git a/src/zir_sema.zig b/src/zir_sema.zig index 8da4fd9eff..864f766f54 100644 --- a/src/zir_sema.zig +++ b/src/zir_sema.zig @@ -1178,7 +1178,6 @@ fn zirErrorValue(mod: *Module, scope: *Scope, inst: *zir.Inst.ErrorValue) InnerE .ty = result_type, .val = try Value.Tag.@"error".create(scope.arena(), .{ .name = entry.key, - .value = entry.value, }), }); } @@ -2215,7 +2214,8 @@ fn zirCmp( } if (rhs.value()) |rval| { if (lhs.value()) |lval| { - return mod.constBool(scope, inst.base.src, (lval.castTag(.@"error").?.data.value == rval.castTag(.@"error").?.data.value) == (op == .eq)); + // TODO optimisation oppurtunity: evaluate if std.mem.eql is faster with the names, or calling to Module.getErrorValue to get the values and then compare them is faster + return mod.constBool(scope, inst.base.src, std.mem.eql(u8, lval.castTag(.@"error").?.data.name, rval.castTag(.@"error").?.data.name) == (op == .eq)); } } return mod.fail(scope, inst.base.src, "TODO implement equality comparison between runtime errors", .{}); From a5a3ad4f956bae1ca0e5a49de2e9ac7145170039 Mon Sep 17 00:00:00 2001 From: Frank Denis Date: Sun, 28 Feb 2021 08:29:38 +0100 Subject: [PATCH 10/13] std/crypto: add AES-OCB OCB has been around for a long time. It's simpler, faster and more secure than AES-GCM. RFC 7253 was published in 2014. OCB also won the CAESAR competition along with AEGIS. It's been implemented in OpenSSL and other libraries for years. So, why isn't everybody using it instead of GCM? And why don't we have it in Zig already? The sad reason for this was patents. GCM was invented only to work around these patents, and for all this time, OCB was that nice thing that everybody knew existed but that couldn't be freely used. That just changed. The OCB patents are now abandoned, and OCB's author just announced that OCB was officially public domain. --- lib/std/crypto.zig | 30 +-- lib/std/crypto/aes_ocb.zig | 343 +++++++++++++++++++++++++++++++++++ lib/std/crypto/benchmark.zig | 2 + 3 files changed, 353 insertions(+), 22 deletions(-) create mode 100644 lib/std/crypto/aes_ocb.zig diff --git a/lib/std/crypto.zig b/lib/std/crypto.zig index da6ec2edf0..24ca549479 100644 --- a/lib/std/crypto.zig +++ b/lib/std/crypto.zig @@ -16,6 +16,11 @@ pub const aead = struct { pub const Aes256Gcm = @import("crypto/aes_gcm.zig").Aes256Gcm; }; + pub const aes_ocb = struct { + pub const Aes128Ocb = @import("crypto/aes_ocb.zig").Aes128Ocb; + pub const Aes256Ocb = @import("crypto/aes_ocb.zig").Aes256Ocb; + }; + pub const Gimli = @import("crypto/gimli.zig").Aead; pub const chacha_poly = struct { @@ -157,30 +162,11 @@ test "crypto" { } } - _ = @import("crypto/aes.zig"); - _ = @import("crypto/bcrypt.zig"); + _ = @import("crypto/aegis.zig"); + _ = @import("crypto/aes_gcm.zig"); + _ = @import("crypto/aes_ocb.zig"); _ = @import("crypto/blake2.zig"); - _ = @import("crypto/blake3.zig"); _ = @import("crypto/chacha20.zig"); - _ = @import("crypto/gimli.zig"); - _ = @import("crypto/hmac.zig"); - _ = @import("crypto/isap.zig"); - _ = @import("crypto/md5.zig"); - _ = @import("crypto/modes.zig"); - _ = @import("crypto/pbkdf2.zig"); - _ = @import("crypto/poly1305.zig"); - _ = @import("crypto/sha1.zig"); - _ = @import("crypto/sha2.zig"); - _ = @import("crypto/sha3.zig"); - _ = @import("crypto/salsa20.zig"); - _ = @import("crypto/siphash.zig"); - _ = @import("crypto/25519/curve25519.zig"); - _ = @import("crypto/25519/ed25519.zig"); - _ = @import("crypto/25519/edwards25519.zig"); - _ = @import("crypto/25519/field.zig"); - _ = @import("crypto/25519/scalar.zig"); - _ = @import("crypto/25519/x25519.zig"); - _ = @import("crypto/25519/ristretto255.zig"); } test "CSPRNG" { diff --git a/lib/std/crypto/aes_ocb.zig b/lib/std/crypto/aes_ocb.zig new file mode 100644 index 0000000000..ab0138f181 --- /dev/null +++ b/lib/std/crypto/aes_ocb.zig @@ -0,0 +1,343 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2015-2021 Zig Contributors +// This file is part of [zig](https://ziglang.org/), which is MIT licensed. +// The MIT license requires this copyright notice to be included in all copies +// and substantial portions of the software. + +const std = @import("std"); +const crypto = std.crypto; +const aes = crypto.core.aes; +const assert = std.debug.assert; +const math = std.math; +const mem = std.mem; + +pub const Aes128Ocb = AesOcb(aes.Aes128); +pub const Aes256Ocb = AesOcb(aes.Aes256); + +const Block = [16]u8; + +/// AES-OCB (RFC 7253 - https://competitions.cr.yp.to/round3/ocbv11.pdf) +fn AesOcb(comptime Aes: anytype) type { + const EncryptCtx = aes.AesEncryptCtx(Aes); + const DecryptCtx = aes.AesDecryptCtx(Aes); + + return struct { + pub const key_length = Aes.key_bits / 8; + pub const nonce_length: usize = 12; + pub const tag_length: usize = 16; + + const Lx = struct { + star: Block align(16), + dol: Block align(16), + table: [56]Block align(16) = undefined, + upto: usize, + + fn double(l: Block) callconv(.Inline) Block { + const l_ = mem.readIntBig(u128, &l); + const l_2 = (l_ << 1) ^ (0x87 & -%(l_ >> 127)); + var l2: Block = undefined; + mem.writeIntBig(u128, &l2, l_2); + return l2; + } + + fn precomp(lx: *Lx, upto: usize) []const Block { + const table = &lx.table; + assert(upto < table.len); + var i = lx.upto; + while (i + 1 <= upto) : (i += 1) { + table[i + 1] = double(table[i]); + } + lx.upto = upto; + return lx.table[0 .. upto + 1]; + } + + fn init(aes_enc_ctx: EncryptCtx) Lx { + const zeros = [_]u8{0} ** 16; + var star: Block = undefined; + aes_enc_ctx.encrypt(&star, &zeros); + const dol = double(star); + var lx = Lx{ .star = star, .dol = dol, .upto = 0 }; + lx.table[0] = double(dol); + return lx; + } + }; + + fn hash(aes_enc_ctx: EncryptCtx, lx: *Lx, a: []const u8) Block { + const full_blocks: usize = a.len / 16; + const x_max = if (full_blocks > 0) math.log2_int(usize, full_blocks) else 0; + const lt = lx.precomp(x_max); + var sum = [_]u8{0} ** 16; + var offset = [_]u8{0} ** 16; + var i: usize = 0; + while (i < full_blocks) : (i += 1) { + xorWith(&offset, lt[@ctz(usize, i + 1)]); + var e = xorBlocks(offset, a[i * 16 ..][0..16].*); + aes_enc_ctx.encrypt(&e, &e); + xorWith(&sum, e); + } + const leftover = a.len % 16; + if (leftover > 0) { + xorWith(&offset, lx.star); + var padded = [_]u8{0} ** 16; + mem.copy(u8, padded[0..leftover], a[i * 16 ..][0..leftover]); + padded[leftover] = 1; + var e = xorBlocks(offset, padded); + aes_enc_ctx.encrypt(&e, &e); + xorWith(&sum, e); + } + return sum; + } + + fn getOffset(aes_enc_ctx: EncryptCtx, npub: [nonce_length]u8) Block { + var nx = [_]u8{0} ** 16; + nx[0] = @intCast(u8, @truncate(u7, tag_length * 8) << 1); + nx[16 - nonce_length - 1] = 1; + mem.copy(u8, nx[16 - nonce_length ..], &npub); + + const bottom = @truncate(u6, nx[15]); + nx[15] &= 0xc0; + var ktop_: Block = undefined; + aes_enc_ctx.encrypt(&ktop_, &nx); + const ktop = mem.readIntBig(u128, &ktop_); + var stretch = (@as(u192, ktop) << 64) | @as(u192, @truncate(u64, ktop >> 64) ^ @truncate(u64, ktop >> 56)); + var offset: Block = undefined; + mem.writeIntBig(u128, &offset, @truncate(u128, stretch >> (64 - @as(u7, bottom)))); + return offset; + } + + const has_aesni = comptime std.Target.x86.featureSetHas(std.Target.current.cpu.features, .aes); + const has_armaes = comptime std.Target.aarch64.featureSetHas(std.Target.current.cpu.features, .aes); + const wb: usize = if ((std.Target.current.cpu.arch == .x86_64 and has_aesni) or (std.Target.current.cpu.arch == .aarch64 and has_armaes)) 4 else 0; + + /// c: ciphertext: output buffer should be of size m.len + /// tag: authentication tag: output MAC + /// m: message + /// ad: Associated Data + /// npub: public nonce + /// k: secret key + pub fn encrypt(c: []u8, tag: *[tag_length]u8, m: []const u8, ad: []const u8, npub: [nonce_length]u8, key: [key_length]u8) void { + assert(c.len == m.len); + + const aes_enc_ctx = Aes.initEnc(key); + const full_blocks: usize = m.len / 16; + const x_max = if (full_blocks > 0) math.log2_int(usize, full_blocks) else 0; + var lx = Lx.init(aes_enc_ctx); + const lt = lx.precomp(x_max); + + var offset = getOffset(aes_enc_ctx, npub); + var sum = [_]u8{0} ** 16; + var i: usize = 0; + + while (wb > 0 and i + wb <= full_blocks) : (i += wb) { + var offsets: [wb]Block align(16) = undefined; + var es: [16 * wb]u8 align(16) = undefined; + var j: usize = 0; + while (j < wb) : (j += 1) { + xorWith(&offset, lt[@ctz(usize, i + 1 + j)]); + offsets[j] = offset; + const p = m[(i + j) * 16 ..][0..16].*; + mem.copy(u8, es[j * 16 ..][0..16], &xorBlocks(p, offsets[j])); + xorWith(&sum, p); + } + aes_enc_ctx.encryptWide(wb, &es, &es); + j = 0; + while (j < wb) : (j += 1) { + const e = es[j * 16 ..][0..16].*; + mem.copy(u8, c[(i + j) * 16 ..][0..16], &xorBlocks(e, offsets[j])); + } + } + while (i < full_blocks) : (i += 1) { + xorWith(&offset, lt[@ctz(usize, i + 1)]); + const p = m[i * 16 ..][0..16].*; + var e = xorBlocks(p, offset); + aes_enc_ctx.encrypt(&e, &e); + mem.copy(u8, c[i * 16 ..][0..16], &xorBlocks(e, offset)); + xorWith(&sum, p); + } + const leftover = m.len % 16; + if (leftover > 0) { + xorWith(&offset, lx.star); + var pad = offset; + aes_enc_ctx.encrypt(&pad, &pad); + for (m[i * 16 ..]) |x, j| { + c[i * 16 + j] = pad[j] ^ x; + } + var e = [_]u8{0} ** 16; + mem.copy(u8, e[0..leftover], m[i * 16 ..][0..leftover]); + e[leftover] = 0x80; + xorWith(&sum, e); + } + var e = xorBlocks(xorBlocks(sum, offset), lx.dol); + aes_enc_ctx.encrypt(&e, &e); + tag.* = xorBlocks(e, hash(aes_enc_ctx, &lx, ad)); + } + + /// m: message: output buffer should be of size c.len + /// c: ciphertext + /// tag: authentication tag + /// ad: Associated Data + /// npub: public nonce + /// k: secret key + pub fn decrypt(m: []u8, c: []const u8, tag: [tag_length]u8, ad: []const u8, npub: [nonce_length]u8, key: [key_length]u8) !void { + assert(c.len == m.len); + + const aes_enc_ctx = Aes.initEnc(key); + const aes_dec_ctx = DecryptCtx.initFromEnc(aes_enc_ctx); + const full_blocks: usize = m.len / 16; + const x_max = if (full_blocks > 0) math.log2_int(usize, full_blocks) else 0; + var lx = Lx.init(aes_enc_ctx); + const lt = lx.precomp(x_max); + + var offset = getOffset(aes_enc_ctx, npub); + var sum = [_]u8{0} ** 16; + var i: usize = 0; + + while (wb > 0 and i + wb <= full_blocks) : (i += wb) { + var offsets: [wb]Block align(16) = undefined; + var es: [16 * wb]u8 align(16) = undefined; + var j: usize = 0; + while (j < wb) : (j += 1) { + xorWith(&offset, lt[@ctz(usize, i + 1 + j)]); + offsets[j] = offset; + const q = c[(i + j) * 16 ..][0..16].*; + mem.copy(u8, es[j * 16 ..][0..16], &xorBlocks(q, offsets[j])); + } + aes_dec_ctx.decryptWide(wb, &es, &es); + j = 0; + while (j < wb) : (j += 1) { + const p = xorBlocks(es[j * 16 ..][0..16].*, offsets[j]); + mem.copy(u8, m[(i + j) * 16 ..][0..16], &p); + xorWith(&sum, p); + } + } + while (i < full_blocks) : (i += 1) { + xorWith(&offset, lt[@ctz(usize, i + 1)]); + const q = c[i * 16 ..][0..16].*; + var e = xorBlocks(q, offset); + aes_dec_ctx.decrypt(&e, &e); + const p = xorBlocks(e, offset); + mem.copy(u8, m[i * 16 ..][0..16], &p); + xorWith(&sum, p); + } + const leftover = m.len % 16; + if (leftover > 0) { + xorWith(&offset, lx.star); + var pad = offset; + aes_enc_ctx.encrypt(&pad, &pad); + for (c[i * 16 ..]) |x, j| { + m[i * 16 + j] = pad[j] ^ x; + } + var e = [_]u8{0} ** 16; + mem.copy(u8, e[0..leftover], m[i * 16 ..][0..leftover]); + e[leftover] = 0x80; + xorWith(&sum, e); + } + var e = xorBlocks(xorBlocks(sum, offset), lx.dol); + aes_enc_ctx.encrypt(&e, &e); + var computed_tag = xorBlocks(e, hash(aes_enc_ctx, &lx, ad)); + const verify = crypto.utils.timingSafeEql([tag_length]u8, computed_tag, tag); + crypto.utils.secureZero(u8, &computed_tag); + if (!verify) { + return error.AuthenticationFailed; + } + } + }; +} + +fn xorBlocks(x: Block, y: Block) callconv(.Inline) Block { + var z: Block = x; + for (z) |*v, i| { + v.* = x[i] ^ y[i]; + } + return z; +} + +fn xorWith(x: *Block, y: Block) callconv(.Inline) void { + for (x) |*v, i| { + v.* ^= y[i]; + } +} + +const hexToBytes = std.fmt.hexToBytes; + +test "AesOcb test vector 1" { + var k: [Aes128Ocb.key_length]u8 = undefined; + var nonce: [Aes128Ocb.nonce_length]u8 = undefined; + var tag: [Aes128Ocb.tag_length]u8 = undefined; + _ = try hexToBytes(&k, "000102030405060708090A0B0C0D0E0F"); + _ = try hexToBytes(&nonce, "BBAA99887766554433221100"); + + var c: [0]u8 = undefined; + Aes128Ocb.encrypt(&c, &tag, "", "", nonce, k); + + var expected_c: [c.len]u8 = undefined; + var expected_tag: [tag.len]u8 = undefined; + _ = try hexToBytes(&expected_tag, "785407BFFFC8AD9EDCC5520AC9111EE6"); + + var m: [0]u8 = undefined; + try Aes128Ocb.decrypt(&m, "", tag, "", nonce, k); +} + +test "AesOcb test vector 2" { + var k: [Aes128Ocb.key_length]u8 = undefined; + var nonce: [Aes128Ocb.nonce_length]u8 = undefined; + var tag: [Aes128Ocb.tag_length]u8 = undefined; + var ad: [40]u8 = undefined; + _ = try hexToBytes(&k, "000102030405060708090A0B0C0D0E0F"); + _ = try hexToBytes(&ad, "000102030405060708090A0B0C0D0E0F101112131415161718191A1B1C1D1E1F2021222324252627"); + _ = try hexToBytes(&nonce, "BBAA9988776655443322110E"); + + var c: [0]u8 = undefined; + Aes128Ocb.encrypt(&c, &tag, "", &ad, nonce, k); + + var expected_tag: [tag.len]u8 = undefined; + _ = try hexToBytes(&expected_tag, "C5CD9D1850C141E358649994EE701B68"); + + var m: [0]u8 = undefined; + try Aes128Ocb.decrypt(&m, &c, tag, &ad, nonce, k); +} + +test "AesOcb test vector 3" { + var k: [Aes128Ocb.key_length]u8 = undefined; + var nonce: [Aes128Ocb.nonce_length]u8 = undefined; + var tag: [Aes128Ocb.tag_length]u8 = undefined; + var m: [40]u8 = undefined; + var c: [m.len]u8 = undefined; + _ = try hexToBytes(&k, "000102030405060708090A0B0C0D0E0F"); + _ = try hexToBytes(&m, "000102030405060708090A0B0C0D0E0F101112131415161718191A1B1C1D1E1F2021222324252627"); + _ = try hexToBytes(&nonce, "BBAA9988776655443322110F"); + + Aes128Ocb.encrypt(&c, &tag, &m, "", nonce, k); + + var expected_c: [c.len]u8 = undefined; + var expected_tag: [tag.len]u8 = undefined; + _ = try hexToBytes(&expected_tag, "479AD363AC366B95A98CA5F3000B1479"); + _ = try hexToBytes(&expected_c, "4412923493C57D5DE0D700F753CCE0D1D2D95060122E9F15A5DDBFC5787E50B5CC55EE507BCB084E"); + + var m2: [m.len]u8 = undefined; + try Aes128Ocb.decrypt(&m2, &c, tag, "", nonce, k); + assert(mem.eql(u8, &m, &m2)); +} + +test "AesOcb test vector 4" { + var k: [Aes128Ocb.key_length]u8 = undefined; + var nonce: [Aes128Ocb.nonce_length]u8 = undefined; + var tag: [Aes128Ocb.tag_length]u8 = undefined; + var m: [40]u8 = undefined; + var ad = m; + var c: [m.len]u8 = undefined; + _ = try hexToBytes(&k, "000102030405060708090A0B0C0D0E0F"); + _ = try hexToBytes(&m, "000102030405060708090A0B0C0D0E0F101112131415161718191A1B1C1D1E1F2021222324252627"); + _ = try hexToBytes(&nonce, "BBAA99887766554433221104"); + + Aes128Ocb.encrypt(&c, &tag, &m, &ad, nonce, k); + + var expected_c: [c.len]u8 = undefined; + var expected_tag: [tag.len]u8 = undefined; + _ = try hexToBytes(&expected_tag, "3AD7A4FF3835B8C5701C1CCEC8FC3358"); + _ = try hexToBytes(&expected_c, "571D535B60B277188BE5147170A9A22C"); + + var m2: [m.len]u8 = undefined; + try Aes128Ocb.decrypt(&m2, &c, tag, &ad, nonce, k); + assert(mem.eql(u8, &m, &m2)); +} diff --git a/lib/std/crypto/benchmark.zig b/lib/std/crypto/benchmark.zig index 476ed86ede..e3ffa62ed1 100644 --- a/lib/std/crypto/benchmark.zig +++ b/lib/std/crypto/benchmark.zig @@ -208,6 +208,8 @@ const aeads = [_]Crypto{ Crypto{ .ty = crypto.aead.aegis.Aegis256, .name = "aegis-256" }, Crypto{ .ty = crypto.aead.aes_gcm.Aes128Gcm, .name = "aes128-gcm" }, Crypto{ .ty = crypto.aead.aes_gcm.Aes256Gcm, .name = "aes256-gcm" }, + Crypto{ .ty = crypto.aead.aes_ocb.Aes128Ocb, .name = "aes128-ocb" }, + Crypto{ .ty = crypto.aead.aes_ocb.Aes256Ocb, .name = "aes256-ocb" }, Crypto{ .ty = crypto.aead.isap.IsapA128A, .name = "isapa128a" }, }; From 45d220cac6e9dbb130b48554915233672c240b7b Mon Sep 17 00:00:00 2001 From: Evan Haas Date: Fri, 15 Jan 2021 19:16:18 -0800 Subject: [PATCH 11/13] translate-c: add support Implement __builtin_expect so C code that uses assert() can be translated. --- lib/std/c/builtins.zig | 6 ++++++ test/run_translated_c.zig | 23 +++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/lib/std/c/builtins.zig b/lib/std/c/builtins.zig index c11bf0a391..2c03c1ceac 100644 --- a/lib/std/c/builtins.zig +++ b/lib/std/c/builtins.zig @@ -182,3 +182,9 @@ pub fn __builtin_memcpy( @memcpy(dst_cast, src_cast, len); return dst; } + +/// The return value of __builtin_expect is `expr`. `c` is the expected value +/// of `expr` and is used as a hint to the compiler in C. Here it is unused. +pub fn __builtin_expect(expr: c_long, c: c_long) callconv(.Inline) c_long { + return expr; +} diff --git a/test/run_translated_c.zig b/test/run_translated_c.zig index 23b0a70395..4eb0f1e50a 100644 --- a/test/run_translated_c.zig +++ b/test/run_translated_c.zig @@ -1106,4 +1106,27 @@ pub fn addCases(cases: *tests.RunTranslatedCContext) void { \\ return 0; \\} , ""); + + cases.add("handle assert.h", + \\#include + \\int main() { + \\ int x = 1; + \\ int *xp = &x; + \\ assert(1); + \\ assert(x != 0); + \\ assert(xp); + \\ assert(*xp); + \\ return 0; + \\} + , ""); + + cases.add("NDEBUG disables assert", + \\#define NDEBUG + \\#include + \\int main() { + \\ assert(0); + \\ assert(NULL); + \\ return 0; + \\} + , ""); } From baab1b2f3135a10ed0adb56e653fc300d4add661 Mon Sep 17 00:00:00 2001 From: fancl20 Date: Mon, 1 Mar 2021 19:38:56 +1100 Subject: [PATCH 12/13] std: Add std.fs.path.joinZ (#7974) * std: Add std.fs.path.joinZ * Merge std.fs.path.join and std.fs.path.joinZZ tests --- lib/std/fs/path.zig | 76 +++++++++++++++++++++++++++------------------ 1 file changed, 46 insertions(+), 30 deletions(-) diff --git a/lib/std/fs/path.zig b/lib/std/fs/path.zig index c04b9dff66..f6ee9122a1 100644 --- a/lib/std/fs/path.zig +++ b/lib/std/fs/path.zig @@ -39,8 +39,8 @@ pub fn isSep(byte: u8) bool { /// This is different from mem.join in that the separator will not be repeated if /// it is found at the end or beginning of a pair of consecutive paths. -fn joinSep(allocator: *Allocator, separator: u8, sepPredicate: fn (u8) bool, paths: []const []const u8) ![]u8 { - if (paths.len == 0) return &[0]u8{}; +fn joinSepMaybeZ(allocator: *Allocator, separator: u8, sepPredicate: fn (u8) bool, paths: []const []const u8, zero: bool) ![]u8 { + if (paths.len == 0) return if (zero) try allocator.dupe(u8, &[1]u8{0}) else &[0]u8{}; const total_len = blk: { var sum: usize = paths[0].len; @@ -53,6 +53,7 @@ fn joinSep(allocator: *Allocator, separator: u8, sepPredicate: fn (u8) bool, pat sum += @boolToInt(!prev_sep and !this_sep); sum += if (prev_sep and this_sep) this_path.len - 1 else this_path.len; } + if (zero) sum += 1; break :blk sum; }; @@ -76,6 +77,8 @@ fn joinSep(allocator: *Allocator, separator: u8, sepPredicate: fn (u8) bool, pat buf_index += adjusted_path.len; } + if (zero) buf[buf.len - 1] = 0; + // No need for shrink since buf is exactly the correct size. return buf; } @@ -83,60 +86,73 @@ fn joinSep(allocator: *Allocator, separator: u8, sepPredicate: fn (u8) bool, pat /// Naively combines a series of paths with the native path seperator. /// Allocates memory for the result, which must be freed by the caller. pub fn join(allocator: *Allocator, paths: []const []const u8) ![]u8 { - return joinSep(allocator, sep, isSep, paths); + return joinSepMaybeZ(allocator, sep, isSep, paths, false); } -fn testJoinWindows(paths: []const []const u8, expected: []const u8) void { +/// Naively combines a series of paths with the native path seperator and null terminator. +/// Allocates memory for the result, which must be freed by the caller. +pub fn joinZ(allocator: *Allocator, paths: []const []const u8) ![:0]u8 { + const out = joinSepMaybeZ(allocator, sep, isSep, paths, true); + return out[0 .. out.len - 1 :0]; +} + +fn testJoinMaybeZWindows(paths: []const []const u8, expected: []const u8, zero: bool) void { const windowsIsSep = struct { fn isSep(byte: u8) bool { return byte == '/' or byte == '\\'; } }.isSep; - const actual = joinSep(testing.allocator, sep_windows, windowsIsSep, paths) catch @panic("fail"); + const actual = joinSepMaybeZ(testing.allocator, sep_windows, windowsIsSep, paths, zero) catch @panic("fail"); defer testing.allocator.free(actual); - testing.expectEqualSlices(u8, expected, actual); + testing.expectEqualSlices(u8, expected, if (zero) actual[0 .. actual.len - 1 :0] else actual); } -fn testJoinPosix(paths: []const []const u8, expected: []const u8) void { +fn testJoinMaybeZPosix(paths: []const []const u8, expected: []const u8, zero: bool) void { const posixIsSep = struct { fn isSep(byte: u8) bool { return byte == '/'; } }.isSep; - const actual = joinSep(testing.allocator, sep_posix, posixIsSep, paths) catch @panic("fail"); + const actual = joinSepMaybeZ(testing.allocator, sep_posix, posixIsSep, paths, zero) catch @panic("fail"); defer testing.allocator.free(actual); - testing.expectEqualSlices(u8, expected, actual); + testing.expectEqualSlices(u8, expected, if (zero) actual[0 .. actual.len - 1 :0] else actual); } test "join" { - testJoinWindows(&[_][]const u8{ "c:\\a\\b", "c" }, "c:\\a\\b\\c"); - testJoinWindows(&[_][]const u8{ "c:\\a\\b", "c" }, "c:\\a\\b\\c"); - testJoinWindows(&[_][]const u8{ "c:\\a\\b\\", "c" }, "c:\\a\\b\\c"); + for (&[_]bool{ false, true }) |zero| { + testJoinMaybeZWindows(&[_][]const u8{}, "", zero); + testJoinMaybeZWindows(&[_][]const u8{ "c:\\a\\b", "c" }, "c:\\a\\b\\c", zero); + testJoinMaybeZWindows(&[_][]const u8{ "c:\\a\\b", "c" }, "c:\\a\\b\\c", zero); + testJoinMaybeZWindows(&[_][]const u8{ "c:\\a\\b\\", "c" }, "c:\\a\\b\\c", zero); - testJoinWindows(&[_][]const u8{ "c:\\", "a", "b\\", "c" }, "c:\\a\\b\\c"); - testJoinWindows(&[_][]const u8{ "c:\\a\\", "b\\", "c" }, "c:\\a\\b\\c"); + testJoinMaybeZWindows(&[_][]const u8{ "c:\\", "a", "b\\", "c" }, "c:\\a\\b\\c", zero); + testJoinMaybeZWindows(&[_][]const u8{ "c:\\a\\", "b\\", "c" }, "c:\\a\\b\\c", zero); - testJoinWindows( - &[_][]const u8{ "c:\\home\\andy\\dev\\zig\\build\\lib\\zig\\std", "io.zig" }, - "c:\\home\\andy\\dev\\zig\\build\\lib\\zig\\std\\io.zig", - ); + testJoinMaybeZWindows( + &[_][]const u8{ "c:\\home\\andy\\dev\\zig\\build\\lib\\zig\\std", "io.zig" }, + "c:\\home\\andy\\dev\\zig\\build\\lib\\zig\\std\\io.zig", + zero, + ); - testJoinWindows(&[_][]const u8{ "c:\\", "a", "b/", "c" }, "c:\\a\\b/c"); - testJoinWindows(&[_][]const u8{ "c:\\a/", "b\\", "/c" }, "c:\\a/b\\c"); + testJoinMaybeZWindows(&[_][]const u8{ "c:\\", "a", "b/", "c" }, "c:\\a\\b/c", zero); + testJoinMaybeZWindows(&[_][]const u8{ "c:\\a/", "b\\", "/c" }, "c:\\a/b\\c", zero); - testJoinPosix(&[_][]const u8{ "/a/b", "c" }, "/a/b/c"); - testJoinPosix(&[_][]const u8{ "/a/b/", "c" }, "/a/b/c"); + testJoinMaybeZPosix(&[_][]const u8{}, "", zero); + testJoinMaybeZPosix(&[_][]const u8{ "/a/b", "c" }, "/a/b/c", zero); + testJoinMaybeZPosix(&[_][]const u8{ "/a/b/", "c" }, "/a/b/c", zero); - testJoinPosix(&[_][]const u8{ "/", "a", "b/", "c" }, "/a/b/c"); - testJoinPosix(&[_][]const u8{ "/a/", "b/", "c" }, "/a/b/c"); + testJoinMaybeZPosix(&[_][]const u8{ "/", "a", "b/", "c" }, "/a/b/c", zero); + testJoinMaybeZPosix(&[_][]const u8{ "/a/", "b/", "c" }, "/a/b/c", zero); - testJoinPosix( - &[_][]const u8{ "/home/andy/dev/zig/build/lib/zig/std", "io.zig" }, - "/home/andy/dev/zig/build/lib/zig/std/io.zig", - ); + testJoinMaybeZPosix( + &[_][]const u8{ "/home/andy/dev/zig/build/lib/zig/std", "io.zig" }, + "/home/andy/dev/zig/build/lib/zig/std/io.zig", + zero, + ); - testJoinPosix(&[_][]const u8{ "a", "/c" }, "a/c"); - testJoinPosix(&[_][]const u8{ "a/", "/c" }, "a/c"); + testJoinMaybeZPosix(&[_][]const u8{ "a", "/c" }, "a/c", zero); + testJoinMaybeZPosix(&[_][]const u8{ "a/", "/c" }, "a/c", zero); + } } pub const isAbsoluteC = @compileError("deprecated: renamed to isAbsoluteZ"); From bee7db77fe65802a41f2812caac4faa7dcb8acd3 Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Mon, 1 Mar 2021 20:06:30 +0100 Subject: [PATCH 13/13] std: Replace lastIndexOf with lastIndexOfScalar This may work around the miscompilation in LLVM 12. --- lib/std/fs/path.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/fs/path.zig b/lib/std/fs/path.zig index f6ee9122a1..776cb4040c 100644 --- a/lib/std/fs/path.zig +++ b/lib/std/fs/path.zig @@ -1226,7 +1226,7 @@ fn testRelativeWindows(from: []const u8, to: []const u8, expected_output: []cons /// pointer address range of `path`, even if it is length zero. pub fn extension(path: []const u8) []const u8 { const filename = basename(path); - const index = mem.lastIndexOf(u8, filename, ".") orelse return path[path.len..]; + const index = mem.lastIndexOfScalar(u8, filename, '.') orelse return path[path.len..]; if (index == 0) return path[path.len..]; return filename[index..]; }