From 1a53c648ed711277232aaa5a0b1082a9f24bd9c3 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 13 Feb 2018 19:06:01 -0500 Subject: [PATCH 01/16] self hosted parser supports builtin fn call with no args --- std/zig/parser.zig | 51 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 792cc2d834..3f74a4affb 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -92,6 +92,8 @@ pub const Parser = struct { FnDef: &ast.NodeFnProto, Block: &ast.NodeBlock, Statement: &ast.NodeBlock, + ExprListItemOrEnd: &ArrayList(&ast.Node), + ExprListCommaOrEnd: &ArrayList(&ast.Node), }; /// Returns an AST tree, allocated with the parser's allocator. @@ -297,6 +299,21 @@ pub const Parser = struct { try stack.append(State.AfterOperand); continue; }, + Token.Id.Builtin => { + const node = try arena.create(ast.NodeBuiltinCall); + *node = ast.NodeBuiltinCall { + .base = ast.Node {.id = ast.Node.Id.BuiltinCall}, + .builtin_token = token, + .params = ArrayList(&ast.Node).init(arena), + }; + try stack.append(State { + .Operand = &node.base + }); + try stack.append(State.AfterOperand); + try stack.append(State {.ExprListItemOrEnd = &node.params }); + try stack.append(State {.ExpectToken = Token.Id.LParen }); + continue; + }, else => return self.parseError(token, "expected primary expression, found {}", @tagName(token.id)), } }, @@ -352,6 +369,29 @@ pub const Parser = struct { } }, + State.ExprListItemOrEnd => |params| { + var token = self.getNextToken(); + switch (token.id) { + Token.Id.RParen => continue, + else => { + self.putBackToken(token); + stack.append(State { .ExprListCommaOrEnd = params }) catch unreachable; + try stack.append(State { .Expression = DestPtr{.List = params} }); + }, + } + }, + + State.ExprListCommaOrEnd => |params| { + var token = self.getNextToken(); + switch (token.id) { + Token.Id.Comma => { + stack.append(State { .ExprListItemOrEnd = params }) catch unreachable; + }, + Token.Id.RParen => continue, + else => return self.parseError(token, "expected ',' or ')', found {}", @tagName(token.id)), + } + }, + State.AddrOfModifiers => |addr_of_info| { var token = self.getNextToken(); switch (token.id) { @@ -539,8 +579,6 @@ pub const Parser = struct { State.PrefixOp => unreachable, State.Operand => unreachable, } - @import("std").debug.panic("{}", @tagName(state)); - //unreachable; } } @@ -988,6 +1026,10 @@ pub const Parser = struct { const float_literal = @fieldParentPtr(ast.NodeFloatLiteral, "base", base); try stream.print("{}", self.tokenizer.getTokenSlice(float_literal.token)); }, + ast.Node.Id.BuiltinCall => { + const builtin_call = @fieldParentPtr(ast.NodeBuiltinCall, "base", base); + try stream.print("{}()", self.tokenizer.getTokenSlice(builtin_call.builtin_token)); + }, else => unreachable, }, RenderState.FnProtoRParen => |fn_proto| { @@ -1098,6 +1140,11 @@ fn testCanonical(source: []const u8) !void { } test "zig fmt" { + try testCanonical( + \\const std = @import(); + \\ + ); + try testCanonical( \\extern fn puts(s: &const u8) c_int; \\ From d790670f4c11687fea50c2fd302ca8a815b32d68 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 14 Feb 2018 13:43:05 -0500 Subject: [PATCH 02/16] self hosted parser: support string literals --- std/zig/ast.zig | 11 +++++++++++ std/zig/parser.zig | 34 ++++++++++++++++++++++++++++++++-- 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 60824b22b8..b6b091973d 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -18,6 +18,7 @@ pub const Node = struct { PrefixOp, IntegerLiteral, FloatLiteral, + StringLiteral, BuiltinCall, }; @@ -33,6 +34,7 @@ pub const Node = struct { Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).iterate(index), Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).iterate(index), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index), + Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).iterate(index), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index), }; } @@ -271,3 +273,12 @@ pub const NodeBuiltinCall = struct { return null; } }; + +pub const NodeStringLiteral = struct { + base: Node, + token: Token, + + pub fn iterate(self: &NodeStringLiteral) ?&Node { + return null; + } +}; diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 3f74a4affb..079a331c6d 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -314,6 +314,18 @@ pub const Parser = struct { try stack.append(State {.ExpectToken = Token.Id.LParen }); continue; }, + Token.Id.StringLiteral => { + const node = try arena.create(ast.NodeStringLiteral); + *node = ast.NodeStringLiteral { + .base = ast.Node {.id = ast.Node.Id.StringLiteral}, + .token = token, + }; + try stack.append(State { + .Operand = &node.base + }); + try stack.append(State.AfterOperand); + continue; + }, else => return self.parseError(token, "expected primary expression, found {}", @tagName(token.id)), } }, @@ -1026,11 +1038,28 @@ pub const Parser = struct { const float_literal = @fieldParentPtr(ast.NodeFloatLiteral, "base", base); try stream.print("{}", self.tokenizer.getTokenSlice(float_literal.token)); }, + ast.Node.Id.StringLiteral => { + const string_literal = @fieldParentPtr(ast.NodeStringLiteral, "base", base); + try stream.print("{}", self.tokenizer.getTokenSlice(string_literal.token)); + }, ast.Node.Id.BuiltinCall => { const builtin_call = @fieldParentPtr(ast.NodeBuiltinCall, "base", base); - try stream.print("{}()", self.tokenizer.getTokenSlice(builtin_call.builtin_token)); + try stream.print("{}(", self.tokenizer.getTokenSlice(builtin_call.builtin_token)); + try stack.append(RenderState { .Text = ")"}); + var i = builtin_call.params.len; + while (i != 0) { + i -= 1; + const param_node = builtin_call.params.at(i); + try stack.append(RenderState { .Expression = param_node}); + if (i != 0) { + try stack.append(RenderState { .Text = ", " }); + } + } }, - else => unreachable, + ast.Node.Id.Root, + ast.Node.Id.VarDecl, + ast.Node.Id.FnProto, + ast.Node.Id.ParamDecl => unreachable, }, RenderState.FnProtoRParen => |fn_proto| { try stream.print(")"); @@ -1141,6 +1170,7 @@ fn testCanonical(source: []const u8) !void { test "zig fmt" { try testCanonical( + \\const std = @import("std"); \\const std = @import(); \\ ); From e8d81c5acf69245f863394f207d63bf07c722bf4 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 14 Feb 2018 13:55:06 -0500 Subject: [PATCH 03/16] fix build broken by previous commit --- std/zig/ast.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index b6b091973d..3466a24d28 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -278,7 +278,7 @@ pub const NodeStringLiteral = struct { base: Node, token: Token, - pub fn iterate(self: &NodeStringLiteral) ?&Node { + pub fn iterate(self: &NodeStringLiteral, index: usize) ?&Node { return null; } }; From 629f134d3805f99938f59a6ee7f6598591abc9d7 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 14 Feb 2018 15:48:15 -0500 Subject: [PATCH 04/16] std.zig.parser understands inferred return type and error inference --- std/zig/ast.zig | 22 +++++++++++-- std/zig/parser.zig | 79 +++++++++++++++++++++++++++++++++++++--------- 2 files changed, 83 insertions(+), 18 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 3466a24d28..3892812882 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -102,7 +102,7 @@ pub const NodeFnProto = struct { fn_token: Token, name_token: ?Token, params: ArrayList(&Node), - return_type: &Node, + return_type: ReturnType, var_args_token: ?Token, extern_token: ?Token, inline_token: ?Token, @@ -111,6 +111,12 @@ pub const NodeFnProto = struct { lib_name: ?&Node, // populated if this is an extern declaration align_expr: ?&Node, // populated if align(A) is present + pub const ReturnType = union(enum) { + Explicit: &Node, + Infer, + InferErrorSet: &Node, + }; + pub fn iterate(self: &NodeFnProto, index: usize) ?&Node { var i = index; @@ -119,8 +125,18 @@ pub const NodeFnProto = struct { i -= 1; } - if (i < 1) return self.return_type; - i -= 1; + switch (self.return_type) { + // TODO allow this and next prong to share bodies since the types are the same + ReturnType.Explicit => |node| { + if (i < 1) return node; + i -= 1; + }, + ReturnType.InferErrorSet => |node| { + if (i < 1) return node; + i -= 1; + }, + ReturnType.Infer => {}, + } if (self.align_expr) |align_expr| { if (i < 1) return align_expr; diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 079a331c6d..0c24fe1410 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -87,6 +87,7 @@ pub const Parser = struct { ExpectToken: @TagType(Token.Id), FnProto: &ast.NodeFnProto, FnProtoAlign: &ast.NodeFnProto, + FnProtoReturnType: &ast.NodeFnProto, ParamDecl: &ast.NodeFnProto, ParamDeclComma, FnDef: &ast.NodeFnProto, @@ -178,7 +179,7 @@ pub const Parser = struct { stack.append(State.TopLevel) catch unreachable; // TODO shouldn't need these casts const fn_proto = try self.createAttachFnProto(arena, &root_node.decls, token, - ctx.extern_token, (?Token)(null), (?Token)(null), (?Token)(null)); + ctx.extern_token, (?Token)(null), ctx.visib_token, (?Token)(null)); try stack.append(State { .FnDef = fn_proto }); try stack.append(State { .FnProto = fn_proto }); continue; @@ -466,11 +467,37 @@ pub const Parser = struct { } self.putBackToken(token); stack.append(State { - .TypeExpr = DestPtr {.Field = &fn_proto.return_type}, + .FnProtoReturnType = fn_proto, }) catch unreachable; continue; }, + State.FnProtoReturnType => |fn_proto| { + const token = self.getNextToken(); + switch (token.id) { + Token.Id.Keyword_var => { + fn_proto.return_type = ast.NodeFnProto.ReturnType.Infer; + }, + Token.Id.Bang => { + fn_proto.return_type = ast.NodeFnProto.ReturnType { .InferErrorSet = undefined }; + stack.append(State { + .TypeExpr = DestPtr {.Field = &fn_proto.return_type.InferErrorSet}, + }) catch unreachable; + }, + else => { + self.putBackToken(token); + fn_proto.return_type = ast.NodeFnProto.ReturnType { .Explicit = undefined }; + stack.append(State { + .TypeExpr = DestPtr {.Field = &fn_proto.return_type.Explicit}, + }) catch unreachable; + }, + } + if (token.id == Token.Id.Keyword_align) { + @panic("TODO fn proto align"); + } + continue; + }, + State.ParamDecl => |fn_proto| { var token = self.getNextToken(); if (token.id == Token.Id.RParen) { @@ -977,19 +1004,23 @@ pub const Parser = struct { }, ast.Node.Id.Block => { const block = @fieldParentPtr(ast.NodeBlock, "base", base); - try stream.write("{"); - try stack.append(RenderState { .Text = "}"}); - try stack.append(RenderState.PrintIndent); - try stack.append(RenderState { .Indent = indent}); - try stack.append(RenderState { .Text = "\n"}); - var i = block.statements.len; - while (i != 0) { - i -= 1; - const statement_node = block.statements.items[i]; - try stack.append(RenderState { .Statement = statement_node}); + if (block.statements.len == 0) { + try stream.write("{}"); + } else { + try stream.write("{"); + try stack.append(RenderState { .Text = "}"}); try stack.append(RenderState.PrintIndent); - try stack.append(RenderState { .Indent = indent + indent_delta}); - try stack.append(RenderState { .Text = "\n" }); + try stack.append(RenderState { .Indent = indent}); + try stack.append(RenderState { .Text = "\n"}); + var i = block.statements.len; + while (i != 0) { + i -= 1; + const statement_node = block.statements.items[i]; + try stack.append(RenderState { .Statement = statement_node}); + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { .Indent = indent + indent_delta}); + try stack.append(RenderState { .Text = "\n" }); + } } }, ast.Node.Id.InfixOp => { @@ -1071,7 +1102,18 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = body_node}); try stack.append(RenderState { .Text = " "}); } - try stack.append(RenderState { .Expression = fn_proto.return_type}); + switch (fn_proto.return_type) { + ast.NodeFnProto.ReturnType.Explicit => |node| { + try stack.append(RenderState { .Expression = node}); + }, + ast.NodeFnProto.ReturnType.Infer => { + try stream.print("var"); + }, + ast.NodeFnProto.ReturnType.InferErrorSet => |node| { + try stream.print("!"); + try stack.append(RenderState { .Expression = node}); + }, + } }, RenderState.Statement => |base| { switch (base.id) { @@ -1169,6 +1211,13 @@ fn testCanonical(source: []const u8) !void { } test "zig fmt" { + try testCanonical( + \\pub fn main() !void {} + \\pub fn main() var {} + \\pub fn main() i32 {} + \\ + ); + try testCanonical( \\const std = @import("std"); \\const std = @import(); From 9fa35adbd4772f55e3e43dd7cc69823415661153 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 14 Feb 2018 16:24:43 -0500 Subject: [PATCH 05/16] fix sometimes not type checking function parameters closes #774 regression introduced in cfb2c676925d77887e46631dcafa783e6c65e61d --- src/ir.cpp | 2 +- test/compile_errors.zig | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/ir.cpp b/src/ir.cpp index 2bb40c7e15..8f6317096c 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -6727,8 +6727,8 @@ static ConstCastOnly types_match_const_cast_only(IrAnalyze *ira, TypeTableEntry result.id = ConstCastResultIdFnReturnType; result.data.return_type = allocate_nonzero(1); *result.data.return_type = child; + return result; } - return result; } if (expected_type->data.fn.fn_type_id.param_count != actual_type->data.fn.fn_type_id.param_count) { result.id = ConstCastResultIdFnArgCount; diff --git a/test/compile_errors.zig b/test/compile_errors.zig index f60705aa31..90b3ff023a 100644 --- a/test/compile_errors.zig +++ b/test/compile_errors.zig @@ -1,6 +1,19 @@ const tests = @import("tests.zig"); pub fn addCases(cases: &tests.CompileErrorContext) void { + cases.add("type checking function pointers", + \\fn a(b: fn (&const u8) void) void { + \\ b('a'); + \\} + \\fn c(d: u8) void { + \\ @import("std").debug.warn("{c}\n", d); + \\} + \\export fn entry() void { + \\ a(c); + \\} + , + ".tmp_source.zig:8:7: error: expected type 'fn(&const u8) void', found 'fn(u8) void'"); + cases.add("no else prong on switch on global error set", \\export fn entry() void { \\ foo(error.A); From ca597e2bfb3c39aecdf3dea2718e84deb749ed07 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 14 Feb 2018 23:00:53 -0500 Subject: [PATCH 06/16] std.zig.parser understands try. zig fmt respects a double line break. --- src/tokenizer.cpp | 2 - src/tokenizer.hpp | 1 - std/debug/index.zig | 4 +- std/zig/ast.zig | 164 ++++++++++++++++++++++++++++++++++++++++-- std/zig/parser.zig | 111 ++++++++++++++++++++++++---- std/zig/tokenizer.zig | 54 +++++++++----- 6 files changed, 296 insertions(+), 40 deletions(-) diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp index 44d838a723..dd60815b7f 100644 --- a/src/tokenizer.cpp +++ b/src/tokenizer.cpp @@ -125,7 +125,6 @@ static const struct ZigKeyword zig_keywords[] = { {"false", TokenIdKeywordFalse}, {"fn", TokenIdKeywordFn}, {"for", TokenIdKeywordFor}, - {"goto", TokenIdKeywordGoto}, {"if", TokenIdKeywordIf}, {"inline", TokenIdKeywordInline}, {"nakedcc", TokenIdKeywordNakedCC}, @@ -1542,7 +1541,6 @@ const char * token_name(TokenId id) { case TokenIdKeywordFalse: return "false"; case TokenIdKeywordFn: return "fn"; case TokenIdKeywordFor: return "for"; - case TokenIdKeywordGoto: return "goto"; case TokenIdKeywordIf: return "if"; case TokenIdKeywordInline: return "inline"; case TokenIdKeywordNakedCC: return "nakedcc"; diff --git a/src/tokenizer.hpp b/src/tokenizer.hpp index 92a3b8de0d..225b75d844 100644 --- a/src/tokenizer.hpp +++ b/src/tokenizer.hpp @@ -66,7 +66,6 @@ enum TokenId { TokenIdKeywordFalse, TokenIdKeywordFn, TokenIdKeywordFor, - TokenIdKeywordGoto, TokenIdKeywordIf, TokenIdKeywordInline, TokenIdKeywordNakedCC, diff --git a/std/debug/index.zig b/std/debug/index.zig index df1a1c5041..cc4832b1ea 100644 --- a/std/debug/index.zig +++ b/std/debug/index.zig @@ -47,7 +47,7 @@ pub fn getSelfDebugInfo() !&ElfStackTrace { pub fn dumpCurrentStackTrace() void { const stderr = getStderrStream() catch return; const debug_info = getSelfDebugInfo() catch |err| { - stderr.print("Unable to open debug info: {}\n", @errorName(err)) catch return; + stderr.print("Unable to dump stack trace: Unable to open debug info: {}\n", @errorName(err)) catch return; return; }; defer debug_info.close(); @@ -61,7 +61,7 @@ pub fn dumpCurrentStackTrace() void { pub fn dumpStackTrace(stack_trace: &const builtin.StackTrace) void { const stderr = getStderrStream() catch return; const debug_info = getSelfDebugInfo() catch |err| { - stderr.print("Unable to open debug info: {}\n", @errorName(err)) catch return; + stderr.print("Unable to dump stack trace: Unable to open debug info: {}\n", @errorName(err)) catch return; return; }; defer debug_info.close(); diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 3892812882..77a2b8cd6a 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -38,11 +38,46 @@ pub const Node = struct { Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index), }; } + + pub fn firstToken(base: &Node) Token { + return switch (base.id) { + Id.Root => @fieldParentPtr(NodeRoot, "base", base).firstToken(), + Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).firstToken(), + Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).firstToken(), + Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).firstToken(), + Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).firstToken(), + Id.Block => @fieldParentPtr(NodeBlock, "base", base).firstToken(), + Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).firstToken(), + Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).firstToken(), + Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).firstToken(), + Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(), + Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(), + Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(), + }; + } + + pub fn lastToken(base: &Node) Token { + return switch (base.id) { + Id.Root => @fieldParentPtr(NodeRoot, "base", base).lastToken(), + Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).lastToken(), + Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).lastToken(), + Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).lastToken(), + Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).lastToken(), + Id.Block => @fieldParentPtr(NodeBlock, "base", base).lastToken(), + Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).lastToken(), + Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).lastToken(), + Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).lastToken(), + Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(), + Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(), + Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(), + }; + } }; pub const NodeRoot = struct { base: Node, decls: ArrayList(&Node), + eof_token: Token, pub fn iterate(self: &NodeRoot, index: usize) ?&Node { if (index < self.decls.len) { @@ -50,6 +85,14 @@ pub const NodeRoot = struct { } return null; } + + pub fn firstToken(self: &NodeRoot) Token { + return if (self.decls.len == 0) self.eof_token else self.decls.at(0).firstToken(); + } + + pub fn lastToken(self: &NodeRoot) Token { + return if (self.decls.len == 0) self.eof_token else self.decls.at(self.decls.len - 1).lastToken(); + } }; pub const NodeVarDecl = struct { @@ -64,6 +107,7 @@ pub const NodeVarDecl = struct { type_node: ?&Node, align_node: ?&Node, init_node: ?&Node, + semicolon_token: Token, pub fn iterate(self: &NodeVarDecl, index: usize) ?&Node { var i = index; @@ -85,6 +129,18 @@ pub const NodeVarDecl = struct { return null; } + + pub fn firstToken(self: &NodeVarDecl) Token { + if (self.visib_token) |visib_token| return visib_token; + if (self.comptime_token) |comptime_token| return comptime_token; + if (self.extern_token) |extern_token| return extern_token; + assert(self.lib_name == null); + return self.mut_token; + } + + pub fn lastToken(self: &NodeVarDecl) Token { + return self.semicolon_token; + } }; pub const NodeIdentifier = struct { @@ -94,6 +150,14 @@ pub const NodeIdentifier = struct { pub fn iterate(self: &NodeIdentifier, index: usize) ?&Node { return null; } + + pub fn firstToken(self: &NodeIdentifier) Token { + return self.name_token; + } + + pub fn lastToken(self: &NodeIdentifier) Token { + return self.name_token; + } }; pub const NodeFnProto = struct { @@ -113,7 +177,7 @@ pub const NodeFnProto = struct { pub const ReturnType = union(enum) { Explicit: &Node, - Infer, + Infer: Token, InferErrorSet: &Node, }; @@ -153,6 +217,25 @@ pub const NodeFnProto = struct { return null; } + + pub fn firstToken(self: &NodeFnProto) Token { + if (self.visib_token) |visib_token| return visib_token; + if (self.extern_token) |extern_token| return extern_token; + assert(self.lib_name == null); + if (self.inline_token) |inline_token| return inline_token; + if (self.cc_token) |cc_token| return cc_token; + return self.fn_token; + } + + pub fn lastToken(self: &NodeFnProto) Token { + if (self.body_node) |body_node| return body_node.lastToken(); + switch (self.return_type) { + // TODO allow this and next prong to share bodies since the types are the same + ReturnType.Explicit => |node| return node.lastToken(), + ReturnType.InferErrorSet => |node| return node.lastToken(), + ReturnType.Infer => |token| return token, + } + } }; pub const NodeParamDecl = struct { @@ -171,6 +254,18 @@ pub const NodeParamDecl = struct { return null; } + + pub fn firstToken(self: &NodeParamDecl) Token { + if (self.comptime_token) |comptime_token| return comptime_token; + if (self.noalias_token) |noalias_token| return noalias_token; + if (self.name_token) |name_token| return name_token; + return self.type_node.firstToken(); + } + + pub fn lastToken(self: &NodeParamDecl) Token { + if (self.var_args_token) |var_args_token| return var_args_token; + return self.type_node.lastToken(); + } }; pub const NodeBlock = struct { @@ -187,6 +282,14 @@ pub const NodeBlock = struct { return null; } + + pub fn firstToken(self: &NodeBlock) Token { + return self.begin_token; + } + + pub fn lastToken(self: &NodeBlock) Token { + return self.end_token; + } }; pub const NodeInfixOp = struct { @@ -199,6 +302,7 @@ pub const NodeInfixOp = struct { const InfixOp = enum { EqualEqual, BangEqual, + Period, }; pub fn iterate(self: &NodeInfixOp, index: usize) ?&Node { @@ -208,8 +312,9 @@ pub const NodeInfixOp = struct { i -= 1; switch (self.op) { - InfixOp.EqualEqual => {}, - InfixOp.BangEqual => {}, + InfixOp.EqualEqual, + InfixOp.BangEqual, + InfixOp.Period => {}, } if (i < 1) return self.rhs; @@ -217,6 +322,14 @@ pub const NodeInfixOp = struct { return null; } + + pub fn firstToken(self: &NodeInfixOp) Token { + return self.lhs.firstToken(); + } + + pub fn lastToken(self: &NodeInfixOp) Token { + return self.rhs.lastToken(); + } }; pub const NodePrefixOp = struct { @@ -227,6 +340,7 @@ pub const NodePrefixOp = struct { const PrefixOp = union(enum) { Return, + Try, AddrOf: AddrOfInfo, }; const AddrOfInfo = struct { @@ -241,7 +355,8 @@ pub const NodePrefixOp = struct { var i = index; switch (self.op) { - PrefixOp.Return => {}, + PrefixOp.Return, + PrefixOp.Try => {}, PrefixOp.AddrOf => |addr_of_info| { if (addr_of_info.align_expr) |align_expr| { if (i < 1) return align_expr; @@ -255,6 +370,14 @@ pub const NodePrefixOp = struct { return null; } + + pub fn firstToken(self: &NodePrefixOp) Token { + return self.op_token; + } + + pub fn lastToken(self: &NodePrefixOp) Token { + return self.rhs.lastToken(); + } }; pub const NodeIntegerLiteral = struct { @@ -264,6 +387,14 @@ pub const NodeIntegerLiteral = struct { pub fn iterate(self: &NodeIntegerLiteral, index: usize) ?&Node { return null; } + + pub fn firstToken(self: &NodeIntegerLiteral) Token { + return self.token; + } + + pub fn lastToken(self: &NodeIntegerLiteral) Token { + return self.token; + } }; pub const NodeFloatLiteral = struct { @@ -273,12 +404,21 @@ pub const NodeFloatLiteral = struct { pub fn iterate(self: &NodeFloatLiteral, index: usize) ?&Node { return null; } + + pub fn firstToken(self: &NodeFloatLiteral) Token { + return self.token; + } + + pub fn lastToken(self: &NodeFloatLiteral) Token { + return self.token; + } }; pub const NodeBuiltinCall = struct { base: Node, builtin_token: Token, params: ArrayList(&Node), + rparen_token: Token, pub fn iterate(self: &NodeBuiltinCall, index: usize) ?&Node { var i = index; @@ -288,6 +428,14 @@ pub const NodeBuiltinCall = struct { return null; } + + pub fn firstToken(self: &NodeBuiltinCall) Token { + return self.builtin_token; + } + + pub fn lastToken(self: &NodeBuiltinCall) Token { + return self.rparen_token; + } }; pub const NodeStringLiteral = struct { @@ -297,4 +445,12 @@ pub const NodeStringLiteral = struct { pub fn iterate(self: &NodeStringLiteral, index: usize) ?&Node { return null; } + + pub fn firstToken(self: &NodeStringLiteral) Token { + return self.token; + } + + pub fn lastToken(self: &NodeStringLiteral) Token { + return self.token; + } }; diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 0c24fe1410..35bba378d4 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -69,6 +69,11 @@ pub const Parser = struct { } }; + const ExpectTokenSave = struct { + id: Token.Id, + ptr: &Token, + }; + const State = union(enum) { TopLevel, TopLevelExtern: ?Token, @@ -85,6 +90,7 @@ pub const Parser = struct { VarDeclAlign: &ast.NodeVarDecl, VarDeclEq: &ast.NodeVarDecl, ExpectToken: @TagType(Token.Id), + ExpectTokenSave: ExpectTokenSave, FnProto: &ast.NodeFnProto, FnProtoAlign: &ast.NodeFnProto, FnProtoReturnType: &ast.NodeFnProto, @@ -136,7 +142,10 @@ pub const Parser = struct { stack.append(State { .TopLevelExtern = token }) catch unreachable; continue; }, - Token.Id.Eof => return Tree {.root_node = root_node, .arena_allocator = arena_allocator}, + Token.Id.Eof => { + root_node.eof_token = token; + return Tree {.root_node = root_node, .arena_allocator = arena_allocator}; + }, else => { self.putBackToken(token); stack.append(State { .TopLevelExtern = null }) catch unreachable; @@ -231,13 +240,19 @@ pub const Parser = struct { const token = self.getNextToken(); if (token.id == Token.Id.Equal) { var_decl.eq_token = token; - stack.append(State { .ExpectToken = Token.Id.Semicolon }) catch unreachable; + stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.Semicolon, + .ptr = &var_decl.semicolon_token, + }, + }) catch unreachable; try stack.append(State { .Expression = DestPtr {.NullableField = &var_decl.init_node}, }); continue; } if (token.id == Token.Id.Semicolon) { + var_decl.semicolon_token = token; continue; } return self.parseError(token, "expected '=' or ';', found {}", @tagName(token.id)); @@ -247,6 +262,11 @@ pub const Parser = struct { continue; }, + State.ExpectTokenSave => |expect_token_save| { + *expect_token_save.ptr = try self.eatToken(expect_token_save.id); + continue; + }, + State.Expression => |dest_ptr| { // save the dest_ptr for later stack.append(state) catch unreachable; @@ -264,6 +284,12 @@ pub const Parser = struct { try stack.append(State.ExpectOperand); continue; }, + Token.Id.Keyword_try => { + try stack.append(State { .PrefixOp = try self.createPrefixOp(arena, token, + ast.NodePrefixOp.PrefixOp.Try) }); + try stack.append(State.ExpectOperand); + continue; + }, Token.Id.Ampersand => { const prefix_op = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp{ .AddrOf = ast.NodePrefixOp.AddrOfInfo { @@ -306,13 +332,19 @@ pub const Parser = struct { .base = ast.Node {.id = ast.Node.Id.BuiltinCall}, .builtin_token = token, .params = ArrayList(&ast.Node).init(arena), + .rparen_token = undefined, }; try stack.append(State { .Operand = &node.base }); try stack.append(State.AfterOperand); try stack.append(State {.ExprListItemOrEnd = &node.params }); - try stack.append(State {.ExpectToken = Token.Id.LParen }); + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.LParen, + .ptr = &node.rparen_token, + }, + }); continue; }, Token.Id.StringLiteral => { @@ -351,6 +383,13 @@ pub const Parser = struct { try stack.append(State.ExpectOperand); continue; }, + Token.Id.Period => { + try stack.append(State { + .InfixOp = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.Period) + }); + try stack.append(State.ExpectOperand); + continue; + }, else => { // no postfix/infix operator after this operand. self.putBackToken(token); @@ -476,7 +515,7 @@ pub const Parser = struct { const token = self.getNextToken(); switch (token.id) { Token.Id.Keyword_var => { - fn_proto.return_type = ast.NodeFnProto.ReturnType.Infer; + fn_proto.return_type = ast.NodeFnProto.ReturnType { .Infer = token }; }, Token.Id.Bang => { fn_proto.return_type = ast.NodeFnProto.ReturnType { .InferErrorSet = undefined }; @@ -627,6 +666,8 @@ pub const Parser = struct { *node = ast.NodeRoot { .base = ast.Node {.id = ast.Node.Id.Root}, .decls = ArrayList(&ast.Node).init(arena), + // initialized when we get the eof token + .eof_token = undefined, }; return node; } @@ -649,6 +690,7 @@ pub const Parser = struct { // initialized later .name_token = undefined, .eq_token = undefined, + .semicolon_token = undefined, }; return node; } @@ -789,11 +831,11 @@ pub const Parser = struct { fn parseError(self: &Parser, token: &const Token, comptime fmt: []const u8, args: ...) (error{ParseError}) { const loc = self.tokenizer.getTokenLocation(token); - warn("{}:{}:{}: error: " ++ fmt ++ "\n", self.source_file_name, loc.line + 1, loc.column + 1, args); + warn("{}:{}:{}: error: " ++ fmt ++ "\n", self.source_file_name, token.line + 1, token.column + 1, args); warn("{}\n", self.tokenizer.buffer[loc.line_start..loc.line_end]); { var i: usize = 0; - while (i < loc.column) : (i += 1) { + while (i < token.column) : (i += 1) { warn(" "); } } @@ -885,11 +927,26 @@ pub const Parser = struct { defer self.deinitUtilityArrayList(stack); { + try stack.append(RenderState { .Text = "\n"}); + var i = root_node.decls.len; while (i != 0) { i -= 1; const decl = root_node.decls.items[i]; try stack.append(RenderState {.TopLevelDecl = decl}); + if (i != 0) { + try stack.append(RenderState { + .Text = blk: { + const prev_node = root_node.decls.at(i - 1); + const prev_line_index = prev_node.lastToken().line; + const this_line_index = decl.firstToken().line; + if (this_line_index - prev_line_index >= 2) { + break :blk "\n\n"; + } + break :blk "\n"; + }, + }); + } } } @@ -919,7 +976,6 @@ pub const Parser = struct { try stream.print("("); - try stack.append(RenderState { .Text = "\n" }); if (fn_proto.body_node == null) { try stack.append(RenderState { .Text = ";" }); } @@ -937,7 +993,6 @@ pub const Parser = struct { }, ast.Node.Id.VarDecl => { const var_decl = @fieldParentPtr(ast.NodeVarDecl, "base", decl); - try stack.append(RenderState { .Text = "\n"}); try stack.append(RenderState { .VarDecl = var_decl}); }, @@ -1019,7 +1074,19 @@ pub const Parser = struct { try stack.append(RenderState { .Statement = statement_node}); try stack.append(RenderState.PrintIndent); try stack.append(RenderState { .Indent = indent + indent_delta}); - try stack.append(RenderState { .Text = "\n" }); + try stack.append(RenderState { + .Text = blk: { + if (i != 0) { + const prev_statement_node = block.statements.items[i - 1]; + const prev_line_index = prev_statement_node.lastToken().line; + const this_line_index = statement_node.firstToken().line; + if (this_line_index - prev_line_index >= 2) { + break :blk "\n\n"; + } + } + break :blk "\n"; + }, + }); } } }, @@ -1033,7 +1100,9 @@ pub const Parser = struct { ast.NodeInfixOp.InfixOp.BangEqual => { try stack.append(RenderState { .Text = " != "}); }, - else => unreachable, + ast.NodeInfixOp.InfixOp.Period => { + try stack.append(RenderState { .Text = "."}); + }, } try stack.append(RenderState { .Expression = prefix_op_node.lhs }); }, @@ -1044,6 +1113,9 @@ pub const Parser = struct { ast.NodePrefixOp.PrefixOp.Return => { try stream.write("return "); }, + ast.NodePrefixOp.PrefixOp.Try => { + try stream.write("try "); + }, ast.NodePrefixOp.PrefixOp.AddrOf => |addr_of_info| { try stream.write("&"); if (addr_of_info.volatile_token != null) { @@ -1058,7 +1130,6 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = align_expr}); } }, - else => unreachable, } }, ast.Node.Id.IntegerLiteral => { @@ -1153,10 +1224,7 @@ pub const Parser = struct { var fixed_buffer_mem: [100 * 1024]u8 = undefined; fn testParse(source: []const u8, allocator: &mem.Allocator) ![]u8 { - var padded_source: [0x100]u8 = undefined; - std.mem.copy(u8, padded_source[0..source.len], source); - - var tokenizer = Tokenizer.init(padded_source[0..source.len]); + var tokenizer = Tokenizer.init(source); var parser = Parser.init(&tokenizer, allocator, "(memory buffer)"); defer parser.deinit(); @@ -1211,6 +1279,19 @@ fn testCanonical(source: []const u8) !void { } test "zig fmt" { + try testCanonical( + \\const std = @import("std"); + \\ + \\pub fn main() !void { + \\ var stdout_file = try std.io.getStdOut; + \\ var stdout_file = try std.io.getStdOut; + \\ + \\ var stdout_file = try std.io.getStdOut; + \\ var stdout_file = try std.io.getStdOut; + \\} + \\ + ); + try testCanonical( \\pub fn main() !void {} \\pub fn main() var {} diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig index de1263ac55..bdd82e4a44 100644 --- a/std/zig/tokenizer.zig +++ b/std/zig/tokenizer.zig @@ -5,6 +5,8 @@ pub const Token = struct { id: Id, start: usize, end: usize, + line: usize, + column: usize, const KeywordId = struct { bytes: []const u8, @@ -16,6 +18,7 @@ pub const Token = struct { KeywordId{.bytes="and", .id = Id.Keyword_and}, KeywordId{.bytes="asm", .id = Id.Keyword_asm}, KeywordId{.bytes="break", .id = Id.Keyword_break}, + KeywordId{.bytes="catch", .id = Id.Keyword_catch}, KeywordId{.bytes="comptime", .id = Id.Keyword_comptime}, KeywordId{.bytes="const", .id = Id.Keyword_const}, KeywordId{.bytes="continue", .id = Id.Keyword_continue}, @@ -28,7 +31,6 @@ pub const Token = struct { KeywordId{.bytes="false", .id = Id.Keyword_false}, KeywordId{.bytes="fn", .id = Id.Keyword_fn}, KeywordId{.bytes="for", .id = Id.Keyword_for}, - KeywordId{.bytes="goto", .id = Id.Keyword_goto}, KeywordId{.bytes="if", .id = Id.Keyword_if}, KeywordId{.bytes="inline", .id = Id.Keyword_inline}, KeywordId{.bytes="nakedcc", .id = Id.Keyword_nakedcc}, @@ -38,12 +40,14 @@ pub const Token = struct { KeywordId{.bytes="packed", .id = Id.Keyword_packed}, KeywordId{.bytes="pub", .id = Id.Keyword_pub}, KeywordId{.bytes="return", .id = Id.Keyword_return}, + KeywordId{.bytes="section", .id = Id.Keyword_section}, KeywordId{.bytes="stdcallcc", .id = Id.Keyword_stdcallcc}, KeywordId{.bytes="struct", .id = Id.Keyword_struct}, KeywordId{.bytes="switch", .id = Id.Keyword_switch}, KeywordId{.bytes="test", .id = Id.Keyword_test}, KeywordId{.bytes="this", .id = Id.Keyword_this}, KeywordId{.bytes="true", .id = Id.Keyword_true}, + KeywordId{.bytes="try", .id = Id.Keyword_try}, KeywordId{.bytes="undefined", .id = Id.Keyword_undefined}, KeywordId{.bytes="union", .id = Id.Keyword_union}, KeywordId{.bytes="unreachable", .id = Id.Keyword_unreachable}, @@ -99,6 +103,7 @@ pub const Token = struct { Keyword_and, Keyword_asm, Keyword_break, + Keyword_catch, Keyword_comptime, Keyword_const, Keyword_continue, @@ -111,7 +116,6 @@ pub const Token = struct { Keyword_false, Keyword_fn, Keyword_for, - Keyword_goto, Keyword_if, Keyword_inline, Keyword_nakedcc, @@ -121,12 +125,14 @@ pub const Token = struct { Keyword_packed, Keyword_pub, Keyword_return, + Keyword_section, Keyword_stdcallcc, Keyword_struct, Keyword_switch, Keyword_test, Keyword_this, Keyword_true, + Keyword_try, Keyword_undefined, Keyword_union, Keyword_unreachable, @@ -140,21 +146,19 @@ pub const Token = struct { pub const Tokenizer = struct { buffer: []const u8, index: usize, + line: usize, + column: usize, pending_invalid_token: ?Token, - pub const Location = struct { - line: usize, - column: usize, + pub const LineLocation = struct { line_start: usize, line_end: usize, }; - pub fn getTokenLocation(self: &Tokenizer, token: &const Token) Location { - var loc = Location { - .line = 0, - .column = 0, + pub fn getTokenLocation(self: &Tokenizer, token: &const Token) LineLocation { + var loc = LineLocation { .line_start = 0, - .line_end = 0, + .line_end = self.buffer.len, }; for (self.buffer) |c, i| { if (i == token.start) { @@ -163,11 +167,7 @@ pub const Tokenizer = struct { return loc; } if (c == '\n') { - loc.line += 1; - loc.column = 0; loc.line_start = i + 1; - } else { - loc.column += 1; } } return loc; @@ -182,6 +182,8 @@ pub const Tokenizer = struct { return Tokenizer { .buffer = buffer, .index = 0, + .line = 0, + .column = 0, .pending_invalid_token = null, }; } @@ -222,13 +224,21 @@ pub const Tokenizer = struct { .id = Token.Id.Eof, .start = self.index, .end = undefined, + .line = self.line, + .column = self.column, }; - while (self.index < self.buffer.len) : (self.index += 1) { + while (self.index < self.buffer.len) { const c = self.buffer[self.index]; switch (state) { State.Start => switch (c) { - ' ', '\n' => { + ' ' => { result.start = self.index + 1; + result.column += 1; + }, + '\n' => { + result.start = self.index + 1; + result.line += 1; + result.column = 0; }, 'c' => { state = State.C; @@ -474,6 +484,8 @@ pub const Tokenizer = struct { result = Token { .id = Token.Id.Eof, .start = self.index + 1, + .column = 0, + .line = self.line + 1, .end = undefined, }; }, @@ -543,6 +555,14 @@ pub const Tokenizer = struct { else => break, }, } + + self.index += 1; + if (c == '\n') { + self.line += 1; + self.column = 0; + } else { + self.column += 1; + } } else if (self.index == self.buffer.len) { switch (state) { State.Start, @@ -622,6 +642,8 @@ pub const Tokenizer = struct { .id = Token.Id.Invalid, .start = self.index, .end = self.index + invalid_length, + .line = self.line, + .column = self.column, }; } From 1c1c0691cc4d47a39f382aec29654ae94cdf524c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 14 Feb 2018 23:12:51 -0500 Subject: [PATCH 07/16] fix crash when doing comptime float rem comptime int closes #776 --- src/ir.cpp | 9 ++++++--- test/cases/math.zig | 9 ++++++++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/ir.cpp b/src/ir.cpp index 8f6317096c..bb9a153894 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -9975,15 +9975,18 @@ static TypeTableEntry *ir_analyze_bin_op_math(IrAnalyze *ira, IrInstructionBinOp ok = bigint_cmp(&rem_result, &mod_result) == CmpEQ; } } else { - if (float_cmp_zero(&op2->value) == CmpEQ) { + IrInstruction *casted_op2 = ir_implicit_cast(ira, op2, resolved_type); + if (casted_op2 == ira->codegen->invalid_instruction) + return ira->codegen->builtin_types.entry_invalid; + if (float_cmp_zero(&casted_op2->value) == CmpEQ) { // the division by zero error will be caught later, but we don't // have a remainder function ambiguity problem ok = true; } else { ConstExprValue rem_result; ConstExprValue mod_result; - float_rem(&rem_result, &op1->value, &op2->value); - float_mod(&mod_result, &op1->value, &op2->value); + float_rem(&rem_result, &op1->value, &casted_op2->value); + float_mod(&mod_result, &op1->value, &casted_op2->value); ok = float_cmp(&rem_result, &mod_result) == CmpEQ; } } diff --git a/test/cases/math.zig b/test/cases/math.zig index 2deef08cd0..574aa39bb1 100644 --- a/test/cases/math.zig +++ b/test/cases/math.zig @@ -394,4 +394,11 @@ fn test_f128() void { fn should_not_be_zero(x: f128) void { assert(x != 0.0); -} \ No newline at end of file +} + +test "comptime float rem int" { + comptime { + var x = f32(1) % 2; + assert(x == 1.0); + } +} From cc26148ba776f713bb81b5ac06fc646eb323e6dc Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 15 Feb 2018 12:14:20 -0500 Subject: [PATCH 08/16] fix compiler crash when struct contains... ptr to another struct which contains original struct --- src/analyze.cpp | 19 ++++++++++------ test/behavior.zig | 1 + .../cases/struct_contains_null_ptr_itself.zig | 22 +++++++++++++++++++ 3 files changed, 35 insertions(+), 7 deletions(-) create mode 100644 test/cases/struct_contains_null_ptr_itself.zig diff --git a/src/analyze.cpp b/src/analyze.cpp index bf7b6e363f..c16a5d462a 100644 --- a/src/analyze.cpp +++ b/src/analyze.cpp @@ -2278,17 +2278,16 @@ static void resolve_struct_zero_bits(CodeGen *g, TypeTableEntry *struct_type) { return; if (struct_type->data.structure.zero_bits_loop_flag) { - // If we get here it's due to recursion. From this we conclude that the struct is - // not zero bits, and if abi_alignment == 0 we further conclude that the first field - // is a pointer to this very struct, or a function pointer with parameters that - // reference such a type. + // If we get here it's due to recursion. This is a design flaw in the compiler, + // we should be able to still figure out alignment, but here we give up and say that + // the alignment is pointer width, then assert that the first field is within that + // alignment struct_type->data.structure.zero_bits_known = true; if (struct_type->data.structure.abi_alignment == 0) { if (struct_type->data.structure.layout == ContainerLayoutPacked) { struct_type->data.structure.abi_alignment = 1; } else { - struct_type->data.structure.abi_alignment = LLVMABIAlignmentOfType(g->target_data_ref, - LLVMPointerType(LLVMInt8Type(), 0)); + struct_type->data.structure.abi_alignment = LLVMABIAlignmentOfType(g->target_data_ref, LLVMPointerType(LLVMInt8Type(), 0)); } } return; @@ -2352,11 +2351,17 @@ static void resolve_struct_zero_bits(CodeGen *g, TypeTableEntry *struct_type) { if (gen_field_index == 0) { if (struct_type->data.structure.layout == ContainerLayoutPacked) { struct_type->data.structure.abi_alignment = 1; - } else { + } else if (struct_type->data.structure.abi_alignment == 0) { // Alignment of structs is the alignment of the first field, for now. // TODO change this when we re-order struct fields (issue #168) struct_type->data.structure.abi_alignment = get_abi_alignment(g, field_type); assert(struct_type->data.structure.abi_alignment != 0); + } else { + // due to a design flaw in the compiler we assumed that alignment was + // pointer width, so we assert that this wasn't violated. + if (get_abi_alignment(g, field_type) > struct_type->data.structure.abi_alignment) { + zig_panic("compiler design flaw: incorrect alignment assumption"); + } } } diff --git a/test/behavior.zig b/test/behavior.zig index 96a323a6c8..e718ba6c86 100644 --- a/test/behavior.zig +++ b/test/behavior.zig @@ -35,6 +35,7 @@ comptime { _ = @import("cases/slice.zig"); _ = @import("cases/struct.zig"); _ = @import("cases/struct_contains_slice_of_itself.zig"); + _ = @import("cases/struct_contains_null_ptr_itself.zig"); _ = @import("cases/switch.zig"); _ = @import("cases/switch_prong_err_enum.zig"); _ = @import("cases/switch_prong_implicit_cast.zig"); diff --git a/test/cases/struct_contains_null_ptr_itself.zig b/test/cases/struct_contains_null_ptr_itself.zig new file mode 100644 index 0000000000..5864ef4038 --- /dev/null +++ b/test/cases/struct_contains_null_ptr_itself.zig @@ -0,0 +1,22 @@ +const std = @import("std"); +const assert = std.debug.assert; + +test "struct contains null pointer which contains original struct" { + var x: ?&NodeLineComment = null; + assert(x == null); +} + +pub const Node = struct { + id: Id, + comment: ?&NodeLineComment, + + pub const Id = enum { + Root, + LineComment, + }; +}; + +pub const NodeLineComment = struct { + base: Node, +}; + From 5f5880979e8c5a7cfa4efdabad6358ceb89cc0e7 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 14 Feb 2018 23:39:20 -0500 Subject: [PATCH 09/16] zig fmt supports simple line comments --- std/zig/ast.zig | 22 ++++++++++++ std/zig/parser.zig | 83 ++++++++++++++++++++++++++++++++++++------- std/zig/tokenizer.zig | 14 ++------ 3 files changed, 95 insertions(+), 24 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 77a2b8cd6a..903dc051e2 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -6,6 +6,7 @@ const mem = std.mem; pub const Node = struct { id: Id, + comment: ?&NodeLineComment, pub const Id = enum { Root, @@ -20,6 +21,7 @@ pub const Node = struct { FloatLiteral, StringLiteral, BuiltinCall, + LineComment, }; pub fn iterate(base: &Node, index: usize) ?&Node { @@ -36,6 +38,7 @@ pub const Node = struct { Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).iterate(index), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index), + Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).iterate(index), }; } @@ -53,6 +56,7 @@ pub const Node = struct { Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(), + Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).firstToken(), }; } @@ -70,6 +74,7 @@ pub const Node = struct { Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(), + Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).lastToken(), }; } }; @@ -454,3 +459,20 @@ pub const NodeStringLiteral = struct { return self.token; } }; + +pub const NodeLineComment = struct { + base: Node, + lines: ArrayList(Token), + + pub fn iterate(self: &NodeLineComment, index: usize) ?&Node { + return null; + } + + pub fn firstToken(self: &NodeLineComment) Token { + return self.lines.at(0); + } + + pub fn lastToken(self: &NodeLineComment) Token { + return self.lines.at(self.lines.len - 1); + } +}; diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 35bba378d4..533ad754ac 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -18,6 +18,7 @@ pub const Parser = struct { put_back_tokens: [2]Token, put_back_count: usize, source_file_name: []const u8, + pending_line_comment_node: ?&ast.NodeLineComment, pub const Tree = struct { root_node: &ast.NodeRoot, @@ -43,6 +44,7 @@ pub const Parser = struct { .put_back_count = 0, .source_file_name = source_file_name, .utility_bytes = []align(utility_bytes_align) u8{}, + .pending_line_comment_node = null, }; } @@ -131,6 +133,33 @@ pub const Parser = struct { // warn("\n"); //} + // look for line comments + while (true) { + const token = self.getNextToken(); + if (token.id == Token.Id.LineComment) { + const node = blk: { + if (self.pending_line_comment_node) |comment_node| { + break :blk comment_node; + } else { + const comment_node = try arena.create(ast.NodeLineComment); + *comment_node = ast.NodeLineComment { + .base = ast.Node { + .id = ast.Node.Id.LineComment, + .comment = null, + }, + .lines = ArrayList(Token).init(arena), + }; + self.pending_line_comment_node = comment_node; + break :blk comment_node; + } + }; + try node.lines.append(token); + continue; + } + self.putBackToken(token); + break; + } + // This gives us 1 free append that can't fail const state = stack.pop(); @@ -329,7 +358,7 @@ pub const Parser = struct { Token.Id.Builtin => { const node = try arena.create(ast.NodeBuiltinCall); *node = ast.NodeBuiltinCall { - .base = ast.Node {.id = ast.Node.Id.BuiltinCall}, + .base = self.initNode(ast.Node.Id.BuiltinCall), .builtin_token = token, .params = ArrayList(&ast.Node).init(arena), .rparen_token = undefined, @@ -350,7 +379,7 @@ pub const Parser = struct { Token.Id.StringLiteral => { const node = try arena.create(ast.NodeStringLiteral); *node = ast.NodeStringLiteral { - .base = ast.Node {.id = ast.Node.Id.StringLiteral}, + .base = self.initNode(ast.Node.Id.StringLiteral), .token = token, }; try stack.append(State { @@ -359,6 +388,7 @@ pub const Parser = struct { try stack.append(State.AfterOperand); continue; }, + else => return self.parseError(token, "expected primary expression, found {}", @tagName(token.id)), } }, @@ -660,11 +690,19 @@ pub const Parser = struct { } } + fn initNode(self: &Parser, id: ast.Node.Id) ast.Node { + if (self.pending_line_comment_node) |comment_node| { + self.pending_line_comment_node = null; + return ast.Node {.id = id, .comment = comment_node}; + } + return ast.Node {.id = id, .comment = null }; + } + fn createRoot(self: &Parser, arena: &mem.Allocator) !&ast.NodeRoot { const node = try arena.create(ast.NodeRoot); *node = ast.NodeRoot { - .base = ast.Node {.id = ast.Node.Id.Root}, + .base = self.initNode(ast.Node.Id.Root), .decls = ArrayList(&ast.Node).init(arena), // initialized when we get the eof token .eof_token = undefined, @@ -678,7 +716,7 @@ pub const Parser = struct { const node = try arena.create(ast.NodeVarDecl); *node = ast.NodeVarDecl { - .base = ast.Node {.id = ast.Node.Id.VarDecl}, + .base = self.initNode(ast.Node.Id.VarDecl), .visib_token = *visib_token, .mut_token = *mut_token, .comptime_token = *comptime_token, @@ -701,7 +739,7 @@ pub const Parser = struct { const node = try arena.create(ast.NodeFnProto); *node = ast.NodeFnProto { - .base = ast.Node {.id = ast.Node.Id.FnProto}, + .base = self.initNode(ast.Node.Id.FnProto), .visib_token = *visib_token, .name_token = null, .fn_token = *fn_token, @@ -722,7 +760,7 @@ pub const Parser = struct { const node = try arena.create(ast.NodeParamDecl); *node = ast.NodeParamDecl { - .base = ast.Node {.id = ast.Node.Id.ParamDecl}, + .base = self.initNode(ast.Node.Id.ParamDecl), .comptime_token = null, .noalias_token = null, .name_token = null, @@ -736,7 +774,7 @@ pub const Parser = struct { const node = try arena.create(ast.NodeBlock); *node = ast.NodeBlock { - .base = ast.Node {.id = ast.Node.Id.Block}, + .base = self.initNode(ast.Node.Id.Block), .begin_token = *begin_token, .end_token = undefined, .statements = ArrayList(&ast.Node).init(arena), @@ -748,7 +786,7 @@ pub const Parser = struct { const node = try arena.create(ast.NodeInfixOp); *node = ast.NodeInfixOp { - .base = ast.Node {.id = ast.Node.Id.InfixOp}, + .base = self.initNode(ast.Node.Id.InfixOp), .op_token = *op_token, .lhs = undefined, .op = *op, @@ -761,7 +799,7 @@ pub const Parser = struct { const node = try arena.create(ast.NodePrefixOp); *node = ast.NodePrefixOp { - .base = ast.Node {.id = ast.Node.Id.PrefixOp}, + .base = self.initNode(ast.Node.Id.PrefixOp), .op_token = *op_token, .op = *op, .rhs = undefined, @@ -773,7 +811,7 @@ pub const Parser = struct { const node = try arena.create(ast.NodeIdentifier); *node = ast.NodeIdentifier { - .base = ast.Node {.id = ast.Node.Id.Identifier}, + .base = self.initNode(ast.Node.Id.Identifier), .name_token = *name_token, }; return node; @@ -783,7 +821,7 @@ pub const Parser = struct { const node = try arena.create(ast.NodeIntegerLiteral); *node = ast.NodeIntegerLiteral { - .base = ast.Node {.id = ast.Node.Id.IntegerLiteral}, + .base = self.initNode(ast.Node.Id.IntegerLiteral), .token = *token, }; return node; @@ -793,7 +831,7 @@ pub const Parser = struct { const node = try arena.create(ast.NodeFloatLiteral); *node = ast.NodeFloatLiteral { - .base = ast.Node {.id = ast.Node.Id.FloatLiteral}, + .base = self.initNode(ast.Node.Id.FloatLiteral), .token = *token, }; return node; @@ -1158,9 +1196,11 @@ pub const Parser = struct { } } }, + ast.Node.Id.FnProto => @panic("TODO fn proto in an expression"), + ast.Node.Id.LineComment => @panic("TODO render line comment in an expression"), + ast.Node.Id.Root, ast.Node.Id.VarDecl, - ast.Node.Id.FnProto, ast.Node.Id.ParamDecl => unreachable, }, RenderState.FnProtoRParen => |fn_proto| { @@ -1187,6 +1227,12 @@ pub const Parser = struct { } }, RenderState.Statement => |base| { + if (base.comment) |comment| { + for (comment.lines.toSliceConst()) |line_token| { + try stream.print("{}\n", self.tokenizer.getTokenSlice(line_token)); + try stream.writeByteNTimes(' ', indent); + } + } switch (base.id) { ast.Node.Id.VarDecl => { const var_decl = @fieldParentPtr(ast.NodeVarDecl, "base", base); @@ -1279,6 +1325,17 @@ fn testCanonical(source: []const u8) !void { } test "zig fmt" { + try testCanonical( + \\const std = @import("std"); + \\ + \\pub fn main() !void { + \\ // If this program is run without stdout attached, exit with an error. + \\ // another comment + \\ var stdout_file = try std.io.getStdOut; + \\} + \\ + ); + try testCanonical( \\const std = @import("std"); \\ diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig index bdd82e4a44..4af6c20cad 100644 --- a/std/zig/tokenizer.zig +++ b/std/zig/tokenizer.zig @@ -99,6 +99,7 @@ pub const Token = struct { AmpersandEqual, IntegerLiteral, FloatLiteral, + LineComment, Keyword_align, Keyword_and, Keyword_asm, @@ -470,7 +471,7 @@ pub const Tokenizer = struct { State.Slash => switch (c) { '/' => { - result.id = undefined; + result.id = Token.Id.LineComment; state = State.LineComment; }, else => { @@ -479,16 +480,7 @@ pub const Tokenizer = struct { }, }, State.LineComment => switch (c) { - '\n' => { - state = State.Start; - result = Token { - .id = Token.Id.Eof, - .start = self.index + 1, - .column = 0, - .line = self.line + 1, - .end = undefined, - }; - }, + '\n' => break, else => self.checkLiteralCharacter(), }, State.Zero => switch (c) { From cbbd6cfa1e85dc5cda38572f7a1d462a678c2adf Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 15 Feb 2018 23:39:35 -0500 Subject: [PATCH 10/16] add an assert to catch #777 asserting is better than segfaulting --- src/codegen.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/codegen.cpp b/src/codegen.cpp index 4f100d75ad..00103be259 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -4373,6 +4373,7 @@ static LLVMValueRef gen_const_val(CodeGen *g, ConstExprValue *const_val, const c } } } + zig_unreachable(); case TypeTableEntryIdErrorUnion: { TypeTableEntry *payload_type = type_entry->data.error_union.payload_type; From 72ca2b214d4ffbaeed4840a806fb63740cf13c05 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 16 Feb 2018 15:22:29 -0500 Subject: [PATCH 11/16] ability to slice an undefined pointer at compile time if the len is 0 --- src/codegen.cpp | 1 + src/ir.cpp | 68 +++++++++++++++++++++++++---------------- std/mem.zig | 7 +++++ test/cases/eval.zig | 7 +++++ test/compile_errors.zig | 7 +++++ 5 files changed, 63 insertions(+), 27 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 00103be259..15648cbdec 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -4201,6 +4201,7 @@ static LLVMValueRef gen_const_val(CodeGen *g, ConstExprValue *const_val, const c continue; } ConstExprValue *field_val = &const_val->data.x_struct.fields[i]; + assert(field_val->type != nullptr); LLVMValueRef val = gen_const_val(g, field_val, ""); fields[type_struct_field->gen_index] = val; make_unnamed_struct = make_unnamed_struct || is_llvm_value_unnamed_type(field_val->type, val); diff --git a/src/ir.cpp b/src/ir.cpp index bb9a153894..7eac9e4d23 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -8183,7 +8183,7 @@ static IrInstruction *ir_get_ref(IrAnalyze *ira, IrInstruction *source_instructi } if (instr_is_comptime(value)) { - ConstExprValue *val = ir_resolve_const(ira, value, UndefBad); + ConstExprValue *val = ir_resolve_const(ira, value, UndefOk); if (!val) return ira->codegen->invalid_instruction; bool final_is_const = (value->value.type->id == TypeTableEntryIdMetaType) ? is_const : true; @@ -14931,6 +14931,7 @@ static TypeTableEntry *ir_analyze_instruction_slice(IrAnalyze *ira, IrInstructio ConstExprValue *parent_ptr; size_t abs_offset; size_t rel_end; + bool ptr_is_undef = false; if (array_type->id == TypeTableEntryIdArray) { array_val = const_ptr_pointee(ira->codegen, &ptr_ptr->value); abs_offset = 0; @@ -14938,7 +14939,12 @@ static TypeTableEntry *ir_analyze_instruction_slice(IrAnalyze *ira, IrInstructio parent_ptr = nullptr; } else if (array_type->id == TypeTableEntryIdPointer) { parent_ptr = const_ptr_pointee(ira->codegen, &ptr_ptr->value); - switch (parent_ptr->data.x_ptr.special) { + if (parent_ptr->special == ConstValSpecialUndef) { + array_val = nullptr; + abs_offset = 0; + rel_end = SIZE_MAX; + ptr_is_undef = true; + } else switch (parent_ptr->data.x_ptr.special) { case ConstPtrSpecialInvalid: case ConstPtrSpecialDiscard: zig_unreachable(); @@ -14992,7 +14998,7 @@ static TypeTableEntry *ir_analyze_instruction_slice(IrAnalyze *ira, IrInstructio } uint64_t start_scalar = bigint_as_unsigned(&casted_start->value.data.x_bigint); - if (start_scalar > rel_end) { + if (!ptr_is_undef && start_scalar > rel_end) { ir_add_error(ira, &instruction->base, buf_sprintf("out of bounds slice")); return ira->codegen->builtin_types.entry_invalid; } @@ -15003,12 +15009,18 @@ static TypeTableEntry *ir_analyze_instruction_slice(IrAnalyze *ira, IrInstructio } else { end_scalar = rel_end; } - if (end_scalar > rel_end) { - ir_add_error(ira, &instruction->base, buf_sprintf("out of bounds slice")); - return ira->codegen->builtin_types.entry_invalid; + if (!ptr_is_undef) { + if (end_scalar > rel_end) { + ir_add_error(ira, &instruction->base, buf_sprintf("out of bounds slice")); + return ira->codegen->builtin_types.entry_invalid; + } + if (start_scalar > end_scalar) { + ir_add_error(ira, &instruction->base, buf_sprintf("slice start is greater than end")); + return ira->codegen->builtin_types.entry_invalid; + } } - if (start_scalar > end_scalar) { - ir_add_error(ira, &instruction->base, buf_sprintf("slice start is greater than end")); + if (ptr_is_undef && start_scalar != end_scalar) { + ir_add_error(ira, &instruction->base, buf_sprintf("non-zero length slice of undefined pointer")); return ira->codegen->builtin_types.entry_invalid; } @@ -15024,25 +15036,27 @@ static TypeTableEntry *ir_analyze_instruction_slice(IrAnalyze *ira, IrInstructio if (array_type->id == TypeTableEntryIdArray) { ptr_val->data.x_ptr.mut = ptr_ptr->value.data.x_ptr.mut; } - } else { - switch (parent_ptr->data.x_ptr.special) { - case ConstPtrSpecialInvalid: - case ConstPtrSpecialDiscard: - zig_unreachable(); - case ConstPtrSpecialRef: - init_const_ptr_ref(ira->codegen, ptr_val, - parent_ptr->data.x_ptr.data.ref.pointee, slice_is_const(return_type)); - break; - case ConstPtrSpecialBaseArray: - zig_unreachable(); - case ConstPtrSpecialBaseStruct: - zig_panic("TODO"); - case ConstPtrSpecialHardCodedAddr: - init_const_ptr_hard_coded_addr(ira->codegen, ptr_val, - parent_ptr->type->data.pointer.child_type, - parent_ptr->data.x_ptr.data.hard_coded_addr.addr + start_scalar, - slice_is_const(return_type)); - } + } else if (ptr_is_undef) { + ptr_val->type = get_pointer_to_type(ira->codegen, parent_ptr->type->data.pointer.child_type, + slice_is_const(return_type)); + ptr_val->special = ConstValSpecialUndef; + } else switch (parent_ptr->data.x_ptr.special) { + case ConstPtrSpecialInvalid: + case ConstPtrSpecialDiscard: + zig_unreachable(); + case ConstPtrSpecialRef: + init_const_ptr_ref(ira->codegen, ptr_val, + parent_ptr->data.x_ptr.data.ref.pointee, slice_is_const(return_type)); + break; + case ConstPtrSpecialBaseArray: + zig_unreachable(); + case ConstPtrSpecialBaseStruct: + zig_panic("TODO"); + case ConstPtrSpecialHardCodedAddr: + init_const_ptr_hard_coded_addr(ira->codegen, ptr_val, + parent_ptr->type->data.pointer.child_type, + parent_ptr->data.x_ptr.data.hard_coded_addr.addr + start_scalar, + slice_is_const(return_type)); } ConstExprValue *len_val = &out_val->data.x_struct.fields[slice_len_index]; diff --git a/std/mem.zig b/std/mem.zig index f40fc9bbea..07521bfcb8 100644 --- a/std/mem.zig +++ b/std/mem.zig @@ -42,6 +42,9 @@ pub const Allocator = struct { fn alignedAlloc(self: &Allocator, comptime T: type, comptime alignment: u29, n: usize) ![]align(alignment) T { + if (n == 0) { + return (&align(alignment) T)(undefined)[0..0]; + } const byte_count = math.mul(usize, @sizeOf(T), n) catch return Error.OutOfMemory; const byte_slice = try self.allocFn(self, byte_count, alignment); assert(byte_slice.len == byte_count); @@ -62,6 +65,10 @@ pub const Allocator = struct { if (old_mem.len == 0) { return self.alloc(T, n); } + if (n == 0) { + self.free(old_mem); + return (&align(alignment) T)(undefined)[0..0]; + } const old_byte_slice = ([]u8)(old_mem); const byte_count = math.mul(usize, @sizeOf(T), n) catch return Error.OutOfMemory; diff --git a/test/cases/eval.zig b/test/cases/eval.zig index e8dd828b4d..fc67d8f135 100644 --- a/test/cases/eval.zig +++ b/test/cases/eval.zig @@ -388,3 +388,10 @@ test "string literal used as comptime slice is memoized" { comptime assert(TypeWithCompTimeSlice(a).Node == TypeWithCompTimeSlice(b).Node); comptime assert(TypeWithCompTimeSlice("link").Node == TypeWithCompTimeSlice("link").Node); } + +test "comptime slice of undefined pointer of length 0" { + const slice1 = (&i32)(undefined)[0..0]; + assert(slice1.len == 0); + const slice2 = (&i32)(undefined)[100..100]; + assert(slice2.len == 0); +} diff --git a/test/compile_errors.zig b/test/compile_errors.zig index 90b3ff023a..940125711b 100644 --- a/test/compile_errors.zig +++ b/test/compile_errors.zig @@ -1,6 +1,13 @@ const tests = @import("tests.zig"); pub fn addCases(cases: &tests.CompileErrorContext) void { + cases.add("comptime slice of undefined pointer non-zero len", + \\export fn entry() void { + \\ const slice = (&i32)(undefined)[0..1]; + \\} + , + ".tmp_source.zig:2:36: error: non-zero length slice of undefined pointer"); + cases.add("type checking function pointers", \\fn a(b: fn (&const u8) void) void { \\ b('a'); From bde15cf0806f2b8d6fb0c90602b42a74863ec515 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 17 Feb 2018 17:53:07 -0500 Subject: [PATCH 12/16] improve std lib linux epoll API --- std/os/linux/index.zig | 19 +++++++++++++++---- std/os/linux/test.zig | 2 +- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/std/os/linux/index.zig b/std/os/linux/index.zig index 113f2ef454..646b1ef300 100644 --- a/std/os/linux/index.zig +++ b/std/os/linux/index.zig @@ -329,6 +329,8 @@ pub const TIOCGPKT = 0x80045438; pub const TIOCGPTLCK = 0x80045439; pub const TIOCGEXCL = 0x80045440; +pub const EPOLL_CLOEXEC = O_CLOEXEC; + pub const EPOLL_CTL_ADD = 1; pub const EPOLL_CTL_DEL = 2; pub const EPOLL_CTL_MOD = 3; @@ -751,22 +753,31 @@ pub fn fstat(fd: i32, stat_buf: &Stat) usize { return arch.syscall2(arch.SYS_fstat, usize(fd), @ptrToInt(stat_buf)); } -pub const epoll_data = u64; +pub const epoll_data = extern union { + ptr: usize, + fd: i32, + @"u32": u32, + @"u64": u64, +}; pub const epoll_event = extern struct { events: u32, - data: epoll_data + data: epoll_data, }; pub fn epoll_create() usize { - return arch.syscall1(arch.SYS_epoll_create, usize(1)); + return epoll_create1(0); +} + +pub fn epoll_create1(flags: usize) usize { + return arch.syscall1(arch.SYS_epoll_create1, flags); } pub fn epoll_ctl(epoll_fd: i32, op: i32, fd: i32, ev: &epoll_event) usize { return arch.syscall4(arch.SYS_epoll_ctl, usize(epoll_fd), usize(op), usize(fd), @ptrToInt(ev)); } -pub fn epoll_wait(epoll_fd: i32, events: &epoll_event, maxevents: i32, timeout: i32) usize { +pub fn epoll_wait(epoll_fd: i32, events: &epoll_event, maxevents: u32, timeout: i32) usize { return arch.syscall4(arch.SYS_epoll_wait, usize(epoll_fd), @ptrToInt(events), usize(maxevents), usize(timeout)); } diff --git a/std/os/linux/test.zig b/std/os/linux/test.zig index c7dbeab67f..e427fd5d59 100644 --- a/std/os/linux/test.zig +++ b/std/os/linux/test.zig @@ -25,7 +25,7 @@ test "timer" { var event = linux.epoll_event { .events = linux.EPOLLIN | linux.EPOLLOUT | linux.EPOLLET, - .data = 0 + .data = linux.epoll_data { .ptr = 0 }, }; err = linux.epoll_ctl(i32(epoll_fd), linux.EPOLL_CTL_ADD, i32(timer_fd), &event); From ab48934e9cefb510d39ba3fe8c0dcf7619bec4cf Mon Sep 17 00:00:00 2001 From: Ben Noordhuis Date: Mon, 19 Feb 2018 23:06:54 +0100 Subject: [PATCH 13/16] add support for stack traces on macosx Add basic address->symbol resolution support. Uses symtab data from the MachO image, not external dSYM data; that's left as a future exercise. The net effect is that we can now map addresses to function names but not much more. File names and line number data will have to wait until a future pull request. Partially fixes #434. --- CMakeLists.txt | 1 + std/debug/index.zig | 148 ++++++++++++++++++++++-------------- std/index.zig | 2 + std/macho.zig | 177 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 271 insertions(+), 57 deletions(-) create mode 100644 std/macho.zig diff --git a/CMakeLists.txt b/CMakeLists.txt index bdc3465830..2856a20606 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -386,6 +386,7 @@ set(ZIG_STD_FILES "index.zig" "io.zig" "linked_list.zig" + "macho.zig" "math/acos.zig" "math/acosh.zig" "math/asin.zig" diff --git a/std/debug/index.zig b/std/debug/index.zig index cc4832b1ea..2418654986 100644 --- a/std/debug/index.zig +++ b/std/debug/index.zig @@ -5,6 +5,7 @@ const io = std.io; const os = std.os; const elf = std.elf; const DW = std.dwarf; +const macho = std.macho; const ArrayList = std.ArrayList; const builtin = @import("builtin"); @@ -180,43 +181,57 @@ pub fn writeCurrentStackTrace(out_stream: var, allocator: &mem.Allocator, } fn printSourceAtAddress(debug_info: &ElfStackTrace, out_stream: var, address: usize) !void { - if (builtin.os == builtin.Os.windows) { - return error.UnsupportedDebugInfo; - } // TODO we really should be able to convert @sizeOf(usize) * 2 to a string literal // at compile time. I'll call it issue #313 const ptr_hex = if (@sizeOf(usize) == 4) "0x{x8}" else "0x{x16}"; - const compile_unit = findCompileUnit(debug_info, address) catch { - try out_stream.print("???:?:?: " ++ DIM ++ ptr_hex ++ " in ??? (???)" ++ RESET ++ "\n ???\n\n", - address); - return; - }; - const compile_unit_name = try compile_unit.die.getAttrString(debug_info, DW.AT_name); - if (getLineNumberInfo(debug_info, compile_unit, address - 1)) |line_info| { - defer line_info.deinit(); - try out_stream.print(WHITE ++ "{}:{}:{}" ++ RESET ++ ": " ++ - DIM ++ ptr_hex ++ " in ??? ({})" ++ RESET ++ "\n", - line_info.file_name, line_info.line, line_info.column, - address, compile_unit_name); - if (printLineFromFile(debug_info.allocator(), out_stream, line_info)) { - if (line_info.column == 0) { - try out_stream.write("\n"); - } else { - {var col_i: usize = 1; while (col_i < line_info.column) : (col_i += 1) { - try out_stream.writeByte(' '); - }} - try out_stream.write(GREEN ++ "^" ++ RESET ++ "\n"); - } - } else |err| switch (err) { - error.EndOfFile => {}, - else => return err, - } - } else |err| switch (err) { - error.MissingDebugInfo, error.InvalidDebugInfo => { - try out_stream.print(ptr_hex ++ " in ??? ({})\n", address, compile_unit_name); + switch (builtin.os) { + builtin.Os.windows => return error.UnsupportedDebugInfo, + builtin.Os.macosx => { + // TODO(bnoordhuis) It's theoretically possible to obtain the + // compilation unit from the symbtab but it's not that useful + // in practice because the compiler dumps everything in a single + // object file. Future improvement: use external dSYM data when + // available. + const unknown = macho.Symbol { .name = "???", .address = address }; + const symbol = debug_info.symbol_table.search(address) ?? &unknown; + try out_stream.print(WHITE ++ "{}" ++ RESET ++ ": " ++ + DIM ++ ptr_hex ++ " in ??? (???)" ++ RESET ++ "\n", + symbol.name, address); + }, + else => { + const compile_unit = findCompileUnit(debug_info, address) catch { + try out_stream.print("???:?:?: " ++ DIM ++ ptr_hex ++ " in ??? (???)" ++ RESET ++ "\n ???\n\n", + address); + return; + }; + const compile_unit_name = try compile_unit.die.getAttrString(debug_info, DW.AT_name); + if (getLineNumberInfo(debug_info, compile_unit, address - 1)) |line_info| { + defer line_info.deinit(); + try out_stream.print(WHITE ++ "{}:{}:{}" ++ RESET ++ ": " ++ + DIM ++ ptr_hex ++ " in ??? ({})" ++ RESET ++ "\n", + line_info.file_name, line_info.line, line_info.column, + address, compile_unit_name); + if (printLineFromFile(debug_info.allocator(), out_stream, line_info)) { + if (line_info.column == 0) { + try out_stream.write("\n"); + } else { + {var col_i: usize = 1; while (col_i < line_info.column) : (col_i += 1) { + try out_stream.writeByte(' '); + }} + try out_stream.write(GREEN ++ "^" ++ RESET ++ "\n"); + } + } else |err| switch (err) { + error.EndOfFile => {}, + else => return err, + } + } else |err| switch (err) { + error.MissingDebugInfo, error.InvalidDebugInfo => { + try out_stream.print(ptr_hex ++ " in ??? ({})\n", address, compile_unit_name); + }, + else => return err, + } }, - else => return err, } } @@ -249,12 +264,22 @@ pub fn openSelfDebugInfo(allocator: &mem.Allocator) !&ElfStackTrace { try scanAllCompileUnits(st); return st; }, + builtin.ObjectFormat.macho => { + var exe_file = try os.openSelfExe(); + defer exe_file.close(); + + const st = try allocator.create(ElfStackTrace); + errdefer allocator.destroy(st); + + *st = ElfStackTrace { + .symbol_table = try macho.loadSymbols(allocator, &io.FileInStream.init(&exe_file)), + }; + + return st; + }, builtin.ObjectFormat.coff => { return error.TodoSupportCoffDebugInfo; }, - builtin.ObjectFormat.macho => { - return error.TodoSupportMachoDebugInfo; - }, builtin.ObjectFormat.wasm => { return error.TodoSupportCOFFDebugInfo; }, @@ -297,31 +322,40 @@ fn printLineFromFile(allocator: &mem.Allocator, out_stream: var, line_info: &con } } -pub const ElfStackTrace = struct { - self_exe_file: os.File, - elf: elf.Elf, - debug_info: &elf.SectionHeader, - debug_abbrev: &elf.SectionHeader, - debug_str: &elf.SectionHeader, - debug_line: &elf.SectionHeader, - debug_ranges: ?&elf.SectionHeader, - abbrev_table_list: ArrayList(AbbrevTableHeader), - compile_unit_list: ArrayList(CompileUnit), +pub const ElfStackTrace = switch (builtin.os) { + builtin.Os.macosx => struct { + symbol_table: macho.SymbolTable, - pub fn allocator(self: &const ElfStackTrace) &mem.Allocator { - return self.abbrev_table_list.allocator; - } + pub fn close(self: &ElfStackTrace) void { + self.symbol_table.deinit(); + } + }, + else => struct { + self_exe_file: os.File, + elf: elf.Elf, + debug_info: &elf.SectionHeader, + debug_abbrev: &elf.SectionHeader, + debug_str: &elf.SectionHeader, + debug_line: &elf.SectionHeader, + debug_ranges: ?&elf.SectionHeader, + abbrev_table_list: ArrayList(AbbrevTableHeader), + compile_unit_list: ArrayList(CompileUnit), - pub fn readString(self: &ElfStackTrace) ![]u8 { - var in_file_stream = io.FileInStream.init(&self.self_exe_file); - const in_stream = &in_file_stream.stream; - return readStringRaw(self.allocator(), in_stream); - } + pub fn allocator(self: &const ElfStackTrace) &mem.Allocator { + return self.abbrev_table_list.allocator; + } - pub fn close(self: &ElfStackTrace) void { - self.self_exe_file.close(); - self.elf.close(); - } + pub fn readString(self: &ElfStackTrace) ![]u8 { + var in_file_stream = io.FileInStream.init(&self.self_exe_file); + const in_stream = &in_file_stream.stream; + return readStringRaw(self.allocator(), in_stream); + } + + pub fn close(self: &ElfStackTrace) void { + self.self_exe_file.close(); + self.elf.close(); + } + }, }; const PcRange = struct { diff --git a/std/index.zig b/std/index.zig index 8d292c2f5c..179eae159e 100644 --- a/std/index.zig +++ b/std/index.zig @@ -21,6 +21,7 @@ pub const endian = @import("endian.zig"); pub const fmt = @import("fmt/index.zig"); pub const heap = @import("heap.zig"); pub const io = @import("io.zig"); +pub const macho = @import("macho.zig"); pub const math = @import("math/index.zig"); pub const mem = @import("mem.zig"); pub const net = @import("net.zig"); @@ -51,6 +52,7 @@ test "std" { _ = @import("endian.zig"); _ = @import("fmt/index.zig"); _ = @import("io.zig"); + _ = @import("macho.zig"); _ = @import("math/index.zig"); _ = @import("mem.zig"); _ = @import("heap.zig"); diff --git a/std/macho.zig b/std/macho.zig new file mode 100644 index 0000000000..05239bf191 --- /dev/null +++ b/std/macho.zig @@ -0,0 +1,177 @@ +const builtin = @import("builtin"); +const std = @import("index.zig"); +const io = std.io; +const mem = std.mem; + +const MH_MAGIC_64 = 0xFEEDFACF; +const MH_PIE = 0x200000; +const LC_SYMTAB = 2; + +const MachHeader64 = packed struct { + magic: u32, + cputype: u32, + cpusubtype: u32, + filetype: u32, + ncmds: u32, + sizeofcmds: u32, + flags: u32, + reserved: u32, +}; + +const LoadCommand = packed struct { + cmd: u32, + cmdsize: u32, +}; + +const SymtabCommand = packed struct { + symoff: u32, + nsyms: u32, + stroff: u32, + strsize: u32, +}; + +const Nlist64 = packed struct { + n_strx: u32, + n_type: u8, + n_sect: u8, + n_desc: u16, + n_value: u64, +}; + +pub const Symbol = struct { + name: []const u8, + address: u64, + + fn addressLessThan(lhs: &const Symbol, rhs: &const Symbol) bool { + return lhs.address < rhs.address; + } +}; + +pub const SymbolTable = struct { + allocator: &mem.Allocator, + symbols: []const Symbol, + strings: []const u8, + + // Doubles as an eyecatcher to calculate the PIE slide, see loadSymbols(). + // Ideally we'd use _mh_execute_header because it's always at 0x100000000 + // in the image but as it's located in a different section than executable + // code, its displacement is different. + pub fn deinit(self: &SymbolTable) void { + self.allocator.free(self.symbols); + self.symbols = []const Symbol {}; + + self.allocator.free(self.strings); + self.strings = []const u8 {}; + } + + pub fn search(self: &const SymbolTable, address: usize) ?&const Symbol { + var min: usize = 0; + var max: usize = self.symbols.len - 1; // Exclude sentinel. + while (min < max) { + const mid = min + (max - min) / 2; + const curr = &self.symbols[mid]; + const next = &self.symbols[mid + 1]; + if (address >= next.address) { + min = mid + 1; + } else if (address < curr.address) { + max = mid; + } else { + return curr; + } + } + return null; + } +}; + +pub fn loadSymbols(allocator: &mem.Allocator, in: &io.FileInStream) !SymbolTable { + var file = in.file; + try file.seekTo(0); + + var hdr: MachHeader64 = undefined; + try readNoEof(in, &hdr); + if (hdr.magic != MH_MAGIC_64) return error.MissingDebugInfo; + const is_pie = MH_PIE == (hdr.flags & MH_PIE); + + var pos: usize = @sizeOf(@typeOf(hdr)); + var ncmd: u32 = hdr.ncmds; + while (ncmd != 0) : (ncmd -= 1) { + try file.seekTo(pos); + var lc: LoadCommand = undefined; + try readNoEof(in, &lc); + if (lc.cmd == LC_SYMTAB) break; + pos += lc.cmdsize; + } else { + return error.MissingDebugInfo; + } + + var cmd: SymtabCommand = undefined; + try readNoEof(in, &cmd); + + try file.seekTo(cmd.symoff); + var syms = try allocator.alloc(Nlist64, cmd.nsyms); + defer allocator.free(syms); + try readNoEof(in, syms); + + try file.seekTo(cmd.stroff); + var strings = try allocator.alloc(u8, cmd.strsize); + errdefer allocator.free(strings); + try in.stream.readNoEof(strings); + + var nsyms: usize = 0; + for (syms) |sym| if (isSymbol(sym)) nsyms += 1; + if (nsyms == 0) return error.MissingDebugInfo; + + var symbols = try allocator.alloc(Symbol, nsyms + 1); // Room for sentinel. + errdefer allocator.free(symbols); + + var pie_slide: usize = 0; + var nsym: usize = 0; + for (syms) |sym| { + if (!isSymbol(sym)) continue; + const start = sym.n_strx; + const end = ??mem.indexOfScalarPos(u8, strings, start, 0); + const name = strings[start..end]; + const address = sym.n_value; + symbols[nsym] = Symbol { .name = name, .address = address }; + nsym += 1; + if (is_pie and mem.eql(u8, name, "_SymbolTable_deinit")) { + pie_slide = @ptrToInt(SymbolTable.deinit) - address; + } + } + + // Effectively a no-op, lld emits symbols in ascending order. + std.sort.insertionSort(Symbol, symbols[0..nsyms], Symbol.addressLessThan); + + // Insert the sentinel. Since we don't know where the last function ends, + // we arbitrarily limit it to the start address + 4 KB. + const top = symbols[nsyms - 1].address + 4096; + symbols[nsyms] = Symbol { .name = "", .address = top }; + + if (pie_slide != 0) { + for (symbols) |*symbol| symbol.address += pie_slide; + } + + return SymbolTable { + .allocator = allocator, + .symbols = symbols, + .strings = strings, + }; +} + +fn readNoEof(in: &io.FileInStream, sink: var) !void { + if (@typeOf(sink) == []Nlist64) { + const T = @typeOf(sink[0]); + const len = @sizeOf(T) * sink.len; + const bytes = @ptrCast(&u8, &sink[0]); + return in.stream.readNoEof(bytes[0..len]); + } else { + const T = @typeOf(*sink); + const len = @sizeOf(T); + const bytes = @ptrCast(&u8, sink); + return in.stream.readNoEof(bytes[0..len]); + } +} + +fn isSymbol(sym: &const Nlist64) bool { + return sym.n_value != 0 and sym.n_desc == 0; +} From 2b35615ffbe238c8ec421654a7e1ae0890477fe0 Mon Sep 17 00:00:00 2001 From: Ben Noordhuis Date: Mon, 19 Feb 2018 23:06:54 +0100 Subject: [PATCH 14/16] fix memory leak in std.debug.openSelfDebugInfo() --- std/debug/index.zig | 1 + 1 file changed, 1 insertion(+) diff --git a/std/debug/index.zig b/std/debug/index.zig index 2418654986..5de201b0e6 100644 --- a/std/debug/index.zig +++ b/std/debug/index.zig @@ -239,6 +239,7 @@ pub fn openSelfDebugInfo(allocator: &mem.Allocator) !&ElfStackTrace { switch (builtin.object_format) { builtin.ObjectFormat.elf => { const st = try allocator.create(ElfStackTrace); + errdefer allocator.destroy(st); *st = ElfStackTrace { .self_exe_file = undefined, .elf = undefined, From 623466762eba820f263a40622d70dc46ba0cb8ab Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 21 Feb 2018 02:00:33 -0500 Subject: [PATCH 15/16] clean up mach-o stack trace code --- std/macho.zig | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/std/macho.zig b/std/macho.zig index 05239bf191..70e2c09788 100644 --- a/std/macho.zig +++ b/std/macho.zig @@ -88,7 +88,7 @@ pub fn loadSymbols(allocator: &mem.Allocator, in: &io.FileInStream) !SymbolTable try file.seekTo(0); var hdr: MachHeader64 = undefined; - try readNoEof(in, &hdr); + try readOneNoEof(in, MachHeader64, &hdr); if (hdr.magic != MH_MAGIC_64) return error.MissingDebugInfo; const is_pie = MH_PIE == (hdr.flags & MH_PIE); @@ -97,7 +97,7 @@ pub fn loadSymbols(allocator: &mem.Allocator, in: &io.FileInStream) !SymbolTable while (ncmd != 0) : (ncmd -= 1) { try file.seekTo(pos); var lc: LoadCommand = undefined; - try readNoEof(in, &lc); + try readOneNoEof(in, LoadCommand, &lc); if (lc.cmd == LC_SYMTAB) break; pos += lc.cmdsize; } else { @@ -105,12 +105,12 @@ pub fn loadSymbols(allocator: &mem.Allocator, in: &io.FileInStream) !SymbolTable } var cmd: SymtabCommand = undefined; - try readNoEof(in, &cmd); + try readOneNoEof(in, SymtabCommand, &cmd); try file.seekTo(cmd.symoff); var syms = try allocator.alloc(Nlist64, cmd.nsyms); defer allocator.free(syms); - try readNoEof(in, syms); + try readNoEof(in, Nlist64, syms); try file.seekTo(cmd.stroff); var strings = try allocator.alloc(u8, cmd.strsize); @@ -158,18 +158,11 @@ pub fn loadSymbols(allocator: &mem.Allocator, in: &io.FileInStream) !SymbolTable }; } -fn readNoEof(in: &io.FileInStream, sink: var) !void { - if (@typeOf(sink) == []Nlist64) { - const T = @typeOf(sink[0]); - const len = @sizeOf(T) * sink.len; - const bytes = @ptrCast(&u8, &sink[0]); - return in.stream.readNoEof(bytes[0..len]); - } else { - const T = @typeOf(*sink); - const len = @sizeOf(T); - const bytes = @ptrCast(&u8, sink); - return in.stream.readNoEof(bytes[0..len]); - } +fn readNoEof(in: &io.FileInStream, comptime T: type, result: []T) !void { + return in.stream.readNoEof(([]u8)(result)); +} +fn readOneNoEof(in: &io.FileInStream, comptime T: type, result: &T) !void { + return readNoEof(in, T, result[0..1]); } fn isSymbol(sym: &const Nlist64) bool { From 0845cbe27783486feb5b4b57b2839326a2c86a6b Mon Sep 17 00:00:00 2001 From: Ben Noordhuis Date: Thu, 22 Feb 2018 17:53:58 +0100 Subject: [PATCH 16/16] name types inside functions after variable Before this commit: fn f() []const u8 { const S = struct {}; return @typeName(S); // "f()", unexpected. } And now: fn f() []const u8 { const S = struct {}; return @typeName(S); // "S", expected. } Fixes #675. --- src/ir.cpp | 6 ++++++ std/fmt/index.zig | 10 ++++------ test/cases/misc.zig | 17 +++++++++++++++++ 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/src/ir.cpp b/src/ir.cpp index 7eac9e4d23..b276abff33 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -4172,7 +4172,13 @@ static IrInstruction *ir_gen_var_decl(IrBuilder *irb, Scope *scope, AstNode *nod buf_sprintf("cannot set section of local variable '%s'", buf_ptr(variable_declaration->symbol))); } + // Temporarily set the name of the IrExecutable to the VariableDeclaration + // so that the struct or enum from the init expression inherits the name. + Buf *old_exec_name = irb->exec->name; + irb->exec->name = variable_declaration->symbol; IrInstruction *init_value = ir_gen_node(irb, variable_declaration->expr, scope); + irb->exec->name = old_exec_name; + if (init_value == irb->codegen->invalid_instruction) return init_value; diff --git a/std/fmt/index.zig b/std/fmt/index.zig index 56b0add86d..bd5b5710e0 100644 --- a/std/fmt/index.zig +++ b/std/fmt/index.zig @@ -550,12 +550,6 @@ test "parse unsigned comptime" { } } -// Dummy field because of https://github.com/zig-lang/zig/issues/557. -// At top level because of https://github.com/zig-lang/zig/issues/675. -const Struct = struct { - unused: u8, -}; - test "fmt.format" { { var buf1: [32]u8 = undefined; @@ -588,6 +582,10 @@ test "fmt.format" { assert(mem.eql(u8, result, "u3: 5\n")); } { + // Dummy field because of https://github.com/zig-lang/zig/issues/557. + const Struct = struct { + unused: u8, + }; var buf1: [32]u8 = undefined; const value = Struct { .unused = 42, diff --git a/test/cases/misc.zig b/test/cases/misc.zig index 964c5babc1..5e453fcbc1 100644 --- a/test/cases/misc.zig +++ b/test/cases/misc.zig @@ -499,12 +499,29 @@ test "@canImplicitCast" { } test "@typeName" { + const Struct = struct { + }; + const Union = union { + unused: u8, + }; + const Enum = enum { + Unused, + }; comptime { assert(mem.eql(u8, @typeName(i64), "i64")); assert(mem.eql(u8, @typeName(&usize), "&usize")); + // https://github.com/zig-lang/zig/issues/675 + assert(mem.eql(u8, @typeName(TypeFromFn(u8)), "TypeFromFn(u8)")); + assert(mem.eql(u8, @typeName(Struct), "Struct")); + assert(mem.eql(u8, @typeName(Union), "Union")); + assert(mem.eql(u8, @typeName(Enum), "Enum")); } } +fn TypeFromFn(comptime T: type) type { + return struct {}; +} + test "volatile load and store" { var number: i32 = 1234; const ptr = (&volatile i32)(&number);