From 5118caf5ab2599ca61b57f68a89aa2d094f51981 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Sat, 31 Mar 2018 00:53:00 +0200 Subject: [PATCH 01/87] Added a lot of test cases --- std/zig/parser.zig | 150 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 149 insertions(+), 1 deletion(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 62c62ed185..3b532d7030 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -1648,6 +1648,13 @@ test "zig fmt" { \\ ); + try testCanonical( + \\extern fn f1(s: [][]align(1) []const []volatile u8) c_int; + \\extern fn f2(s: []align(1) const []align(1) volatile []const volatile u8) c_int; + \\extern fn f3(s: []align(1) const volatile u8) c_int; + \\ + ); + try testCanonical( \\fn f1(a: bool, b: bool) bool { \\ a != b; @@ -1716,7 +1723,7 @@ test "zig fmt" { try testCanonical( \\test "prefix operators" { - \\ --%~??!*&0; + \\ try return --%~??!*&0; \\} \\ ); @@ -1730,4 +1737,145 @@ test "zig fmt" { \\} \\ ); + + try testCanonical( + \\test "test index" { + \\ a[0]; + \\ a[0 + 5]; + \\ a[0..]; + \\ a[0..5]; + \\} + \\ + ); + + try testCanonical( + \\test "test array" { + \\ const a : [2]u8 = [2]u8{ 1, 2 }; + \\ const a : [2]u8 = []u8{ 1, 2 }; + \\ const a : [0]u8 = []u8{}; + \\} + \\ + ); + +// PrimaryExpression = Integer | Float | String | CharLiteral | KeywordLiteral | GroupedExpression | BlockExpression(BlockOrExpression) | Symbol | ("@" Symbol FnCallExpression) | ArrayType | FnProto | AsmExpression | ContainerDecl | ("continue" option(":" Symbol)) | ErrorSetDecl | PromiseType + try testCanonical( + \\test "values" { + \\ 1; + \\ 1.0; + \\ "string"; + \\ c"cstring"; + \\ \\ Multi + \\ \\ line + \\ \\ string + \\ ; + \\ 'c'; + \\ true; + \\ false; + \\ null; + \\ undefined; + \\ error; + \\ this; + \\ unreachable; + \\ suspend; + \\} + \\ + ); + + try testCanonical( + \\test "percendence" { + \\ a!b(); + \\ (a!b)(); + \\ !a!b; + \\ !(a!b); + \\ !a{}; + \\ !(a{}); + \\ a + b{}; + \\ (a + b){}; + \\ a << b + c; + \\ (a << b) + c; + \\ a & b << c; + \\ (a & b) << c; + \\ a ^ b & c; + \\ (a ^ b) & c; + \\ a | b ^ c; + \\ (a | b) ^ c; + \\ a == b | c; + \\ (a == b) | c; + \\ a and b == c; + \\ (a and b) == c; + \\ a or b and c; + \\ (a or b) and c; + \\ (a or b) and c; + \\ a = b or c; + \\ (a = b) or c; + \\} + \\ + ); + + try testCanonical( + \\const S = struct { + \\ const Self = this; + \\ f1: u8, + \\ + \\ fn method(self: &Self) Self { + \\ return *self; + \\ } + \\ + \\ f2: u8, + \\}; + \\ + \\const Ps = packed struct { + \\ a: u8, + \\ b: u8, + \\ + \\ c: u8, + \\}; + \\ + \\const Es = extern struct { + \\ a: u8, + \\ b: u8, + \\ + \\ c: u8, + \\}; + \\ + ); + + try testCanonical( + \\const E = enum { + \\ Ok, + \\ SomethingElse = 0, + \\}; + \\ + \\const E2 = enum(u8) { + \\ Ok, + \\ SomethingElse = 255, + \\ SomethingThird, + \\}; + \\ + \\const Ee = extern enum { + \\ Ok, + \\ SomethingElse, + \\ SomethingThird, + \\}; + \\ + \\const Ep = packed enum { + \\ Ok, + \\ SomethingElse, + \\ SomethingThird, + \\}; + \\ + ); + + try testCanonical( + \\const a1 = []u8{ }; + \\const a2 = []u8{ 1, 2, 3, 4 }; + \\const s1 = S{ }; + \\const s2 = S{ .a = 1, .b = 2, }; + \\ + ); + + try testCanonical(@embedFile("ast.zig")); + try testCanonical(@embedFile("index.zig")); + try testCanonical(@embedFile("parser.zig")); + try testCanonical(@embedFile("tokenizer.zig")); } From 596f4b6002d3ed29adc8777b0417952cb79ad888 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Sat, 31 Mar 2018 14:00:49 +0200 Subject: [PATCH 02/87] Fixed review commented code --- std/zig/parser.zig | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 3b532d7030..b9246033dd 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -1750,9 +1750,9 @@ test "zig fmt" { try testCanonical( \\test "test array" { - \\ const a : [2]u8 = [2]u8{ 1, 2 }; - \\ const a : [2]u8 = []u8{ 1, 2 }; - \\ const a : [0]u8 = []u8{}; + \\ const a: [2]u8 = [2]u8{ 1, 2 }; + \\ const a: [2]u8 = []u8{ 1, 2 }; + \\ const a: [0]u8 = []u8{}; \\} \\ ); @@ -1782,7 +1782,7 @@ test "zig fmt" { ); try testCanonical( - \\test "percendence" { + \\test "precendence" { \\ a!b(); \\ (a!b)(); \\ !a!b; From 26e56f2fab7c56829d51db1afb10acb0306a101f Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Sat, 31 Mar 2018 14:18:09 +0200 Subject: [PATCH 03/87] Each test now have it's own test name --- std/zig/parser.zig | 122 +++++++++++++++++++++++++++++---------------- 1 file changed, 78 insertions(+), 44 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index b9246033dd..9b493d3b4b 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -1557,7 +1557,7 @@ fn testCanonical(source: []const u8) !void { } } -test "zig fmt" { +test "zig fmt: get stdout or fail" { try testCanonical( \\const std = @import("std"); \\ @@ -1568,7 +1568,9 @@ test "zig fmt" { \\} \\ ); +} +test "zig fmt: preserve spacing" { try testCanonical( \\const std = @import("std"); \\ @@ -1581,25 +1583,33 @@ test "zig fmt" { \\} \\ ); +} +test "zig fmt: return types" { try testCanonical( \\pub fn main() !void {} \\pub fn main() var {} \\pub fn main() i32 {} \\ ); +} +test "zig fmt: imports" { try testCanonical( \\const std = @import("std"); \\const std = @import(); \\ ); +} +test "zig fmt: extern function" { try testCanonical( \\extern fn puts(s: &const u8) c_int; \\ ); +} +test "zig fmt: global declarations" { try testCanonical( \\const a = b; \\pub const a = b; @@ -1611,66 +1621,71 @@ test "zig fmt" { \\pub var a: i32 = b; \\ ); +} +test "zig fmt: extern declaration" { try testCanonical( \\extern var foo: c_int; \\ ); +} - try testCanonical( +test "zig fmt: alignment" { + try testCanonical( \\var foo: c_int align(1); \\ ); +} +test "zig fmt: C main" { try testCanonical( \\fn main(argc: c_int, argv: &&u8) c_int { \\ const a = b; \\} \\ ); +} +test "zig fmt: return" { try testCanonical( \\fn foo(argc: c_int, argv: &&u8) c_int { \\ return 0; \\} \\ ); +} +test "zig fmt: pointer attributes" { try testCanonical( \\extern fn f1(s: &align(&u8) u8) c_int; + \\extern fn f2(s: &&align(1) &const &volatile u8) c_int; + \\extern fn f3(s: &align(1) const &align(1) volatile &const volatile u8) c_int; + \\extern fn f4(s: &align(1) const volatile u8) c_int; \\ ); +} +test "zig fmt: slice attributes" { try testCanonical( - \\extern fn f1(s: &&align(1) &const &volatile u8) c_int; - \\extern fn f2(s: &align(1) const &align(1) volatile &const volatile u8) c_int; - \\extern fn f3(s: &align(1) const volatile u8) c_int; + \\extern fn f1(s: &align(&u8) u8) c_int; + \\extern fn f2(s: &&align(1) &const &volatile u8) c_int; + \\extern fn f3(s: &align(1) const &align(1) volatile &const volatile u8) c_int; + \\extern fn f4(s: &align(1) const volatile u8) c_int; \\ ); +} - try testCanonical( - \\extern fn f1(s: [][]align(1) []const []volatile u8) c_int; - \\extern fn f2(s: []align(1) const []align(1) volatile []const volatile u8) c_int; - \\extern fn f3(s: []align(1) const volatile u8) c_int; - \\ - ); - - try testCanonical( - \\fn f1(a: bool, b: bool) bool { - \\ a != b; - \\ return a == b; - \\} - \\ - ); - - try testCanonical( +test "zig fmt: test declaration" { + try testCanonical( \\test "test name" { \\ const a = 1; \\ var b = 1; \\} \\ ); +} +test "zig fmt: infix operators" { try testCanonical( \\test "infix operators" { \\ var i = undefined; @@ -1720,14 +1735,18 @@ test "zig fmt" { \\} \\ ); +} +test "zig fmt: prefix operators" { try testCanonical( \\test "prefix operators" { \\ try return --%~??!*&0; \\} \\ ); +} +test "zig fmt: call expression" { try testCanonical( \\test "test calls" { \\ a(); @@ -1737,27 +1756,9 @@ test "zig fmt" { \\} \\ ); +} - try testCanonical( - \\test "test index" { - \\ a[0]; - \\ a[0 + 5]; - \\ a[0..]; - \\ a[0..5]; - \\} - \\ - ); - - try testCanonical( - \\test "test array" { - \\ const a: [2]u8 = [2]u8{ 1, 2 }; - \\ const a: [2]u8 = []u8{ 1, 2 }; - \\ const a: [0]u8 = []u8{}; - \\} - \\ - ); - -// PrimaryExpression = Integer | Float | String | CharLiteral | KeywordLiteral | GroupedExpression | BlockExpression(BlockOrExpression) | Symbol | ("@" Symbol FnCallExpression) | ArrayType | FnProto | AsmExpression | ContainerDecl | ("continue" option(":" Symbol)) | ErrorSetDecl | PromiseType +test "zig fmt: values" { try testCanonical( \\test "values" { \\ 1; @@ -1780,9 +1781,34 @@ test "zig fmt" { \\} \\ ); +} +test "zig fmt: indexing" { try testCanonical( - \\test "precendence" { + \\test "test index" { + \\ a[0]; + \\ a[0 + 5]; + \\ a[0..]; + \\ a[0..5]; + \\} + \\ + ); +} + +test "zig fmt: arrays" { + try testCanonical( + \\test "test array" { + \\ const a: [2]u8 = [2]u8{ 1, 2 }; + \\ const a: [2]u8 = []u8{ 1, 2 }; + \\ const a: [0]u8 = []u8{}; + \\} + \\ + ); +} + +test "zig fmt: precedence" { + try testCanonical( + \\test "precedence" { \\ a!b(); \\ (a!b)(); \\ !a!b; @@ -1811,7 +1837,9 @@ test "zig fmt" { \\} \\ ); +} +test "zig fmt: struct declaration" { try testCanonical( \\const S = struct { \\ const Self = this; @@ -1839,8 +1867,10 @@ test "zig fmt" { \\}; \\ ); +} - try testCanonical( +test "zig fmt: enum declaration" { + try testCanonical( \\const E = enum { \\ Ok, \\ SomethingElse = 0, @@ -1865,7 +1895,9 @@ test "zig fmt" { \\}; \\ ); +} +test "zig fmt: container initializers" { try testCanonical( \\const a1 = []u8{ }; \\const a2 = []u8{ 1, 2, 3, 4 }; @@ -1873,9 +1905,11 @@ test "zig fmt" { \\const s2 = S{ .a = 1, .b = 2, }; \\ ); +} +test "zig fmt: zig fmt" { try testCanonical(@embedFile("ast.zig")); try testCanonical(@embedFile("index.zig")); try testCanonical(@embedFile("parser.zig")); try testCanonical(@embedFile("tokenizer.zig")); -} +} \ No newline at end of file From cda35093537221abe7c148c57b3a28892046b8b2 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Sat, 31 Mar 2018 15:39:51 +0200 Subject: [PATCH 04/87] Added test cases to cover all of zigs syntax --- std/zig/parser.zig | 352 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 345 insertions(+), 7 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 9b493d3b4b..22e87dd178 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -1602,13 +1602,6 @@ test "zig fmt: imports" { ); } -test "zig fmt: extern function" { - try testCanonical( - \\extern fn puts(s: &const u8) c_int; - \\ - ); -} - test "zig fmt: global declarations" { try testCanonical( \\const a = b; @@ -1758,6 +1751,21 @@ test "zig fmt: call expression" { ); } +test "zig fmt: var args" { + try testCanonical( + \\fn print(args: ...) void {} + \\ + ); +} + +test "zig fmt: extern function" { + try testCanonical( + \\extern fn puts(s: &const u8) c_int; + \\extern "c" fn puts(s: &const u8) c_int; + \\ + ); +} + test "zig fmt: values" { try testCanonical( \\test "values" { @@ -1897,6 +1905,41 @@ test "zig fmt: enum declaration" { ); } +test "zig fmt: union declaration" { + try testCanonical( + \\const U = union { + \\ Int: u8, + \\ Float: f32, + \\ Bool: bool, + \\}; + \\ + \\const Ue = union(enum) { + \\ Int: u8, + \\ Float: f32, + \\ Bool: bool, + \\}; + \\ + \\const E = enum { + \\ Int, + \\ Float, + \\ Bool, + \\}; + \\ + \\const Ue2 = union(E) { + \\ Int: u8, + \\ Float: f32, + \\ Bool: bool, + \\}; + \\ + \\const Eu = extern union { + \\ Int: u8, + \\ Float: f32, + \\ Bool: bool, + \\}; + \\ + ); +} + test "zig fmt: container initializers" { try testCanonical( \\const a1 = []u8{ }; @@ -1907,6 +1950,301 @@ test "zig fmt: container initializers" { ); } +test "zig fmt: switch" { + try testCanonical( + \\test "switch" { + \\ switch (0) { + \\ 0 => {}, + \\ 1 => unreachable, + \\ 2, 3 => {}, + \\ 4 ... 7 => {}, + \\ 1 + 4 * 3 + 22 => {}, + \\ else => { + \\ const a = 1; + \\ const b = a; + \\ }, + \\ } + \\ + \\ const res = switch (0) { + \\ 0 => 0, + \\ 1 => 2, + \\ else => 4, + \\ }; + \\ + \\ const Union = union(enum) { + \\ Int: i64, + \\ Float: f64, + \\ }; + \\ + \\ const u = Union { .Int = 0 }; + \\ switch (u) { + \\ Union.Int => |int| {}, + \\ Union.Float => |*float| unreachable, + \\ } + \\} + \\ + ); +} + +test "zig fmt: while" { + try testCanonical( + \\test "while" { + \\ while (10 < 1) { + \\ unreachable; + \\ } + \\ + \\ while (10 < 1) + \\ unreachable; + \\ + \\ var i: usize = 0; + \\ while (i < 10) : (i += 1) { + \\ continue; + \\ } + \\ + \\ i = 0; + \\ while (i < 10) : (i += 1) + \\ continue; + \\ + \\ i = 0; + \\ var j usize = 0; + \\ while (i < 10) : ({ i += 1; j += 1; }) { + \\ continue; + \\ } + \\ + \\ var a: ?u8 = 2; + \\ while (a) |v| : (a = null) { + \\ continue; + \\ } + \\ + \\ while (a) |v| : (a = null) + \\ unreachable; + \\ + \\ label: while (10 < 0) { + \\ unreachable; + \\ } + \\ + \\ const res = while (0 < 10) { + \\ break 7; + \\ } else { + \\ unreachable; + \\ } + \\ + \\ var a: error!u8 = 0; + \\ while (a) |v| { + \\ a = error.Err; + \\ } else |err| { + \\ i = 1; + \\ } + \\ + \\ comptime var k: usize = 0; + \\ inline while (i < 10) (i += 1) + \\ j += 2; + \\} + \\ + ); +} + +test "zig fmt: for" { + try testCanonical( + \\test "for" { + \\ const a = []u8{ 1, 2, 3 }; + \\ for (a) |v| { + \\ continue; + \\ } + \\ + \\ for (a) |v| + \\ continue; + \\ + \\ for (a) |*v| + \\ continue; + \\ + \\ for (a) |v, i| { + \\ continue; + \\ } + \\ + \\ for (a) |v, i| + \\ continue; + \\ + \\ const res = for (a) |v, i| { + \\ breal v; + \\ } else { + \\ unreachable; + \\ } + \\ + \\ var num: usize = 0; + \\ inline for (a) |v, i| { + \\ num += v; + \\ num += i; + \\ } + \\} + \\ + ); +} + +test "zig fmt: if" { + try testCanonical( + \\test "if" { + \\ if (10 < 0) { + \\ unreachable; + \\ } + \\ + \\ if (10 < 0) + \\ unreachable; + \\ + \\ if (10 < 0) { + \\ unreachable; + \\ } else { + \\ const a = 20; + \\ } + \\ + \\ if (10 < 0) { + \\ unreachable; + \\ } else if (5 < 0) { + \\ unreachable; + \\ } else { + \\ const a = 20; + \\ } + \\ + \\ const is_world_broken = if (10 < 0) true else false; + \\ + \\ const a: ?u8 = 10; + \\ const b: ?u8 = null; + \\ if (a) |v| { + \\ const some = v; + \\ } else if (b) |*v| { + \\ unreachable; + \\ } else { + \\ const some = 10; + \\ } + \\ + \\ const non_null_a = if (a) |v| v else 0; + \\ + \\ const a_err: error!u8 = 0; + \\ if (a_err) |v| { + \\ const p = v; + \\ } else |err| { + \\ unreachable; + \\ } + \\} + \\ + ); +} + +test "zig fmt: defer" { + try testCanonical( + \\test "defer" { + \\ var i: usize = 0; + \\ defer i = 1; + \\ defer { + \\ i += 2; + \\ i *= i; + \\ } + \\ + \\ errdefer i += 3; + \\ errdefer { + \\ i += 2; + \\ i /= i; + \\ } + \\} + \\ + ); +} + +test "zig fmt: catch" { + try testCanonical( + \\test "catch" { + \\ const a: error!u8 = 0; + \\ _ = a catch return; + \\ _ = a catch |err| return; + \\} + \\ + ); +} + +test "zig fmt: comptime" { + try testCanonical( + \\fn a() u8 { + \\ return 5; + \\} + \\ + \\fn b(comptime i: u8) u8 { + \\ return i; + \\} + \\ + \\const av = comptime a(); + \\const av2 = comptime blk: { + \\ var res = a(); + \\ res *= b(2); + \\ break :blk res; + \\}; + \\ + \\comptime { + \\ _ = a(); + \\} + \\ + \\test "comptime" { + \\ const av3 = comptime a(); + \\ const av4 = comptime blk: { + \\ var res = a(); + \\ res *= a(); + \\ break :blk res; + \\ }; + \\ + \\ comptime var i = 0; + \\ comptime { + \\ i = a(); + \\ i += b(i); + \\ } + \\} + \\ + ); +} + +test "zig fmt: fn type" { + try testCanonical( + \\fn a(i: u8) u8 { + \\ return i + 1; + \\} + \\ + \\const ap: fn(u8) u8 = a; + \\ + ); +} + +test "zig fmt: inline asm" { + try testCanonical( + \\pub fn syscall1(number: usize, arg1: usize) usize { + \\ return asm volatile ("syscall" + \\ : [ret] "={rax}" (-> usize) + \\ : [number] "{rax}" (number), + \\ [arg1] "{rdi}" (arg1) + \\ : "rcx", "r11"); + \\} + \\ + ); +} + +test "zig fmt: coroutines" { + try testCanonical( + \\async fn simpleAsyncFn() void { + \\ x += 1; + \\ suspend; + \\ x += 1; + \\ suspend |p| { + \\ } + \\ const p = async simpleAsyncFn() cache unreachable; + \\ await p; + \\} + \\ + \\test "coroutine suspend, resume, cancel" { + \\ const p = try async testAsyncSeq(); + \\ resume p; + \\ cancel p; + \\} + \\ + ); +} + test "zig fmt: zig fmt" { try testCanonical(@embedFile("ast.zig")); try testCanonical(@embedFile("index.zig")); From 4793c3397e17169317d2feded9ca6901ff0e99e8 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Sat, 31 Mar 2018 17:46:29 +0200 Subject: [PATCH 05/87] std.zig.parser now handles lib name for extern var and fn --- std/zig/ast.zig | 3 +- std/zig/parser.zig | 162 ++++++++++++++++++++++++++++----------------- 2 files changed, 102 insertions(+), 63 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 715a333c0f..e64e5931c3 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -624,7 +624,7 @@ pub const NodeLineComment = struct { pub const NodeTestDecl = struct { base: Node, test_token: Token, - name_token: Token, + name: &Node, body_node: &Node, pub fn iterate(self: &NodeTestDecl, index: usize) ?&Node { @@ -644,4 +644,3 @@ pub const NodeTestDecl = struct { return self.body_node.lastToken(); } }; - diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 22e87dd178..1bad72aea4 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -55,6 +55,7 @@ pub const Parser = struct { const TopLevelDeclCtx = struct { visib_token: ?Token, extern_token: ?Token, + lib_name: ?&ast.Node, }; const DestPtr = union(enum) { @@ -183,8 +184,9 @@ pub const Parser = struct { if (lbrace.id != Token.Id.LBrace) return self.parseError(token, "expected {}, found {}", @tagName(Token.Id.LBrace), @tagName(name_token.id)); + const name = try self.createStringLiteral(arena, name_token); const block = try self.createBlock(arena, token); - const test_decl = try self.createAttachTestDecl(arena, &root_node.decls, token, name_token, block); + const test_decl = try self.createAttachTestDecl(arena, &root_node.decls, token, &name.base, block); try stack.append(State { .Block = block }); continue; }, @@ -202,10 +204,22 @@ pub const Parser = struct { State.TopLevelExtern => |visib_token| { const token = self.getNextToken(); if (token.id == Token.Id.Keyword_extern) { + const lib_name_token = self.getNextToken(); + const lib_name = blk: { + if (lib_name_token.id == Token.Id.StringLiteral) { + const res = try self.createStringLiteral(arena, lib_name_token); + break :blk &res.base; + } else { + self.putBackToken(lib_name_token); + break :blk null; + } + }; + stack.append(State { .TopLevelDecl = TopLevelDeclCtx { .visib_token = visib_token, .extern_token = token, + .lib_name = lib_name, }, }) catch unreachable; continue; @@ -215,6 +229,7 @@ pub const Parser = struct { .TopLevelDecl = TopLevelDeclCtx { .visib_token = visib_token, .extern_token = null, + .lib_name = null, }, }) catch unreachable; continue; @@ -226,7 +241,7 @@ pub const Parser = struct { stack.append(State.TopLevel) catch unreachable; // TODO shouldn't need these casts const var_decl_node = try self.createAttachVarDecl(arena, &root_node.decls, ctx.visib_token, - token, (?Token)(null), ctx.extern_token); + token, (?Token)(null), ctx.extern_token, ctx.lib_name); try stack.append(State { .VarDecl = var_decl_node }); continue; }, @@ -234,20 +249,17 @@ pub const Parser = struct { stack.append(State.TopLevel) catch unreachable; // TODO shouldn't need these casts const fn_proto = try self.createAttachFnProto(arena, &root_node.decls, token, - ctx.extern_token, (?Token)(null), ctx.visib_token, (?Token)(null)); + ctx.extern_token, ctx.lib_name, (?Token)(null), ctx.visib_token, (?Token)(null)); try stack.append(State { .FnDef = fn_proto }); try stack.append(State { .FnProto = fn_proto }); continue; }, - Token.Id.StringLiteral => { - @panic("TODO extern with string literal"); - }, Token.Id.Keyword_nakedcc, Token.Id.Keyword_stdcallcc => { stack.append(State.TopLevel) catch unreachable; const fn_token = try self.eatToken(Token.Id.Keyword_fn); // TODO shouldn't need this cast const fn_proto = try self.createAttachFnProto(arena, &root_node.decls, fn_token, - ctx.extern_token, (?Token)(token), (?Token)(null), (?Token)(null)); + ctx.extern_token, ctx.lib_name, (?Token)(token), (?Token)(null), (?Token)(null)); try stack.append(State { .FnDef = fn_proto }); try stack.append(State { .FnProto = fn_proto }); continue; @@ -437,11 +449,7 @@ pub const Parser = struct { continue; }, Token.Id.StringLiteral => { - const node = try arena.create(ast.NodeStringLiteral); - *node = ast.NodeStringLiteral { - .base = self.initNode(ast.Node.Id.StringLiteral), - .token = token, - }; + const node = try self.createStringLiteral(arena, token); try stack.append(State { .Operand = &node.base }); @@ -722,7 +730,7 @@ pub const Parser = struct { if (mut_token.id == Token.Id.Keyword_var or mut_token.id == Token.Id.Keyword_const) { // TODO shouldn't need these casts const var_decl = try self.createAttachVarDecl(arena, &block.statements, (?Token)(null), - mut_token, (?Token)(comptime_token), (?Token)(null)); + mut_token, (?Token)(comptime_token), (?Token)(null), null); try stack.append(State { .VarDecl = var_decl }); continue; } @@ -736,7 +744,7 @@ pub const Parser = struct { if (mut_token.id == Token.Id.Keyword_var or mut_token.id == Token.Id.Keyword_const) { // TODO shouldn't need these casts const var_decl = try self.createAttachVarDecl(arena, &block.statements, (?Token)(null), - mut_token, (?Token)(null), (?Token)(null)); + mut_token, (?Token)(null), (?Token)(null), null); try stack.append(State { .VarDecl = var_decl }); continue; } @@ -852,7 +860,7 @@ pub const Parser = struct { } fn createVarDecl(self: &Parser, arena: &mem.Allocator, visib_token: &const ?Token, mut_token: &const Token, - comptime_token: &const ?Token, extern_token: &const ?Token) !&ast.NodeVarDecl + comptime_token: &const ?Token, extern_token: &const ?Token, lib_name: ?&ast.Node) !&ast.NodeVarDecl { const node = try arena.create(ast.NodeVarDecl); @@ -865,7 +873,7 @@ pub const Parser = struct { .type_node = null, .align_node = null, .init_node = null, - .lib_name = null, + .lib_name = lib_name, // initialized later .name_token = undefined, .eq_token = undefined, @@ -874,7 +882,18 @@ pub const Parser = struct { return node; } - fn createTestDecl(self: &Parser, arena: &mem.Allocator, test_token: &const Token, name_token: &const Token, + fn createStringLiteral(self: &Parser, arena: &mem.Allocator, token: &const Token) !&ast.NodeStringLiteral { + const node = try arena.create(ast.NodeStringLiteral); + + assert(token.id == Token.Id.StringLiteral); + *node = ast.NodeStringLiteral { + .base = self.initNode(ast.Node.Id.StringLiteral), + .token = *token, + }; + return node; + } + + fn createTestDecl(self: &Parser, arena: &mem.Allocator, test_token: &const Token, name: &ast.Node, block: &ast.NodeBlock) !&ast.NodeTestDecl { const node = try arena.create(ast.NodeTestDecl); @@ -882,14 +901,14 @@ pub const Parser = struct { *node = ast.NodeTestDecl { .base = self.initNode(ast.Node.Id.TestDecl), .test_token = *test_token, - .name_token = *name_token, + .name = name, .body_node = &block.base, }; return node; } fn createFnProto(self: &Parser, arena: &mem.Allocator, fn_token: &const Token, extern_token: &const ?Token, - cc_token: &const ?Token, visib_token: &const ?Token, inline_token: &const ?Token) !&ast.NodeFnProto + lib_name: ?&ast.Node, cc_token: &const ?Token, visib_token: &const ?Token, inline_token: &const ?Token) !&ast.NodeFnProto { const node = try arena.create(ast.NodeFnProto); @@ -905,7 +924,7 @@ pub const Parser = struct { .inline_token = *inline_token, .cc_token = *cc_token, .body_node = null, - .lib_name = null, + .lib_name = lib_name, .align_expr = null, }; return node; @@ -1015,27 +1034,27 @@ pub const Parser = struct { } fn createAttachFnProto(self: &Parser, arena: &mem.Allocator, list: &ArrayList(&ast.Node), fn_token: &const Token, - extern_token: &const ?Token, cc_token: &const ?Token, visib_token: &const ?Token, + extern_token: &const ?Token, lib_name: ?&ast.Node, cc_token: &const ?Token, visib_token: &const ?Token, inline_token: &const ?Token) !&ast.NodeFnProto { - const node = try self.createFnProto(arena, fn_token, extern_token, cc_token, visib_token, inline_token); + const node = try self.createFnProto(arena, fn_token, extern_token, lib_name, cc_token, visib_token, inline_token); try list.append(&node.base); return node; } fn createAttachVarDecl(self: &Parser, arena: &mem.Allocator, list: &ArrayList(&ast.Node), visib_token: &const ?Token, mut_token: &const Token, comptime_token: &const ?Token, - extern_token: &const ?Token) !&ast.NodeVarDecl + extern_token: &const ?Token, lib_name: ?&ast.Node) !&ast.NodeVarDecl { - const node = try self.createVarDecl(arena, visib_token, mut_token, comptime_token, extern_token); + const node = try self.createVarDecl(arena, visib_token, mut_token, comptime_token, extern_token, lib_name); try list.append(&node.base); return node; } fn createAttachTestDecl(self: &Parser, arena: &mem.Allocator, list: &ArrayList(&ast.Node), - test_token: &const Token, name_token: &const Token, block: &ast.NodeBlock) !&ast.NodeTestDecl + test_token: &const Token, name: &ast.Node, block: &ast.NodeBlock) !&ast.NodeTestDecl { - const node = try self.createTestDecl(arena, test_token, name_token, block); + const node = try self.createTestDecl(arena, test_token, name, block); try list.append(&node.base); return node; } @@ -1169,23 +1188,6 @@ pub const Parser = struct { switch (decl.id) { ast.Node.Id.FnProto => { const fn_proto = @fieldParentPtr(ast.NodeFnProto, "base", decl); - if (fn_proto.visib_token) |visib_token| { - switch (visib_token.id) { - Token.Id.Keyword_pub => try stream.print("pub "), - Token.Id.Keyword_export => try stream.print("export "), - else => unreachable, - } - } - if (fn_proto.extern_token) |extern_token| { - try stream.print("{} ", self.tokenizer.getTokenSlice(extern_token)); - } - try stream.print("fn"); - - if (fn_proto.name_token) |name_token| { - try stream.print(" {}", self.tokenizer.getTokenSlice(name_token)); - } - - try stream.print("("); if (fn_proto.body_node == null) { try stack.append(RenderState { .Text = ";" }); @@ -1201,6 +1203,27 @@ pub const Parser = struct { try stack.append(RenderState { .Text = ", " }); } } + + try stack.append(RenderState { .Text = "(" }); + if (fn_proto.name_token) |name_token| { + try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(name_token) }); + } + + try stack.append(RenderState { .Text = "fn " }); + if (fn_proto.lib_name) |lib_name| { + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Expression = lib_name }); + } + if (fn_proto.extern_token) |extern_token| { + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(extern_token) }); + } + + if (fn_proto.visib_token) |visib_token| { + assert(visib_token.id == Token.Id.Keyword_pub or visib_token.id == Token.Id.Keyword_export); + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(visib_token) }); + } }, ast.Node.Id.VarDecl => { const var_decl = @fieldParentPtr(ast.NodeVarDecl, "base", decl); @@ -1208,29 +1231,16 @@ pub const Parser = struct { }, ast.Node.Id.TestDecl => { const test_decl = @fieldParentPtr(ast.NodeTestDecl, "base", decl); - try stream.print("test {} ", self.tokenizer.getTokenSlice(test_decl.name_token)); + try stream.print("test "); try stack.append(RenderState { .Expression = test_decl.body_node }); + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Expression = test_decl.name }); }, else => unreachable, } }, RenderState.VarDecl => |var_decl| { - if (var_decl.visib_token) |visib_token| { - try stream.print("{} ", self.tokenizer.getTokenSlice(visib_token)); - } - if (var_decl.extern_token) |extern_token| { - try stream.print("{} ", self.tokenizer.getTokenSlice(extern_token)); - if (var_decl.lib_name != null) { - @panic("TODO"); - } - } - if (var_decl.comptime_token) |comptime_token| { - try stream.print("{} ", self.tokenizer.getTokenSlice(comptime_token)); - } - try stream.print("{} ", self.tokenizer.getTokenSlice(var_decl.mut_token)); - try stream.print("{}", self.tokenizer.getTokenSlice(var_decl.name_token)); - try stack.append(RenderState { .Text = ";" }); if (var_decl.init_node) |init_node| { try stack.append(RenderState { .Expression = init_node }); @@ -1242,8 +1252,30 @@ pub const Parser = struct { try stack.append(RenderState { .Text = " align(" }); } if (var_decl.type_node) |type_node| { - try stream.print(": "); try stack.append(RenderState { .Expression = type_node }); + try stack.append(RenderState { .Text = ": " }); + } + try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(var_decl.name_token) }); + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(var_decl.mut_token) }); + + if (var_decl.comptime_token) |comptime_token| { + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(comptime_token) }); + } + + if (var_decl.extern_token) |extern_token| { + if (var_decl.lib_name != null) { + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Expression = ??var_decl.lib_name }); + } + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(extern_token) }); + } + + if (var_decl.visib_token) |visib_token| { + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(visib_token) }); } }, @@ -1612,6 +1644,14 @@ test "zig fmt: global declarations" { \\pub const a: i32 = b; \\var a: i32 = b; \\pub var a: i32 = b; + \\extern const a: i32 = b; + \\pub extern const a: i32 = b; + \\extern var a: i32 = b; + \\pub extern var a: i32 = b; + \\extern "a" const a: i32 = b; + \\pub extern "a" const a: i32 = b; + \\extern "a" var a: i32 = b; + \\pub extern "a" var a: i32 = b; \\ ); } @@ -2232,7 +2272,7 @@ test "zig fmt: coroutines" { \\ x += 1; \\ suspend |p| { \\ } - \\ const p = async simpleAsyncFn() cache unreachable; + \\ const p = async simpleAsyncFn() catch unreachable; \\ await p; \\} \\ From 4d8f9e2295bb672f70550b3bb5a4cb667f68bb70 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Sat, 31 Mar 2018 21:04:54 +0200 Subject: [PATCH 06/87] std.zig.parser now parses multi line strings --- std/zig/ast.zig | 21 +++++++++++++++++ std/zig/parser.zig | 52 +++++++++++++++++++++++++++++++++++++++---- std/zig/tokenizer.zig | 40 ++++++++++++++++++++++++++++++++- 3 files changed, 108 insertions(+), 5 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index e64e5931c3..75003b06cd 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -20,6 +20,7 @@ pub const Node = struct { IntegerLiteral, FloatLiteral, StringLiteral, + MultilineStringLiteral, UndefinedLiteral, BuiltinCall, Call, @@ -40,6 +41,7 @@ pub const Node = struct { Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).iterate(index), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).iterate(index), + Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).iterate(index), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).iterate(index), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index), Id.Call => @fieldParentPtr(NodeCall, "base", base).iterate(index), @@ -61,6 +63,7 @@ pub const Node = struct { Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).firstToken(), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(), + Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).firstToken(), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).firstToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(), Id.Call => @fieldParentPtr(NodeCall, "base", base).firstToken(), @@ -82,6 +85,7 @@ pub const Node = struct { Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).lastToken(), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(), + Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).lastToken(), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).lastToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(), Id.Call => @fieldParentPtr(NodeCall, "base", base).lastToken(), @@ -587,6 +591,23 @@ pub const NodeStringLiteral = struct { } }; +pub const NodeMultilineStringLiteral = struct { + base: Node, + tokens: ArrayList(Token), + + pub fn iterate(self: &NodeMultilineStringLiteral, index: usize) ?&Node { + return null; + } + + pub fn firstToken(self: &NodeMultilineStringLiteral) Token { + return self.tokens.at(0); + } + + pub fn lastToken(self: &NodeMultilineStringLiteral) Token { + return self.tokens.at(self.tokens.len - 1); + } +}; + pub const NodeUndefinedLiteral = struct { base: Node, token: Token, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 1bad72aea4..fa09a26dbb 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -456,6 +456,30 @@ pub const Parser = struct { try stack.append(State.AfterOperand); continue; }, + Token.Id.MultilineStringLiteralLine => { + const node = try arena.create(ast.NodeMultilineStringLiteral); + *node = ast.NodeMultilineStringLiteral { + .base = self.initNode(ast.Node.Id.MultilineStringLiteral), + .tokens = ArrayList(Token).init(arena), + }; + try node.tokens.append(token); + + while (true) { + const multiline_str = self.getNextToken(); + if (multiline_str.id != Token.Id.MultilineStringLiteralLine) { + self.putBackToken(multiline_str); + break; + } + + try node.tokens.append(multiline_str); + } + + try stack.append(State { + .Operand = &node.base + }); + try stack.append(State.AfterOperand); + continue; + }, else => return self.parseError(token, "expected primary expression, found {}", @tagName(token.id)), } @@ -1427,6 +1451,20 @@ pub const Parser = struct { const string_literal = @fieldParentPtr(ast.NodeStringLiteral, "base", base); try stream.print("{}", self.tokenizer.getTokenSlice(string_literal.token)); }, + ast.Node.Id.MultilineStringLiteral => { + const multiline_str_literal = @fieldParentPtr(ast.NodeMultilineStringLiteral, "base", base); + try stream.print("\n"); + + var i : usize = 0; + indent += 4; + while (i < multiline_str_literal.tokens.len) : (i += 1) { + const t = multiline_str_literal.tokens.at(i); + try stream.writeByteNTimes(' ', indent); + try stream.print("{}", self.tokenizer.getTokenSlice(t)); + } + try stream.writeByteNTimes(' ', indent); + indent -= 4; + }, ast.Node.Id.UndefinedLiteral => { const undefined_literal = @fieldParentPtr(ast.NodeUndefinedLiteral, "base", base); try stream.print("{}", self.tokenizer.getTokenSlice(undefined_literal.token)); @@ -1806,6 +1844,16 @@ test "zig fmt: extern function" { ); } +test "zig fmt: multiline string" { + try testCanonical( + \\const s = + \\ \\ something + \\ \\ something else + \\ ; + \\ + ); +} + test "zig fmt: values" { try testCanonical( \\test "values" { @@ -1813,10 +1861,6 @@ test "zig fmt: values" { \\ 1.0; \\ "string"; \\ c"cstring"; - \\ \\ Multi - \\ \\ line - \\ \\ string - \\ ; \\ 'c'; \\ true; \\ false; diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig index 7a13d89975..5647fcb866 100644 --- a/std/zig/tokenizer.zig +++ b/std/zig/tokenizer.zig @@ -72,6 +72,7 @@ pub const Token = struct { Invalid, Identifier, StringLiteral: StrLitKind, + MultilineStringLiteralLine: StrLitKind, StringIdentifier, Eof, Builtin, @@ -225,6 +226,9 @@ pub const Tokenizer = struct { C, StringLiteral, StringLiteralBackslash, + MultilineStringLiteralLine, + MultilineStringLiteralLineBackslash, + Backslash, Equal, Bang, Pipe, @@ -352,6 +356,10 @@ pub const Tokenizer = struct { '^' => { state = State.Caret; }, + '\\' => { + state = State.Backslash; + result.id = Token.Id { .MultilineStringLiteralLine = Token.StrLitKind.Normal }; + }, '{' => { result.id = Token.Id.LBrace; self.index += 1; @@ -532,8 +540,17 @@ pub const Tokenizer = struct { 'a'...'z', 'A'...'Z', '_', '0'...'9' => {}, else => break, }, + State.Backslash => switch (c) { + '\\' => { + state = State.MultilineStringLiteralLine; + }, + else => break, + }, State.C => switch (c) { - '\\' => @panic("TODO"), + '\\' => { + state = State.Backslash; + result.id = Token.Id { .MultilineStringLiteralLine = Token.StrLitKind.C }; + }, '"' => { state = State.StringLiteral; result.id = Token.Id { .StringLiteral = Token.StrLitKind.C }; @@ -562,6 +579,24 @@ pub const Tokenizer = struct { }, }, + State.MultilineStringLiteralLine => switch (c) { + '\\' => { + state = State.MultilineStringLiteralLineBackslash; + }, + '\n' => { + self.index += 1; + break; + }, + else => self.checkLiteralCharacter(), + }, + + State.MultilineStringLiteralLineBackslash => switch (c) { + '\n' => break, // Look for this error later. + else => { + state = State.MultilineStringLiteralLine; + }, + }, + State.Bang => switch (c) { '=' => { result.id = Token.Id.BangEqual; @@ -811,6 +846,7 @@ pub const Tokenizer = struct { State.FloatFraction, State.FloatExponentNumber, State.StringLiteral, // find this error later + State.MultilineStringLiteralLine, State.Builtin => {}, State.Identifier => { @@ -825,6 +861,8 @@ pub const Tokenizer = struct { State.NumberDot, State.FloatExponentUnsigned, State.SawAtSign, + State.Backslash, + State.MultilineStringLiteralLineBackslash, State.StringLiteralBackslash => { result.id = Token.Id.Invalid; }, From 975dc5a390490794df367b6a9869609a8b14d8f8 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Sat, 31 Mar 2018 21:28:40 +0200 Subject: [PATCH 07/87] std.zig.parser now parses char literals --- std/zig/ast.zig | 21 ++++++++++++++++++++ std/zig/parser.zig | 16 +++++++++++++++ std/zig/tokenizer.zig | 46 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 75003b06cd..1c84e12642 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -21,6 +21,7 @@ pub const Node = struct { FloatLiteral, StringLiteral, MultilineStringLiteral, + CharLiteral, UndefinedLiteral, BuiltinCall, Call, @@ -42,6 +43,7 @@ pub const Node = struct { Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).iterate(index), Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).iterate(index), + Id.CharLiteral => @fieldParentPtr(NodeCharLiteral, "base", base).iterate(index), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).iterate(index), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index), Id.Call => @fieldParentPtr(NodeCall, "base", base).iterate(index), @@ -64,6 +66,7 @@ pub const Node = struct { Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(), Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).firstToken(), + Id.CharLiteral => @fieldParentPtr(NodeCharLiteral, "base", base).firstToken(), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).firstToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(), Id.Call => @fieldParentPtr(NodeCall, "base", base).firstToken(), @@ -86,6 +89,7 @@ pub const Node = struct { Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(), Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).lastToken(), + Id.CharLiteral => @fieldParentPtr(NodeCharLiteral, "base", base).lastToken(), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).lastToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(), Id.Call => @fieldParentPtr(NodeCall, "base", base).lastToken(), @@ -608,6 +612,23 @@ pub const NodeMultilineStringLiteral = struct { } }; +pub const NodeCharLiteral = struct { + base: Node, + token: Token, + + pub fn iterate(self: &NodeCharLiteral, index: usize) ?&Node { + return null; + } + + pub fn firstToken(self: &NodeCharLiteral) Token { + return self.token; + } + + pub fn lastToken(self: &NodeCharLiteral) Token { + return self.token; + } +}; + pub const NodeUndefinedLiteral = struct { base: Node, token: Token, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index fa09a26dbb..a05328660b 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -456,6 +456,18 @@ pub const Parser = struct { try stack.append(State.AfterOperand); continue; }, + Token.Id.CharLiteral => { + const node = try arena.create(ast.NodeCharLiteral); + *node = ast.NodeCharLiteral { + .base = self.initNode(ast.Node.Id.CharLiteral), + .token = token, + }; + try stack.append(State { + .Operand = &node.base + }); + try stack.append(State.AfterOperand); + continue; + }, Token.Id.MultilineStringLiteralLine => { const node = try arena.create(ast.NodeMultilineStringLiteral); *node = ast.NodeMultilineStringLiteral { @@ -1451,6 +1463,10 @@ pub const Parser = struct { const string_literal = @fieldParentPtr(ast.NodeStringLiteral, "base", base); try stream.print("{}", self.tokenizer.getTokenSlice(string_literal.token)); }, + ast.Node.Id.CharLiteral => { + const char_literal = @fieldParentPtr(ast.NodeCharLiteral, "base", base); + try stream.print("{}", self.tokenizer.getTokenSlice(char_literal.token)); + }, ast.Node.Id.MultilineStringLiteral => { const multiline_str_literal = @fieldParentPtr(ast.NodeMultilineStringLiteral, "base", base); try stream.print("\n"); diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig index 5647fcb866..64eb627590 100644 --- a/std/zig/tokenizer.zig +++ b/std/zig/tokenizer.zig @@ -73,6 +73,7 @@ pub const Token = struct { Identifier, StringLiteral: StrLitKind, MultilineStringLiteralLine: StrLitKind, + CharLiteral, StringIdentifier, Eof, Builtin, @@ -228,6 +229,9 @@ pub const Tokenizer = struct { StringLiteralBackslash, MultilineStringLiteralLine, MultilineStringLiteralLineBackslash, + CharLiteral, + CharLiteralBackslash, + CharLiteralEnd, Backslash, Equal, Bang, @@ -294,6 +298,9 @@ pub const Tokenizer = struct { state = State.StringLiteral; result.id = Token.Id { .StringLiteral = Token.StrLitKind.Normal }; }, + '\'' => { + state = State.CharLiteral; + }, 'a'...'b', 'd'...'z', 'A'...'Z', '_' => { state = State.Identifier; result.id = Token.Id.Identifier; @@ -579,6 +586,35 @@ pub const Tokenizer = struct { }, }, + State.CharLiteral => switch (c) { + '\\' => { + state = State.CharLiteralBackslash; + }, + '\'' => break, // Look for this error later. + else => { + if (c < 0x20 or c == 0x7f) + break; // Look for this error later. + + state = State.CharLiteralEnd; + } + }, + + State.CharLiteralBackslash => switch (c) { + '\n' => break, // Look for this error later. + else => { + state = State.CharLiteralEnd; + }, + }, + + State.CharLiteralEnd => switch (c) { + '\'' => { + result.id = Token.Id.CharLiteral; + self.index += 1; + break; + }, + else => break, // Look for this error later. + }, + State.MultilineStringLiteralLine => switch (c) { '\\' => { state = State.MultilineStringLiteralLineBackslash; @@ -847,6 +883,7 @@ pub const Tokenizer = struct { State.FloatExponentNumber, State.StringLiteral, // find this error later State.MultilineStringLiteralLine, + State.CharLiteralEnd, State.Builtin => {}, State.Identifier => { @@ -863,6 +900,8 @@ pub const Tokenizer = struct { State.SawAtSign, State.Backslash, State.MultilineStringLiteralLineBackslash, + State.CharLiteral, + State.CharLiteralBackslash, State.StringLiteralBackslash => { result.id = Token.Id.Invalid; }, @@ -1006,9 +1045,16 @@ test "tokenizer" { }); } +test "tokenizer - chars" { + testTokenize("'c'", []Token.Id {Token.Id.CharLiteral}); +} + test "tokenizer - invalid token characters" { testTokenize("#", []Token.Id{Token.Id.Invalid}); testTokenize("`", []Token.Id{Token.Id.Invalid}); + testTokenize("'c", []Token.Id {Token.Id.Invalid}); + testTokenize("'", []Token.Id {Token.Id.Invalid}); + testTokenize("''", []Token.Id {Token.Id.Invalid}); } test "tokenizer - invalid literal/comment characters" { From aabf7cf57e62379d5f21aa013d0735c5427eace2 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Sat, 31 Mar 2018 22:10:49 +0200 Subject: [PATCH 08/87] std.zig.parser now parses null and bool literals --- std/zig/ast.zig | 42 ++++++++++++++++++++++++++++++++++++++++++ std/zig/parser.zig | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 1c84e12642..6c06b5423d 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -22,6 +22,8 @@ pub const Node = struct { StringLiteral, MultilineStringLiteral, CharLiteral, + BoolLiteral, + NullLiteral, UndefinedLiteral, BuiltinCall, Call, @@ -44,6 +46,8 @@ pub const Node = struct { Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).iterate(index), Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).iterate(index), Id.CharLiteral => @fieldParentPtr(NodeCharLiteral, "base", base).iterate(index), + Id.BoolLiteral => @fieldParentPtr(NodeBoolLiteral, "base", base).iterate(index), + Id.NullLiteral => @fieldParentPtr(NodeNullLiteral, "base", base).iterate(index), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).iterate(index), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index), Id.Call => @fieldParentPtr(NodeCall, "base", base).iterate(index), @@ -67,6 +71,8 @@ pub const Node = struct { Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(), Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).firstToken(), Id.CharLiteral => @fieldParentPtr(NodeCharLiteral, "base", base).firstToken(), + Id.BoolLiteral => @fieldParentPtr(NodeBoolLiteral, "base", base).firstToken(), + Id.NullLiteral => @fieldParentPtr(NodeNullLiteral, "base", base).firstToken(), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).firstToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(), Id.Call => @fieldParentPtr(NodeCall, "base", base).firstToken(), @@ -90,6 +96,8 @@ pub const Node = struct { Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(), Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).lastToken(), Id.CharLiteral => @fieldParentPtr(NodeCharLiteral, "base", base).lastToken(), + Id.BoolLiteral => @fieldParentPtr(NodeBoolLiteral, "base", base).lastToken(), + Id.NullLiteral => @fieldParentPtr(NodeNullLiteral, "base", base).lastToken(), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).lastToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(), Id.Call => @fieldParentPtr(NodeCall, "base", base).lastToken(), @@ -629,6 +637,40 @@ pub const NodeCharLiteral = struct { } }; +pub const NodeBoolLiteral = struct { + base: Node, + token: Token, + + pub fn iterate(self: &NodeBoolLiteral, index: usize) ?&Node { + return null; + } + + pub fn firstToken(self: &NodeBoolLiteral) Token { + return self.token; + } + + pub fn lastToken(self: &NodeBoolLiteral) Token { + return self.token; + } +}; + +pub const NodeNullLiteral = struct { + base: Node, + token: Token, + + pub fn iterate(self: &NodeNullLiteral, index: usize) ?&Node { + return null; + } + + pub fn firstToken(self: &NodeNullLiteral) Token { + return self.token; + } + + pub fn lastToken(self: &NodeNullLiteral) Token { + return self.token; + } +}; + pub const NodeUndefinedLiteral = struct { base: Node, token: Token, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index a05328660b..6c0a1f3f19 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -427,6 +427,30 @@ pub const Parser = struct { try stack.append(State.AfterOperand); continue; }, + Token.Id.Keyword_true, Token.Id.Keyword_false => { + const node = try arena.create(ast.NodeBoolLiteral); + *node = ast.NodeBoolLiteral { + .base = self.initNode(ast.Node.Id.BoolLiteral), + .token = token, + }; + try stack.append(State { + .Operand = &node.base + }); + try stack.append(State.AfterOperand); + continue; + }, + Token.Id.Keyword_null => { + const node = try arena.create(ast.NodeNullLiteral); + *node = ast.NodeNullLiteral { + .base = self.initNode(ast.Node.Id.NullLiteral), + .token = token, + }; + try stack.append(State { + .Operand = &node.base + }); + try stack.append(State.AfterOperand); + continue; + }, Token.Id.Builtin => { const node = try arena.create(ast.NodeBuiltinCall); *node = ast.NodeBuiltinCall { @@ -1467,6 +1491,14 @@ pub const Parser = struct { const char_literal = @fieldParentPtr(ast.NodeCharLiteral, "base", base); try stream.print("{}", self.tokenizer.getTokenSlice(char_literal.token)); }, + ast.Node.Id.BoolLiteral => { + const bool_literal = @fieldParentPtr(ast.NodeCharLiteral, "base", base); + try stream.print("{}", self.tokenizer.getTokenSlice(bool_literal.token)); + }, + ast.Node.Id.NullLiteral => { + const null_literal = @fieldParentPtr(ast.NodeNullLiteral, "base", base); + try stream.print("{}", self.tokenizer.getTokenSlice(null_literal.token)); + }, ast.Node.Id.MultilineStringLiteral => { const multiline_str_literal = @fieldParentPtr(ast.NodeMultilineStringLiteral, "base", base); try stream.print("\n"); From df09c01f7f141a384010d17ab23db3b36316f5b6 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Sat, 31 Mar 2018 22:48:12 +0200 Subject: [PATCH 09/87] std.zig.parser now parses error, this and unreachable --- std/zig/ast.zig | 63 ++++++++++++++++++++++++++++++++++++++++++++++ std/zig/parser.zig | 48 +++++++++++++++++++++++++++++++++++ 2 files changed, 111 insertions(+) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 6c06b5423d..8a9a072d67 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -25,6 +25,9 @@ pub const Node = struct { BoolLiteral, NullLiteral, UndefinedLiteral, + ThisLiteral, + Unreachable, + ErrorType, BuiltinCall, Call, LineComment, @@ -49,6 +52,9 @@ pub const Node = struct { Id.BoolLiteral => @fieldParentPtr(NodeBoolLiteral, "base", base).iterate(index), Id.NullLiteral => @fieldParentPtr(NodeNullLiteral, "base", base).iterate(index), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).iterate(index), + Id.ThisLiteral => @fieldParentPtr(NodeThisLiteral, "base", base).iterate(index), + Id.Unreachable => @fieldParentPtr(NodeUnreachable, "base", base).iterate(index), + Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).iterate(index), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index), Id.Call => @fieldParentPtr(NodeCall, "base", base).iterate(index), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).iterate(index), @@ -74,6 +80,9 @@ pub const Node = struct { Id.BoolLiteral => @fieldParentPtr(NodeBoolLiteral, "base", base).firstToken(), Id.NullLiteral => @fieldParentPtr(NodeNullLiteral, "base", base).firstToken(), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).firstToken(), + Id.Unreachable => @fieldParentPtr(NodeUnreachable, "base", base).firstToken(), + Id.ThisLiteral => @fieldParentPtr(NodeThisLiteral, "base", base).firstToken(), + Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).firstToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(), Id.Call => @fieldParentPtr(NodeCall, "base", base).firstToken(), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).firstToken(), @@ -99,6 +108,9 @@ pub const Node = struct { Id.BoolLiteral => @fieldParentPtr(NodeBoolLiteral, "base", base).lastToken(), Id.NullLiteral => @fieldParentPtr(NodeNullLiteral, "base", base).lastToken(), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).lastToken(), + Id.ThisLiteral => @fieldParentPtr(NodeThisLiteral, "base", base).lastToken(), + Id.Unreachable => @fieldParentPtr(NodeUnreachable, "base", base).lastToken(), + Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).lastToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(), Id.Call => @fieldParentPtr(NodeCall, "base", base).lastToken(), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).lastToken(), @@ -688,6 +700,57 @@ pub const NodeUndefinedLiteral = struct { } }; +pub const NodeThisLiteral = struct { + base: Node, + token: Token, + + pub fn iterate(self: &NodeThisLiteral, index: usize) ?&Node { + return null; + } + + pub fn firstToken(self: &NodeThisLiteral) Token { + return self.token; + } + + pub fn lastToken(self: &NodeThisLiteral) Token { + return self.token; + } +}; + +pub const NodeUnreachable = struct { + base: Node, + token: Token, + + pub fn iterate(self: &NodeUnreachable, index: usize) ?&Node { + return null; + } + + pub fn firstToken(self: &NodeUnreachable) Token { + return self.token; + } + + pub fn lastToken(self: &NodeUnreachable) Token { + return self.token; + } +}; + +pub const NodeErrorType = struct { + base: Node, + token: Token, + + pub fn iterate(self: &NodeErrorType, index: usize) ?&Node { + return null; + } + + pub fn firstToken(self: &NodeErrorType) Token { + return self.token; + } + + pub fn lastToken(self: &NodeErrorType) Token { + return self.token; + } +}; + pub const NodeLineComment = struct { base: Node, lines: ArrayList(Token), diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 6c0a1f3f19..0b8a15d3dd 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -451,6 +451,42 @@ pub const Parser = struct { try stack.append(State.AfterOperand); continue; }, + Token.Id.Keyword_this => { + const node = try arena.create(ast.NodeThisLiteral); + *node = ast.NodeThisLiteral { + .base = self.initNode(ast.Node.Id.ThisLiteral), + .token = token, + }; + try stack.append(State { + .Operand = &node.base + }); + try stack.append(State.AfterOperand); + continue; + }, + Token.Id.Keyword_unreachable => { + const node = try arena.create(ast.NodeUnreachable); + *node = ast.NodeUnreachable { + .base = self.initNode(ast.Node.Id.Unreachable), + .token = token, + }; + try stack.append(State { + .Operand = &node.base + }); + try stack.append(State.AfterOperand); + continue; + }, + Token.Id.Keyword_error => { + const node = try arena.create(ast.NodeErrorType); + *node = ast.NodeErrorType { + .base = self.initNode(ast.Node.Id.ErrorType), + .token = token, + }; + try stack.append(State { + .Operand = &node.base + }); + try stack.append(State.AfterOperand); + continue; + }, Token.Id.Builtin => { const node = try arena.create(ast.NodeBuiltinCall); *node = ast.NodeBuiltinCall { @@ -1499,6 +1535,18 @@ pub const Parser = struct { const null_literal = @fieldParentPtr(ast.NodeNullLiteral, "base", base); try stream.print("{}", self.tokenizer.getTokenSlice(null_literal.token)); }, + ast.Node.Id.ThisLiteral => { + const this_literal = @fieldParentPtr(ast.NodeThisLiteral, "base", base); + try stream.print("{}", self.tokenizer.getTokenSlice(this_literal.token)); + }, + ast.Node.Id.Unreachable => { + const unreachable_node = @fieldParentPtr(ast.NodeUnreachable, "base", base); + try stream.print("{}", self.tokenizer.getTokenSlice(unreachable_node.token)); + }, + ast.Node.Id.ErrorType => { + const error_type = @fieldParentPtr(ast.NodeErrorType, "base", base); + try stream.print("{}", self.tokenizer.getTokenSlice(error_type.token)); + }, ast.Node.Id.MultilineStringLiteral => { const multiline_str_literal = @fieldParentPtr(ast.NodeMultilineStringLiteral, "base", base); try stream.print("\n"); From b9093185f748293064e7eeaaa07e7479099420ed Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Sun, 1 Apr 2018 22:02:51 +0200 Subject: [PATCH 10/87] std.zig.parser now parses slicing and array access --- std/zig/ast.zig | 66 +++++++++++++++++++++++++++++++ std/zig/parser.zig | 92 +++++++++++++++++++++++++++++++++++++++++++ std/zig/tokenizer.zig | 12 ++++++ 3 files changed, 170 insertions(+) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 8a9a072d67..680e617962 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -30,6 +30,8 @@ pub const Node = struct { ErrorType, BuiltinCall, Call, + ArrayAccess, + SliceExpression, LineComment, TestDecl, }; @@ -57,6 +59,8 @@ pub const Node = struct { Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).iterate(index), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index), Id.Call => @fieldParentPtr(NodeCall, "base", base).iterate(index), + Id.ArrayAccess => @fieldParentPtr(NodeArrayAccess, "base", base).iterate(index), + Id.SliceExpression => @fieldParentPtr(NodeSliceExpression, "base", base).iterate(index), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).iterate(index), Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).iterate(index), }; @@ -85,6 +89,8 @@ pub const Node = struct { Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).firstToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(), Id.Call => @fieldParentPtr(NodeCall, "base", base).firstToken(), + Id.ArrayAccess => @fieldParentPtr(NodeArrayAccess, "base", base).firstToken(), + Id.SliceExpression => @fieldParentPtr(NodeSliceExpression, "base", base).firstToken(), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).firstToken(), Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).firstToken(), }; @@ -113,6 +119,8 @@ pub const Node = struct { Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).lastToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(), Id.Call => @fieldParentPtr(NodeCall, "base", base).lastToken(), + Id.ArrayAccess => @fieldParentPtr(NodeArrayAccess, "base", base).lastToken(), + Id.SliceExpression => @fieldParentPtr(NodeSliceExpression, "base", base).lastToken(), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).lastToken(), Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).lastToken(), }; @@ -598,6 +606,64 @@ pub const NodeCall = struct { } }; +pub const NodeArrayAccess = struct { + base: Node, + expr: &Node, + index: &Node, + rbracket_token: Token, + + pub fn iterate(self: &NodeArrayAccess, index: usize) ?&Node { + var i = index; + + if (i < 1) return self.expr; + i -= 1; + + if (i < 1) return self.index; + i -= 1; + + return null; + } + + pub fn firstToken(self: &NodeArrayAccess) Token { + return self.expr.firstToken(); + } + + pub fn lastToken(self: &NodeArrayAccess) Token { + return self.rbracket_token; + } +}; + +pub const NodeSliceExpression = struct { + base: Node, + expr: &Node, + start: &Node, + end: ?&Node, + rbracket_token: Token, + + pub fn iterate(self: &NodeSliceExpression, index: usize) ?&Node { + var i = index; + + if (i < 1) return self.callee; + i -= 1; + + if (i < 1) return self.start; + i -= 1; + + if (i < 1) return self.end; + i -= 1; + + return null; + } + + pub fn firstToken(self: &NodeSliceExpression) Token { + return self.expr.firstToken(); + } + + pub fn lastToken(self: &NodeSliceExpression) Token { + return self.rbracket_token; + } +}; + pub const NodeStringLiteral = struct { base: Node, token: Token, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 0b8a15d3dd..d0ab7e17d8 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -88,6 +88,7 @@ pub const Parser = struct { InfixOp: &ast.NodeInfixOp, PrefixOp: &ast.NodePrefixOp, SuffixOp: &ast.Node, + SliceOrArrayAccess, AddrOfModifiers: &ast.NodePrefixOp.AddrOfInfo, TypeExpr: DestPtr, VarDecl: &ast.NodeVarDecl, @@ -590,6 +591,11 @@ pub const Parser = struct { }); continue; + } else if (token.id == Token.Id.LBracket) { + try stack.append(State.SliceOrArrayAccess); + try stack.append(State.ExpectOperand); + continue; + // TODO: Parse postfix operator } else { // no postfix/infix operator after this operand. @@ -603,6 +609,53 @@ pub const Parser = struct { try dest_ptr.store(expression); break; }, + State.SliceOrArrayAccess => { + var rbracket_or_ellipsis2_token = self.getNextToken(); + + switch (rbracket_or_ellipsis2_token.id) { + Token.Id.Ellipsis2 => { + const node = try arena.create(ast.NodeSliceExpression); + *node = ast.NodeSliceExpression { + .base = self.initNode(ast.Node.Id.SliceExpression), + .expr = undefined, + .start = expression, + .end = null, + .rbracket_token = undefined, + }; + + try stack.append(State { .SuffixOp = &node.base }); + try stack.append(State.AfterOperand); + + const rbracket_token = self.getNextToken(); + if (rbracket_token.id != Token.Id.RBracket) { + self.putBackToken(rbracket_token); + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.RBracket, + .ptr = &node.rbracket_token, + } + }); + try stack.append(State { .Expression = DestPtr { .NullableField = &node.end } }); + } else { + node.rbracket_token = rbracket_token; + } + break; + }, + Token.Id.RBracket => { + const node = try arena.create(ast.NodeArrayAccess); + *node = ast.NodeArrayAccess { + .base = self.initNode(ast.Node.Id.ArrayAccess), + .expr = undefined, + .index = expression, + .rbracket_token = token, + }; + try stack.append(State { .SuffixOp = &node.base }); + try stack.append(State.AfterOperand); + break; + }, + else => return self.parseError(token, "expected ']' or '..', found {}", @tagName(token.id)) + } + }, State.InfixOp => |infix_op| { infix_op.rhs = expression; infix_op.lhs = popSuffixOp(&stack); @@ -857,6 +910,7 @@ pub const Parser = struct { State.PrefixOp => unreachable, State.SuffixOp => unreachable, State.Operand => unreachable, + State.SliceOrArrayAccess => unreachable, } } } @@ -874,6 +928,18 @@ pub const Parser = struct { left_leaf_ptr = &call.callee; continue; }, + ast.Node.Id.ArrayAccess => { + const arr_access = @fieldParentPtr(ast.NodeArrayAccess, "base", suffix_op); + *left_leaf_ptr = &arr_access.base; + left_leaf_ptr = &arr_access.expr; + continue; + }, + ast.Node.Id.SliceExpression => { + const slice_expr = @fieldParentPtr(ast.NodeSliceExpression, "base", suffix_op); + *left_leaf_ptr = &slice_expr.base; + left_leaf_ptr = &slice_expr.expr; + continue; + }, else => unreachable, } }, @@ -1594,6 +1660,24 @@ pub const Parser = struct { try stack.append(RenderState { .Text = "("}); try stack.append(RenderState { .Expression = call.callee }); }, + ast.Node.Id.ArrayAccess => { + const arr_access = @fieldParentPtr(ast.NodeArrayAccess, "base", base); + try stack.append(RenderState { .Text = "]"}); + try stack.append(RenderState { .Expression = arr_access.index}); + try stack.append(RenderState { .Text = "["}); + try stack.append(RenderState { .Expression = arr_access.expr }); + }, + ast.Node.Id.SliceExpression => { + const slice_expr = @fieldParentPtr(ast.NodeSliceExpression, "base", base); + try stack.append(RenderState { .Text = "]"}); + if (slice_expr.end) |end| { + try stack.append(RenderState { .Expression = end}); + } + try stack.append(RenderState { .Text = ".."}); + try stack.append(RenderState { .Expression = slice_expr.start}); + try stack.append(RenderState { .Text = "["}); + try stack.append(RenderState { .Expression = slice_expr.expr}); + }, ast.Node.Id.FnProto => @panic("TODO fn proto in an expression"), ast.Node.Id.LineComment => @panic("TODO render line comment in an expression"), @@ -1978,6 +2062,14 @@ test "zig fmt: indexing" { \\ a[0 + 5]; \\ a[0..]; \\ a[0..5]; + \\ a[a[0]]; + \\ a[a[0..]]; + \\ a[a[0..5]]; + \\ a[a[0]..]; + \\ a[a[0..5]..]; + \\ a[a[0]..a[0]]; + \\ a[a[0..5]..a[0]]; + \\ a[a[0..5]..a[0..5]]; \\} \\ ); diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig index 64eb627590..1014bacb30 100644 --- a/std/zig/tokenizer.zig +++ b/std/zig/tokenizer.zig @@ -91,6 +91,8 @@ pub const Token = struct { PercentEqual, LBrace, RBrace, + LBracket, + RBracket, Period, Ellipsis2, Ellipsis3, @@ -327,6 +329,16 @@ pub const Tokenizer = struct { self.index += 1; break; }, + '[' => { + result.id = Token.Id.LBracket; + self.index += 1; + break; + }, + ']' => { + result.id = Token.Id.RBracket; + self.index += 1; + break; + }, ';' => { result.id = Token.Id.Semicolon; self.index += 1; From a2330d0ea366e0ced7fa917d3bf1ccd022e932c3 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 3 Apr 2018 10:54:19 +0200 Subject: [PATCH 11/87] std.zig.parser now parses slice and array types --- std/zig/ast.zig | 5 +++++ std/zig/parser.zig | 50 +++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 680e617962..1e62bd26a8 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -475,9 +475,12 @@ pub const NodePrefixOp = struct { Negation, NegationWrap, Return, + ArrayType: &Node, + SliceType: AddrOfInfo, Try, UnwrapMaybe, }; + const AddrOfInfo = struct { align_expr: ?&Node, bit_offset_start_token: ?Token, @@ -502,6 +505,8 @@ pub const NodePrefixOp = struct { PrefixOp.Negation, PrefixOp.NegationWrap, PrefixOp.Return, + PrefixOp.ArrayType, + PrefixOp.SliceType, PrefixOp.Try, PrefixOp.UnwrapMaybe => {}, } diff --git a/std/zig/parser.zig b/std/zig/parser.zig index d0ab7e17d8..671f454f5a 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -385,6 +385,35 @@ pub const Parser = struct { try stack.append(State.ExpectOperand); continue; }, + Token.Id.LBracket => { + const rbracket_token = self.getNextToken(); + if (rbracket_token.id == Token.Id.RBracket) { + const prefix_op = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp{ + .SliceType = ast.NodePrefixOp.AddrOfInfo { + .align_expr = null, + .bit_offset_start_token = null, + .bit_offset_end_token = null, + .const_token = null, + .volatile_token = null, + } + }); + try stack.append(State { .PrefixOp = prefix_op }); + try stack.append(State.ExpectOperand); + try stack.append(State { .AddrOfModifiers = &prefix_op.op.AddrOf }); + continue; + } + + self.putBackToken(rbracket_token); + + const prefix_op = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp{ + .ArrayType = undefined, + }); + try stack.append(State { .PrefixOp = prefix_op }); + try stack.append(State.ExpectOperand); + try stack.append(State { .ExpectToken = Token.Id.RBracket }); + try stack.append(State { .Expression = DestPtr { .Field = &prefix_op.op.ArrayType } }); + + }, Token.Id.Ampersand => { const prefix_op = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp{ .AddrOf = ast.NodePrefixOp.AddrOfInfo { @@ -1567,6 +1596,25 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = align_expr}); } }, + ast.NodePrefixOp.PrefixOp.SliceType => |addr_of_info| { + try stream.write("[]"); + if (addr_of_info.volatile_token != null) { + try stack.append(RenderState { .Text = "volatile "}); + } + if (addr_of_info.const_token != null) { + try stack.append(RenderState { .Text = "const "}); + } + if (addr_of_info.align_expr) |align_expr| { + try stream.print("align("); + try stack.append(RenderState { .Text = ") "}); + try stack.append(RenderState { .Expression = align_expr}); + } + }, + ast.NodePrefixOp.PrefixOp.ArrayType => |array_index| { + try stack.append(RenderState { .Text = "]"}); + try stack.append(RenderState { .Expression = array_index}); + try stack.append(RenderState { .Text = "["}); + }, ast.NodePrefixOp.PrefixOp.BitNot => try stream.write("~"), ast.NodePrefixOp.PrefixOp.BoolNot => try stream.write("!"), ast.NodePrefixOp.PrefixOp.Deref => try stream.write("*"), @@ -2522,4 +2570,4 @@ test "zig fmt: zig fmt" { try testCanonical(@embedFile("index.zig")); try testCanonical(@embedFile("parser.zig")); try testCanonical(@embedFile("tokenizer.zig")); -} \ No newline at end of file +} From 22e38ffb54775c2e523665ac739f2a895d8757ad Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 3 Apr 2018 11:18:18 +0200 Subject: [PATCH 12/87] std.zig.tokenizer fixed tokens having wrong column and line --- std/zig/tokenizer.zig | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig index 1014bacb30..2e40999920 100644 --- a/std/zig/tokenizer.zig +++ b/std/zig/tokenizer.zig @@ -271,6 +271,7 @@ pub const Tokenizer = struct { self.pending_invalid_token = null; return token; } + const start_index = self.index; var state = State.Start; var result = Token { .id = Token.Id.Eof, @@ -279,7 +280,7 @@ pub const Tokenizer = struct { .line = self.line, .column = self.column, }; - while (self.index < self.buffer.len) { + while (self.index < self.buffer.len) : (self.index += 1) { const c = self.buffer[self.index]; switch (state) { State.Start => switch (c) { @@ -877,14 +878,6 @@ pub const Tokenizer = struct { else => break, }, } - - self.index += 1; - if (c == '\n') { - self.line += 1; - self.column = 0; - } else { - self.column += 1; - } } else if (self.index == self.buffer.len) { switch (state) { State.Start, @@ -983,6 +976,16 @@ pub const Tokenizer = struct { }, } } + + for (self.buffer[start_index..self.index]) |c| { + if (c == '\n') { + self.line += 1; + self.column = 0; + } else { + self.column += 1; + } + } + if (result.id == Token.Id.Eof) { if (self.pending_invalid_token) |token| { self.pending_invalid_token = null; From b424cd75ab9df69f44fe47c10ded3fd25c2d27bb Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 3 Apr 2018 12:33:06 +0200 Subject: [PATCH 13/87] std.zig.parser refactored call, slice and array access to be suffix op --- std/zig/ast.zig | 183 +++++++++++++++++++++------------------------ std/zig/parser.zig | 161 +++++++++++++++++++-------------------- 2 files changed, 160 insertions(+), 184 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 1e62bd26a8..380dd95aee 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -17,6 +17,7 @@ pub const Node = struct { Block, InfixOp, PrefixOp, + SuffixOp, IntegerLiteral, FloatLiteral, StringLiteral, @@ -29,9 +30,6 @@ pub const Node = struct { Unreachable, ErrorType, BuiltinCall, - Call, - ArrayAccess, - SliceExpression, LineComment, TestDecl, }; @@ -46,6 +44,7 @@ pub const Node = struct { Id.Block => @fieldParentPtr(NodeBlock, "base", base).iterate(index), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).iterate(index), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).iterate(index), + Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).iterate(index), Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).iterate(index), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).iterate(index), @@ -58,9 +57,6 @@ pub const Node = struct { Id.Unreachable => @fieldParentPtr(NodeUnreachable, "base", base).iterate(index), Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).iterate(index), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index), - Id.Call => @fieldParentPtr(NodeCall, "base", base).iterate(index), - Id.ArrayAccess => @fieldParentPtr(NodeArrayAccess, "base", base).iterate(index), - Id.SliceExpression => @fieldParentPtr(NodeSliceExpression, "base", base).iterate(index), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).iterate(index), Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).iterate(index), }; @@ -76,6 +72,7 @@ pub const Node = struct { Id.Block => @fieldParentPtr(NodeBlock, "base", base).firstToken(), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).firstToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).firstToken(), + Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).firstToken(), Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).firstToken(), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(), @@ -88,9 +85,6 @@ pub const Node = struct { Id.ThisLiteral => @fieldParentPtr(NodeThisLiteral, "base", base).firstToken(), Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).firstToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(), - Id.Call => @fieldParentPtr(NodeCall, "base", base).firstToken(), - Id.ArrayAccess => @fieldParentPtr(NodeArrayAccess, "base", base).firstToken(), - Id.SliceExpression => @fieldParentPtr(NodeSliceExpression, "base", base).firstToken(), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).firstToken(), Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).firstToken(), }; @@ -106,6 +100,7 @@ pub const Node = struct { Id.Block => @fieldParentPtr(NodeBlock, "base", base).lastToken(), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).lastToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).lastToken(), + Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).lastToken(), Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).lastToken(), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(), @@ -118,9 +113,6 @@ pub const Node = struct { Id.Unreachable => @fieldParentPtr(NodeUnreachable, "base", base).lastToken(), Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).lastToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(), - Id.Call => @fieldParentPtr(NodeCall, "base", base).lastToken(), - Id.ArrayAccess => @fieldParentPtr(NodeArrayAccess, "base", base).lastToken(), - Id.SliceExpression => @fieldParentPtr(NodeSliceExpression, "base", base).lastToken(), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).lastToken(), Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).lastToken(), }; @@ -493,20 +485,28 @@ pub const NodePrefixOp = struct { var i = index; switch (self.op) { + PrefixOp.SliceType => |addr_of_info| { + if (addr_of_info.align_expr) |align_expr| { + if (i < 1) return align_expr; + i -= 1; + } + }, PrefixOp.AddrOf => |addr_of_info| { if (addr_of_info.align_expr) |align_expr| { if (i < 1) return align_expr; i -= 1; } }, + PrefixOp.ArrayType => |size_expr| { + if (i < 1) return size_expr; + i -= 1; + }, PrefixOp.BitNot, PrefixOp.BoolNot, PrefixOp.Deref, PrefixOp.Negation, PrefixOp.NegationWrap, PrefixOp.Return, - PrefixOp.ArrayType, - PrefixOp.SliceType, PrefixOp.Try, PrefixOp.UnwrapMaybe => {}, } @@ -526,6 +526,76 @@ pub const NodePrefixOp = struct { } }; +pub const NodeSuffixOp = struct { + base: Node, + lhs: &Node, + op: SuffixOp, + rtoken: Token, + + const SuffixOp = union(enum) { + Call: CallInfo, + ArrayAccess: &Node, + Slice: SliceRange, + ArrayInitializer: ArrayList(&Node), + StructInitializer: ArrayList(&Node), + }; + + const CallInfo = struct { + params: ArrayList(&Node), + is_async: bool, + }; + + const SliceRange = struct { + start: &Node, + end: ?&Node, + }; + + pub fn iterate(self: &NodeSuffixOp, index: usize) ?&Node { + var i = index; + + if (i < 1) return self.lhs; + i -= 1; + + switch (self.op) { + SuffixOp.Call => |call_info| { + if (i < call_info.params.len) return call_info.params.at(i); + i -= call_info.params.len; + }, + SuffixOp.ArrayAccess => |index_expr| { + if (i < 1) return index_expr; + i -= 1; + }, + SuffixOp.Slice => |range| { + if (i < 1) return range.start; + i -= 1; + + if (range.end) |end| { + if (i < 1) return end; + i -= 1; + } + }, + SuffixOp.ArrayInitializer => |exprs| { + if (i < exprs.len) return exprs.at(i); + i -= exprs.len; + }, + SuffixOp.StructInitializer => |fields| { + if (i < fields.len) return fields.at(i); + i -= fields.len; + }, + } + + return null; + } + + pub fn firstToken(self: &NodeSuffixOp) Token { + return self.lhs.firstToken(); + } + + pub fn lastToken(self: &NodeSuffixOp) Token { + return self.rtoken; + } +}; + pub const NodeIntegerLiteral = struct { base: Node, token: Token, @@ -584,91 +654,6 @@ pub const NodeBuiltinCall = struct { } }; -pub const NodeCall = struct { - base: Node, - callee: &Node, - params: ArrayList(&Node), - rparen_token: Token, - - pub fn iterate(self: &NodeCall, index: usize) ?&Node { - var i = index; - - if (i < 1) return self.callee; - i -= 1; - - if (i < self.params.len) return self.params.at(i); - i -= self.params.len; - - return null; - } - - pub fn firstToken(self: &NodeCall) Token { - return self.callee.firstToken(); - } - - pub fn lastToken(self: &NodeCall) Token { - return self.rparen_token; - } -}; - -pub const NodeArrayAccess = struct { - base: Node, - expr: &Node, - index: &Node, - rbracket_token: Token, - - pub fn iterate(self: &NodeArrayAccess, index: usize) ?&Node { - var i = index; - - if (i < 1) return self.expr; - i -= 1; - - if (i < 1) return self.index; - i -= 1; - - return null; - } - - pub fn firstToken(self: &NodeArrayAccess) Token { - return self.expr.firstToken(); - } - - pub fn lastToken(self: &NodeArrayAccess) Token { - return self.rbracket_token; - } -}; - -pub const NodeSliceExpression = struct { - base: Node, - expr: &Node, - start: &Node, - end: ?&Node, - rbracket_token: Token, - - pub fn iterate(self: &NodeSliceExpression, index: usize) ?&Node { - var i = index; - - if (i < 1) return self.callee; - i -= 1; - - if (i < 1) return self.start; - i -= 1; - - if (i < 1) return self.end; - i -= 1; - - return null; - } - - pub fn firstToken(self: &NodeSliceExpression) Token { - return self.expr.firstToken(); - } - - pub fn lastToken(self: &NodeSliceExpression) Token { - return self.rbracket_token; - } -}; - pub const NodeStringLiteral = struct { base: Node, token: Token, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 671f454f5a..6d5ec54bc8 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -87,7 +87,7 @@ pub const Parser = struct { AfterOperand, InfixOp: &ast.NodeInfixOp, PrefixOp: &ast.NodePrefixOp, - SuffixOp: &ast.Node, + SuffixOp: &ast.NodeSuffixOp, SliceOrArrayAccess, AddrOfModifiers: &ast.NodePrefixOp.AddrOfInfo, TypeExpr: DestPtr, @@ -602,20 +602,19 @@ pub const Parser = struct { } else if (token.id == Token.Id.LParen) { self.putBackToken(token); - const node = try arena.create(ast.NodeCall); - *node = ast.NodeCall { - .base = self.initNode(ast.Node.Id.Call), - .callee = undefined, - .params = ArrayList(&ast.Node).init(arena), - .rparen_token = undefined, - }; - try stack.append(State { .SuffixOp = &node.base }); + const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { + .Call = ast.NodeSuffixOp.CallInfo { + .params = ArrayList(&ast.Node).init(arena), + .is_async = false, // TODO: ASYNC + } + }); + try stack.append(State { .SuffixOp = node }); try stack.append(State.AfterOperand); - try stack.append(State {.ExprListItemOrEnd = &node.params }); + try stack.append(State {.ExprListItemOrEnd = &node.op.Call.params }); try stack.append(State { .ExpectTokenSave = ExpectTokenSave { .id = Token.Id.LParen, - .ptr = &node.rparen_token, + .ptr = &node.rtoken, }, }); continue; @@ -643,16 +642,14 @@ pub const Parser = struct { switch (rbracket_or_ellipsis2_token.id) { Token.Id.Ellipsis2 => { - const node = try arena.create(ast.NodeSliceExpression); - *node = ast.NodeSliceExpression { - .base = self.initNode(ast.Node.Id.SliceExpression), - .expr = undefined, - .start = expression, - .end = null, - .rbracket_token = undefined, - }; + const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { + .Slice = ast.NodeSuffixOp.SliceRange { + .start = expression, + .end = null, + } + }); - try stack.append(State { .SuffixOp = &node.base }); + try stack.append(State { .SuffixOp = node }); try stack.append(State.AfterOperand); const rbracket_token = self.getNextToken(); @@ -661,24 +658,21 @@ pub const Parser = struct { try stack.append(State { .ExpectTokenSave = ExpectTokenSave { .id = Token.Id.RBracket, - .ptr = &node.rbracket_token, + .ptr = &node.rtoken, } }); - try stack.append(State { .Expression = DestPtr { .NullableField = &node.end } }); + try stack.append(State { .Expression = DestPtr { .NullableField = &node.op.Slice.end } }); } else { - node.rbracket_token = rbracket_token; + node.rtoken = rbracket_token; } break; }, Token.Id.RBracket => { - const node = try arena.create(ast.NodeArrayAccess); - *node = ast.NodeArrayAccess { - .base = self.initNode(ast.Node.Id.ArrayAccess), - .expr = undefined, - .index = expression, - .rbracket_token = token, - }; - try stack.append(State { .SuffixOp = &node.base }); + const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { + .ArrayAccess = expression + }); + node.rtoken = token; + try stack.append(State { .SuffixOp = node }); try stack.append(State.AfterOperand); break; }, @@ -950,27 +944,8 @@ pub const Parser = struct { while (true) { switch (stack.pop()) { State.SuffixOp => |suffix_op| { - switch (suffix_op.id) { - ast.Node.Id.Call => { - const call = @fieldParentPtr(ast.NodeCall, "base", suffix_op); - *left_leaf_ptr = &call.base; - left_leaf_ptr = &call.callee; - continue; - }, - ast.Node.Id.ArrayAccess => { - const arr_access = @fieldParentPtr(ast.NodeArrayAccess, "base", suffix_op); - *left_leaf_ptr = &arr_access.base; - left_leaf_ptr = &arr_access.expr; - continue; - }, - ast.Node.Id.SliceExpression => { - const slice_expr = @fieldParentPtr(ast.NodeSliceExpression, "base", suffix_op); - *left_leaf_ptr = &slice_expr.base; - left_leaf_ptr = &slice_expr.expr; - continue; - }, - else => unreachable, - } + *left_leaf_ptr = &suffix_op.base; + left_leaf_ptr = &suffix_op.lhs; }, State.Operand => |operand| { *left_leaf_ptr = operand; @@ -1172,6 +1147,18 @@ pub const Parser = struct { return node; } + fn createSuffixOp(self: &Parser, arena: &mem.Allocator, op: &const ast.NodeSuffixOp.SuffixOp) !&ast.NodeSuffixOp { + const node = try arena.create(ast.NodeSuffixOp); + + *node = ast.NodeSuffixOp { + .base = self.initNode(ast.Node.Id.SuffixOp), + .lhs = undefined, + .op = *op, + .rtoken = undefined, + }; + return node; + } + fn createIdentifier(self: &Parser, arena: &mem.Allocator, name_token: &const Token) !&ast.NodeIdentifier { const node = try arena.create(ast.NodeIdentifier); @@ -1625,6 +1612,43 @@ pub const Parser = struct { ast.NodePrefixOp.PrefixOp.UnwrapMaybe => try stream.write("??"), } }, + ast.Node.Id.SuffixOp => { + const suffix_op = @fieldParentPtr(ast.NodeSuffixOp, "base", base); + + switch (suffix_op.op) { + ast.NodeSuffixOp.SuffixOp.Call => |call_info| { + try stack.append(RenderState { .Text = ")"}); + var i = call_info.params.len; + while (i != 0) { + i -= 1; + const param_node = call_info.params.at(i); + try stack.append(RenderState { .Expression = param_node}); + if (i != 0) { + try stack.append(RenderState { .Text = ", " }); + } + } + try stack.append(RenderState { .Text = "("}); + }, + ast.NodeSuffixOp.SuffixOp.ArrayAccess => |index_expr| { + try stack.append(RenderState { .Text = "]"}); + try stack.append(RenderState { .Expression = index_expr}); + try stack.append(RenderState { .Text = "["}); + }, + ast.NodeSuffixOp.SuffixOp.Slice => |range| { + try stack.append(RenderState { .Text = "]"}); + if (range.end) |end| { + try stack.append(RenderState { .Expression = end}); + } + try stack.append(RenderState { .Text = ".."}); + try stack.append(RenderState { .Expression = range.start}); + try stack.append(RenderState { .Text = "["}); + }, + ast.NodeSuffixOp.SuffixOp.StructInitializer => @panic("TODO: StructInitializer"), + ast.NodeSuffixOp.SuffixOp.ArrayInitializer => @panic("TODO: ArrayInitializer"), + } + + try stack.append(RenderState { .Expression = suffix_op.lhs }); + }, ast.Node.Id.IntegerLiteral => { const integer_literal = @fieldParentPtr(ast.NodeIntegerLiteral, "base", base); try stream.print("{}", self.tokenizer.getTokenSlice(integer_literal.token)); @@ -1693,39 +1717,6 @@ pub const Parser = struct { } } }, - ast.Node.Id.Call => { - const call = @fieldParentPtr(ast.NodeCall, "base", base); - try stack.append(RenderState { .Text = ")"}); - var i = call.params.len; - while (i != 0) { - i -= 1; - const param_node = call.params.at(i); - try stack.append(RenderState { .Expression = param_node}); - if (i != 0) { - try stack.append(RenderState { .Text = ", " }); - } - } - try stack.append(RenderState { .Text = "("}); - try stack.append(RenderState { .Expression = call.callee }); - }, - ast.Node.Id.ArrayAccess => { - const arr_access = @fieldParentPtr(ast.NodeArrayAccess, "base", base); - try stack.append(RenderState { .Text = "]"}); - try stack.append(RenderState { .Expression = arr_access.index}); - try stack.append(RenderState { .Text = "["}); - try stack.append(RenderState { .Expression = arr_access.expr }); - }, - ast.Node.Id.SliceExpression => { - const slice_expr = @fieldParentPtr(ast.NodeSliceExpression, "base", base); - try stack.append(RenderState { .Text = "]"}); - if (slice_expr.end) |end| { - try stack.append(RenderState { .Expression = end}); - } - try stack.append(RenderState { .Text = ".."}); - try stack.append(RenderState { .Expression = slice_expr.start}); - try stack.append(RenderState { .Text = "["}); - try stack.append(RenderState { .Expression = slice_expr.expr}); - }, ast.Node.Id.FnProto => @panic("TODO fn proto in an expression"), ast.Node.Id.LineComment => @panic("TODO render line comment in an expression"), From 0b9247fb636a47add80f7c4ec194df436629df91 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 3 Apr 2018 14:20:34 +0200 Subject: [PATCH 14/87] std.zig.parser Refactor: * Slice/Array access is now not parsed in the expr contruction loop * State.ExprListItemOrEnd now takes a token id for the end token --- std/zig/ast.zig | 30 ++++++++- std/zig/parser.zig | 149 +++++++++++++++++++++++++++------------------ 2 files changed, 118 insertions(+), 61 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 380dd95aee..2ec305429c 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -18,6 +18,7 @@ pub const Node = struct { InfixOp, PrefixOp, SuffixOp, + FieldInitializer, IntegerLiteral, FloatLiteral, StringLiteral, @@ -45,6 +46,7 @@ pub const Node = struct { Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).iterate(index), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).iterate(index), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).iterate(index), + Id.FieldInitializer => @fieldParentPtr(NodeFieldInitializer, "base", base).iterate(index), Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).iterate(index), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).iterate(index), @@ -73,6 +75,7 @@ pub const Node = struct { Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).firstToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).firstToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).firstToken(), + Id.FieldInitializer => @fieldParentPtr(NodeFieldInitializer, "base", base).firstToken(), Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).firstToken(), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(), @@ -101,6 +104,7 @@ pub const Node = struct { Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).lastToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).lastToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).lastToken(), + Id.FieldInitializer => @fieldParentPtr(NodeFieldInitializer, "base", base).lastToken(), Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).lastToken(), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(), @@ -526,6 +530,30 @@ pub const NodePrefixOp = struct { } }; +pub const NodeFieldInitializer = struct { + base: Node, + dot_token: Token, + name_token: Token, + expr: &Node, + + pub fn iterate(self: &NodeFieldInitializer, index: usize) ?&Node { + var i = index; + + if (i < 1) return self.expr; + i -= 1; + + return null; + } + + pub fn firstToken(self: &NodeFieldInitializer) Token { + return self.dot_token; + } + + pub fn lastToken(self: &NodeFieldInitializer) Token { + return self.expr.lastToken(); + } +}; + pub const NodeSuffixOp = struct { base: Node, lhs: &Node, @@ -537,7 +565,7 @@ pub const NodeSuffixOp = struct { ArrayAccess: &Node, Slice: SliceRange, ArrayInitializer: ArrayList(&Node), - StructInitializer: ArrayList(&Node), + StructInitializer: ArrayList(&NodeFieldInitializer), }; const CallInfo = struct { diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 6d5ec54bc8..43cc35fb6d 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -77,6 +77,12 @@ pub const Parser = struct { ptr: &Token, }; + const ExprListState = struct { + list: &ArrayList(&ast.Node), + end: Token.Id, + ptr: &Token, + }; + const State = union(enum) { TopLevel, TopLevelExtern: ?Token, @@ -88,7 +94,7 @@ pub const Parser = struct { InfixOp: &ast.NodeInfixOp, PrefixOp: &ast.NodePrefixOp, SuffixOp: &ast.NodeSuffixOp, - SliceOrArrayAccess, + SliceOrArrayAccess: &ast.NodeSuffixOp, AddrOfModifiers: &ast.NodePrefixOp.AddrOfInfo, TypeExpr: DestPtr, VarDecl: &ast.NodeVarDecl, @@ -104,8 +110,8 @@ pub const Parser = struct { FnDef: &ast.NodeFnProto, Block: &ast.NodeBlock, Statement: &ast.NodeBlock, - ExprListItemOrEnd: &ArrayList(&ast.Node), - ExprListCommaOrEnd: &ArrayList(&ast.Node), + ExprListItemOrEnd: ExprListState, + ExprListCommaOrEnd: ExprListState, }; /// Returns an AST tree, allocated with the parser's allocator. @@ -529,13 +535,14 @@ pub const Parser = struct { .Operand = &node.base }); try stack.append(State.AfterOperand); - try stack.append(State {.ExprListItemOrEnd = &node.params }); try stack.append(State { - .ExpectTokenSave = ExpectTokenSave { - .id = Token.Id.LParen, + .ExprListItemOrEnd = ExprListState { + .list = &node.params, + .end = Token.Id.RParen, .ptr = &node.rparen_token, - }, + } }); + try stack.append(State { .ExpectToken = Token.Id.LParen, }); continue; }, Token.Id.StringLiteral => { @@ -587,6 +594,46 @@ pub const Parser = struct { } }, + State.SliceOrArrayAccess => |node| { + var token = self.getNextToken(); + + switch (token.id) { + Token.Id.Ellipsis2 => { + const start = node.op.ArrayAccess; + node.op = ast.NodeSuffixOp.SuffixOp { + .Slice = ast.NodeSuffixOp.SliceRange { + .start = start, + .end = undefined, + } + }; + try stack.append(State { .SuffixOp = node }); + try stack.append(State.AfterOperand); + + const rbracket_token = self.getNextToken(); + if (rbracket_token.id != Token.Id.RBracket) { + self.putBackToken(rbracket_token); + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.RBracket, + .ptr = &node.rtoken, + } + }); + try stack.append(State { .Expression = DestPtr { .NullableField = &node.op.Slice.end } }); + } else { + node.rtoken = rbracket_token; + } + continue; + }, + Token.Id.RBracket => { + node.rtoken = token; + try stack.append(State { .SuffixOp = node }); + try stack.append(State.AfterOperand); + continue; + }, + else => return self.parseError(token, "expected ']' or '..', found {}", @tagName(token.id)) + } + }, + State.AfterOperand => { // we'll either get an infix operator (like != or ^), // or a postfix operator (like () or {}), @@ -610,7 +657,13 @@ pub const Parser = struct { }); try stack.append(State { .SuffixOp = node }); try stack.append(State.AfterOperand); - try stack.append(State {.ExprListItemOrEnd = &node.op.Call.params }); + try stack.append(State { + .ExprListItemOrEnd = ExprListState { + .list = &node.op.Call.params, + .end = Token.Id.RParen, + .ptr = &node.rtoken, + } + }); try stack.append(State { .ExpectTokenSave = ExpectTokenSave { .id = Token.Id.LParen, @@ -620,8 +673,17 @@ pub const Parser = struct { continue; } else if (token.id == Token.Id.LBracket) { - try stack.append(State.SliceOrArrayAccess); - try stack.append(State.ExpectOperand); + const node = try arena.create(ast.NodeSuffixOp); + *node = ast.NodeSuffixOp { + .base = self.initNode(ast.Node.Id.SuffixOp), + .lhs = undefined, + .op = ast.NodeSuffixOp.SuffixOp { + .ArrayAccess = undefined, + }, + .rtoken = undefined, + }; + try stack.append(State { .SliceOrArrayAccess = node }); + try stack.append(State { .Expression = DestPtr { .Field = &node.op.ArrayAccess }}); continue; // TODO: Parse postfix operator @@ -637,48 +699,6 @@ pub const Parser = struct { try dest_ptr.store(expression); break; }, - State.SliceOrArrayAccess => { - var rbracket_or_ellipsis2_token = self.getNextToken(); - - switch (rbracket_or_ellipsis2_token.id) { - Token.Id.Ellipsis2 => { - const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { - .Slice = ast.NodeSuffixOp.SliceRange { - .start = expression, - .end = null, - } - }); - - try stack.append(State { .SuffixOp = node }); - try stack.append(State.AfterOperand); - - const rbracket_token = self.getNextToken(); - if (rbracket_token.id != Token.Id.RBracket) { - self.putBackToken(rbracket_token); - try stack.append(State { - .ExpectTokenSave = ExpectTokenSave { - .id = Token.Id.RBracket, - .ptr = &node.rtoken, - } - }); - try stack.append(State { .Expression = DestPtr { .NullableField = &node.op.Slice.end } }); - } else { - node.rtoken = rbracket_token; - } - break; - }, - Token.Id.RBracket => { - const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { - .ArrayAccess = expression - }); - node.rtoken = token; - try stack.append(State { .SuffixOp = node }); - try stack.append(State.AfterOperand); - break; - }, - else => return self.parseError(token, "expected ']' or '..', found {}", @tagName(token.id)) - } - }, State.InfixOp => |infix_op| { infix_op.rhs = expression; infix_op.lhs = popSuffixOp(&stack); @@ -697,26 +717,31 @@ pub const Parser = struct { } }, - State.ExprListItemOrEnd => |params| { + State.ExprListItemOrEnd => |expr_list_state| { var token = self.getNextToken(); switch (token.id) { Token.Id.RParen => continue, else => { self.putBackToken(token); - stack.append(State { .ExprListCommaOrEnd = params }) catch unreachable; - try stack.append(State { .Expression = DestPtr{.List = params} }); + stack.append(State { .ExprListCommaOrEnd = expr_list_state }) catch unreachable; + try stack.append(State { .Expression = DestPtr{.List = expr_list_state.list} }); }, } }, - State.ExprListCommaOrEnd => |params| { + State.ExprListCommaOrEnd => |expr_list_state| { var token = self.getNextToken(); switch (token.id) { Token.Id.Comma => { - stack.append(State { .ExprListItemOrEnd = params }) catch unreachable; + stack.append(State { .ExprListItemOrEnd = expr_list_state }) catch unreachable; + }, + else => { + const IdTag = @TagType(Token.Id); + if (IdTag(expr_list_state.end) == token.id) + continue; + + return self.parseError(token, "expected ',' or {}, found {}", @tagName(expr_list_state.end), @tagName(token.id)); }, - Token.Id.RParen => continue, - else => return self.parseError(token, "expected ',' or ')', found {}", @tagName(token.id)), } }, @@ -933,7 +958,6 @@ pub const Parser = struct { State.PrefixOp => unreachable, State.SuffixOp => unreachable, State.Operand => unreachable, - State.SliceOrArrayAccess => unreachable, } } } @@ -1649,6 +1673,11 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = suffix_op.lhs }); }, + ast.Node.Id.FieldInitializer => { + const field_init = @fieldParentPtr(ast.NodeFieldInitializer, "base", base); + try stream.print(".{} = ", self.tokenizer.getTokenSlice(field_init.name_token)); + try stack.append(RenderState { .Expression = field_init.expr }); + }, ast.Node.Id.IntegerLiteral => { const integer_literal = @fieldParentPtr(ast.NodeIntegerLiteral, "base", base); try stream.print("{}", self.tokenizer.getTokenSlice(integer_literal.token)); From 5c82ed2ea9360ea0931ba789a4ffb73d689b6a72 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 3 Apr 2018 14:53:27 +0200 Subject: [PATCH 15/87] std.zig.parser now parses initializers... Or, it would, if it worked --- std/zig/ast.zig | 4 +- std/zig/parser.zig | 149 +++++++++++++++++++++++++++++++++++++-------- 2 files changed, 125 insertions(+), 28 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 2ec305429c..c4b4ef983a 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -532,7 +532,7 @@ pub const NodePrefixOp = struct { pub const NodeFieldInitializer = struct { base: Node, - dot_token: Token, + period_token: Token, name_token: Token, expr: &Node, @@ -546,7 +546,7 @@ pub const NodeFieldInitializer = struct { } pub fn firstToken(self: &NodeFieldInitializer) Token { - return self.dot_token; + return self.period_token; } pub fn lastToken(self: &NodeFieldInitializer) Token { diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 43cc35fb6d..19ed4af4bf 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -75,13 +75,16 @@ pub const Parser = struct { const ExpectTokenSave = struct { id: Token.Id, ptr: &Token, + }; - const ExprListState = struct { - list: &ArrayList(&ast.Node), - end: Token.Id, - ptr: &Token, - }; + fn ListState(comptime T: type) type { + return struct { + list: &ArrayList(T), + end: Token.Id, + ptr: &Token, + }; + } const State = union(enum) { TopLevel, @@ -110,8 +113,10 @@ pub const Parser = struct { FnDef: &ast.NodeFnProto, Block: &ast.NodeBlock, Statement: &ast.NodeBlock, - ExprListItemOrEnd: ExprListState, - ExprListCommaOrEnd: ExprListState, + ExprListItemOrEnd: ListState(&ast.Node), + ExprListCommaOrEnd: ListState(&ast.Node), + FieldInitListItemOrEnd: ListState(&ast.NodeFieldInitializer), + FieldInitListCommaOrEnd: ListState(&ast.NodeFieldInitializer), }; /// Returns an AST tree, allocated with the parser's allocator. @@ -536,7 +541,7 @@ pub const Parser = struct { }); try stack.append(State.AfterOperand); try stack.append(State { - .ExprListItemOrEnd = ExprListState { + .ExprListItemOrEnd = ListState(&ast.Node) { .list = &node.params, .end = Token.Id.RParen, .ptr = &node.rparen_token, @@ -647,8 +652,6 @@ pub const Parser = struct { continue; } else if (token.id == Token.Id.LParen) { - self.putBackToken(token); - const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { .Call = ast.NodeSuffixOp.CallInfo { .params = ArrayList(&ast.Node).init(arena), @@ -658,18 +661,12 @@ pub const Parser = struct { try stack.append(State { .SuffixOp = node }); try stack.append(State.AfterOperand); try stack.append(State { - .ExprListItemOrEnd = ExprListState { + .ExprListItemOrEnd = ListState(&ast.Node) { .list = &node.op.Call.params, .end = Token.Id.RParen, .ptr = &node.rtoken, } }); - try stack.append(State { - .ExpectTokenSave = ExpectTokenSave { - .id = Token.Id.LParen, - .ptr = &node.rtoken, - }, - }); continue; } else if (token.id == Token.Id.LBracket) { @@ -686,6 +683,47 @@ pub const Parser = struct { try stack.append(State { .Expression = DestPtr { .Field = &node.op.ArrayAccess }}); continue; + // TODO: This is the initializer parsing code. It doesn't work because of + // the ambiguity between function bodies and initializers: + // fn main() void {} or fn main() (void {}) + } else if (false) { //(token.id == Token.Id.LBrace) { + const next = self.getNextToken(); + + switch (next.id) { + Token.Id.Period => { + self.putBackToken(token); + + const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { + .StructInitializer = ArrayList(&ast.NodeFieldInitializer).init(arena), + }); + + try stack.append(State { + .FieldInitListItemOrEnd = ListState(&ast.NodeFieldInitializer) { + .list = &node.op.StructInitializer, + .end = Token.Id.RBrace, + .ptr = &node.rtoken, + } + }); + continue; + }, + else => { + self.putBackToken(token); + + const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { + .ArrayInitializer = ArrayList(&ast.Node).init(arena), + }); + + try stack.append(State { + .ExprListItemOrEnd = ListState(&ast.Node) { + .list = &node.op.ArrayInitializer, + .end = Token.Id.RBrace, + .ptr = &node.rtoken, + } + }); + continue; + }, + } + // TODO: Parse postfix operator } else { // no postfix/infix operator after this operand. @@ -717,30 +755,89 @@ pub const Parser = struct { } }, - State.ExprListItemOrEnd => |expr_list_state| { + State.ExprListItemOrEnd => |list_state| { + var token = self.getNextToken(); + + const IdTag = @TagType(Token.Id); + if (IdTag(list_state.end) == token.id) { + *list_state.ptr = token; + continue; + } + + self.putBackToken(token); + stack.append(State { .ExprListCommaOrEnd = list_state }) catch unreachable; + try stack.append(State { .Expression = DestPtr{.List = list_state.list} }); + }, + + State.ExprListCommaOrEnd => |list_state| { var token = self.getNextToken(); switch (token.id) { - Token.Id.RParen => continue, + Token.Id.Comma => { + stack.append(State { .ExprListItemOrEnd = list_state }) catch unreachable; + }, else => { - self.putBackToken(token); - stack.append(State { .ExprListCommaOrEnd = expr_list_state }) catch unreachable; - try stack.append(State { .Expression = DestPtr{.List = expr_list_state.list} }); + const IdTag = @TagType(Token.Id); + if (IdTag(list_state.end) == token.id) { + *list_state.ptr = token; + continue; + } + + return self.parseError(token, "expected ',' or {}, found {}", @tagName(list_state.end), @tagName(token.id)); }, } }, - State.ExprListCommaOrEnd => |expr_list_state| { + State.FieldInitListItemOrEnd => |list_state| { + var token = self.getNextToken(); + + const IdTag = @TagType(Token.Id); + if (IdTag(list_state.end) == token.id){ + *list_state.ptr = token; + continue; + } + + self.putBackToken(token); + + const node = try arena.create(ast.NodeFieldInitializer); + *node = ast.NodeFieldInitializer { + .base = self.initNode(ast.Node.Id.FieldInitializer), + .period_token = undefined, + .name_token = undefined, + .expr = undefined, + }; + try list_state.list.append(node); + + stack.append(State { .FieldInitListCommaOrEnd = list_state }) catch unreachable; + try stack.append(State { .Expression = DestPtr{.Field = &node.expr} }); + try stack.append(State { .ExpectToken = Token.Id.Equal }); + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.Identifier, + .ptr = &node.name_token, + } + }); + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.Period, + .ptr = &node.period_token, + } + }); + }, + + State.FieldInitListCommaOrEnd => |list_state| { var token = self.getNextToken(); switch (token.id) { Token.Id.Comma => { - stack.append(State { .ExprListItemOrEnd = expr_list_state }) catch unreachable; + stack.append(State { .FieldInitListItemOrEnd = list_state }) catch unreachable; }, else => { const IdTag = @TagType(Token.Id); - if (IdTag(expr_list_state.end) == token.id) + if (IdTag(list_state.end) == token.id) { + *list_state.ptr = token; continue; + } - return self.parseError(token, "expected ',' or {}, found {}", @tagName(expr_list_state.end), @tagName(token.id)); + return self.parseError(token, "expected ',' or {}, found {}", @tagName(list_state.end), @tagName(token.id)); }, } }, From 9d69e94bbad0b1ff23999584f5f632cfe8db3656 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 3 Apr 2018 15:16:32 +0200 Subject: [PATCH 16/87] std.zig.parser now parses grouped expressions * I also moved some tests down, as they fail in ways I can't fix yet --- std/zig/ast.zig | 28 +++++++++++ std/zig/parser.zig | 115 ++++++++++++++++++++++++++++----------------- 2 files changed, 99 insertions(+), 44 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index c4b4ef983a..32a8a7f110 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -18,6 +18,7 @@ pub const Node = struct { InfixOp, PrefixOp, SuffixOp, + GroupedExpression, FieldInitializer, IntegerLiteral, FloatLiteral, @@ -46,6 +47,7 @@ pub const Node = struct { Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).iterate(index), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).iterate(index), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).iterate(index), + Id.GroupedExpression => @fieldParentPtr(NodeGroupedExpression, "base", base).iterate(index), Id.FieldInitializer => @fieldParentPtr(NodeFieldInitializer, "base", base).iterate(index), Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).iterate(index), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index), @@ -75,6 +77,7 @@ pub const Node = struct { Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).firstToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).firstToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).firstToken(), + Id.GroupedExpression => @fieldParentPtr(NodeGroupedExpression, "base", base).firstToken(), Id.FieldInitializer => @fieldParentPtr(NodeFieldInitializer, "base", base).firstToken(), Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).firstToken(), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(), @@ -104,6 +107,7 @@ pub const Node = struct { Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).lastToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).lastToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).lastToken(), + Id.GroupedExpression => @fieldParentPtr(NodeGroupedExpression, "base", base).lastToken(), Id.FieldInitializer => @fieldParentPtr(NodeFieldInitializer, "base", base).lastToken(), Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).lastToken(), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(), @@ -624,6 +628,30 @@ pub const NodeSuffixOp = struct { } }; +pub const NodeGroupedExpression = struct { + base: Node, + lparen: Token, + expr: &Node, + rparen: Token, + + pub fn iterate(self: &NodeGroupedExpression, index: usize) ?&Node { + var i = index; + + if (i < 1) return self.expr; + i -= 1; + + return null; + } + + pub fn firstToken(self: &NodeGroupedExpression) Token { + return self.lparen; + } + + pub fn lastToken(self: &NodeGroupedExpression) Token { + return self.rparen; + } +}; + pub const NodeIntegerLiteral = struct { base: Node, token: Token, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 19ed4af4bf..61552f2a3d 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -594,6 +594,27 @@ pub const Parser = struct { try stack.append(State.AfterOperand); continue; }, + Token.Id.LParen => { + const node = try arena.create(ast.NodeGroupedExpression); + *node = ast.NodeGroupedExpression { + .base = self.initNode(ast.Node.Id.GroupedExpression), + .lparen = token, + .expr = undefined, + .rparen = undefined, + }; + try stack.append(State { + .Operand = &node.base + }); + try stack.append(State.AfterOperand); + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.RParen, + .ptr = &node.rparen, + } + }); + try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); + continue; + }, else => return self.parseError(token, "expected primary expression, found {}", @tagName(token.id)), } @@ -1770,6 +1791,12 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = suffix_op.lhs }); }, + ast.Node.Id.GroupedExpression => { + const grouped_expr = @fieldParentPtr(ast.NodeGroupedExpression, "base", base); + try stack.append(RenderState { .Text = ")"}); + try stack.append(RenderState { .Expression = grouped_expr.expr }); + try stack.append(RenderState { .Text = "("}); + }, ast.Node.Id.FieldInitializer => { const field_init = @fieldParentPtr(ast.NodeFieldInitializer, "base", base); try stream.print(".{} = ", self.tokenizer.getTokenSlice(field_init.name_token)); @@ -2240,50 +2267,6 @@ test "zig fmt: indexing" { ); } -test "zig fmt: arrays" { - try testCanonical( - \\test "test array" { - \\ const a: [2]u8 = [2]u8{ 1, 2 }; - \\ const a: [2]u8 = []u8{ 1, 2 }; - \\ const a: [0]u8 = []u8{}; - \\} - \\ - ); -} - -test "zig fmt: precedence" { - try testCanonical( - \\test "precedence" { - \\ a!b(); - \\ (a!b)(); - \\ !a!b; - \\ !(a!b); - \\ !a{}; - \\ !(a{}); - \\ a + b{}; - \\ (a + b){}; - \\ a << b + c; - \\ (a << b) + c; - \\ a & b << c; - \\ (a & b) << c; - \\ a ^ b & c; - \\ (a ^ b) & c; - \\ a | b ^ c; - \\ (a | b) ^ c; - \\ a == b | c; - \\ (a == b) | c; - \\ a and b == c; - \\ (a and b) == c; - \\ a or b and c; - \\ (a or b) and c; - \\ (a or b) and c; - \\ a = b or c; - \\ (a = b) or c; - \\} - \\ - ); -} - test "zig fmt: struct declaration" { try testCanonical( \\const S = struct { @@ -2682,6 +2665,50 @@ test "zig fmt: coroutines" { ); } +test "zig fmt: arrays" { + try testCanonical( + \\test "test array" { + \\ const a: [2]u8 = [2]u8{ 1, 2 }; + \\ const a: [2]u8 = []u8{ 1, 2 }; + \\ const a: [0]u8 = []u8{}; + \\} + \\ + ); +} + +test "zig fmt: precedence" { + try testCanonical( + \\test "precedence" { + \\ a!b(); + \\ (a!b)(); + \\ !a!b; + \\ !(a!b); + \\ !a{}; + \\ !(a{}); + \\ a + b{}; + \\ (a + b){}; + \\ a << b + c; + \\ (a << b) + c; + \\ a & b << c; + \\ (a & b) << c; + \\ a ^ b & c; + \\ (a ^ b) & c; + \\ a | b ^ c; + \\ (a | b) ^ c; + \\ a == b | c; + \\ (a == b) | c; + \\ a and b == c; + \\ (a and b) == c; + \\ a or b and c; + \\ (a or b) and c; + \\ (a or b) and c; + \\ a = b or c; + \\ (a = b) or c; + \\} + \\ + ); +} + test "zig fmt: zig fmt" { try testCanonical(@embedFile("ast.zig")); try testCanonical(@embedFile("index.zig")); From 40f35e997a2155df8141ec309e43d6dad9785965 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 3 Apr 2018 15:17:26 +0200 Subject: [PATCH 17/87] std.zig.parser moved container initializer tests down --- std/zig/parser.zig | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 61552f2a3d..bcb66a6e62 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -2360,16 +2360,6 @@ test "zig fmt: union declaration" { ); } -test "zig fmt: container initializers" { - try testCanonical( - \\const a1 = []u8{ }; - \\const a2 = []u8{ 1, 2, 3, 4 }; - \\const s1 = S{ }; - \\const s2 = S{ .a = 1, .b = 2, }; - \\ - ); -} - test "zig fmt: switch" { try testCanonical( \\test "switch" { @@ -2676,6 +2666,16 @@ test "zig fmt: arrays" { ); } +test "zig fmt: container initializers" { + try testCanonical( + \\const a1 = []u8{ }; + \\const a2 = []u8{ 1, 2, 3, 4 }; + \\const s1 = S{ }; + \\const s2 = S{ .a = 1, .b = 2, }; + \\ + ); +} + test "zig fmt: precedence" { try testCanonical( \\test "precedence" { From 4fae452684c108275e9d3003b4927c8c8d41c59b Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 3 Apr 2018 15:33:22 +0200 Subject: [PATCH 18/87] std.zig.parser Refactored top level decl parsing * Now, the arraylist from the root node is passed through the states. * This allows us to reuse the code for enums, unions and structs --- std/zig/parser.zig | 41 +++++++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index bcb66a6e62..99ea89bf97 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -53,6 +53,7 @@ pub const Parser = struct { } const TopLevelDeclCtx = struct { + decls: &ArrayList(&ast.Node), visib_token: ?Token, extern_token: ?Token, lib_name: ?&ast.Node, @@ -75,7 +76,6 @@ pub const Parser = struct { const ExpectTokenSave = struct { id: Token.Id, ptr: &Token, - }; fn ListState(comptime T: type) type { @@ -88,7 +88,7 @@ pub const Parser = struct { const State = union(enum) { TopLevel, - TopLevelExtern: ?Token, + TopLevelExtern: TopLevelDeclCtx, TopLevelDecl: TopLevelDeclCtx, Expression: DestPtr, ExpectOperand, @@ -182,7 +182,14 @@ pub const Parser = struct { const token = self.getNextToken(); switch (token.id) { Token.Id.Keyword_pub, Token.Id.Keyword_export => { - stack.append(State { .TopLevelExtern = token }) catch unreachable; + stack.append(State { + .TopLevelExtern = TopLevelDeclCtx { + .decls = &root_node.decls, + .visib_token = token, + .extern_token = null, + .lib_name = null, + } + }) catch unreachable; continue; }, Token.Id.Keyword_test => { @@ -208,12 +215,19 @@ pub const Parser = struct { }, else => { self.putBackToken(token); - stack.append(State { .TopLevelExtern = null }) catch unreachable; + stack.append(State { + .TopLevelExtern = TopLevelDeclCtx { + .decls = &root_node.decls, + .visib_token = null, + .extern_token = null, + .lib_name = null, + } + }) catch unreachable; continue; }, } }, - State.TopLevelExtern => |visib_token| { + State.TopLevelExtern => |ctx| { const token = self.getNextToken(); if (token.id == Token.Id.Keyword_extern) { const lib_name_token = self.getNextToken(); @@ -229,7 +243,8 @@ pub const Parser = struct { stack.append(State { .TopLevelDecl = TopLevelDeclCtx { - .visib_token = visib_token, + .decls = ctx.decls, + .visib_token = ctx.visib_token, .extern_token = token, .lib_name = lib_name, }, @@ -237,13 +252,7 @@ pub const Parser = struct { continue; } self.putBackToken(token); - stack.append(State { - .TopLevelDecl = TopLevelDeclCtx { - .visib_token = visib_token, - .extern_token = null, - .lib_name = null, - }, - }) catch unreachable; + stack.append(State { .TopLevelDecl = ctx }) catch unreachable; continue; }, State.TopLevelDecl => |ctx| { @@ -252,7 +261,7 @@ pub const Parser = struct { Token.Id.Keyword_var, Token.Id.Keyword_const => { stack.append(State.TopLevel) catch unreachable; // TODO shouldn't need these casts - const var_decl_node = try self.createAttachVarDecl(arena, &root_node.decls, ctx.visib_token, + const var_decl_node = try self.createAttachVarDecl(arena, ctx.decls, ctx.visib_token, token, (?Token)(null), ctx.extern_token, ctx.lib_name); try stack.append(State { .VarDecl = var_decl_node }); continue; @@ -260,7 +269,7 @@ pub const Parser = struct { Token.Id.Keyword_fn => { stack.append(State.TopLevel) catch unreachable; // TODO shouldn't need these casts - const fn_proto = try self.createAttachFnProto(arena, &root_node.decls, token, + const fn_proto = try self.createAttachFnProto(arena, ctx.decls, token, ctx.extern_token, ctx.lib_name, (?Token)(null), ctx.visib_token, (?Token)(null)); try stack.append(State { .FnDef = fn_proto }); try stack.append(State { .FnProto = fn_proto }); @@ -270,7 +279,7 @@ pub const Parser = struct { stack.append(State.TopLevel) catch unreachable; const fn_token = try self.eatToken(Token.Id.Keyword_fn); // TODO shouldn't need this cast - const fn_proto = try self.createAttachFnProto(arena, &root_node.decls, fn_token, + const fn_proto = try self.createAttachFnProto(arena, ctx.decls, fn_token, ctx.extern_token, ctx.lib_name, (?Token)(token), (?Token)(null), (?Token)(null)); try stack.append(State { .FnDef = fn_proto }); try stack.append(State { .FnProto = fn_proto }); From d602f12df8f12c3a363abec8c1976f8bc84eb2be Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 3 Apr 2018 15:59:14 +0200 Subject: [PATCH 19/87] std.zig.ast Added ContainerDecl --- std/zig/ast.zig | 69 +++++++++++++++++++++++++++++++++++++++++++++- std/zig/parser.zig | 4 +++ 2 files changed, 72 insertions(+), 1 deletion(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 32a8a7f110..767a797cab 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -11,6 +11,7 @@ pub const Node = struct { pub const Id = enum { Root, VarDecl, + ContainerDecl, Identifier, FnProto, ParamDecl, @@ -40,6 +41,7 @@ pub const Node = struct { return switch (base.id) { Id.Root => @fieldParentPtr(NodeRoot, "base", base).iterate(index), Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).iterate(index), + Id.ContainerDecl => @fieldParentPtr(NodeContainerDecl, "base", base).iterate(index), Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).iterate(index), Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).iterate(index), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).iterate(index), @@ -70,6 +72,7 @@ pub const Node = struct { return switch (base.id) { Id.Root => @fieldParentPtr(NodeRoot, "base", base).firstToken(), Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).firstToken(), + Id.ContainerDecl => @fieldParentPtr(NodeContainerDecl, "base", base).firstToken(), Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).firstToken(), Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).firstToken(), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).firstToken(), @@ -100,6 +103,7 @@ pub const Node = struct { return switch (base.id) { Id.Root => @fieldParentPtr(NodeRoot, "base", base).lastToken(), Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).lastToken(), + Id.ContainerDecl => @fieldParentPtr(NodeContainerDecl, "base", base).lastToken(), Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).lastToken(), Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).lastToken(), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).lastToken(), @@ -196,6 +200,69 @@ pub const NodeVarDecl = struct { } }; +pub const NodeContainerDecl = struct { + base: Node, + kind_token: Token, + init_arg_expr: InitArg, + kind: Kind, + decls: ArrayList(&Node), + rbrace_token: Token, + + // TODO: Different array lists for each kind. + const Kind = union(enum) { + Struct: ArrayList(&Node), + Enum: ArrayList(&Node), + Union: ArrayList(&Node), + }; + + const InitArg = union(enum) { + None, + Enum, + Type: &Node, + }; + + pub fn iterate(self: &NodeContainerDecl, index: usize) ?&Node { + var i = index; + + switch (self.init_arg_expr) { + InitArg.Type => |t| { + if (i < 1) return t; + i -= 1; + }, + InitArg.None, + InitArg.Enum => { } + } + + if (i < self.decls.len) return self.decls.at(i); + i -= self.decls.len; + + switch (self.kind) { + Kind.Struct => |fields| { + if (i < fields.len) return fields.at(i); + i -= fields.len; + }, + Kind.Enum => |tags| { + if (i < tags.len) return tags.at(i); + i -= tags.len; + }, + Kind.Union => |tags| { + if (i < tags.len) return tags.at(i); + i -= tags.len; + }, + } + + return null; + } + + pub fn firstToken(self: &NodeContainerDecl) Token { + return self.kind_token; + } + + pub fn lastToken(self: &NodeContainerDecl) Token { + return self.rbrace_token; + } +}; + pub const NodeIdentifier = struct { base: Node, name_token: Token, @@ -611,7 +678,7 @@ pub const NodeSuffixOp = struct { i -= exprs.len; }, SuffixOp.StructInitializer => |fields| { - if (i < fields.len) return fields.at(i); + if (i < fields.len) return &fields.at(i).base; i -= fields.len; }, } diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 99ea89bf97..0b4e55913c 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -1806,6 +1806,10 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = grouped_expr.expr }); try stack.append(RenderState { .Text = "("}); }, + ast.Node.Id.ContainerDecl => { + const container_decl = @fieldParentPtr(ast.NodeGroupedExpression, "base", base); + @panic("TODO: ContainerDecl"); + }, ast.Node.Id.FieldInitializer => { const field_init = @fieldParentPtr(ast.NodeFieldInitializer, "base", base); try stream.print(".{} = ", self.tokenizer.getTokenSlice(field_init.name_token)); From ec611bf8b47af0db5244af644040907bfcb63945 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 3 Apr 2018 20:00:02 +0200 Subject: [PATCH 20/87] std.zig.parser now parses regular enums, unions and struct * Still missing packed, and extern --- std/zig/ast.zig | 106 ++++++++++++++-- std/zig/parser.zig | 305 +++++++++++++++++++++++++++++++++++++-------- 2 files changed, 351 insertions(+), 60 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 767a797cab..484bc59f16 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -12,6 +12,9 @@ pub const Node = struct { Root, VarDecl, ContainerDecl, + StructField, + UnionTag, + EnumTag, Identifier, FnProto, ParamDecl, @@ -42,6 +45,9 @@ pub const Node = struct { Id.Root => @fieldParentPtr(NodeRoot, "base", base).iterate(index), Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).iterate(index), Id.ContainerDecl => @fieldParentPtr(NodeContainerDecl, "base", base).iterate(index), + Id.StructField => @fieldParentPtr(NodeStructField, "base", base).iterate(index), + Id.UnionTag => @fieldParentPtr(NodeUnionTag, "base", base).iterate(index), + Id.EnumTag => @fieldParentPtr(NodeEnumTag, "base", base).iterate(index), Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).iterate(index), Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).iterate(index), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).iterate(index), @@ -73,6 +79,9 @@ pub const Node = struct { Id.Root => @fieldParentPtr(NodeRoot, "base", base).firstToken(), Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).firstToken(), Id.ContainerDecl => @fieldParentPtr(NodeContainerDecl, "base", base).firstToken(), + Id.StructField => @fieldParentPtr(NodeStructField, "base", base).firstToken(), + Id.UnionTag => @fieldParentPtr(NodeUnionTag, "base", base).firstToken(), + Id.EnumTag => @fieldParentPtr(NodeEnumTag, "base", base).firstToken(), Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).firstToken(), Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).firstToken(), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).firstToken(), @@ -104,6 +113,9 @@ pub const Node = struct { Id.Root => @fieldParentPtr(NodeRoot, "base", base).lastToken(), Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).lastToken(), Id.ContainerDecl => @fieldParentPtr(NodeContainerDecl, "base", base).lastToken(), + Id.StructField => @fieldParentPtr(NodeStructField, "base", base).lastToken(), + Id.UnionTag => @fieldParentPtr(NodeUnionTag, "base", base).lastToken(), + Id.EnumTag => @fieldParentPtr(NodeEnumTag, "base", base).lastToken(), Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).lastToken(), Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).lastToken(), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).lastToken(), @@ -203,16 +215,15 @@ pub const NodeVarDecl = struct { pub const NodeContainerDecl = struct { base: Node, kind_token: Token, - init_arg_expr: InitArg, kind: Kind, - decls: ArrayList(&Node), + init_arg_expr: InitArg, + fields_and_decls: ArrayList(&Node), rbrace_token: Token, - // TODO: Different array lists for each kind. - const Kind = union(enum) { - Struct: ArrayList(&Node), - Enum: ArrayList(&Node), - Union: ArrayList(&Node), + const Kind = enum { + Struct, + Enum, + Union, }; const InitArg = union(enum) { @@ -263,6 +274,87 @@ pub const NodeContainerDecl = struct { } }; +pub const NodeStructField = struct { + base: Node, + name_token: Token, + type_expr: &Node, + + pub fn iterate(self: &NodeStructField, index: usize) ?&Node { + var i = index; + + if (i < 1) return self.type_expr; + i -= 1; + + return null; + } + + pub fn firstToken(self: &NodeStructField) Token { + return self.name_token; + } + + pub fn lastToken(self: &NodeStructField) Token { + return self.type_expr.lastToken(); + } +}; + +pub const NodeUnionTag = struct { + base: Node, + name_token: Token, + type_expr: ?&Node, + + pub fn iterate(self: &NodeUnionTag, index: usize) ?&Node { + var i = index; + + if (self.type_expr) |type_expr| { + if (i < 1) return type_expr; + i -= 1; + } + + return null; + } + + pub fn firstToken(self: &NodeUnionTag) Token { + return self.name_token; + } + + pub fn lastToken(self: &NodeUnionTag) Token { + if (self.type_expr) |type_expr| { + return type_expr.lastToken(); + } + + return self.name_token; + } +}; + +pub const NodeEnumTag = struct { + base: Node, + name_token: Token, + value: ?&Node, + + pub fn iterate(self: &NodeEnumTag, index: usize) ?&Node { + var i = index; + + if (self.value) |value| { + if (i < 1) return value; + i -= 1; + } + + return null; + } + + pub fn firstToken(self: &NodeEnumTag) Token { + return self.name_token; + } + + pub fn lastToken(self: &NodeEnumTag) Token { + if (self.value) |value| { + return value.lastToken(); + } + + return self.name_token; + } +}; + pub const NodeIdentifier = struct { base: Node, name_token: Token, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 0b4e55913c..fc00ba5f4e 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -90,6 +90,7 @@ pub const Parser = struct { TopLevel, TopLevelExtern: TopLevelDeclCtx, TopLevelDecl: TopLevelDeclCtx, + ContainerDecl: &ast.NodeContainerDecl, Expression: DestPtr, ExpectOperand, Operand: &ast.Node, @@ -117,6 +118,7 @@ pub const Parser = struct { ExprListCommaOrEnd: ListState(&ast.Node), FieldInitListItemOrEnd: ListState(&ast.NodeFieldInitializer), FieldInitListCommaOrEnd: ListState(&ast.NodeFieldInitializer), + FieldListCommaOrEnd: &ast.NodeContainerDecl, }; /// Returns an AST tree, allocated with the parser's allocator. @@ -181,17 +183,6 @@ pub const Parser = struct { State.TopLevel => { const token = self.getNextToken(); switch (token.id) { - Token.Id.Keyword_pub, Token.Id.Keyword_export => { - stack.append(State { - .TopLevelExtern = TopLevelDeclCtx { - .decls = &root_node.decls, - .visib_token = token, - .extern_token = null, - .lib_name = null, - } - }) catch unreachable; - continue; - }, Token.Id.Keyword_test => { stack.append(State.TopLevel) catch unreachable; @@ -213,16 +204,29 @@ pub const Parser = struct { root_node.eof_token = token; return Tree {.root_node = root_node, .arena_allocator = arena_allocator}; }, + Token.Id.Keyword_pub, Token.Id.Keyword_export => { + stack.append(State.TopLevel) catch unreachable; + try stack.append(State { + .TopLevelExtern = TopLevelDeclCtx { + .decls = &root_node.decls, + .visib_token = token, + .extern_token = null, + .lib_name = null, + } + }); + continue; + }, else => { self.putBackToken(token); - stack.append(State { + stack.append(State.TopLevel) catch unreachable; + try stack.append(State { .TopLevelExtern = TopLevelDeclCtx { .decls = &root_node.decls, .visib_token = null, .extern_token = null, .lib_name = null, } - }) catch unreachable; + }); continue; }, } @@ -259,7 +263,6 @@ pub const Parser = struct { const token = self.getNextToken(); switch (token.id) { Token.Id.Keyword_var, Token.Id.Keyword_const => { - stack.append(State.TopLevel) catch unreachable; // TODO shouldn't need these casts const var_decl_node = try self.createAttachVarDecl(arena, ctx.decls, ctx.visib_token, token, (?Token)(null), ctx.extern_token, ctx.lib_name); @@ -267,7 +270,6 @@ pub const Parser = struct { continue; }, Token.Id.Keyword_fn => { - stack.append(State.TopLevel) catch unreachable; // TODO shouldn't need these casts const fn_proto = try self.createAttachFnProto(arena, ctx.decls, token, ctx.extern_token, ctx.lib_name, (?Token)(null), ctx.visib_token, (?Token)(null)); @@ -276,7 +278,6 @@ pub const Parser = struct { continue; }, Token.Id.Keyword_nakedcc, Token.Id.Keyword_stdcallcc => { - stack.append(State.TopLevel) catch unreachable; const fn_token = try self.eatToken(Token.Id.Keyword_fn); // TODO shouldn't need this cast const fn_proto = try self.createAttachFnProto(arena, ctx.decls, fn_token, @@ -336,6 +337,101 @@ pub const Parser = struct { } return self.parseError(token, "expected '=' or ';', found {}", @tagName(token.id)); }, + + State.ContainerDecl => |container_decl| { + const token = self.getNextToken(); + + switch (token.id) { + Token.Id.Identifier => { + switch (container_decl.kind) { + ast.NodeContainerDecl.Kind.Struct => { + const node = try arena.create(ast.NodeStructField); + *node = ast.NodeStructField { + .base = self.initNode(ast.Node.Id.StructField), + .name_token = token, + .type_expr = undefined, + }; + try container_decl.fields_and_decls.append(&node.base); + + try stack.append(State { .FieldListCommaOrEnd = container_decl }); + try stack.append(State { .Expression = DestPtr { .Field = &node.type_expr } }); + try stack.append(State { .ExpectToken = Token.Id.Colon }); + continue; + }, + ast.NodeContainerDecl.Kind.Union => { + const node = try arena.create(ast.NodeUnionTag); + *node = ast.NodeUnionTag { + .base = self.initNode(ast.Node.Id.UnionTag), + .name_token = token, + .type_expr = null, + }; + try container_decl.fields_and_decls.append(&node.base); + + try stack.append(State { .FieldListCommaOrEnd = container_decl }); + + const next = self.getNextToken(); + if (next.id != Token.Id.Colon) { + self.putBackToken(next); + continue; + } + + try stack.append(State { .Expression = DestPtr { .NullableField = &node.type_expr } }); + continue; + }, + ast.NodeContainerDecl.Kind.Enum => { + const node = try arena.create(ast.NodeEnumTag); + *node = ast.NodeEnumTag { + .base = self.initNode(ast.Node.Id.EnumTag), + .name_token = token, + .value = null, + }; + try container_decl.fields_and_decls.append(&node.base); + + try stack.append(State { .FieldListCommaOrEnd = container_decl }); + + const next = self.getNextToken(); + if (next.id != Token.Id.Equal) { + self.putBackToken(next); + continue; + } + + try stack.append(State { .Expression = DestPtr { .NullableField = &node.value } }); + continue; + }, + } + }, + Token.Id.Keyword_pub, Token.Id.Keyword_export => { + stack.append(State{ .ContainerDecl = container_decl }) catch unreachable; + try stack.append(State { + .TopLevelExtern = TopLevelDeclCtx { + .decls = &container_decl.fields_and_decls, + .visib_token = token, + .extern_token = null, + .lib_name = null, + } + }); + continue; + }, + Token.Id.RBrace => { + container_decl.rbrace_token = token; + continue; + }, + else => { + self.putBackToken(token); + stack.append(State{ .ContainerDecl = container_decl }) catch unreachable; + try stack.append(State { + .TopLevelExtern = TopLevelDeclCtx { + .decls = &container_decl.fields_and_decls, + .visib_token = null, + .extern_token = null, + .lib_name = null, + } + }); + continue; + } + } + }, + State.ExpectToken => |token_id| { _ = try self.eatToken(token_id); continue; @@ -537,6 +633,53 @@ pub const Parser = struct { try stack.append(State.AfterOperand); continue; }, + Token.Id.Keyword_struct, Token.Id.Keyword_union, Token.Id.Keyword_enum => { + const node = try arena.create(ast.NodeContainerDecl); + *node = ast.NodeContainerDecl { + .base = self.initNode(ast.Node.Id.ContainerDecl), + .kind_token = token, + .kind = switch (token.id) { + Token.Id.Keyword_struct => ast.NodeContainerDecl.Kind.Struct, + Token.Id.Keyword_union => ast.NodeContainerDecl.Kind.Union, + Token.Id.Keyword_enum => ast.NodeContainerDecl.Kind.Enum, + else => unreachable, + }, + .init_arg_expr = undefined, + .fields_and_decls = ArrayList(&ast.Node).init(arena), + .rbrace_token = undefined, + }; + + try stack.append(State { .Operand = &node.base }); + try stack.append(State.AfterOperand); + try stack.append(State { .ContainerDecl = node }); + try stack.append(State { .ExpectToken = Token.Id.LBrace }); + + const lparen = self.getNextToken(); + if (lparen.id != Token.Id.LParen) { + self.putBackToken(lparen); + node.init_arg_expr = ast.NodeContainerDecl.InitArg.None; + continue; + } + + try stack.append(State { .ExpectToken = Token.Id.RParen }); + + const init_arg_token = self.getNextToken(); + switch (init_arg_token.id) { + Token.Id.Keyword_enum => { + node.init_arg_expr = ast.NodeContainerDecl.InitArg.Enum; + }, + else => { + self.putBackToken(lparen); + node.init_arg_expr = ast.NodeContainerDecl.InitArg { .Type = undefined }; + try stack.append(State { + .Expression = DestPtr { + .Field = &node.init_arg_expr.Type + } + }); + }, + } + continue; + }, Token.Id.Builtin => { const node = try arena.create(ast.NodeBuiltinCall); *node = ast.NodeBuiltinCall { @@ -799,24 +942,6 @@ pub const Parser = struct { try stack.append(State { .Expression = DestPtr{.List = list_state.list} }); }, - State.ExprListCommaOrEnd => |list_state| { - var token = self.getNextToken(); - switch (token.id) { - Token.Id.Comma => { - stack.append(State { .ExprListItemOrEnd = list_state }) catch unreachable; - }, - else => { - const IdTag = @TagType(Token.Id); - if (IdTag(list_state.end) == token.id) { - *list_state.ptr = token; - continue; - } - - return self.parseError(token, "expected ',' or {}, found {}", @tagName(list_state.end), @tagName(token.id)); - }, - } - }, - State.FieldInitListItemOrEnd => |list_state| { var token = self.getNextToken(); @@ -854,22 +979,20 @@ pub const Parser = struct { }); }, - State.FieldInitListCommaOrEnd => |list_state| { - var token = self.getNextToken(); - switch (token.id) { - Token.Id.Comma => { - stack.append(State { .FieldInitListItemOrEnd = list_state }) catch unreachable; - }, - else => { - const IdTag = @TagType(Token.Id); - if (IdTag(list_state.end) == token.id) { - *list_state.ptr = token; - continue; - } + State.ExprListCommaOrEnd => |list_state| { + try self.commaOrEnd(&stack, list_state.end, list_state.ptr, State { .ExprListItemOrEnd = list_state }); + continue; + }, - return self.parseError(token, "expected ',' or {}, found {}", @tagName(list_state.end), @tagName(token.id)); - }, - } + State.FieldInitListCommaOrEnd => |list_state| { + try self.commaOrEnd(&stack, list_state.end, list_state.ptr, State { .FieldInitListItemOrEnd = list_state }); + continue; + }, + + State.FieldListCommaOrEnd => |container_decl| { + try self.commaOrEnd(&stack, Token.Id.RBrace, &container_decl.rbrace_token, + State { .ContainerDecl = container_decl }); + continue; }, State.AddrOfModifiers => |addr_of_info| { @@ -1089,6 +1212,24 @@ pub const Parser = struct { } } + fn commaOrEnd(self: &Parser, stack: &ArrayList(State), end: &const Token.Id, ptr: &Token, state_after_comma: &const State) !void { + var token = self.getNextToken(); + switch (token.id) { + Token.Id.Comma => { + stack.append(state_after_comma) catch unreachable; + }, + else => { + const IdTag = @TagType(Token.Id); + if (IdTag(*end) == token.id) { + *ptr = token; + return; + } + + return self.parseError(token, "expected ',' or {}, found {}", @tagName(*end), @tagName(token.id)); + }, + } + } + fn popSuffixOp(stack: &ArrayList(State)) &ast.Node { var expression: &ast.Node = undefined; var left_leaf_ptr: &&ast.Node = &expression; @@ -1806,10 +1947,6 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = grouped_expr.expr }); try stack.append(RenderState { .Text = "("}); }, - ast.Node.Id.ContainerDecl => { - const container_decl = @fieldParentPtr(ast.NodeGroupedExpression, "base", base); - @panic("TODO: ContainerDecl"); - }, ast.Node.Id.FieldInitializer => { const field_init = @fieldParentPtr(ast.NodeFieldInitializer, "base", base); try stream.print(".{} = ", self.tokenizer.getTokenSlice(field_init.name_token)); @@ -1851,6 +1988,46 @@ pub const Parser = struct { const error_type = @fieldParentPtr(ast.NodeErrorType, "base", base); try stream.print("{}", self.tokenizer.getTokenSlice(error_type.token)); }, + ast.Node.Id.ContainerDecl => { + const container_decl = @fieldParentPtr(ast.NodeContainerDecl, "base", base); + try stream.print("{} {{", self.tokenizer.getTokenSlice(container_decl.kind_token)); + try stack.append(RenderState { .Text = "}"}); + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { .Indent = indent }); + + const fields_and_decls = container_decl.fields_and_decls.toSliceConst(); + var i = fields_and_decls.len; + while (i != 0) { + i -= 1; + const node = fields_and_decls[i]; + if (i != 0) { + switch (node.id) { + ast.Node.Id.StructField, + ast.Node.Id.UnionTag, + ast.Node.Id.EnumTag => { + try stack.append(RenderState { .Text = "," }); + }, + else => { } + } + } + try stack.append(RenderState { .Expression = node}); + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { + .Text = blk: { + if (i != 0) { + const prev_node = fields_and_decls[i - 1]; + const prev_line_index = prev_node.lastToken().line; + const this_line_index = node.firstToken().line; + if (this_line_index - prev_line_index >= 2) { + break :blk "\n\n"; + } + } + break :blk "\n"; + }, + }); + } + try stack.append(RenderState { .Indent = indent + indent_delta}); + }, ast.Node.Id.MultilineStringLiteral => { const multiline_str_literal = @fieldParentPtr(ast.NodeMultilineStringLiteral, "base", base); try stream.print("\n"); @@ -1883,6 +2060,28 @@ pub const Parser = struct { } } }, + ast.Node.Id.StructField => { + const field = @fieldParentPtr(ast.NodeStructField, "base", base); + try stream.print("{}:", self.tokenizer.getTokenSlice(field.name_token)); + }, + ast.Node.Id.UnionTag => { + const tag = @fieldParentPtr(ast.NodeUnionTag, "base", base); + try stream.print("{}", self.tokenizer.getTokenSlice(tag.name_token)); + + if (tag.type_expr) |type_expr| { + try stream.print(": "); + try stack.append(RenderState { .Expression = type_expr}); + } + }, + ast.Node.Id.EnumTag => { + const tag = @fieldParentPtr(ast.NodeEnumTag, "base", base); + try stream.print("{}", self.tokenizer.getTokenSlice(tag.name_token)); + + if (tag.value) |value| { + try stream.print(" = "); + try stack.append(RenderState { .Expression = value}); + } + }, ast.Node.Id.FnProto => @panic("TODO fn proto in an expression"), ast.Node.Id.LineComment => @panic("TODO render line comment in an expression"), From 09cf82361999c1a22819467e02fc68fd22ca188e Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Wed, 4 Apr 2018 09:57:37 +0200 Subject: [PATCH 21/87] std.zig.parser now parses container decls --- std/zig/ast.zig | 11 +- std/zig/parser.zig | 262 +++++++++++++++++++++++++++++---------------- 2 files changed, 177 insertions(+), 96 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 484bc59f16..1d42d721d9 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -214,12 +214,19 @@ pub const NodeVarDecl = struct { pub const NodeContainerDecl = struct { base: Node, - kind_token: Token, + ltoken: Token, + layout: Layout, kind: Kind, init_arg_expr: InitArg, fields_and_decls: ArrayList(&Node), rbrace_token: Token, + const Layout = enum { + Auto, + Extern, + Packed, + }; + const Kind = enum { Struct, Enum, @@ -266,7 +273,7 @@ pub const NodeContainerDecl = struct { } pub fn firstToken(self: &NodeContainerDecl) Token { - return self.kind_token; + return self.ltoken; } pub fn lastToken(self: &NodeContainerDecl) Token { diff --git a/std/zig/parser.zig b/std/zig/parser.zig index fc00ba5f4e..8a4999a4ee 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -59,6 +59,11 @@ pub const Parser = struct { lib_name: ?&ast.Node, }; + const ContainerExternCtx = struct { + ltoken: Token, + layout: ast.NodeContainerDecl.Layout, + }; + const DestPtr = union(enum) { Field: &&ast.Node, NullableField: &?&ast.Node, @@ -90,6 +95,7 @@ pub const Parser = struct { TopLevel, TopLevelExtern: TopLevelDeclCtx, TopLevelDecl: TopLevelDeclCtx, + ContainerExtern: ContainerExternCtx, ContainerDecl: &ast.NodeContainerDecl, Expression: DestPtr, ExpectOperand, @@ -338,6 +344,63 @@ pub const Parser = struct { return self.parseError(token, "expected '=' or ';', found {}", @tagName(token.id)); }, + State.ContainerExtern => |ctx| { + const token = self.getNextToken(); + + const node = try arena.create(ast.NodeContainerDecl); + *node = ast.NodeContainerDecl { + .base = self.initNode(ast.Node.Id.ContainerDecl), + .ltoken = ctx.ltoken, + .layout = ctx.layout, + .kind = switch (token.id) { + Token.Id.Keyword_struct => ast.NodeContainerDecl.Kind.Struct, + Token.Id.Keyword_union => ast.NodeContainerDecl.Kind.Union, + Token.Id.Keyword_enum => ast.NodeContainerDecl.Kind.Enum, + else => { + return self.parseError(token, "expected {}, {} or {}, found {}", + @tagName(Token.Id.Keyword_struct), + @tagName(Token.Id.Keyword_union), + @tagName(Token.Id.Keyword_enum), + @tagName(token.id)); + }, + }, + .init_arg_expr = undefined, + .fields_and_decls = ArrayList(&ast.Node).init(arena), + .rbrace_token = undefined, + }; + + try stack.append(State { .Operand = &node.base }); + try stack.append(State.AfterOperand); + try stack.append(State { .ContainerDecl = node }); + try stack.append(State { .ExpectToken = Token.Id.LBrace }); + + const lparen = self.getNextToken(); + if (lparen.id != Token.Id.LParen) { + self.putBackToken(lparen); + node.init_arg_expr = ast.NodeContainerDecl.InitArg.None; + continue; + } + + try stack.append(State { .ExpectToken = Token.Id.RParen }); + + const init_arg_token = self.getNextToken(); + switch (init_arg_token.id) { + Token.Id.Keyword_enum => { + node.init_arg_expr = ast.NodeContainerDecl.InitArg.Enum; + }, + else => { + self.putBackToken(init_arg_token); + node.init_arg_expr = ast.NodeContainerDecl.InitArg { .Type = undefined }; + try stack.append(State { + .Expression = DestPtr { + .Field = &node.init_arg_expr.Type + } + }); + }, + } + continue; + }, + State.ContainerDecl => |container_decl| { const token = self.getNextToken(); @@ -633,52 +696,30 @@ pub const Parser = struct { try stack.append(State.AfterOperand); continue; }, + Token.Id.Keyword_packed => { + try stack.append(State { + .ContainerExtern = ContainerExternCtx { + .ltoken = token, + .layout = ast.NodeContainerDecl.Layout.Packed, + }, + }); + }, + Token.Id.Keyword_extern => { + try stack.append(State { + .ContainerExtern = ContainerExternCtx { + .ltoken = token, + .layout = ast.NodeContainerDecl.Layout.Extern, + }, + }); + }, Token.Id.Keyword_struct, Token.Id.Keyword_union, Token.Id.Keyword_enum => { - const node = try arena.create(ast.NodeContainerDecl); - *node = ast.NodeContainerDecl { - .base = self.initNode(ast.Node.Id.ContainerDecl), - .kind_token = token, - .kind = switch (token.id) { - Token.Id.Keyword_struct => ast.NodeContainerDecl.Kind.Struct, - Token.Id.Keyword_union => ast.NodeContainerDecl.Kind.Union, - Token.Id.Keyword_enum => ast.NodeContainerDecl.Kind.Enum, - else => unreachable, + self.putBackToken(token); + try stack.append(State { + .ContainerExtern = ContainerExternCtx { + .ltoken = token, + .layout = ast.NodeContainerDecl.Layout.Auto, }, - .init_arg_expr = undefined, - .fields_and_decls = ArrayList(&ast.Node).init(arena), - .rbrace_token = undefined, - }; - - try stack.append(State { .Operand = &node.base }); - try stack.append(State.AfterOperand); - try stack.append(State { .ContainerDecl = node }); - try stack.append(State { .ExpectToken = Token.Id.LBrace }); - - const lparen = self.getNextToken(); - if (lparen.id != Token.Id.LParen) { - self.putBackToken(lparen); - node.init_arg_expr = ast.NodeContainerDecl.InitArg.None; - continue; - } - - try stack.append(State { .ExpectToken = Token.Id.RParen }); - - const init_arg_token = self.getNextToken(); - switch (init_arg_token.id) { - Token.Id.Keyword_enum => { - node.init_arg_expr = ast.NodeContainerDecl.InitArg.Enum; - }, - else => { - self.putBackToken(lparen); - node.init_arg_expr = ast.NodeContainerDecl.InitArg { .Type = undefined }; - try stack.append(State { - .Expression = DestPtr { - .Field = &node.init_arg_expr.Type - } - }); - }, - } - continue; + }); }, Token.Id.Builtin => { const node = try arena.create(ast.NodeBuiltinCall); @@ -1706,6 +1747,29 @@ pub const Parser = struct { try stack.append(RenderState { .Text = " " }); try stack.append(RenderState { .Expression = test_decl.name }); }, + ast.Node.Id.StructField => { + const field = @fieldParentPtr(ast.NodeStructField, "base", decl); + try stream.print("{}: ", self.tokenizer.getTokenSlice(field.name_token)); + try stack.append(RenderState { .Expression = field.type_expr}); + }, + ast.Node.Id.UnionTag => { + const tag = @fieldParentPtr(ast.NodeUnionTag, "base", decl); + try stream.print("{}", self.tokenizer.getTokenSlice(tag.name_token)); + + if (tag.type_expr) |type_expr| { + try stream.print(": "); + try stack.append(RenderState { .Expression = type_expr}); + } + }, + ast.Node.Id.EnumTag => { + const tag = @fieldParentPtr(ast.NodeEnumTag, "base", decl); + try stream.print("{}", self.tokenizer.getTokenSlice(tag.name_token)); + + if (tag.value) |value| { + try stream.print(" = "); + try stack.append(RenderState { .Expression = value}); + } + }, else => unreachable, } }, @@ -1990,27 +2054,30 @@ pub const Parser = struct { }, ast.Node.Id.ContainerDecl => { const container_decl = @fieldParentPtr(ast.NodeContainerDecl, "base", base); - try stream.print("{} {{", self.tokenizer.getTokenSlice(container_decl.kind_token)); + + switch (container_decl.layout) { + ast.NodeContainerDecl.Layout.Packed => try stream.print("packed "), + ast.NodeContainerDecl.Layout.Extern => try stream.print("extern "), + ast.NodeContainerDecl.Layout.Auto => { }, + } + + switch (container_decl.kind) { + ast.NodeContainerDecl.Kind.Struct => try stream.print("struct"), + ast.NodeContainerDecl.Kind.Enum => try stream.print("enum"), + ast.NodeContainerDecl.Kind.Union => try stream.print("union"), + } + try stack.append(RenderState { .Text = "}"}); try stack.append(RenderState.PrintIndent); try stack.append(RenderState { .Indent = indent }); + try stack.append(RenderState { .Text = "\n"}); const fields_and_decls = container_decl.fields_and_decls.toSliceConst(); var i = fields_and_decls.len; while (i != 0) { i -= 1; const node = fields_and_decls[i]; - if (i != 0) { - switch (node.id) { - ast.Node.Id.StructField, - ast.Node.Id.UnionTag, - ast.Node.Id.EnumTag => { - try stack.append(RenderState { .Text = "," }); - }, - else => { } - } - } - try stack.append(RenderState { .Expression = node}); + try stack.append(RenderState { .TopLevelDecl = node}); try stack.append(RenderState.PrintIndent); try stack.append(RenderState { .Text = blk: { @@ -2025,22 +2092,43 @@ pub const Parser = struct { break :blk "\n"; }, }); + + if (i != 0) { + const prev_node = fields_and_decls[i - 1]; + switch (prev_node.id) { + ast.Node.Id.StructField, + ast.Node.Id.UnionTag, + ast.Node.Id.EnumTag => { + try stack.append(RenderState { .Text = "," }); + }, + else => { } + } + } } try stack.append(RenderState { .Indent = indent + indent_delta}); + try stack.append(RenderState { .Text = "{"}); + + switch (container_decl.init_arg_expr) { + ast.NodeContainerDecl.InitArg.None => try stack.append(RenderState { .Text = " "}), + ast.NodeContainerDecl.InitArg.Enum => try stack.append(RenderState { .Text = "(enum) "}), + ast.NodeContainerDecl.InitArg.Type => |type_expr| { + try stack.append(RenderState { .Text = ") "}); + try stack.append(RenderState { .Expression = type_expr}); + try stack.append(RenderState { .Text = "("}); + }, + } }, ast.Node.Id.MultilineStringLiteral => { const multiline_str_literal = @fieldParentPtr(ast.NodeMultilineStringLiteral, "base", base); try stream.print("\n"); var i : usize = 0; - indent += 4; while (i < multiline_str_literal.tokens.len) : (i += 1) { const t = multiline_str_literal.tokens.at(i); - try stream.writeByteNTimes(' ', indent); + try stream.writeByteNTimes(' ', indent + indent_delta); try stream.print("{}", self.tokenizer.getTokenSlice(t)); } - try stream.writeByteNTimes(' ', indent); - indent -= 4; + try stream.writeByteNTimes(' ', indent + indent_delta); }, ast.Node.Id.UndefinedLiteral => { const undefined_literal = @fieldParentPtr(ast.NodeUndefinedLiteral, "base", base); @@ -2060,31 +2148,12 @@ pub const Parser = struct { } } }, - ast.Node.Id.StructField => { - const field = @fieldParentPtr(ast.NodeStructField, "base", base); - try stream.print("{}:", self.tokenizer.getTokenSlice(field.name_token)); - }, - ast.Node.Id.UnionTag => { - const tag = @fieldParentPtr(ast.NodeUnionTag, "base", base); - try stream.print("{}", self.tokenizer.getTokenSlice(tag.name_token)); - - if (tag.type_expr) |type_expr| { - try stream.print(": "); - try stack.append(RenderState { .Expression = type_expr}); - } - }, - ast.Node.Id.EnumTag => { - const tag = @fieldParentPtr(ast.NodeEnumTag, "base", base); - try stream.print("{}", self.tokenizer.getTokenSlice(tag.name_token)); - - if (tag.value) |value| { - try stream.print(" = "); - try stack.append(RenderState { .Expression = value}); - } - }, ast.Node.Id.FnProto => @panic("TODO fn proto in an expression"), ast.Node.Id.LineComment => @panic("TODO render line comment in an expression"), + ast.Node.Id.StructField, + ast.Node.Id.UnionTag, + ast.Node.Id.EnumTag, ast.Node.Id.Root, ast.Node.Id.VarDecl, ast.Node.Id.TestDecl, @@ -2489,21 +2558,21 @@ test "zig fmt: struct declaration" { \\ return *self; \\ } \\ - \\ f2: u8, + \\ f2: u8 \\}; \\ \\const Ps = packed struct { \\ a: u8, \\ b: u8, \\ - \\ c: u8, + \\ c: u8 \\}; \\ \\const Es = extern struct { \\ a: u8, \\ b: u8, \\ - \\ c: u8, + \\ c: u8 \\}; \\ ); @@ -2513,25 +2582,25 @@ test "zig fmt: enum declaration" { try testCanonical( \\const E = enum { \\ Ok, - \\ SomethingElse = 0, + \\ SomethingElse = 0 \\}; \\ \\const E2 = enum(u8) { \\ Ok, \\ SomethingElse = 255, - \\ SomethingThird, + \\ SomethingThird \\}; \\ \\const Ee = extern enum { \\ Ok, \\ SomethingElse, - \\ SomethingThird, + \\ SomethingThird \\}; \\ \\const Ep = packed enum { \\ Ok, \\ SomethingElse, - \\ SomethingThird, + \\ SomethingThird \\}; \\ ); @@ -2542,31 +2611,36 @@ test "zig fmt: union declaration" { \\const U = union { \\ Int: u8, \\ Float: f32, - \\ Bool: bool, + \\ None, + \\ Bool: bool \\}; \\ \\const Ue = union(enum) { \\ Int: u8, \\ Float: f32, - \\ Bool: bool, + \\ None, + \\ Bool: bool \\}; \\ \\const E = enum { \\ Int, \\ Float, - \\ Bool, + \\ None, + \\ Bool \\}; \\ \\const Ue2 = union(E) { \\ Int: u8, \\ Float: f32, - \\ Bool: bool, + \\ None, + \\ Bool: bool \\}; \\ \\const Eu = extern union { \\ Int: u8, \\ Float: f32, - \\ Bool: bool, + \\ None, + \\ Bool: bool \\}; \\ ); From 020724cfa0677bf42f48957d0ca7a474f6fe31e0 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Wed, 4 Apr 2018 10:27:38 +0200 Subject: [PATCH 22/87] std.zig.tokenizer Tokens now don't contain a line and column field. * Instead, this information is optained by asking the tokenizer. * getTokenLocation takes a start_index, so relative loc can be optained --- std/zig/parser.zig | 23 ++++++++++------------ std/zig/tokenizer.zig | 44 +++++++++++++++---------------------------- 2 files changed, 25 insertions(+), 42 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 8a4999a4ee..f7db8fd93d 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -1571,12 +1571,12 @@ pub const Parser = struct { } fn parseError(self: &Parser, token: &const Token, comptime fmt: []const u8, args: ...) (error{ParseError}) { - const loc = self.tokenizer.getTokenLocation(token); - warn("{}:{}:{}: error: " ++ fmt ++ "\n", self.source_file_name, token.line + 1, token.column + 1, args); + const loc = self.tokenizer.getTokenLocation(0, token); + warn("{}:{}:{}: error: " ++ fmt ++ "\n", self.source_file_name, loc.line + 1, loc.column + 1, args); warn("{}\n", self.tokenizer.buffer[loc.line_start..loc.line_end]); { var i: usize = 0; - while (i < token.column) : (i += 1) { + while (i < loc.column) : (i += 1) { warn(" "); } } @@ -1679,9 +1679,8 @@ pub const Parser = struct { try stack.append(RenderState { .Text = blk: { const prev_node = root_node.decls.at(i - 1); - const prev_line_index = prev_node.lastToken().line; - const this_line_index = decl.firstToken().line; - if (this_line_index - prev_line_index >= 2) { + const loc = self.tokenizer.getTokenLocation(prev_node.lastToken().end, decl.firstToken()); + if (loc.line >= 2) { break :blk "\n\n"; } break :blk "\n"; @@ -1858,10 +1857,9 @@ pub const Parser = struct { try stack.append(RenderState { .Text = blk: { if (i != 0) { - const prev_statement_node = block.statements.items[i - 1]; - const prev_line_index = prev_statement_node.lastToken().line; - const this_line_index = statement_node.firstToken().line; - if (this_line_index - prev_line_index >= 2) { + const prev_node = block.statements.items[i - 1]; + const loc = self.tokenizer.getTokenLocation(prev_node.lastToken().end, statement_node.firstToken()); + if (loc.line >= 2) { break :blk "\n\n"; } } @@ -2083,9 +2081,8 @@ pub const Parser = struct { .Text = blk: { if (i != 0) { const prev_node = fields_and_decls[i - 1]; - const prev_line_index = prev_node.lastToken().line; - const this_line_index = node.firstToken().line; - if (this_line_index - prev_line_index >= 2) { + const loc = self.tokenizer.getTokenLocation(prev_node.lastToken().end, node.firstToken()); + if (loc.line >= 2) { break :blk "\n\n"; } } diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig index 2e40999920..3427678341 100644 --- a/std/zig/tokenizer.zig +++ b/std/zig/tokenizer.zig @@ -5,8 +5,6 @@ pub const Token = struct { id: Id, start: usize, end: usize, - line: usize, - column: usize, const KeywordId = struct { bytes: []const u8, @@ -180,28 +178,34 @@ pub const Token = struct { pub const Tokenizer = struct { buffer: []const u8, index: usize, - line: usize, - column: usize, pending_invalid_token: ?Token, - pub const LineLocation = struct { + pub const Location = struct { + line: usize, + column: usize, line_start: usize, line_end: usize, }; - pub fn getTokenLocation(self: &Tokenizer, token: &const Token) LineLocation { - var loc = LineLocation { - .line_start = 0, + pub fn getTokenLocation(self: &Tokenizer, start_index: usize, token: &const Token) Location { + var loc = Location { + .line = 0, + .column = 0, + .line_start = start_index, .line_end = self.buffer.len, }; - for (self.buffer) |c, i| { - if (i == token.start) { - loc.line_end = i; + for (self.buffer[start_index..]) |c, i| { + if (i + start_index == token.start) { + loc.line_end = i + start_index; while (loc.line_end < self.buffer.len and self.buffer[loc.line_end] != '\n') : (loc.line_end += 1) {} return loc; } if (c == '\n') { + loc.line += 1; + loc.column = 0; loc.line_start = i + 1; + } else { + loc.column += 1; } } return loc; @@ -216,8 +220,6 @@ pub const Tokenizer = struct { return Tokenizer { .buffer = buffer, .index = 0, - .line = 0, - .column = 0, .pending_invalid_token = null, }; } @@ -277,8 +279,6 @@ pub const Tokenizer = struct { .id = Token.Id.Eof, .start = self.index, .end = undefined, - .line = self.line, - .column = self.column, }; while (self.index < self.buffer.len) : (self.index += 1) { const c = self.buffer[self.index]; @@ -286,12 +286,9 @@ pub const Tokenizer = struct { State.Start => switch (c) { ' ' => { result.start = self.index + 1; - result.column += 1; }, '\n' => { result.start = self.index + 1; - result.line += 1; - result.column = 0; }, 'c' => { state = State.C; @@ -977,15 +974,6 @@ pub const Tokenizer = struct { } } - for (self.buffer[start_index..self.index]) |c| { - if (c == '\n') { - self.line += 1; - self.column = 0; - } else { - self.column += 1; - } - } - if (result.id == Token.Id.Eof) { if (self.pending_invalid_token) |token| { self.pending_invalid_token = null; @@ -1009,8 +997,6 @@ pub const Tokenizer = struct { .id = Token.Id.Invalid, .start = self.index, .end = self.index + invalid_length, - .line = self.line, - .column = self.column, }; } From ca0085c46dc5acb6ea93e35192b7b52294381d75 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Wed, 4 Apr 2018 10:54:48 +0200 Subject: [PATCH 23/87] std.zig.parser now parses error set declarations --- std/zig/ast.zig | 47 ++++++++++++------- std/zig/parser.zig | 111 +++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 137 insertions(+), 21 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 1d42d721d9..efb959c7f2 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -11,6 +11,7 @@ pub const Node = struct { pub const Id = enum { Root, VarDecl, + ErrorSetDecl, ContainerDecl, StructField, UnionTag, @@ -44,6 +45,7 @@ pub const Node = struct { return switch (base.id) { Id.Root => @fieldParentPtr(NodeRoot, "base", base).iterate(index), Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).iterate(index), + Id.ErrorSetDecl => @fieldParentPtr(NodeErrorSetDecl, "base", base).iterate(index), Id.ContainerDecl => @fieldParentPtr(NodeContainerDecl, "base", base).iterate(index), Id.StructField => @fieldParentPtr(NodeStructField, "base", base).iterate(index), Id.UnionTag => @fieldParentPtr(NodeUnionTag, "base", base).iterate(index), @@ -78,6 +80,7 @@ pub const Node = struct { return switch (base.id) { Id.Root => @fieldParentPtr(NodeRoot, "base", base).firstToken(), Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).firstToken(), + Id.ErrorSetDecl => @fieldParentPtr(NodeErrorSetDecl, "base", base).firstToken(), Id.ContainerDecl => @fieldParentPtr(NodeContainerDecl, "base", base).firstToken(), Id.StructField => @fieldParentPtr(NodeStructField, "base", base).firstToken(), Id.UnionTag => @fieldParentPtr(NodeUnionTag, "base", base).firstToken(), @@ -112,6 +115,7 @@ pub const Node = struct { return switch (base.id) { Id.Root => @fieldParentPtr(NodeRoot, "base", base).lastToken(), Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).lastToken(), + Id.ErrorSetDecl => @fieldParentPtr(NodeErrorSetDecl, "base", base).lastToken(), Id.ContainerDecl => @fieldParentPtr(NodeContainerDecl, "base", base).lastToken(), Id.StructField => @fieldParentPtr(NodeStructField, "base", base).lastToken(), Id.UnionTag => @fieldParentPtr(NodeUnionTag, "base", base).lastToken(), @@ -212,6 +216,30 @@ pub const NodeVarDecl = struct { } }; +pub const NodeErrorSetDecl = struct { + base: Node, + error_token: Token, + decls: ArrayList(&NodeIdentifier), + rbrace_token: Token, + + pub fn iterate(self: &NodeErrorSetDecl, index: usize) ?&Node { + var i = index; + + if (i < self.decls.len) return self.decls.at(i); + i -= self.decls.len; + + return null; + } + + pub fn firstToken(self: &NodeErrorSetDecl) Token { + return self.error_token; + } + + pub fn lastToken(self: &NodeErrorSetDecl) Token { + return self.rbrace_token; + } +}; + pub const NodeContainerDecl = struct { base: Node, ltoken: Token, @@ -251,23 +279,8 @@ pub const NodeContainerDecl = struct { InitArg.Enum => { } } - if (i < self.decls.len) return self.decls.at(i); - i -= self.decls.len; - - switch (self.kind) { - Kind.Struct => |fields| { - if (i < fields.len) return fields.at(i); - i -= fields.len; - }, - Kind.Enum => |tags| { - if (i < tags.len) return tags.at(i); - i -= tags.len; - }, - Kind.Union => |tags| { - if (i < tags.len) return tags.at(i); - i -= tags.len; - }, - } + if (i < self.fields_and_decls.len) return self.fields_and_decls.at(i); + i -= self.fields_and_decls.len; return null; } diff --git a/std/zig/parser.zig b/std/zig/parser.zig index f7db8fd93d..8177700fb5 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -685,11 +685,66 @@ pub const Parser = struct { continue; }, Token.Id.Keyword_error => { - const node = try arena.create(ast.NodeErrorType); - *node = ast.NodeErrorType { - .base = self.initNode(ast.Node.Id.ErrorType), - .token = token, + const next = self.getNextToken(); + + if (next.id != Token.Id.LBrace) { + self.putBackToken(next); + const node = try arena.create(ast.NodeErrorType); + *node = ast.NodeErrorType { + .base = self.initNode(ast.Node.Id.ErrorType), + .token = token, + }; + try stack.append(State { + .Operand = &node.base + }); + try stack.append(State.AfterOperand); + continue; + } + + const node = try arena.create(ast.NodeErrorSetDecl); + *node = ast.NodeErrorSetDecl { + .base = self.initNode(ast.Node.Id.ErrorSetDecl), + .error_token = token, + .decls = ArrayList(&ast.NodeIdentifier).init(arena), + .rbrace_token = undefined, }; + + while (true) { + const t = self.getNextToken(); + switch (t.id) { + Token.Id.RBrace => { + node.rbrace_token = t; + break; + }, + Token.Id.Identifier => { + try node.decls.append( + try self.createIdentifier(arena, t) + ); + }, + else => { + return self.parseError(token, "expected {} or {}, found {}", + @tagName(Token.Id.RBrace), + @tagName(Token.Id.Identifier), + @tagName(token.id)); + } + } + + const t2 = self.getNextToken(); + switch (t2.id) { + Token.Id.RBrace => { + node.rbrace_token = t; + break; + }, + Token.Id.Comma => continue, + else => { + return self.parseError(token, "expected {} or {}, found {}", + @tagName(Token.Id.RBrace), + @tagName(Token.Id.Comma), + @tagName(token.id)); + } + } + } + try stack.append(State { .Operand = &node.base }); @@ -2115,6 +2170,42 @@ pub const Parser = struct { }, } }, + ast.Node.Id.ErrorSetDecl => { + const err_set_decl = @fieldParentPtr(ast.NodeErrorSetDecl, "base", base); + try stream.print("error "); + + try stack.append(RenderState { .Text = "}"}); + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { .Indent = indent }); + try stack.append(RenderState { .Text = "\n"}); + + const decls = err_set_decl.decls.toSliceConst(); + var i = decls.len; + while (i != 0) { + i -= 1; + const node = decls[i]; + try stack.append(RenderState { .Expression = &node.base}); + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { + .Text = blk: { + if (i != 0) { + const prev_node = decls[i - 1]; + const loc = self.tokenizer.getTokenLocation(prev_node.lastToken().end, node.firstToken()); + if (loc.line >= 2) { + break :blk "\n\n"; + } + } + break :blk "\n"; + }, + }); + + if (i != 0) { + try stack.append(RenderState { .Text = "," }); + } + } + try stack.append(RenderState { .Indent = indent + indent_delta}); + try stack.append(RenderState { .Text = "{"}); + }, ast.Node.Id.MultilineStringLiteral => { const multiline_str_literal = @fieldParentPtr(ast.NodeMultilineStringLiteral, "base", base); try stream.print("\n"); @@ -2643,6 +2734,18 @@ test "zig fmt: union declaration" { ); } +test "zig fmt: error set declaration" { + try testCanonical( + \\const E = error { + \\ A, + \\ B, + \\ + \\ C + \\}; + \\ + ); +} + test "zig fmt: switch" { try testCanonical( \\test "switch" { From 744416ce0cd23c84e33965fc8171c0d1aea0659c Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Wed, 4 Apr 2018 14:58:51 +0200 Subject: [PATCH 24/87] std.zig.parser should now parse operators with precedence. * This haven't been tested yet --- std/zig/parser.zig | 227 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 177 insertions(+), 50 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 8177700fb5..3f7b5f4e31 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -1000,24 +1000,58 @@ pub const Parser = struct { var expression = popSuffixOp(&stack); while (true) { - switch (stack.pop()) { - State.Expression => |dest_ptr| { - // we're done - try dest_ptr.store(expression); + const s = stack.pop(); + if (s == State.Expression) { + const dest_ptr = s.Expression; + // we're done + try dest_ptr.store(expression); + break; + } + + var placement_ptr = &expression; + var rhs : &&ast.Node = undefined; + const node = blk: { + switch (s) { + State.InfixOp => |infix_op| { + infix_op.lhs = popSuffixOp(&stack); + infix_op.rhs = expression; + rhs = &infix_op.rhs; + break :blk &infix_op.base; + }, + State.PrefixOp => |prefix_op| { + prefix_op.rhs = expression; + rhs = &prefix_op.rhs; + break :blk &prefix_op.base; + }, + else => unreachable, + } + }; + const node_perc = precedence(node); + + while (true) { + const perc = precedence(*placement_ptr); + + if (node_perc > perc) { + *placement_ptr = node; break; - }, - State.InfixOp => |infix_op| { - infix_op.rhs = expression; - infix_op.lhs = popSuffixOp(&stack); - expression = &infix_op.base; - continue; - }, - State.PrefixOp => |prefix_op| { - prefix_op.rhs = expression; - expression = &prefix_op.base; - continue; - }, - else => unreachable, + } + + switch ((*placement_ptr).id) { + ast.Node.Id.SuffixOp => { + const suffix_op = @fieldParentPtr(ast.NodeSuffixOp, "base", *placement_ptr); + placement_ptr = &suffix_op.lhs; + *rhs = suffix_op.lhs; + }, + ast.Node.Id.InfixOp => { + const infix_op = @fieldParentPtr(ast.NodeInfixOp, "base", *placement_ptr); + placement_ptr = &infix_op.lhs; + *rhs = infix_op.lhs; + }, + else => { + *placement_ptr = node; + break; + }, + } } } continue; @@ -1308,6 +1342,99 @@ pub const Parser = struct { } } + fn precedence(node: &ast.Node) u8 { + switch (node.id) { + ast.Node.Id.PrefixOp => { + const prefix_op = @fieldParentPtr(ast.NodePrefixOp, "base", node); + switch (prefix_op.op) { + ast.NodePrefixOp.PrefixOp.ArrayType, + ast.NodePrefixOp.PrefixOp.SliceType => return 1, + + ast.NodePrefixOp.PrefixOp.BoolNot, + ast.NodePrefixOp.PrefixOp.Negation, + ast.NodePrefixOp.PrefixOp.NegationWrap, + ast.NodePrefixOp.PrefixOp.BitNot, + ast.NodePrefixOp.PrefixOp.Deref, + ast.NodePrefixOp.PrefixOp.AddrOf, + ast.NodePrefixOp.PrefixOp.UnwrapMaybe => return 3, + + ast.NodePrefixOp.PrefixOp.Try, + ast.NodePrefixOp.PrefixOp.Return => return 255, + } + }, + ast.Node.Id.SuffixOp => { + const suffix_op = @fieldParentPtr(ast.NodeSuffixOp, "base", node); + switch (suffix_op.op) { + ast.NodeSuffixOp.SuffixOp.Call, + ast.NodeSuffixOp.SuffixOp.Slice, + ast.NodeSuffixOp.SuffixOp.ArrayAccess => return 2, + + ast.NodeSuffixOp.SuffixOp.ArrayInitializer, + ast.NodeSuffixOp.SuffixOp.StructInitializer => return 5, + } + }, + ast.Node.Id.InfixOp => { + const infix_op = @fieldParentPtr(ast.NodeInfixOp, "base", node); + switch (infix_op.op) { + ast.NodeInfixOp.InfixOp.Period => return 2, + + ast.NodeInfixOp.InfixOp.ErrorUnion => return 4, + + ast.NodeInfixOp.InfixOp.Div, + ast.NodeInfixOp.InfixOp.ArrayMult, + ast.NodeInfixOp.InfixOp.Mod, + ast.NodeInfixOp.InfixOp.Mult, + ast.NodeInfixOp.InfixOp.MultWrap => return 6, + + ast.NodeInfixOp.InfixOp.Add, + ast.NodeInfixOp.InfixOp.AddWrap, + ast.NodeInfixOp.InfixOp.ArrayCat, + ast.NodeInfixOp.InfixOp.Sub, + ast.NodeInfixOp.InfixOp.SubWrap => return 7, + + ast.NodeInfixOp.InfixOp.BitShiftLeft, + ast.NodeInfixOp.InfixOp.BitShiftRight => return 8, + + ast.NodeInfixOp.InfixOp.BitAnd => return 9, + + ast.NodeInfixOp.InfixOp.BitXor => return 10, + + ast.NodeInfixOp.InfixOp.BitOr => return 11, + + ast.NodeInfixOp.InfixOp.EqualEqual, + ast.NodeInfixOp.InfixOp.BangEqual, + ast.NodeInfixOp.InfixOp.GreaterOrEqual, + ast.NodeInfixOp.InfixOp.GreaterThan, + ast.NodeInfixOp.InfixOp.LessOrEqual, + ast.NodeInfixOp.InfixOp.LessThan => return 12, + + ast.NodeInfixOp.InfixOp.BoolAnd => return 13, + + ast.NodeInfixOp.InfixOp.BoolOr => return 14, + + ast.NodeInfixOp.InfixOp.UnwrapMaybe => return 15, + + ast.NodeInfixOp.InfixOp.Assign, + ast.NodeInfixOp.InfixOp.AssignBitAnd, + ast.NodeInfixOp.InfixOp.AssignBitOr, + ast.NodeInfixOp.InfixOp.AssignBitShiftLeft, + ast.NodeInfixOp.InfixOp.AssignBitShiftRight, + ast.NodeInfixOp.InfixOp.AssignBitXor, + ast.NodeInfixOp.InfixOp.AssignDiv, + ast.NodeInfixOp.InfixOp.AssignMinus, + ast.NodeInfixOp.InfixOp.AssignMinusWrap, + ast.NodeInfixOp.InfixOp.AssignMod, + ast.NodeInfixOp.InfixOp.AssignPlus, + ast.NodeInfixOp.InfixOp.AssignPlusWrap, + ast.NodeInfixOp.InfixOp.AssignTimes, + ast.NodeInfixOp.InfixOp.AssignTimesWarp, + ast.NodeInfixOp.InfixOp.MergeErrorSets => return 16, + } + }, + else => return 0, + } + } + fn commaOrEnd(self: &Parser, stack: &ArrayList(State), end: &const Token.Id, ptr: &Token, state_after_comma: &const State) !void { var token = self.getNextToken(); switch (token.id) { @@ -2549,6 +2676,39 @@ test "zig fmt: infix operators" { ); } +test "zig fmt: precedence" { + try testCanonical( + \\test "precedence" { + \\ a!b(); + \\ (a!b)(); + \\ !a!b; + \\ !(a!b); + \\ a << b + c; + \\ (a << b) + c; + \\ a & b << c; + \\ (a & b) << c; + \\ a ^ b & c; + \\ (a ^ b) & c; + \\ a | b ^ c; + \\ (a | b) ^ c; + \\ a == b | c; + \\ (a == b) | c; + \\ a and b == c; + \\ (a and b) == c; + \\ a or b and c; + \\ (a or b) and c; + \\ (a or b) and c; + \\ a = b or c; + \\ (a = b) or c; + \\} + \\ + //\\ !a{}; + //\\ !(a{}); + //\\ a + b{}; + //\\ (a + b){}; + ); +} + test "zig fmt: prefix operators" { try testCanonical( \\test "prefix operators" { @@ -3062,39 +3222,6 @@ test "zig fmt: container initializers" { ); } -test "zig fmt: precedence" { - try testCanonical( - \\test "precedence" { - \\ a!b(); - \\ (a!b)(); - \\ !a!b; - \\ !(a!b); - \\ !a{}; - \\ !(a{}); - \\ a + b{}; - \\ (a + b){}; - \\ a << b + c; - \\ (a << b) + c; - \\ a & b << c; - \\ (a & b) << c; - \\ a ^ b & c; - \\ (a ^ b) & c; - \\ a | b ^ c; - \\ (a | b) ^ c; - \\ a == b | c; - \\ (a == b) | c; - \\ a and b == c; - \\ (a and b) == c; - \\ a or b and c; - \\ (a or b) and c; - \\ (a or b) and c; - \\ a = b or c; - \\ (a = b) or c; - \\} - \\ - ); -} - test "zig fmt: zig fmt" { try testCanonical(@embedFile("ast.zig")); try testCanonical(@embedFile("index.zig")); From 779247ba114492a28287bc2026719091b7022e1d Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Wed, 4 Apr 2018 23:36:55 +0200 Subject: [PATCH 25/87] std.zig Major Refactor * parser now parses expression like the C++ compiler does * This makes initializers work * Added control flow expression (only return is parsed) * Added catch parsing (It doesn't quite work) * The parse can now specify states as optional. * The parse will roll back on error if states are optional * This can be overriden by State.Required --- std/zig/ast.zig | 60 +- std/zig/parser.zig | 1691 ++++++++++++++++++++++++----------------- std/zig/tokenizer.zig | 10 + 3 files changed, 1077 insertions(+), 684 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index efb959c7f2..4aaeef8dab 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -24,6 +24,7 @@ pub const Node = struct { PrefixOp, SuffixOp, GroupedExpression, + ControlFlowExpression, FieldInitializer, IntegerLiteral, FloatLiteral, @@ -58,6 +59,7 @@ pub const Node = struct { Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).iterate(index), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).iterate(index), Id.GroupedExpression => @fieldParentPtr(NodeGroupedExpression, "base", base).iterate(index), + Id.ControlFlowExpression => @fieldParentPtr(NodeControlFlowExpression, "base", base).iterate(index), Id.FieldInitializer => @fieldParentPtr(NodeFieldInitializer, "base", base).iterate(index), Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).iterate(index), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index), @@ -93,6 +95,7 @@ pub const Node = struct { Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).firstToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).firstToken(), Id.GroupedExpression => @fieldParentPtr(NodeGroupedExpression, "base", base).firstToken(), + Id.ControlFlowExpression => @fieldParentPtr(NodeControlFlowExpression, "base", base).firstToken(), Id.FieldInitializer => @fieldParentPtr(NodeFieldInitializer, "base", base).firstToken(), Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).firstToken(), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(), @@ -128,6 +131,7 @@ pub const Node = struct { Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).lastToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).lastToken(), Id.GroupedExpression => @fieldParentPtr(NodeGroupedExpression, "base", base).lastToken(), + Id.ControlFlowExpression => @fieldParentPtr(NodeControlFlowExpression, "base", base).lastToken(), Id.FieldInitializer => @fieldParentPtr(NodeFieldInitializer, "base", base).lastToken(), Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).lastToken(), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(), @@ -531,7 +535,7 @@ pub const NodeInfixOp = struct { op: InfixOp, rhs: &Node, - const InfixOp = enum { + const InfixOp = union(enum) { Add, AddWrap, ArrayCat, @@ -558,6 +562,7 @@ pub const NodeInfixOp = struct { BitXor, BoolAnd, BoolOr, + Catch: ?&NodeIdentifier, Div, EqualEqual, ErrorUnion, @@ -651,9 +656,9 @@ pub const NodePrefixOp = struct { BitNot, BoolNot, Deref, + MaybeType, Negation, NegationWrap, - Return, ArrayType: &Node, SliceType: AddrOfInfo, Try, @@ -754,6 +759,7 @@ pub const NodeSuffixOp = struct { const CallInfo = struct { params: ArrayList(&Node), is_async: bool, + allocator: ?&Node, }; const SliceRange = struct { @@ -831,6 +837,56 @@ pub const NodeGroupedExpression = struct { } }; +pub const NodeControlFlowExpression = struct { + base: Node, + ltoken: Token, + kind: Kind, + rhs: ?&Node, + + const Kind = union(enum) { + Break: ?Token, + Continue: ?Token, + Return, + }; + + pub fn iterate(self: &NodeControlFlowExpression, index: usize) ?&Node { + var i = index; + + if (self.rhs) |rhs| { + if (i < 1) return rhs; + i -= 1; + } + + return null; + } + + pub fn firstToken(self: &NodeControlFlowExpression) Token { + return self.ltoken; + } + + pub fn lastToken(self: &NodeControlFlowExpression) Token { + if (self.rhs) |rhs| { + return rhs.lastToken(); + } + + switch (self.kind) { + Kind.Break => |maybe_blk_token| { + if (maybe_blk_token) |blk_token| { + return blk_token; + } + }, + Kind.Continue => |maybe_blk_token| { + if (maybe_blk_token) |blk_token| { + return blk_token; + } + }, + Kind.Return => return self.ltoken, + } + + return self.ltoken; + } +}; + pub const NodeIntegerLiteral = struct { base: Node, token: Token, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 3f7b5f4e31..1a8e7bde88 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -60,6 +60,7 @@ pub const Parser = struct { }; const ContainerExternCtx = struct { + dest_ptr: DestPtr, ltoken: Token, layout: ast.NodeContainerDecl.Layout, }; @@ -67,13 +68,18 @@ pub const Parser = struct { const DestPtr = union(enum) { Field: &&ast.Node, NullableField: &?&ast.Node, - List: &ArrayList(&ast.Node), - pub fn store(self: &const DestPtr, value: &ast.Node) !void { + pub fn store(self: &const DestPtr, value: &ast.Node) void { switch (*self) { DestPtr.Field => |ptr| *ptr = value, DestPtr.NullableField => |ptr| *ptr = value, - DestPtr.List => |list| try list.append(value), + } + } + + pub fn get(self: &const DestPtr) &ast.Node { + switch (*self) { + DestPtr.Field => |ptr| return *ptr, + DestPtr.NullableField => |ptr| return ??*ptr, } } }; @@ -97,19 +103,13 @@ pub const Parser = struct { TopLevelDecl: TopLevelDeclCtx, ContainerExtern: ContainerExternCtx, ContainerDecl: &ast.NodeContainerDecl, - Expression: DestPtr, - ExpectOperand, - Operand: &ast.Node, - AfterOperand, - InfixOp: &ast.NodeInfixOp, - PrefixOp: &ast.NodePrefixOp, - SuffixOp: &ast.NodeSuffixOp, SliceOrArrayAccess: &ast.NodeSuffixOp, AddrOfModifiers: &ast.NodePrefixOp.AddrOfInfo, - TypeExpr: DestPtr, VarDecl: &ast.NodeVarDecl, VarDeclAlign: &ast.NodeVarDecl, VarDeclEq: &ast.NodeVarDecl, + IfToken: @TagType(Token.Id), + IfTokenSave: ExpectTokenSave, ExpectToken: @TagType(Token.Id), ExpectTokenSave: ExpectTokenSave, FnProto: &ast.NodeFnProto, @@ -125,6 +125,48 @@ pub const Parser = struct { FieldInitListItemOrEnd: ListState(&ast.NodeFieldInitializer), FieldInitListCommaOrEnd: ListState(&ast.NodeFieldInitializer), FieldListCommaOrEnd: &ast.NodeContainerDecl, + + /// A state that can be appended before any other State. If an error occures, + /// the parser will first try looking for the closest optional state. If an + /// optional state is found, the parser will revert to the state it was in + /// when the optional was added. This will polute the arena allocator with + /// "leaked" nodes. TODO: Figure out if it's nessesary to handle leaked nodes. + Optional: Parser, + + /// Optional can be reverted by adding the Required state to the stack. + Required, + + Expression: DestPtr, + AssignmentExpressionBegin: DestPtr, + AssignmentExpressionEnd: DestPtr, + UnwrapExpressionBegin: DestPtr, + UnwrapExpressionEnd: DestPtr, + BoolOrExpressionBegin: DestPtr, + BoolOrExpressionEnd: DestPtr, + BoolAndExpressionBegin: DestPtr, + BoolAndExpressionEnd: DestPtr, + ComparisonExpressionBegin: DestPtr, + ComparisonExpressionEnd: DestPtr, + BinaryOrExpressionBegin: DestPtr, + BinaryOrExpressionEnd: DestPtr, + BinaryXorExpressionBegin: DestPtr, + BinaryXorExpressionEnd: DestPtr, + BinaryAndExpressionBegin: DestPtr, + BinaryAndExpressionEnd: DestPtr, + BitShiftExpressionBegin: DestPtr, + BitShiftExpressionEnd: DestPtr, + AdditionExpressionBegin: DestPtr, + AdditionExpressionEnd: DestPtr, + MultiplyExpressionBegin: DestPtr, + MultiplyExpressionEnd: DestPtr, + CurlySuffixExpressionBegin: DestPtr, + CurlySuffixExpressionEnd: DestPtr, + TypeExprBegin: DestPtr, + TypeExprEnd: DestPtr, + PrefixOpExpression: DestPtr, + SuffixOpExpressionBegin: DestPtr, + SuffixOpExpressionEnd: DestPtr, + PrimaryExpression: DestPtr, }; /// Returns an AST tree, allocated with the parser's allocator. @@ -193,12 +235,16 @@ pub const Parser = struct { stack.append(State.TopLevel) catch unreachable; const name_token = self.getNextToken(); - if (name_token.id != Token.Id.StringLiteral) - return self.parseError(token, "expected {}, found {}", @tagName(Token.Id.StringLiteral), @tagName(name_token.id)); + if (name_token.id != Token.Id.StringLiteral) { + try self.parseError(&stack, token, "expected {}, found {}", @tagName(Token.Id.StringLiteral), @tagName(name_token.id)); + continue; + } const lbrace = self.getNextToken(); - if (lbrace.id != Token.Id.LBrace) - return self.parseError(token, "expected {}, found {}", @tagName(Token.Id.LBrace), @tagName(name_token.id)); + if (lbrace.id != Token.Id.LBrace) { + try self.parseError(&stack, token, "expected {}, found {}", @tagName(Token.Id.LBrace), @tagName(name_token.id)); + continue; + } const name = try self.createStringLiteral(arena, name_token); const block = try self.createBlock(arena, token); @@ -284,28 +330,35 @@ pub const Parser = struct { continue; }, Token.Id.Keyword_nakedcc, Token.Id.Keyword_stdcallcc => { - const fn_token = try self.eatToken(Token.Id.Keyword_fn); // TODO shouldn't need this cast - const fn_proto = try self.createAttachFnProto(arena, ctx.decls, fn_token, + const fn_proto = try self.createAttachFnProto(arena, ctx.decls, undefined, ctx.extern_token, ctx.lib_name, (?Token)(token), (?Token)(null), (?Token)(null)); try stack.append(State { .FnDef = fn_proto }); try stack.append(State { .FnProto = fn_proto }); + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.Keyword_fn, + .ptr = &fn_proto.fn_token, + } + }); + continue; + }, + else => { + try self.parseError(&stack, token, "expected variable declaration or function, found {}", @tagName(token.id)); continue; }, - else => return self.parseError(token, "expected variable declaration or function, found {}", @tagName(token.id)), } }, State.VarDecl => |var_decl| { - var_decl.name_token = try self.eatToken(Token.Id.Identifier); stack.append(State { .VarDeclAlign = var_decl }) catch unreachable; - - const next_token = self.getNextToken(); - if (next_token.id == Token.Id.Colon) { - try stack.append(State { .TypeExpr = DestPtr {.NullableField = &var_decl.type_node} }); - continue; - } - - self.putBackToken(next_token); + try stack.append(State { .TypeExprBegin = DestPtr {.NullableField = &var_decl.type_node} }); + try stack.append(State { .IfToken = Token.Id.Colon }); + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.Identifier, + .ptr = &var_decl.name_token, + } + }); continue; }, State.VarDeclAlign => |var_decl| { @@ -313,9 +366,9 @@ pub const Parser = struct { const next_token = self.getNextToken(); if (next_token.id == Token.Id.Keyword_align) { - _ = try self.eatToken(Token.Id.LParen); try stack.append(State { .ExpectToken = Token.Id.RParen }); try stack.append(State { .Expression = DestPtr{.NullableField = &var_decl.align_node} }); + try stack.append(State { .ExpectToken = Token.Id.LParen }); continue; } @@ -341,7 +394,8 @@ pub const Parser = struct { var_decl.semicolon_token = token; continue; } - return self.parseError(token, "expected '=' or ';', found {}", @tagName(token.id)); + try self.parseError(&stack, token, "expected '=' or ';', found {}", @tagName(token.id)); + continue; }, State.ContainerExtern => |ctx| { @@ -357,20 +411,20 @@ pub const Parser = struct { Token.Id.Keyword_union => ast.NodeContainerDecl.Kind.Union, Token.Id.Keyword_enum => ast.NodeContainerDecl.Kind.Enum, else => { - return self.parseError(token, "expected {}, {} or {}, found {}", + try self.parseError(&stack, token, "expected {}, {} or {}, found {}", @tagName(Token.Id.Keyword_struct), @tagName(Token.Id.Keyword_union), @tagName(Token.Id.Keyword_enum), @tagName(token.id)); + continue; }, }, .init_arg_expr = undefined, .fields_and_decls = ArrayList(&ast.Node).init(arena), .rbrace_token = undefined, }; + ctx.dest_ptr.store(&node.base); - try stack.append(State { .Operand = &node.base }); - try stack.append(State.AfterOperand); try stack.append(State { .ContainerDecl = node }); try stack.append(State { .ExpectToken = Token.Id.LBrace }); @@ -496,144 +550,569 @@ pub const Parser = struct { }, State.ExpectToken => |token_id| { - _ = try self.eatToken(token_id); + _ = (try self.eatToken(&stack, token_id)) ?? continue; continue; }, State.ExpectTokenSave => |expect_token_save| { - *expect_token_save.ptr = try self.eatToken(expect_token_save.id); + *expect_token_save.ptr = (try self.eatToken(&stack, expect_token_save.id)) ?? continue; continue; }, - State.Expression => |dest_ptr| { - // save the dest_ptr for later - stack.append(state) catch unreachable; - try stack.append(State.ExpectOperand); + State.IfToken => |token_id| { + const token = self.getNextToken(); + if (@TagType(Token.Id)(token.id) != token_id) { + self.putBackToken(token); + _ = stack.pop(); + continue; + } continue; }, - State.ExpectOperand => { - // we'll either get an operand (like 1 or x), - // or a prefix operator (like ~ or return). + + State.IfTokenSave => |if_token_save| { + const token = self.getNextToken(); + if (@TagType(Token.Id)(token.id) != if_token_save.id) { + self.putBackToken(token); + _ = stack.pop(); + continue; + } + + *if_token_save.ptr = token; + continue; + }, + + State.Optional, + State.Required => { }, + + State.Expression => |dest_ptr| { const token = self.getNextToken(); switch (token.id) { - Token.Id.Keyword_return => { - try stack.append(State { .PrefixOp = try self.createPrefixOp(arena, token, - ast.NodePrefixOp.PrefixOp.Return) }); - try stack.append(State.ExpectOperand); - continue; - }, Token.Id.Keyword_try => { - try stack.append(State { .PrefixOp = try self.createPrefixOp(arena, token, - ast.NodePrefixOp.PrefixOp.Try) }); - try stack.append(State.ExpectOperand); + const node = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp.Try); + dest_ptr.store(&node.base); + + stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }) catch unreachable; continue; }, - Token.Id.Minus => { - try stack.append(State { .PrefixOp = try self.createPrefixOp(arena, token, - ast.NodePrefixOp.PrefixOp.Negation) }); - try stack.append(State.ExpectOperand); + Token.Id.Keyword_return => { + const node = try self.createControlFlowExpr(arena, token, ast.NodeControlFlowExpression.Kind.Return); + dest_ptr.store(&node.base); + + stack.append(State { .Optional = *self }) catch unreachable; + try stack.append(State { .Expression = DestPtr { .NullableField = &node.rhs } }); continue; }, - Token.Id.MinusPercent => { - try stack.append(State { .PrefixOp = try self.createPrefixOp(arena, token, - ast.NodePrefixOp.PrefixOp.NegationWrap) }); - try stack.append(State.ExpectOperand); - continue; + Token.Id.Keyword_break => { + @panic("TODO: break"); }, - Token.Id.Tilde => { - try stack.append(State { .PrefixOp = try self.createPrefixOp(arena, token, - ast.NodePrefixOp.PrefixOp.BitNot) }); - try stack.append(State.ExpectOperand); - continue; + Token.Id.Keyword_continue => { + @panic("TODO: break"); }, - Token.Id.QuestionMarkQuestionMark => { - try stack.append(State { .PrefixOp = try self.createPrefixOp(arena, token, - ast.NodePrefixOp.PrefixOp.UnwrapMaybe) }); - try stack.append(State.ExpectOperand); - continue; + Token.Id.Keyword_cancel => { + @panic("TODO: cancel"); }, - Token.Id.Bang => { - try stack.append(State { .PrefixOp = try self.createPrefixOp(arena, token, - ast.NodePrefixOp.PrefixOp.BoolNot) }); - try stack.append(State.ExpectOperand); - continue; + Token.Id.Keyword_resume => { + @panic("TODO: resume"); }, - Token.Id.Asterisk => { - try stack.append(State { .PrefixOp = try self.createPrefixOp(arena, token, - ast.NodePrefixOp.PrefixOp.Deref) }); - try stack.append(State.ExpectOperand); - continue; + Token.Id.Keyword_await => { + @panic("TODO: await"); }, - Token.Id.LBracket => { - const rbracket_token = self.getNextToken(); - if (rbracket_token.id == Token.Id.RBracket) { - const prefix_op = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp{ - .SliceType = ast.NodePrefixOp.AddrOfInfo { - .align_expr = null, - .bit_offset_start_token = null, - .bit_offset_end_token = null, - .const_token = null, - .volatile_token = null, - } - }); - try stack.append(State { .PrefixOp = prefix_op }); - try stack.append(State.ExpectOperand); - try stack.append(State { .AddrOfModifiers = &prefix_op.op.AddrOf }); + Token.Id.Keyword_suspend => { + @panic("TODO: suspend"); + }, + else => { + self.putBackToken(token); + stack.append(State { .AssignmentExpressionBegin = dest_ptr }) catch unreachable; + continue; + } + } + }, + + State.AssignmentExpressionBegin => |dest_ptr| { + try stack.append(State { .AssignmentExpressionEnd = dest_ptr }); + stack.append(State { .UnwrapExpressionBegin = dest_ptr }) catch unreachable; + continue; + }, + + State.AssignmentExpressionEnd => |dest_ptr| { + const token = self.getNextToken(); + if (tokenIdToAssignment(token.id)) |ass_id| { + const node = try self.createInfixOp(arena, token, ass_id); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .AssignmentExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .UnwrapExpressionBegin = DestPtr { .Field = &node.rhs } }); + continue; + } else { + self.putBackToken(token); + continue; + } + }, + + State.UnwrapExpressionBegin => |dest_ptr| { + stack.append(State { .UnwrapExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .BoolOrExpressionBegin = dest_ptr }); + continue; + }, + + State.UnwrapExpressionEnd => |dest_ptr| { + const token = self.getNextToken(); + switch (token.id) { + Token.Id.Keyword_catch => { + const node = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp { .Catch = null }); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .UnwrapExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }); + + const next = self.getNextToken(); + if (next.id != Token.Id.Pipe) { + self.putBackToken(next); continue; } - self.putBackToken(rbracket_token); - - const prefix_op = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp{ - .ArrayType = undefined, - }); - try stack.append(State { .PrefixOp = prefix_op }); - try stack.append(State.ExpectOperand); - try stack.append(State { .ExpectToken = Token.Id.RBracket }); - try stack.append(State { .Expression = DestPtr { .Field = &prefix_op.op.ArrayType } }); - - }, - Token.Id.Ampersand => { - const prefix_op = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp{ - .AddrOf = ast.NodePrefixOp.AddrOfInfo { - .align_expr = null, - .bit_offset_start_token = null, - .bit_offset_end_token = null, - .const_token = null, - .volatile_token = null, + node.op.Catch = try self.createIdentifier(arena, undefined); + try stack.append(State { .ExpectToken = Token.Id.Pipe }); + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.Identifier, + .ptr = &(??node.op.Catch).name_token } }); - try stack.append(State { .PrefixOp = prefix_op }); - try stack.append(State.ExpectOperand); - try stack.append(State { .AddrOfModifiers = &prefix_op.op.AddrOf }); continue; }, - Token.Id.Identifier => { - try stack.append(State { - .Operand = &(try self.createIdentifier(arena, token)).base + Token.Id.QuestionMarkQuestionMark => { + const node = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.UnwrapMaybe); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .UnwrapExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }); + continue; + }, + else => { + self.putBackToken(token); + continue; + }, + } + }, + + State.BoolOrExpressionBegin => |dest_ptr| { + stack.append(State { .BoolOrExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .BoolAndExpressionBegin = dest_ptr }); + continue; + }, + + State.BoolOrExpressionEnd => |dest_ptr| { + const token = self.getNextToken(); + switch (token.id) { + Token.Id.Keyword_or => { + const node = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.BoolOr); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .BoolOrExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .BoolAndExpressionBegin = DestPtr { .Field = &node.rhs } }); + continue; + }, + else => { + self.putBackToken(token); + continue; + }, + } + }, + + State.BoolAndExpressionBegin => |dest_ptr| { + stack.append(State { .BoolAndExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .ComparisonExpressionBegin = dest_ptr }); + continue; + }, + + State.BoolAndExpressionEnd => |dest_ptr| { + const token = self.getNextToken(); + switch (token.id) { + Token.Id.Keyword_and => { + const node = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.BoolAnd); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .BoolAndExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .ComparisonExpressionBegin = DestPtr { .Field = &node.rhs } }); + continue; + }, + else => { + self.putBackToken(token); + continue; + }, + } + }, + + State.ComparisonExpressionBegin => |dest_ptr| { + stack.append(State { .ComparisonExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .BinaryOrExpressionBegin = dest_ptr }); + continue; + }, + + State.ComparisonExpressionEnd => |dest_ptr| { + const token = self.getNextToken(); + if (tokenIdToComparison(token.id)) |comp_id| { + const node = try self.createInfixOp(arena, token, comp_id); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .ComparisonExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .BinaryOrExpressionBegin = DestPtr { .Field = &node.rhs } }); + continue; + } else { + self.putBackToken(token); + continue; + } + }, + + State.BinaryOrExpressionBegin => |dest_ptr| { + stack.append(State { .BinaryOrExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .BinaryXorExpressionBegin = dest_ptr }); + continue; + }, + + State.BinaryOrExpressionEnd => |dest_ptr| { + const token = self.getNextToken(); + switch (token.id) { + Token.Id.Pipe => { + const node = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.BitOr); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .BinaryOrExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .BinaryXorExpressionBegin = DestPtr { .Field = &node.rhs } }); + continue; + }, + else => { + self.putBackToken(token); + continue; + }, + } + }, + + State.BinaryXorExpressionBegin => |dest_ptr| { + stack.append(State { .BinaryXorExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .BinaryAndExpressionBegin = dest_ptr }); + continue; + }, + + State.BinaryXorExpressionEnd => |dest_ptr| { + const token = self.getNextToken(); + switch (token.id) { + Token.Id.Caret => { + const node = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.BitXor); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .BinaryXorExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .BinaryAndExpressionBegin = DestPtr { .Field = &node.rhs } }); + continue; + }, + else => { + self.putBackToken(token); + continue; + }, + } + }, + + State.BinaryAndExpressionBegin => |dest_ptr| { + stack.append(State { .BinaryAndExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .BitShiftExpressionBegin = dest_ptr }); + continue; + }, + + State.BinaryAndExpressionEnd => |dest_ptr| { + const token = self.getNextToken(); + switch (token.id) { + Token.Id.Ampersand => { + const node = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.BitAnd); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .BinaryAndExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .BitShiftExpressionBegin = DestPtr { .Field = &node.rhs } }); + continue; + }, + else => { + self.putBackToken(token); + continue; + }, + } + }, + + State.BitShiftExpressionBegin => |dest_ptr| { + stack.append(State { .BitShiftExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .AdditionExpressionBegin = dest_ptr }); + continue; + }, + + State.BitShiftExpressionEnd => |dest_ptr| { + const token = self.getNextToken(); + if (tokenIdToBitShift(token.id)) |bitshift_id| { + const node = try self.createInfixOp(arena, token, bitshift_id); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .BitShiftExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .AdditionExpressionBegin = DestPtr { .Field = &node.rhs } }); + continue; + } else { + self.putBackToken(token); + continue; + } + }, + + State.AdditionExpressionBegin => |dest_ptr| { + stack.append(State { .AdditionExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .MultiplyExpressionBegin = dest_ptr }); + continue; + }, + + State.AdditionExpressionEnd => |dest_ptr| { + const token = self.getNextToken(); + if (tokenIdToAddition(token.id)) |add_id| { + const node = try self.createInfixOp(arena, token, add_id); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .AdditionExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .MultiplyExpressionBegin = DestPtr { .Field = &node.rhs } }); + continue; + } else { + self.putBackToken(token); + continue; + } + }, + + State.MultiplyExpressionBegin => |dest_ptr| { + stack.append(State { .MultiplyExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .CurlySuffixExpressionBegin = dest_ptr }); + continue; + }, + + State.MultiplyExpressionEnd => |dest_ptr| { + const token = self.getNextToken(); + if (tokenIdToMultiply(token.id)) |mult_id| { + const node = try self.createInfixOp(arena, token, mult_id); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .MultiplyExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .CurlySuffixExpressionBegin = DestPtr { .Field = &node.rhs } }); + continue; + } else { + self.putBackToken(token); + continue; + } + }, + + State.CurlySuffixExpressionBegin => |dest_ptr| { + stack.append(State { .CurlySuffixExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .TypeExprBegin = dest_ptr }); + continue; + }, + + State.CurlySuffixExpressionEnd => |dest_ptr| { + const token = self.getNextToken(); + if (token.id != Token.Id.LBrace) { + self.putBackToken(token); + continue; + } + + const next = self.getNextToken(); + self.putBackToken(token); + switch (next.id) { + Token.Id.Period => { + const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { + .StructInitializer = ArrayList(&ast.NodeFieldInitializer).init(arena), + }); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .CurlySuffixExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { + .FieldInitListItemOrEnd = ListState(&ast.NodeFieldInitializer) { + .list = &node.op.StructInitializer, + .end = Token.Id.RBrace, + .ptr = &node.rtoken, + } }); - try stack.append(State.AfterOperand); continue; }, + else => { + const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { + .ArrayInitializer = ArrayList(&ast.Node).init(arena), + }); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .CurlySuffixExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { + .ExprListItemOrEnd = ListState(&ast.Node) { + .list = &node.op.ArrayInitializer, + .end = Token.Id.RBrace, + .ptr = &node.rtoken, + } + }); + continue; + }, + } + }, + + State.TypeExprBegin => |dest_ptr| { + const token = self.getNextToken(); + if (token.id == Token.Id.Keyword_var) { + @panic("TODO param with type var"); + } + self.putBackToken(token); + + stack.append(State { .TypeExprEnd = dest_ptr }) catch unreachable; + try stack.append(State { .PrefixOpExpression = dest_ptr }); + continue; + }, + + State.TypeExprEnd => |dest_ptr| { + const token = self.getNextToken(); + switch (token.id) { + Token.Id.Bang => { + const node = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.ErrorUnion); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .TypeExprEnd = dest_ptr }) catch unreachable; + try stack.append(State { .PrefixOpExpression = DestPtr { .Field = &node.rhs } }); + continue; + }, + else => { + self.putBackToken(token); + continue; + }, + } + }, + + State.PrefixOpExpression => |dest_ptr| { + const token = self.getNextToken(); + if (tokenIdToPrefixOp(token.id)) |prefix_id| { + const node = try self.createPrefixOp(arena, token, prefix_id); + dest_ptr.store(&node.base); + + stack.append(State { .TypeExprBegin = DestPtr { .Field = &node.rhs } }) catch unreachable; + if (node.op == ast.NodePrefixOp.PrefixOp.AddrOf) { + try stack.append(State { .AddrOfModifiers = &node.op.AddrOf }); + } + continue; + } else { + self.putBackToken(token); + stack.append(State { .SuffixOpExpressionBegin = dest_ptr }) catch unreachable; + continue; + } + }, + + State.SuffixOpExpressionBegin => |dest_ptr| { + const token = self.getNextToken(); + switch (token.id) { + Token.Id.Keyword_async => { + @panic("TODO: Parse async"); + }, + else => { + self.putBackToken(token); + stack.append(State { .SuffixOpExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .PrimaryExpression = dest_ptr }); + continue; + } + } + }, + + State.SuffixOpExpressionEnd => |dest_ptr| { + const token = self.getNextToken(); + switch (token.id) { + Token.Id.LParen => { + const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { + .Call = ast.NodeSuffixOp.CallInfo { + .params = ArrayList(&ast.Node).init(arena), + .is_async = false, // TODO: ASYNC + .allocator = null, + } + }); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .SuffixOpExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { + .ExprListItemOrEnd = ListState(&ast.Node) { + .list = &node.op.Call.params, + .end = Token.Id.RParen, + .ptr = &node.rtoken, + } + }); + continue; + }, + Token.Id.LBracket => { + const node = try arena.create(ast.NodeSuffixOp); + *node = ast.NodeSuffixOp { + .base = self.initNode(ast.Node.Id.SuffixOp), + .lhs = undefined, + .op = ast.NodeSuffixOp.SuffixOp { + .ArrayAccess = undefined, + }, + .rtoken = undefined, + }; + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .SuffixOpExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .SliceOrArrayAccess = node }); + try stack.append(State { .Expression = DestPtr { .Field = &node.op.ArrayAccess }}); + continue; + }, + Token.Id.Period => { + const node = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.Period); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .SuffixOpExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .SuffixOpExpressionBegin = DestPtr { .Field = &node.rhs }}); + continue; + }, + else => { + self.putBackToken(token); + continue; + }, + } + }, + + State.PrimaryExpression => |dest_ptr| { + const token = self.getNextToken(); + switch (token.id) { Token.Id.IntegerLiteral => { - try stack.append(State { - .Operand = &(try self.createIntegerLiteral(arena, token)).base - }); - try stack.append(State.AfterOperand); + dest_ptr.store(&(try self.createIntegerLiteral(arena, token)).base); continue; }, Token.Id.FloatLiteral => { - try stack.append(State { - .Operand = &(try self.createFloatLiteral(arena, token)).base - }); - try stack.append(State.AfterOperand); + dest_ptr.store(&(try self.createFloatLiteral(arena, token)).base); + continue; + }, + Token.Id.StringLiteral => { + dest_ptr.store(&(try self.createStringLiteral(arena, token)).base); + continue; + }, + Token.Id.CharLiteral => { + const node = try arena.create(ast.NodeCharLiteral); + *node = ast.NodeCharLiteral { + .base = self.initNode(ast.Node.Id.CharLiteral), + .token = token, + }; + dest_ptr.store(&node.base); continue; }, Token.Id.Keyword_undefined => { - try stack.append(State { - .Operand = &(try self.createUndefined(arena, token)).base - }); - try stack.append(State.AfterOperand); + dest_ptr.store(&(try self.createUndefined(arena, token)).base); continue; }, Token.Id.Keyword_true, Token.Id.Keyword_false => { @@ -642,10 +1121,7 @@ pub const Parser = struct { .base = self.initNode(ast.Node.Id.BoolLiteral), .token = token, }; - try stack.append(State { - .Operand = &node.base - }); - try stack.append(State.AfterOperand); + dest_ptr.store(&node.base); continue; }, Token.Id.Keyword_null => { @@ -654,10 +1130,7 @@ pub const Parser = struct { .base = self.initNode(ast.Node.Id.NullLiteral), .token = token, }; - try stack.append(State { - .Operand = &node.base - }); - try stack.append(State.AfterOperand); + dest_ptr.store(&node.base); continue; }, Token.Id.Keyword_this => { @@ -666,10 +1139,7 @@ pub const Parser = struct { .base = self.initNode(ast.Node.Id.ThisLiteral), .token = token, }; - try stack.append(State { - .Operand = &node.base - }); - try stack.append(State.AfterOperand); + dest_ptr.store(&node.base); continue; }, Token.Id.Keyword_unreachable => { @@ -678,12 +1148,97 @@ pub const Parser = struct { .base = self.initNode(ast.Node.Id.Unreachable), .token = token, }; - try stack.append(State { - .Operand = &node.base - }); - try stack.append(State.AfterOperand); + dest_ptr.store(&node.base); continue; }, + Token.Id.MultilineStringLiteralLine => { + const node = try arena.create(ast.NodeMultilineStringLiteral); + *node = ast.NodeMultilineStringLiteral { + .base = self.initNode(ast.Node.Id.MultilineStringLiteral), + .tokens = ArrayList(Token).init(arena), + }; + dest_ptr.store(&node.base); + try node.tokens.append(token); + + while (true) { + const multiline_str = self.getNextToken(); + if (multiline_str.id != Token.Id.MultilineStringLiteralLine) { + self.putBackToken(multiline_str); + break; + } + + try node.tokens.append(multiline_str); + } + continue; + }, + Token.Id.LParen => { + const node = try arena.create(ast.NodeGroupedExpression); + *node = ast.NodeGroupedExpression { + .base = self.initNode(ast.Node.Id.GroupedExpression), + .lparen = token, + .expr = undefined, + .rparen = undefined, + }; + dest_ptr.store(&node.base); + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.RParen, + .ptr = &node.rparen, + } + }); + try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); + continue; + }, + Token.Id.Identifier => { + dest_ptr.store(&(try self.createIdentifier(arena, token)).base); + continue; + }, + Token.Id.Builtin => { + const node = try arena.create(ast.NodeBuiltinCall); + *node = ast.NodeBuiltinCall { + .base = self.initNode(ast.Node.Id.BuiltinCall), + .builtin_token = token, + .params = ArrayList(&ast.Node).init(arena), + .rparen_token = undefined, + }; + dest_ptr.store(&node.base); + try stack.append(State { + .ExprListItemOrEnd = ListState(&ast.Node) { + .list = &node.params, + .end = Token.Id.RParen, + .ptr = &node.rparen_token, + } + }); + try stack.append(State { .ExpectToken = Token.Id.LParen, }); + continue; + }, + Token.Id.LBracket => { + const rbracket_token = self.getNextToken(); + if (rbracket_token.id == Token.Id.RBracket) { + const node = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp{ + .SliceType = ast.NodePrefixOp.AddrOfInfo { + .align_expr = null, + .bit_offset_start_token = null, + .bit_offset_end_token = null, + .const_token = null, + .volatile_token = null, + } + }); + dest_ptr.store(&node.base); + try stack.append(State { .AddrOfModifiers = &node.op.AddrOf }); + continue; + } + + self.putBackToken(rbracket_token); + + const node = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp{ + .ArrayType = undefined, + }); + dest_ptr.store(&node.base); + try stack.append(State { .ExpectToken = Token.Id.RBracket }); + try stack.append(State { .Expression = DestPtr { .Field = &node.op.ArrayType } }); + + }, Token.Id.Keyword_error => { const next = self.getNextToken(); @@ -694,10 +1249,7 @@ pub const Parser = struct { .base = self.initNode(ast.Node.Id.ErrorType), .token = token, }; - try stack.append(State { - .Operand = &node.base - }); - try stack.append(State.AfterOperand); + dest_ptr.store(&node.base); continue; } @@ -708,6 +1260,7 @@ pub const Parser = struct { .decls = ArrayList(&ast.NodeIdentifier).init(arena), .rbrace_token = undefined, }; + dest_ptr.store(&node.base); while (true) { const t = self.getNextToken(); @@ -722,10 +1275,11 @@ pub const Parser = struct { ); }, else => { - return self.parseError(token, "expected {} or {}, found {}", + try self.parseError(&stack, token, "expected {} or {}, found {}", @tagName(Token.Id.RBrace), @tagName(Token.Id.Identifier), @tagName(token.id)); + continue; } } @@ -737,23 +1291,20 @@ pub const Parser = struct { }, Token.Id.Comma => continue, else => { - return self.parseError(token, "expected {} or {}, found {}", + try self.parseError(&stack, token, "expected {} or {}, found {}", @tagName(Token.Id.RBrace), @tagName(Token.Id.Comma), @tagName(token.id)); + continue; } } } - - try stack.append(State { - .Operand = &node.base - }); - try stack.append(State.AfterOperand); continue; }, Token.Id.Keyword_packed => { try stack.append(State { .ContainerExtern = ContainerExternCtx { + .dest_ptr = dest_ptr, .ltoken = token, .layout = ast.NodeContainerDecl.Layout.Packed, }, @@ -762,6 +1313,7 @@ pub const Parser = struct { Token.Id.Keyword_extern => { try stack.append(State { .ContainerExtern = ContainerExternCtx { + .dest_ptr = dest_ptr, .ltoken = token, .layout = ast.NodeContainerDecl.Layout.Extern, }, @@ -771,100 +1323,25 @@ pub const Parser = struct { self.putBackToken(token); try stack.append(State { .ContainerExtern = ContainerExternCtx { + .dest_ptr = dest_ptr, .ltoken = token, .layout = ast.NodeContainerDecl.Layout.Auto, }, }); }, - Token.Id.Builtin => { - const node = try arena.create(ast.NodeBuiltinCall); - *node = ast.NodeBuiltinCall { - .base = self.initNode(ast.Node.Id.BuiltinCall), - .builtin_token = token, - .params = ArrayList(&ast.Node).init(arena), - .rparen_token = undefined, - }; - try stack.append(State { - .Operand = &node.base - }); - try stack.append(State.AfterOperand); - try stack.append(State { - .ExprListItemOrEnd = ListState(&ast.Node) { - .list = &node.params, - .end = Token.Id.RParen, - .ptr = &node.rparen_token, - } - }); - try stack.append(State { .ExpectToken = Token.Id.LParen, }); - continue; + Token.Id.LBrace => { + @panic("TODO: Block expr"); }, - Token.Id.StringLiteral => { - const node = try self.createStringLiteral(arena, token); - try stack.append(State { - .Operand = &node.base - }); - try stack.append(State.AfterOperand); - continue; + Token.Id.Keyword_fn => { + @panic("TODO: fn proto"); }, - Token.Id.CharLiteral => { - const node = try arena.create(ast.NodeCharLiteral); - *node = ast.NodeCharLiteral { - .base = self.initNode(ast.Node.Id.CharLiteral), - .token = token, - }; - try stack.append(State { - .Operand = &node.base - }); - try stack.append(State.AfterOperand); - continue; + Token.Id.Keyword_asm => { + @panic("TODO: inline asm"); }, - Token.Id.MultilineStringLiteralLine => { - const node = try arena.create(ast.NodeMultilineStringLiteral); - *node = ast.NodeMultilineStringLiteral { - .base = self.initNode(ast.Node.Id.MultilineStringLiteral), - .tokens = ArrayList(Token).init(arena), - }; - try node.tokens.append(token); - - while (true) { - const multiline_str = self.getNextToken(); - if (multiline_str.id != Token.Id.MultilineStringLiteralLine) { - self.putBackToken(multiline_str); - break; - } - - try node.tokens.append(multiline_str); - } - - try stack.append(State { - .Operand = &node.base - }); - try stack.append(State.AfterOperand); + else => { + try self.parseError(&stack, token, "expected primary expression, found {}", @tagName(token.id)); continue; - }, - Token.Id.LParen => { - const node = try arena.create(ast.NodeGroupedExpression); - *node = ast.NodeGroupedExpression { - .base = self.initNode(ast.Node.Id.GroupedExpression), - .lparen = token, - .expr = undefined, - .rparen = undefined, - }; - try stack.append(State { - .Operand = &node.base - }); - try stack.append(State.AfterOperand); - try stack.append(State { - .ExpectTokenSave = ExpectTokenSave { - .id = Token.Id.RParen, - .ptr = &node.rparen, - } - }); - try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); - continue; - }, - - else => return self.parseError(token, "expected primary expression, found {}", @tagName(token.id)), + } } }, @@ -880,8 +1357,6 @@ pub const Parser = struct { .end = undefined, } }; - try stack.append(State { .SuffixOp = node }); - try stack.append(State.AfterOperand); const rbracket_token = self.getNextToken(); if (rbracket_token.id != Token.Id.RBracket) { @@ -900,161 +1375,12 @@ pub const Parser = struct { }, Token.Id.RBracket => { node.rtoken = token; - try stack.append(State { .SuffixOp = node }); - try stack.append(State.AfterOperand); continue; }, - else => return self.parseError(token, "expected ']' or '..', found {}", @tagName(token.id)) - } - }, - - State.AfterOperand => { - // we'll either get an infix operator (like != or ^), - // or a postfix operator (like () or {}), - // otherwise this expression is done (like on a ; or else). - var token = self.getNextToken(); - if (tokenIdToInfixOp(token.id)) |infix_id| { - try stack.append(State { - .InfixOp = try self.createInfixOp(arena, token, infix_id) - }); - try stack.append(State.ExpectOperand); + else => { + try self.parseError(&stack, token, "expected ']' or '..', found {}", @tagName(token.id)); continue; - - } else if (token.id == Token.Id.LParen) { - const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { - .Call = ast.NodeSuffixOp.CallInfo { - .params = ArrayList(&ast.Node).init(arena), - .is_async = false, // TODO: ASYNC - } - }); - try stack.append(State { .SuffixOp = node }); - try stack.append(State.AfterOperand); - try stack.append(State { - .ExprListItemOrEnd = ListState(&ast.Node) { - .list = &node.op.Call.params, - .end = Token.Id.RParen, - .ptr = &node.rtoken, - } - }); - continue; - - } else if (token.id == Token.Id.LBracket) { - const node = try arena.create(ast.NodeSuffixOp); - *node = ast.NodeSuffixOp { - .base = self.initNode(ast.Node.Id.SuffixOp), - .lhs = undefined, - .op = ast.NodeSuffixOp.SuffixOp { - .ArrayAccess = undefined, - }, - .rtoken = undefined, - }; - try stack.append(State { .SliceOrArrayAccess = node }); - try stack.append(State { .Expression = DestPtr { .Field = &node.op.ArrayAccess }}); - continue; - - // TODO: This is the initializer parsing code. It doesn't work because of - // the ambiguity between function bodies and initializers: - // fn main() void {} or fn main() (void {}) - } else if (false) { //(token.id == Token.Id.LBrace) { - const next = self.getNextToken(); - - switch (next.id) { - Token.Id.Period => { - self.putBackToken(token); - - const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { - .StructInitializer = ArrayList(&ast.NodeFieldInitializer).init(arena), - }); - - try stack.append(State { - .FieldInitListItemOrEnd = ListState(&ast.NodeFieldInitializer) { - .list = &node.op.StructInitializer, - .end = Token.Id.RBrace, - .ptr = &node.rtoken, - } - }); - continue; - }, - else => { - self.putBackToken(token); - - const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { - .ArrayInitializer = ArrayList(&ast.Node).init(arena), - }); - - try stack.append(State { - .ExprListItemOrEnd = ListState(&ast.Node) { - .list = &node.op.ArrayInitializer, - .end = Token.Id.RBrace, - .ptr = &node.rtoken, - } - }); - continue; - }, } - - // TODO: Parse postfix operator - } else { - // no postfix/infix operator after this operand. - self.putBackToken(token); - - var expression = popSuffixOp(&stack); - while (true) { - const s = stack.pop(); - if (s == State.Expression) { - const dest_ptr = s.Expression; - // we're done - try dest_ptr.store(expression); - break; - } - - var placement_ptr = &expression; - var rhs : &&ast.Node = undefined; - const node = blk: { - switch (s) { - State.InfixOp => |infix_op| { - infix_op.lhs = popSuffixOp(&stack); - infix_op.rhs = expression; - rhs = &infix_op.rhs; - break :blk &infix_op.base; - }, - State.PrefixOp => |prefix_op| { - prefix_op.rhs = expression; - rhs = &prefix_op.rhs; - break :blk &prefix_op.base; - }, - else => unreachable, - } - }; - const node_perc = precedence(node); - - while (true) { - const perc = precedence(*placement_ptr); - - if (node_perc > perc) { - *placement_ptr = node; - break; - } - - switch ((*placement_ptr).id) { - ast.Node.Id.SuffixOp => { - const suffix_op = @fieldParentPtr(ast.NodeSuffixOp, "base", *placement_ptr); - placement_ptr = &suffix_op.lhs; - *rhs = suffix_op.lhs; - }, - ast.Node.Id.InfixOp => { - const infix_op = @fieldParentPtr(ast.NodeInfixOp, "base", *placement_ptr); - placement_ptr = &infix_op.lhs; - *rhs = infix_op.lhs; - }, - else => { - *placement_ptr = node; - break; - }, - } - } - } - continue; } }, @@ -1069,7 +1395,7 @@ pub const Parser = struct { self.putBackToken(token); stack.append(State { .ExprListCommaOrEnd = list_state }) catch unreachable; - try stack.append(State { .Expression = DestPtr{.List = list_state.list} }); + try stack.append(State { .Expression = DestPtr{ .Field = try list_state.list.addOne() } }); }, State.FieldInitListItemOrEnd => |list_state| { @@ -1130,21 +1456,30 @@ pub const Parser = struct { switch (token.id) { Token.Id.Keyword_align => { stack.append(state) catch unreachable; - if (addr_of_info.align_expr != null) return self.parseError(token, "multiple align qualifiers"); - _ = try self.eatToken(Token.Id.LParen); + if (addr_of_info.align_expr != null) { + try self.parseError(&stack, token, "multiple align qualifiers"); + continue; + } try stack.append(State { .ExpectToken = Token.Id.RParen }); try stack.append(State { .Expression = DestPtr{.NullableField = &addr_of_info.align_expr} }); + try stack.append(State { .ExpectToken = Token.Id.LParen }); continue; }, Token.Id.Keyword_const => { stack.append(state) catch unreachable; - if (addr_of_info.const_token != null) return self.parseError(token, "duplicate qualifier: const"); + if (addr_of_info.const_token != null) { + try self.parseError(&stack, token, "duplicate qualifier: const"); + continue; + } addr_of_info.const_token = token; continue; }, Token.Id.Keyword_volatile => { stack.append(state) catch unreachable; - if (addr_of_info.volatile_token != null) return self.parseError(token, "duplicate qualifier: volatile"); + if (addr_of_info.volatile_token != null) { + try self.parseError(&stack, token, "duplicate qualifier: volatile"); + continue; + } addr_of_info.volatile_token = token; continue; }, @@ -1155,17 +1490,6 @@ pub const Parser = struct { } }, - State.TypeExpr => |dest_ptr| { - const token = self.getNextToken(); - if (token.id == Token.Id.Keyword_var) { - @panic("TODO param with type var"); - } - self.putBackToken(token); - - stack.append(State { .Expression = dest_ptr }) catch unreachable; - continue; - }, - State.FnProto => |fn_proto| { stack.append(State { .FnProtoAlign = fn_proto }) catch unreachable; try stack.append(State { .ParamDecl = fn_proto }); @@ -1201,14 +1525,14 @@ pub const Parser = struct { Token.Id.Bang => { fn_proto.return_type = ast.NodeFnProto.ReturnType { .InferErrorSet = undefined }; stack.append(State { - .TypeExpr = DestPtr {.Field = &fn_proto.return_type.InferErrorSet}, + .TypeExprBegin = DestPtr {.Field = &fn_proto.return_type.InferErrorSet}, }) catch unreachable; }, else => { self.putBackToken(token); fn_proto.return_type = ast.NodeFnProto.ReturnType { .Explicit = undefined }; stack.append(State { - .TypeExpr = DestPtr {.Field = &fn_proto.return_type.Explicit}, + .TypeExprBegin = DestPtr {.Field = &fn_proto.return_type.Explicit}, }) catch unreachable; }, } @@ -1251,7 +1575,7 @@ pub const Parser = struct { stack.append(State { .ParamDecl = fn_proto }) catch unreachable; try stack.append(State.ParamDeclComma); try stack.append(State { - .TypeExpr = DestPtr {.Field = ¶m_decl.type_node} + .TypeExprBegin = DestPtr {.Field = ¶m_decl.type_node} }); continue; }, @@ -1264,7 +1588,10 @@ pub const Parser = struct { continue; }, Token.Id.Comma => continue, - else => return self.parseError(token, "expected ',' or ')', found {}", @tagName(token.id)), + else => { + try self.parseError(&stack, token, "expected ',' or ')', found {}", @tagName(token.id)); + continue; + }, } }, @@ -1278,7 +1605,10 @@ pub const Parser = struct { continue; }, Token.Id.Semicolon => continue, - else => return self.parseError(token, "expected ';' or '{{', found {}", @tagName(token.id)), + else => { + try self.parseError(&stack, token, "expected ';' or '{{', found {}", @tagName(token.id)); + continue; + }, } }, @@ -1329,112 +1659,13 @@ pub const Parser = struct { } stack.append(State { .ExpectToken = Token.Id.Semicolon }) catch unreachable; - try stack.append(State { .Expression = DestPtr{.List = &block.statements} }); + try stack.append(State { .Expression = DestPtr{.Field = try block.statements.addOne() } }); continue; }, - - // These are data, not control flow. - State.InfixOp => unreachable, - State.PrefixOp => unreachable, - State.SuffixOp => unreachable, - State.Operand => unreachable, } } } - fn precedence(node: &ast.Node) u8 { - switch (node.id) { - ast.Node.Id.PrefixOp => { - const prefix_op = @fieldParentPtr(ast.NodePrefixOp, "base", node); - switch (prefix_op.op) { - ast.NodePrefixOp.PrefixOp.ArrayType, - ast.NodePrefixOp.PrefixOp.SliceType => return 1, - - ast.NodePrefixOp.PrefixOp.BoolNot, - ast.NodePrefixOp.PrefixOp.Negation, - ast.NodePrefixOp.PrefixOp.NegationWrap, - ast.NodePrefixOp.PrefixOp.BitNot, - ast.NodePrefixOp.PrefixOp.Deref, - ast.NodePrefixOp.PrefixOp.AddrOf, - ast.NodePrefixOp.PrefixOp.UnwrapMaybe => return 3, - - ast.NodePrefixOp.PrefixOp.Try, - ast.NodePrefixOp.PrefixOp.Return => return 255, - } - }, - ast.Node.Id.SuffixOp => { - const suffix_op = @fieldParentPtr(ast.NodeSuffixOp, "base", node); - switch (suffix_op.op) { - ast.NodeSuffixOp.SuffixOp.Call, - ast.NodeSuffixOp.SuffixOp.Slice, - ast.NodeSuffixOp.SuffixOp.ArrayAccess => return 2, - - ast.NodeSuffixOp.SuffixOp.ArrayInitializer, - ast.NodeSuffixOp.SuffixOp.StructInitializer => return 5, - } - }, - ast.Node.Id.InfixOp => { - const infix_op = @fieldParentPtr(ast.NodeInfixOp, "base", node); - switch (infix_op.op) { - ast.NodeInfixOp.InfixOp.Period => return 2, - - ast.NodeInfixOp.InfixOp.ErrorUnion => return 4, - - ast.NodeInfixOp.InfixOp.Div, - ast.NodeInfixOp.InfixOp.ArrayMult, - ast.NodeInfixOp.InfixOp.Mod, - ast.NodeInfixOp.InfixOp.Mult, - ast.NodeInfixOp.InfixOp.MultWrap => return 6, - - ast.NodeInfixOp.InfixOp.Add, - ast.NodeInfixOp.InfixOp.AddWrap, - ast.NodeInfixOp.InfixOp.ArrayCat, - ast.NodeInfixOp.InfixOp.Sub, - ast.NodeInfixOp.InfixOp.SubWrap => return 7, - - ast.NodeInfixOp.InfixOp.BitShiftLeft, - ast.NodeInfixOp.InfixOp.BitShiftRight => return 8, - - ast.NodeInfixOp.InfixOp.BitAnd => return 9, - - ast.NodeInfixOp.InfixOp.BitXor => return 10, - - ast.NodeInfixOp.InfixOp.BitOr => return 11, - - ast.NodeInfixOp.InfixOp.EqualEqual, - ast.NodeInfixOp.InfixOp.BangEqual, - ast.NodeInfixOp.InfixOp.GreaterOrEqual, - ast.NodeInfixOp.InfixOp.GreaterThan, - ast.NodeInfixOp.InfixOp.LessOrEqual, - ast.NodeInfixOp.InfixOp.LessThan => return 12, - - ast.NodeInfixOp.InfixOp.BoolAnd => return 13, - - ast.NodeInfixOp.InfixOp.BoolOr => return 14, - - ast.NodeInfixOp.InfixOp.UnwrapMaybe => return 15, - - ast.NodeInfixOp.InfixOp.Assign, - ast.NodeInfixOp.InfixOp.AssignBitAnd, - ast.NodeInfixOp.InfixOp.AssignBitOr, - ast.NodeInfixOp.InfixOp.AssignBitShiftLeft, - ast.NodeInfixOp.InfixOp.AssignBitShiftRight, - ast.NodeInfixOp.InfixOp.AssignBitXor, - ast.NodeInfixOp.InfixOp.AssignDiv, - ast.NodeInfixOp.InfixOp.AssignMinus, - ast.NodeInfixOp.InfixOp.AssignMinusWrap, - ast.NodeInfixOp.InfixOp.AssignMod, - ast.NodeInfixOp.InfixOp.AssignPlus, - ast.NodeInfixOp.InfixOp.AssignPlusWrap, - ast.NodeInfixOp.InfixOp.AssignTimes, - ast.NodeInfixOp.InfixOp.AssignTimesWarp, - ast.NodeInfixOp.InfixOp.MergeErrorSets => return 16, - } - }, - else => return 0, - } - } - fn commaOrEnd(self: &Parser, stack: &ArrayList(State), end: &const Token.Id, ptr: &Token, state_after_comma: &const State) !void { var token = self.getNextToken(); switch (token.id) { @@ -1448,74 +1679,104 @@ pub const Parser = struct { return; } - return self.parseError(token, "expected ',' or {}, found {}", @tagName(*end), @tagName(token.id)); + try self.parseError(stack, token, "expected ',' or {}, found {}", @tagName(*end), @tagName(token.id)); }, } } - fn popSuffixOp(stack: &ArrayList(State)) &ast.Node { - var expression: &ast.Node = undefined; - var left_leaf_ptr: &&ast.Node = &expression; - while (true) { - switch (stack.pop()) { - State.SuffixOp => |suffix_op| { - *left_leaf_ptr = &suffix_op.base; - left_leaf_ptr = &suffix_op.lhs; - }, - State.Operand => |operand| { - *left_leaf_ptr = operand; - break; - }, - else => unreachable, - } - } - - return expression; + fn tokenIdToAssignment(id: &const Token.Id) ?ast.NodeInfixOp.InfixOp { + // TODO: We have to cast all cases because of this: + // error: expected type '?InfixOp', found '?@TagType(InfixOp)' + return switch (*id) { + Token.Id.AmpersandEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignBitAnd), + Token.Id.AngleBracketAngleBracketLeftEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignBitShiftLeft), + Token.Id.AngleBracketAngleBracketRightEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignBitShiftRight), + Token.Id.AsteriskEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignTimes), + Token.Id.AsteriskPercentEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignTimesWarp), + Token.Id.CaretEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignBitXor), + Token.Id.Equal => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.Assign), + Token.Id.MinusEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignMinus), + Token.Id.MinusPercentEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignMinusWrap), + Token.Id.PercentEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignMod), + Token.Id.PipeEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignBitOr), + Token.Id.PlusEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignPlus), + Token.Id.PlusPercentEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignPlusWrap), + Token.Id.SlashEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignDiv), + else => null, + }; } - fn tokenIdToInfixOp(id: &const Token.Id) ?ast.NodeInfixOp.InfixOp { + fn tokenIdToComparison(id: &const Token.Id) ?ast.NodeInfixOp.InfixOp { + // TODO: We have to cast all cases because of this: + // error: expected type '?InfixOp', found '?@TagType(InfixOp)' return switch (*id) { - Token.Id.Ampersand => ast.NodeInfixOp.InfixOp.BitAnd, - Token.Id.AmpersandEqual => ast.NodeInfixOp.InfixOp.AssignBitAnd, - Token.Id.AngleBracketAngleBracketLeft => ast.NodeInfixOp.InfixOp.BitShiftLeft, - Token.Id.AngleBracketAngleBracketLeftEqual => ast.NodeInfixOp.InfixOp.AssignBitShiftLeft, - Token.Id.AngleBracketAngleBracketRight => ast.NodeInfixOp.InfixOp.BitShiftRight, - Token.Id.AngleBracketAngleBracketRightEqual => ast.NodeInfixOp.InfixOp.AssignBitShiftRight, - Token.Id.AngleBracketLeft => ast.NodeInfixOp.InfixOp.LessThan, - Token.Id.AngleBracketLeftEqual => ast.NodeInfixOp.InfixOp.LessOrEqual, - Token.Id.AngleBracketRight => ast.NodeInfixOp.InfixOp.GreaterThan, - Token.Id.AngleBracketRightEqual => ast.NodeInfixOp.InfixOp.GreaterOrEqual, - Token.Id.Asterisk => ast.NodeInfixOp.InfixOp.Mult, - Token.Id.AsteriskAsterisk => ast.NodeInfixOp.InfixOp.ArrayMult, - Token.Id.AsteriskEqual => ast.NodeInfixOp.InfixOp.AssignTimes, - Token.Id.AsteriskPercent => ast.NodeInfixOp.InfixOp.MultWrap, - Token.Id.AsteriskPercentEqual => ast.NodeInfixOp.InfixOp.AssignTimesWarp, - Token.Id.Bang => ast.NodeInfixOp.InfixOp.ErrorUnion, - Token.Id.BangEqual => ast.NodeInfixOp.InfixOp.BangEqual, - Token.Id.Caret => ast.NodeInfixOp.InfixOp.BitXor, - Token.Id.CaretEqual => ast.NodeInfixOp.InfixOp.AssignBitXor, - Token.Id.Equal => ast.NodeInfixOp.InfixOp.Assign, - Token.Id.EqualEqual => ast.NodeInfixOp.InfixOp.EqualEqual, - Token.Id.Keyword_and => ast.NodeInfixOp.InfixOp.BoolAnd, - Token.Id.Keyword_or => ast.NodeInfixOp.InfixOp.BoolOr, - Token.Id.Minus => ast.NodeInfixOp.InfixOp.Sub, - Token.Id.MinusEqual => ast.NodeInfixOp.InfixOp.AssignMinus, - Token.Id.MinusPercent => ast.NodeInfixOp.InfixOp.SubWrap, - Token.Id.MinusPercentEqual => ast.NodeInfixOp.InfixOp.AssignMinusWrap, - Token.Id.Percent => ast.NodeInfixOp.InfixOp.Mod, - Token.Id.PercentEqual => ast.NodeInfixOp.InfixOp.AssignMod, - Token.Id.Period => ast.NodeInfixOp.InfixOp.Period, - Token.Id.Pipe => ast.NodeInfixOp.InfixOp.BitOr, - Token.Id.PipeEqual => ast.NodeInfixOp.InfixOp.AssignBitOr, - Token.Id.PipePipe => ast.NodeInfixOp.InfixOp.MergeErrorSets, - Token.Id.Plus => ast.NodeInfixOp.InfixOp.Add, - Token.Id.PlusEqual => ast.NodeInfixOp.InfixOp.AssignPlus, - Token.Id.PlusPercent => ast.NodeInfixOp.InfixOp.AddWrap, - Token.Id.PlusPercentEqual => ast.NodeInfixOp.InfixOp.AssignPlusWrap, - Token.Id.PlusPlus => ast.NodeInfixOp.InfixOp.ArrayCat, - Token.Id.QuestionMarkQuestionMark => ast.NodeInfixOp.InfixOp.UnwrapMaybe, - Token.Id.Slash => ast.NodeInfixOp.InfixOp.Div, - Token.Id.SlashEqual => ast.NodeInfixOp.InfixOp.AssignDiv, + Token.Id.BangEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.BangEqual), + Token.Id.EqualEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.EqualEqual), + Token.Id.AngleBracketLeft => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.LessThan), + Token.Id.AngleBracketLeftEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.LessOrEqual), + Token.Id.AngleBracketRight => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.GreaterThan), + Token.Id.AngleBracketRightEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.GreaterOrEqual), + else => null, + }; + } + + fn tokenIdToBitShift(id: &const Token.Id) ?ast.NodeInfixOp.InfixOp { + // TODO: We have to cast all cases because of this: + // error: expected type '?InfixOp', found '?@TagType(InfixOp)' + return switch (*id) { + Token.Id.AngleBracketAngleBracketLeft => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.BitShiftLeft), + Token.Id.AngleBracketAngleBracketRight => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.BitShiftRight), + else => null, + }; + } + + fn tokenIdToAddition(id: &const Token.Id) ?ast.NodeInfixOp.InfixOp { + // TODO: We have to cast all cases because of this: + // error: expected type '?InfixOp', found '?@TagType(InfixOp)' + return switch (*id) { + Token.Id.Minus => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.Sub), + Token.Id.MinusPercent => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.SubWrap), + Token.Id.Plus => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.Add), + Token.Id.PlusPercent => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AddWrap), + Token.Id.PlusPlus => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.ArrayCat), + else => null, + }; + } + + fn tokenIdToMultiply(id: &const Token.Id) ?ast.NodeInfixOp.InfixOp { + // TODO: We have to cast all cases because of this: + // error: expected type '?InfixOp', found '?@TagType(InfixOp)' + return switch (*id) { + Token.Id.Slash => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.Div), + Token.Id.Asterisk => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.Mult), + Token.Id.AsteriskAsterisk => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.ArrayMult), + Token.Id.AsteriskPercent => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.MultWrap), + Token.Id.Percent => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.Mod), + Token.Id.PipePipe => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.MergeErrorSets), + else => null, + }; + } + + fn tokenIdToPrefixOp(id: &const Token.Id) ?ast.NodePrefixOp.PrefixOp { + // TODO: We have to cast all cases because of this: + // error: expected type '?InfixOp', found '?@TagType(InfixOp)' + return switch (*id) { + Token.Id.Bang => (ast.NodePrefixOp.PrefixOp)(ast.NodePrefixOp.PrefixOp.BoolNot), + Token.Id.Tilde => (ast.NodePrefixOp.PrefixOp)(ast.NodePrefixOp.PrefixOp.BitNot), + Token.Id.Minus => (ast.NodePrefixOp.PrefixOp)(ast.NodePrefixOp.PrefixOp.Negation), + Token.Id.MinusPercent => (ast.NodePrefixOp.PrefixOp)(ast.NodePrefixOp.PrefixOp.NegationWrap), + Token.Id.Asterisk => (ast.NodePrefixOp.PrefixOp)(ast.NodePrefixOp.PrefixOp.Deref), + Token.Id.Ampersand => ast.NodePrefixOp.PrefixOp { + .AddrOf = ast.NodePrefixOp.AddrOfInfo { + .align_expr = null, + .bit_offset_start_token = null, + .bit_offset_end_token = null, + .const_token = null, + .volatile_token = null, + }, + }, + Token.Id.QuestionMark => (ast.NodePrefixOp.PrefixOp)(ast.NodePrefixOp.PrefixOp.MaybeType), + Token.Id.QuestionMarkQuestionMark => (ast.NodePrefixOp.PrefixOp)(ast.NodePrefixOp.PrefixOp.UnwrapMaybe), else => null, }; } @@ -1637,6 +1898,19 @@ pub const Parser = struct { return node; } + fn createControlFlowExpr(self: &Parser, arena: &mem.Allocator, ltoken: &const Token, + kind: &const ast.NodeControlFlowExpression.Kind) !&ast.NodeControlFlowExpression + { + const node = try arena.create(ast.NodeControlFlowExpression); + *node = ast.NodeControlFlowExpression { + .base = self.initNode(ast.Node.Id.ControlFlowExpression), + .ltoken = *ltoken, + .kind = *kind, + .rhs = undefined, + }; + return node; + } + fn createInfixOp(self: &Parser, arena: &mem.Allocator, op_token: &const Token, op: &const ast.NodeInfixOp.InfixOp) !&ast.NodeInfixOp { const node = try arena.create(ast.NodeInfixOp); @@ -1752,39 +2026,56 @@ pub const Parser = struct { return node; } - fn parseError(self: &Parser, token: &const Token, comptime fmt: []const u8, args: ...) (error{ParseError}) { - const loc = self.tokenizer.getTokenLocation(0, token); - warn("{}:{}:{}: error: " ++ fmt ++ "\n", self.source_file_name, loc.line + 1, loc.column + 1, args); - warn("{}\n", self.tokenizer.buffer[loc.line_start..loc.line_end]); - { - var i: usize = 0; - while (i < loc.column) : (i += 1) { - warn(" "); + fn parseError(self: &Parser, stack: &ArrayList(State), token: &const Token, comptime fmt: []const u8, args: ...) !void { + // Before reporting an error. We pop the stack to see if our state was optional + self.revertIfOptional(stack) catch { + const loc = self.tokenizer.getTokenLocation(0, token); + warn("{}:{}:{}: error: " ++ fmt ++ "\n", self.source_file_name, loc.line + 1, loc.column + 1, args); + warn("{}\n", self.tokenizer.buffer[loc.line_start..loc.line_end]); + { + var i: usize = 0; + while (i < loc.column) : (i += 1) { + warn(" "); + } } - } - { - const caret_count = token.end - token.start; - var i: usize = 0; - while (i < caret_count) : (i += 1) { - warn("~"); + { + const caret_count = token.end - token.start; + var i: usize = 0; + while (i < caret_count) : (i += 1) { + warn("~"); + } } - } - warn("\n"); - return error.ParseError; + warn("\n"); + return error.ParseError; + }; } - fn expectToken(self: &Parser, token: &const Token, id: @TagType(Token.Id)) !void { - if (token.id != id) { - return self.parseError(token, "expected {}, found {}", @tagName(id), @tagName(token.id)); - } - } - - fn eatToken(self: &Parser, id: @TagType(Token.Id)) !Token { + fn eatToken(self: &Parser, stack: &ArrayList(State), id: @TagType(Token.Id)) !?Token { const token = self.getNextToken(); - try self.expectToken(token, id); + if (token.id != id) { + try self.parseError(stack, token, "expected {}, found {}", @tagName(id), @tagName(token.id)); + return null; + } return token; } + fn revertIfOptional(self: &Parser, stack: &ArrayList(State)) !void { + while (stack.popOrNull()) |state| { + switch (state) { + State.Optional => |revert| { + *self = state.Optional; + return; + }, + State.Required => { + return error.StateRequired; + }, + else => { } + } + } + + return error.NoOptionalStateFound; + } + fn putBackToken(self: &Parser, token: &const Token) void { self.put_back_tokens[self.put_back_count] = *token; self.put_back_count += 1; @@ -2054,51 +2345,62 @@ pub const Parser = struct { ast.Node.Id.InfixOp => { const prefix_op_node = @fieldParentPtr(ast.NodeInfixOp, "base", base); try stack.append(RenderState { .Expression = prefix_op_node.rhs }); - const text = switch (prefix_op_node.op) { - ast.NodeInfixOp.InfixOp.Add => " + ", - ast.NodeInfixOp.InfixOp.AddWrap => " +% ", - ast.NodeInfixOp.InfixOp.ArrayCat => " ++ ", - ast.NodeInfixOp.InfixOp.ArrayMult => " ** ", - ast.NodeInfixOp.InfixOp.Assign => " = ", - ast.NodeInfixOp.InfixOp.AssignBitAnd => " &= ", - ast.NodeInfixOp.InfixOp.AssignBitOr => " |= ", - ast.NodeInfixOp.InfixOp.AssignBitShiftLeft => " <<= ", - ast.NodeInfixOp.InfixOp.AssignBitShiftRight => " >>= ", - ast.NodeInfixOp.InfixOp.AssignBitXor => " ^= ", - ast.NodeInfixOp.InfixOp.AssignDiv => " /= ", - ast.NodeInfixOp.InfixOp.AssignMinus => " -= ", - ast.NodeInfixOp.InfixOp.AssignMinusWrap => " -%= ", - ast.NodeInfixOp.InfixOp.AssignMod => " %= ", - ast.NodeInfixOp.InfixOp.AssignPlus => " += ", - ast.NodeInfixOp.InfixOp.AssignPlusWrap => " +%= ", - ast.NodeInfixOp.InfixOp.AssignTimes => " *= ", - ast.NodeInfixOp.InfixOp.AssignTimesWarp => " *%= ", - ast.NodeInfixOp.InfixOp.BangEqual => " != ", - ast.NodeInfixOp.InfixOp.BitAnd => " & ", - ast.NodeInfixOp.InfixOp.BitOr => " | ", - ast.NodeInfixOp.InfixOp.BitShiftLeft => " << ", - ast.NodeInfixOp.InfixOp.BitShiftRight => " >> ", - ast.NodeInfixOp.InfixOp.BitXor => " ^ ", - ast.NodeInfixOp.InfixOp.BoolAnd => " and ", - ast.NodeInfixOp.InfixOp.BoolOr => " or ", - ast.NodeInfixOp.InfixOp.Div => " / ", - ast.NodeInfixOp.InfixOp.EqualEqual => " == ", - ast.NodeInfixOp.InfixOp.ErrorUnion => "!", - ast.NodeInfixOp.InfixOp.GreaterOrEqual => " >= ", - ast.NodeInfixOp.InfixOp.GreaterThan => " > ", - ast.NodeInfixOp.InfixOp.LessOrEqual => " <= ", - ast.NodeInfixOp.InfixOp.LessThan => " < ", - ast.NodeInfixOp.InfixOp.MergeErrorSets => " || ", - ast.NodeInfixOp.InfixOp.Mod => " % ", - ast.NodeInfixOp.InfixOp.Mult => " * ", - ast.NodeInfixOp.InfixOp.MultWrap => " *% ", - ast.NodeInfixOp.InfixOp.Period => ".", - ast.NodeInfixOp.InfixOp.Sub => " - ", - ast.NodeInfixOp.InfixOp.SubWrap => " -% ", - ast.NodeInfixOp.InfixOp.UnwrapMaybe => " ?? ", - }; - try stack.append(RenderState { .Text = text }); + if (prefix_op_node.op == ast.NodeInfixOp.InfixOp.Catch) { + if (prefix_op_node.op.Catch) |payload| { + try stack.append(RenderState { .Text = "|" }); + try stack.append(RenderState { .Expression = &payload.base }); + try stack.append(RenderState { .Text = "|" }); + } + try stack.append(RenderState { .Text = " catch " }); + } else { + const text = switch (prefix_op_node.op) { + ast.NodeInfixOp.InfixOp.Add => " + ", + ast.NodeInfixOp.InfixOp.AddWrap => " +% ", + ast.NodeInfixOp.InfixOp.ArrayCat => " ++ ", + ast.NodeInfixOp.InfixOp.ArrayMult => " ** ", + ast.NodeInfixOp.InfixOp.Assign => " = ", + ast.NodeInfixOp.InfixOp.AssignBitAnd => " &= ", + ast.NodeInfixOp.InfixOp.AssignBitOr => " |= ", + ast.NodeInfixOp.InfixOp.AssignBitShiftLeft => " <<= ", + ast.NodeInfixOp.InfixOp.AssignBitShiftRight => " >>= ", + ast.NodeInfixOp.InfixOp.AssignBitXor => " ^= ", + ast.NodeInfixOp.InfixOp.AssignDiv => " /= ", + ast.NodeInfixOp.InfixOp.AssignMinus => " -= ", + ast.NodeInfixOp.InfixOp.AssignMinusWrap => " -%= ", + ast.NodeInfixOp.InfixOp.AssignMod => " %= ", + ast.NodeInfixOp.InfixOp.AssignPlus => " += ", + ast.NodeInfixOp.InfixOp.AssignPlusWrap => " +%= ", + ast.NodeInfixOp.InfixOp.AssignTimes => " *= ", + ast.NodeInfixOp.InfixOp.AssignTimesWarp => " *%= ", + ast.NodeInfixOp.InfixOp.BangEqual => " != ", + ast.NodeInfixOp.InfixOp.BitAnd => " & ", + ast.NodeInfixOp.InfixOp.BitOr => " | ", + ast.NodeInfixOp.InfixOp.BitShiftLeft => " << ", + ast.NodeInfixOp.InfixOp.BitShiftRight => " >> ", + ast.NodeInfixOp.InfixOp.BitXor => " ^ ", + ast.NodeInfixOp.InfixOp.BoolAnd => " and ", + ast.NodeInfixOp.InfixOp.BoolOr => " or ", + ast.NodeInfixOp.InfixOp.Div => " / ", + ast.NodeInfixOp.InfixOp.EqualEqual => " == ", + ast.NodeInfixOp.InfixOp.ErrorUnion => "!", + ast.NodeInfixOp.InfixOp.GreaterOrEqual => " >= ", + ast.NodeInfixOp.InfixOp.GreaterThan => " > ", + ast.NodeInfixOp.InfixOp.LessOrEqual => " <= ", + ast.NodeInfixOp.InfixOp.LessThan => " < ", + ast.NodeInfixOp.InfixOp.MergeErrorSets => " || ", + ast.NodeInfixOp.InfixOp.Mod => " % ", + ast.NodeInfixOp.InfixOp.Mult => " * ", + ast.NodeInfixOp.InfixOp.MultWrap => " *% ", + ast.NodeInfixOp.InfixOp.Period => ".", + ast.NodeInfixOp.InfixOp.Sub => " - ", + ast.NodeInfixOp.InfixOp.SubWrap => " -% ", + ast.NodeInfixOp.InfixOp.UnwrapMaybe => " ?? ", + else => unreachable, + }; + + try stack.append(RenderState { .Text = text }); + } try stack.append(RenderState { .Expression = prefix_op_node.lhs }); }, ast.Node.Id.PrefixOp => { @@ -2143,9 +2445,9 @@ pub const Parser = struct { ast.NodePrefixOp.PrefixOp.Deref => try stream.write("*"), ast.NodePrefixOp.PrefixOp.Negation => try stream.write("-"), ast.NodePrefixOp.PrefixOp.NegationWrap => try stream.write("-%"), - ast.NodePrefixOp.PrefixOp.Return => try stream.write("return "), ast.NodePrefixOp.PrefixOp.Try => try stream.write("try "), ast.NodePrefixOp.PrefixOp.UnwrapMaybe => try stream.write("??"), + ast.NodePrefixOp.PrefixOp.MaybeType => try stream.write("?"), } }, ast.Node.Id.SuffixOp => { @@ -2185,6 +2487,32 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = suffix_op.lhs }); }, + ast.Node.Id.ControlFlowExpression => { + const flow_expr = @fieldParentPtr(ast.NodeControlFlowExpression, "base", base); + switch (flow_expr.kind) { + ast.NodeControlFlowExpression.Kind.Break => |maybe_blk_token| { + try stream.print("break"); + if (maybe_blk_token) |blk_token| { + try stream.print(" :{}", self.tokenizer.getTokenSlice(blk_token)); + } + }, + ast.NodeControlFlowExpression.Kind.Continue => |maybe_blk_token| { + try stream.print("continue"); + if (maybe_blk_token) |blk_token| { + try stream.print(" :{}", self.tokenizer.getTokenSlice(blk_token)); + } + }, + ast.NodeControlFlowExpression.Kind.Return => { + try stream.print("return"); + }, + + } + + if (flow_expr.rhs) |rhs| { + try stream.print(" "); + try stack.append(RenderState { .Expression = rhs }); + } + }, ast.Node.Id.GroupedExpression => { const grouped_expr = @fieldParentPtr(ast.NodeGroupedExpression, "base", base); try stack.append(RenderState { .Text = ")"}); @@ -2770,7 +3098,6 @@ test "zig fmt: values" { \\ error; \\ this; \\ unreachable; - \\ suspend; \\} \\ ); @@ -2906,6 +3233,38 @@ test "zig fmt: error set declaration" { ); } +test "zig fmt: catch" { + try testCanonical( + \\test "catch" { + \\ const a: error!u8 = 0; + \\ _ = a catch return; + \\ _ = a catch |err| return; + \\} + \\ + ); +} + +test "zig fmt: arrays" { + try testCanonical( + \\test "test array" { + \\ const a: [2]u8 = [2]u8{ 1, 2 }; + \\ const a: [2]u8 = []u8{ 1, 2 }; + \\ const a: [0]u8 = []u8{}; + \\} + \\ + ); +} + +test "zig fmt: container initializers" { + try testCanonical( + \\const a1 = []u8{ }; + \\const a2 = []u8{ 1, 2, 3, 4 }; + \\const s1 = S{ }; + \\const s2 = S{ .a = 1, .b = 2, }; + \\ + ); +} + test "zig fmt: switch" { try testCanonical( \\test "switch" { @@ -3106,17 +3465,6 @@ test "zig fmt: defer" { ); } -test "zig fmt: catch" { - try testCanonical( - \\test "catch" { - \\ const a: error!u8 = 0; - \\ _ = a catch return; - \\ _ = a catch |err| return; - \\} - \\ - ); -} - test "zig fmt: comptime" { try testCanonical( \\fn a() u8 { @@ -3201,27 +3549,6 @@ test "zig fmt: coroutines" { ); } -test "zig fmt: arrays" { - try testCanonical( - \\test "test array" { - \\ const a: [2]u8 = [2]u8{ 1, 2 }; - \\ const a: [2]u8 = []u8{ 1, 2 }; - \\ const a: [0]u8 = []u8{}; - \\} - \\ - ); -} - -test "zig fmt: container initializers" { - try testCanonical( - \\const a1 = []u8{ }; - \\const a2 = []u8{ 1, 2, 3, 4 }; - \\const s1 = S{ }; - \\const s2 = S{ .a = 1, .b = 2, }; - \\ - ); -} - test "zig fmt: zig fmt" { try testCanonical(@embedFile("ast.zig")); try testCanonical(@embedFile("index.zig")); diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig index 3427678341..87597adff1 100644 --- a/std/zig/tokenizer.zig +++ b/std/zig/tokenizer.zig @@ -15,8 +15,11 @@ pub const Token = struct { KeywordId{.bytes="align", .id = Id.Keyword_align}, KeywordId{.bytes="and", .id = Id.Keyword_and}, KeywordId{.bytes="asm", .id = Id.Keyword_asm}, + KeywordId{.bytes="async", .id = Id.Keyword_async}, + KeywordId{.bytes="await", .id = Id.Keyword_await}, KeywordId{.bytes="break", .id = Id.Keyword_break}, KeywordId{.bytes="catch", .id = Id.Keyword_catch}, + KeywordId{.bytes="cancel", .id = Id.Keyword_cancel}, KeywordId{.bytes="comptime", .id = Id.Keyword_comptime}, KeywordId{.bytes="const", .id = Id.Keyword_const}, KeywordId{.bytes="continue", .id = Id.Keyword_continue}, @@ -37,10 +40,12 @@ pub const Token = struct { KeywordId{.bytes="or", .id = Id.Keyword_or}, KeywordId{.bytes="packed", .id = Id.Keyword_packed}, KeywordId{.bytes="pub", .id = Id.Keyword_pub}, + KeywordId{.bytes="resume", .id = Id.Keyword_resume}, KeywordId{.bytes="return", .id = Id.Keyword_return}, KeywordId{.bytes="section", .id = Id.Keyword_section}, KeywordId{.bytes="stdcallcc", .id = Id.Keyword_stdcallcc}, KeywordId{.bytes="struct", .id = Id.Keyword_struct}, + KeywordId{.bytes="suspend", .id = Id.Keyword_suspend}, KeywordId{.bytes="switch", .id = Id.Keyword_switch}, KeywordId{.bytes="test", .id = Id.Keyword_test}, KeywordId{.bytes="this", .id = Id.Keyword_this}, @@ -134,7 +139,10 @@ pub const Token = struct { Keyword_align, Keyword_and, Keyword_asm, + Keyword_async, + Keyword_await, Keyword_break, + Keyword_cancel, Keyword_catch, Keyword_comptime, Keyword_const, @@ -156,10 +164,12 @@ pub const Token = struct { Keyword_or, Keyword_packed, Keyword_pub, + Keyword_resume, Keyword_return, Keyword_section, Keyword_stdcallcc, Keyword_struct, + Keyword_suspend, Keyword_switch, Keyword_test, Keyword_this, From 9e8519b7a2c765b427f85f0aaa456256785eceb7 Mon Sep 17 00:00:00 2001 From: Ben Noordhuis Date: Thu, 5 Apr 2018 23:26:06 +0200 Subject: [PATCH 26/87] fix use-after-free in BufMap.set() closes #879 --- std/buf_map.zig | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/std/buf_map.zig b/std/buf_map.zig index a58df4b2db..7e2ea99f1a 100644 --- a/std/buf_map.zig +++ b/std/buf_map.zig @@ -31,8 +31,8 @@ pub const BufMap = struct { if (self.hash_map.get(key)) |entry| { const value_copy = try self.copy(value); errdefer self.free(value_copy); - _ = try self.hash_map.put(key, value_copy); - self.free(entry.value); + const old_value = ??(try self.hash_map.put(key, value_copy)); + self.free(old_value); } else { const key_copy = try self.copy(key); errdefer self.free(key_copy); @@ -71,3 +71,29 @@ pub const BufMap = struct { return result; } }; + +const assert = @import("debug/index.zig").assert; +const heap = @import("heap.zig"); + +test "BufMap" { + var direct_allocator = heap.DirectAllocator.init(); + defer direct_allocator.deinit(); + + var bufmap = BufMap.init(&direct_allocator.allocator); + defer bufmap.deinit(); + + try bufmap.set("x", "1"); + assert(mem.eql(u8, ??bufmap.get("x"), "1")); + assert(1 == bufmap.count()); + + try bufmap.set("x", "2"); + assert(mem.eql(u8, ??bufmap.get("x"), "2")); + assert(1 == bufmap.count()); + + try bufmap.set("x", "3"); + assert(mem.eql(u8, ??bufmap.get("x"), "3")); + assert(1 == bufmap.count()); + + bufmap.delete("x"); + assert(0 == bufmap.count()); +} From 8980281184a5d60098f36f3db605181f08aa1caa Mon Sep 17 00:00:00 2001 From: Ben Noordhuis Date: Fri, 6 Apr 2018 00:10:15 +0200 Subject: [PATCH 27/87] fix llvm assert on version string with git sha LLVM's CodeViewDebug pass misparses the version string when it contains a git revision so stop doing that. This only affected Windows builds. closes #898 --- src/codegen.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index bdd28b86fd..0dafcb9502 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -6207,7 +6207,9 @@ static void init(CodeGen *g) { g->builder = LLVMCreateBuilder(); g->dbuilder = ZigLLVMCreateDIBuilder(g->module, true); - Buf *producer = buf_sprintf("zig %s", ZIG_VERSION_STRING); + // Don't use ZIG_VERSION_STRING here, llvm misparses it when it includes + // the git revision. + Buf *producer = buf_sprintf("zig %d.%d.%d", ZIG_VERSION_MAJOR, ZIG_VERSION_MINOR, ZIG_VERSION_PATCH); const char *flags = ""; unsigned runtime_version = 0; ZigLLVMDIFile *compile_unit_file = ZigLLVMCreateFile(g->dbuilder, buf_ptr(g->root_out_name), From e45de607d6437428d82f69711a8cc8f338d019c8 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Fri, 6 Apr 2018 08:56:28 +0200 Subject: [PATCH 28/87] std.zig.parser: Initializers are now parsed and fmt correctly --- std/zig/parser.zig | 127 +++++++++++++++++++++++++++++---------------- 1 file changed, 82 insertions(+), 45 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 1a8e7bde88..5d2a30caee 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -249,7 +249,7 @@ pub const Parser = struct { const name = try self.createStringLiteral(arena, name_token); const block = try self.createBlock(arena, token); const test_decl = try self.createAttachTestDecl(arena, &root_node.decls, token, &name.base, block); - try stack.append(State { .Block = block }); + stack.append(State { .Block = block }) catch unreachable; continue; }, Token.Id.Eof => { @@ -318,14 +318,14 @@ pub const Parser = struct { // TODO shouldn't need these casts const var_decl_node = try self.createAttachVarDecl(arena, ctx.decls, ctx.visib_token, token, (?Token)(null), ctx.extern_token, ctx.lib_name); - try stack.append(State { .VarDecl = var_decl_node }); + stack.append(State { .VarDecl = var_decl_node }) catch unreachable; continue; }, Token.Id.Keyword_fn => { // TODO shouldn't need these casts const fn_proto = try self.createAttachFnProto(arena, ctx.decls, token, ctx.extern_token, ctx.lib_name, (?Token)(null), ctx.visib_token, (?Token)(null)); - try stack.append(State { .FnDef = fn_proto }); + stack.append(State { .FnDef = fn_proto }) catch unreachable; try stack.append(State { .FnProto = fn_proto }); continue; }, @@ -333,7 +333,7 @@ pub const Parser = struct { // TODO shouldn't need this cast const fn_proto = try self.createAttachFnProto(arena, ctx.decls, undefined, ctx.extern_token, ctx.lib_name, (?Token)(token), (?Token)(null), (?Token)(null)); - try stack.append(State { .FnDef = fn_proto }); + stack.append(State { .FnDef = fn_proto }) catch unreachable; try stack.append(State { .FnProto = fn_proto }); try stack.append(State { .ExpectTokenSave = ExpectTokenSave { @@ -425,7 +425,7 @@ pub const Parser = struct { }; ctx.dest_ptr.store(&node.base); - try stack.append(State { .ContainerDecl = node }); + stack.append(State { .ContainerDecl = node }) catch unreachable; try stack.append(State { .ExpectToken = Token.Id.LBrace }); const lparen = self.getNextToken(); @@ -470,7 +470,7 @@ pub const Parser = struct { }; try container_decl.fields_and_decls.append(&node.base); - try stack.append(State { .FieldListCommaOrEnd = container_decl }); + stack.append(State { .FieldListCommaOrEnd = container_decl }) catch unreachable; try stack.append(State { .Expression = DestPtr { .Field = &node.type_expr } }); try stack.append(State { .ExpectToken = Token.Id.Colon }); continue; @@ -484,7 +484,7 @@ pub const Parser = struct { }; try container_decl.fields_and_decls.append(&node.base); - try stack.append(State { .FieldListCommaOrEnd = container_decl }); + stack.append(State { .FieldListCommaOrEnd = container_decl }) catch unreachable; const next = self.getNextToken(); if (next.id != Token.Id.Colon) { @@ -504,7 +504,7 @@ pub const Parser = struct { }; try container_decl.fields_and_decls.append(&node.base); - try stack.append(State { .FieldListCommaOrEnd = container_decl }); + stack.append(State { .FieldListCommaOrEnd = container_decl }) catch unreachable; const next = self.getNextToken(); if (next.id != Token.Id.Equal) { @@ -629,8 +629,8 @@ pub const Parser = struct { }, State.AssignmentExpressionBegin => |dest_ptr| { - try stack.append(State { .AssignmentExpressionEnd = dest_ptr }); - stack.append(State { .UnwrapExpressionBegin = dest_ptr }) catch unreachable; + stack.append(State { .AssignmentExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .UnwrapExpressionBegin = dest_ptr }); continue; }, @@ -926,7 +926,6 @@ pub const Parser = struct { } const next = self.getNextToken(); - self.putBackToken(token); switch (next.id) { Token.Id.Period => { const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { @@ -943,6 +942,7 @@ pub const Parser = struct { .ptr = &node.rtoken, } }); + self.putBackToken(next); continue; }, else => { @@ -960,6 +960,7 @@ pub const Parser = struct { .ptr = &node.rtoken, } }); + self.putBackToken(next); continue; }, } @@ -1180,12 +1181,12 @@ pub const Parser = struct { .rparen = undefined, }; dest_ptr.store(&node.base); - try stack.append(State { + stack.append(State { .ExpectTokenSave = ExpectTokenSave { .id = Token.Id.RParen, .ptr = &node.rparen, } - }); + }) catch unreachable; try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); continue; }, @@ -1202,17 +1203,18 @@ pub const Parser = struct { .rparen_token = undefined, }; dest_ptr.store(&node.base); - try stack.append(State { + stack.append(State { .ExprListItemOrEnd = ListState(&ast.Node) { .list = &node.params, .end = Token.Id.RParen, .ptr = &node.rparen_token, } - }); + }) catch unreachable; try stack.append(State { .ExpectToken = Token.Id.LParen, }); continue; }, Token.Id.LBracket => { + // TODO: option("align" "(" Expression option(":" Integer ":" Integer) ")")) option("const") option("volatile") const rbracket_token = self.getNextToken(); if (rbracket_token.id == Token.Id.RBracket) { const node = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp{ @@ -1225,7 +1227,8 @@ pub const Parser = struct { } }); dest_ptr.store(&node.base); - try stack.append(State { .AddrOfModifiers = &node.op.AddrOf }); + stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }) catch unreachable; + try stack.append(State { .AddrOfModifiers = &node.op.SliceType }); continue; } @@ -1235,6 +1238,7 @@ pub const Parser = struct { .ArrayType = undefined, }); dest_ptr.store(&node.base); + stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }) catch unreachable; try stack.append(State { .ExpectToken = Token.Id.RBracket }); try stack.append(State { .Expression = DestPtr { .Field = &node.op.ArrayType } }); @@ -1302,32 +1306,32 @@ pub const Parser = struct { continue; }, Token.Id.Keyword_packed => { - try stack.append(State { + stack.append(State { .ContainerExtern = ContainerExternCtx { .dest_ptr = dest_ptr, .ltoken = token, .layout = ast.NodeContainerDecl.Layout.Packed, }, - }); + }) catch unreachable; }, Token.Id.Keyword_extern => { - try stack.append(State { + stack.append(State { .ContainerExtern = ContainerExternCtx { .dest_ptr = dest_ptr, .ltoken = token, .layout = ast.NodeContainerDecl.Layout.Extern, }, - }); + }) catch unreachable; }, Token.Id.Keyword_struct, Token.Id.Keyword_union, Token.Id.Keyword_enum => { self.putBackToken(token); - try stack.append(State { + stack.append(State { .ContainerExtern = ContainerExternCtx { .dest_ptr = dest_ptr, .ltoken = token, .layout = ast.NodeContainerDecl.Layout.Auto, }, - }); + }) catch unreachable; }, Token.Id.LBrace => { @panic("TODO: Block expr"); @@ -1361,12 +1365,12 @@ pub const Parser = struct { const rbracket_token = self.getNextToken(); if (rbracket_token.id != Token.Id.RBracket) { self.putBackToken(rbracket_token); - try stack.append(State { + stack.append(State { .ExpectTokenSave = ExpectTokenSave { .id = Token.Id.RBracket, .ptr = &node.rtoken, } - }); + }) catch unreachable; try stack.append(State { .Expression = DestPtr { .NullableField = &node.op.Slice.end } }); } else { node.rtoken = rbracket_token; @@ -1638,7 +1642,7 @@ pub const Parser = struct { // TODO shouldn't need these casts const var_decl = try self.createAttachVarDecl(arena, &block.statements, (?Token)(null), mut_token, (?Token)(comptime_token), (?Token)(null), null); - try stack.append(State { .VarDecl = var_decl }); + stack.append(State { .VarDecl = var_decl }) catch unreachable; continue; } self.putBackToken(mut_token); @@ -1652,7 +1656,7 @@ pub const Parser = struct { // TODO shouldn't need these casts const var_decl = try self.createAttachVarDecl(arena, &block.statements, (?Token)(null), mut_token, (?Token)(null), (?Token)(null), null); - try stack.append(State { .VarDecl = var_decl }); + stack.append(State { .VarDecl = var_decl }) catch unreachable; continue; } self.putBackToken(mut_token); @@ -2132,6 +2136,7 @@ pub const Parser = struct { Expression: &ast.Node, VarDecl: &ast.NodeVarDecl, Statement: &ast.Node, + FieldInitializer: &ast.NodeFieldInitializer, PrintIndent, Indent: usize, }; @@ -2246,6 +2251,12 @@ pub const Parser = struct { } }, + RenderState.FieldInitializer => |field_init| { + try stream.print(".{}", self.tokenizer.getTokenSlice(field_init.name_token)); + try stream.print(" = "); + try stack.append(RenderState { .Expression = field_init.expr }); + }, + RenderState.VarDecl => |var_decl| { try stack.append(RenderState { .Text = ";" }); if (var_decl.init_node) |init_node| { @@ -2481,8 +2492,34 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = range.start}); try stack.append(RenderState { .Text = "["}); }, - ast.NodeSuffixOp.SuffixOp.StructInitializer => @panic("TODO: StructInitializer"), - ast.NodeSuffixOp.SuffixOp.ArrayInitializer => @panic("TODO: ArrayInitializer"), + ast.NodeSuffixOp.SuffixOp.StructInitializer => |field_inits| { + try stack.append(RenderState { .Text = " }"}); + var i = field_inits.len; + while (i != 0) { + i -= 1; + const field_init = field_inits.at(i); + try stack.append(RenderState { .FieldInitializer = field_init }); + try stack.append(RenderState { .Text = " " }); + if (i != 0) { + try stack.append(RenderState { .Text = "," }); + } + } + try stack.append(RenderState { .Text = "{"}); + }, + ast.NodeSuffixOp.SuffixOp.ArrayInitializer => |exprs| { + try stack.append(RenderState { .Text = " }"}); + var i = exprs.len; + while (i != 0) { + i -= 1; + const expr = exprs.at(i); + try stack.append(RenderState { .Expression = expr }); + try stack.append(RenderState { .Text = " " }); + if (i != 0) { + try stack.append(RenderState { .Text = "," }); + } + } + try stack.append(RenderState { .Text = "{"}); + }, } try stack.append(RenderState { .Expression = suffix_op.lhs }); @@ -3011,6 +3048,10 @@ test "zig fmt: precedence" { \\ (a!b)(); \\ !a!b; \\ !(a!b); + \\ !a{ }; + \\ !(a{ }); + \\ a + b{ }; + \\ (a + b){ }; \\ a << b + c; \\ (a << b) + c; \\ a & b << c; @@ -3030,10 +3071,6 @@ test "zig fmt: precedence" { \\ (a = b) or c; \\} \\ - //\\ !a{}; - //\\ !(a{}); - //\\ a + b{}; - //\\ (a + b){}; ); } @@ -3233,23 +3270,12 @@ test "zig fmt: error set declaration" { ); } -test "zig fmt: catch" { - try testCanonical( - \\test "catch" { - \\ const a: error!u8 = 0; - \\ _ = a catch return; - \\ _ = a catch |err| return; - \\} - \\ - ); -} - test "zig fmt: arrays" { try testCanonical( \\test "test array" { \\ const a: [2]u8 = [2]u8{ 1, 2 }; \\ const a: [2]u8 = []u8{ 1, 2 }; - \\ const a: [0]u8 = []u8{}; + \\ const a: [0]u8 = []u8{ }; \\} \\ ); @@ -3260,7 +3286,18 @@ test "zig fmt: container initializers" { \\const a1 = []u8{ }; \\const a2 = []u8{ 1, 2, 3, 4 }; \\const s1 = S{ }; - \\const s2 = S{ .a = 1, .b = 2, }; + \\const s2 = S{ .a = 1, .b = 2 }; + \\ + ); +} + +test "zig fmt: catch" { + try testCanonical( + \\test "catch" { + \\ const a: error!u8 = 0; + \\ _ = a catch return; + \\ _ = a catch |err| return; + \\} \\ ); } From f667744d44b0fc3599c85c01d3fcf3b63c4e68a6 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Fri, 6 Apr 2018 09:36:11 +0200 Subject: [PATCH 29/87] std.zig.parser Fixed: * Parsing of the optional expression in contrl flow expr * Rendering of catch expressions --- std/zig/parser.zig | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 5d2a30caee..dfc3b2f82c 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -89,6 +89,15 @@ pub const Parser = struct { ptr: &Token, }; + const RevertState = struct { + parser: Parser, + tokenizer: Tokenizer, + + // We expect, that if something is optional, then there is a field, + // that needs to be set to null, when we revert. + ptr: &?&ast.Node, + }; + fn ListState(comptime T: type) type { return struct { list: &ArrayList(T), @@ -131,7 +140,7 @@ pub const Parser = struct { /// optional state is found, the parser will revert to the state it was in /// when the optional was added. This will polute the arena allocator with /// "leaked" nodes. TODO: Figure out if it's nessesary to handle leaked nodes. - Optional: Parser, + Optional: RevertState, /// Optional can be reverted by adding the Required state to the stack. Required, @@ -598,7 +607,13 @@ pub const Parser = struct { const node = try self.createControlFlowExpr(arena, token, ast.NodeControlFlowExpression.Kind.Return); dest_ptr.store(&node.base); - stack.append(State { .Optional = *self }) catch unreachable; + stack.append(State { + .Optional = RevertState { + .parser = *self, + .tokenizer = *self.tokenizer, + .ptr = &node.rhs, + } + }) catch unreachable; try stack.append(State { .Expression = DestPtr { .NullableField = &node.rhs } }); continue; }, @@ -673,7 +688,7 @@ pub const Parser = struct { continue; } - node.op.Catch = try self.createIdentifier(arena, undefined); + node.op.Catch = try self.createIdentifier(arena, Token(undefined)); try stack.append(State { .ExpectToken = Token.Id.Pipe }); try stack.append(State { .ExpectTokenSave = ExpectTokenSave { @@ -1910,7 +1925,7 @@ pub const Parser = struct { .base = self.initNode(ast.Node.Id.ControlFlowExpression), .ltoken = *ltoken, .kind = *kind, - .rhs = undefined, + .rhs = null, }; return node; } @@ -2067,7 +2082,9 @@ pub const Parser = struct { while (stack.popOrNull()) |state| { switch (state) { State.Optional => |revert| { - *self = state.Optional; + *self = revert.parser; + *self.tokenizer = revert.tokenizer; + *revert.ptr = null; return; }, State.Required => { @@ -2359,7 +2376,7 @@ pub const Parser = struct { if (prefix_op_node.op == ast.NodeInfixOp.InfixOp.Catch) { if (prefix_op_node.op.Catch) |payload| { - try stack.append(RenderState { .Text = "|" }); + try stack.append(RenderState { .Text = "| " }); try stack.append(RenderState { .Expression = &payload.base }); try stack.append(RenderState { .Text = "|" }); } @@ -2956,6 +2973,10 @@ test "zig fmt: return" { \\ return 0; \\} \\ + \\fn bar() void { + \\ return; + \\} + \\ ); } From c34ce2cbc6ca6f623e2e6e1d444c2b51e43dc493 Mon Sep 17 00:00:00 2001 From: Marc Tiehuis Date: Fri, 6 Apr 2018 23:10:54 +1200 Subject: [PATCH 30/87] Add common hash/checksum functions - SipHash64, SipHash128 - Crc32 (fast + small variants) - Adler32 - Fnv1a (32, 64 and 128 bit variants) --- CMakeLists.txt | 5 + std/hash/adler.zig | 112 +++++++++++++++ std/hash/crc.zig | 180 ++++++++++++++++++++++++ std/hash/fnv.zig | 60 ++++++++ std/hash/index.zig | 22 +++ std/hash/siphash.zig | 320 +++++++++++++++++++++++++++++++++++++++++++ std/index.zig | 2 + 7 files changed, 701 insertions(+) create mode 100644 std/hash/adler.zig create mode 100644 std/hash/crc.zig create mode 100644 std/hash/fnv.zig create mode 100644 std/hash/index.zig create mode 100644 std/hash/siphash.zig diff --git a/CMakeLists.txt b/CMakeLists.txt index cea302cb06..2bb9bf517c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -438,6 +438,11 @@ set(ZIG_STD_FILES "fmt/errol/lookup.zig" "fmt/index.zig" "hash_map.zig" + "hash/index.zig" + "hash/adler.zig" + "hash/crc.zig" + "hash/fnv.zig" + "hash/siphash.zig" "heap.zig" "index.zig" "io.zig" diff --git a/std/hash/adler.zig b/std/hash/adler.zig new file mode 100644 index 0000000000..c77a5aaf50 --- /dev/null +++ b/std/hash/adler.zig @@ -0,0 +1,112 @@ +// Adler32 checksum. +// +// https://tools.ietf.org/html/rfc1950#section-9 +// https://github.com/madler/zlib/blob/master/adler32.c + +const std = @import("../index.zig"); +const debug = std.debug; + +pub const Adler32 = struct { + const base = 65521; + const nmax = 5552; + + adler: u32, + + pub fn init() Adler32 { + return Adler32 { + .adler = 1, + }; + } + + // This fast variant is taken from zlib. It reduces the required modulos and unrolls longer + // buffer inputs and should be much quicker. + pub fn update(self: &Adler32, input: []const u8) void { + var s1 = self.adler & 0xffff; + var s2 = (self.adler >> 16) & 0xffff; + + if (input.len == 1) { + s1 +%= input[0]; + if (s1 >= base) { + s1 -= base; + } + s2 +%= s1; + if (s2 >= base) { + s2 -= base; + } + } + else if (input.len < 16) { + for (input) |b| { + s1 +%= b; + s2 +%= s1; + } + if (s1 >= base) { + s1 -= base; + } + + s2 %= base; + } + else { + var i: usize = 0; + while (i + nmax <= input.len) : (i += nmax) { + const n = nmax / 16; // note: 16 | nmax + + var rounds: usize = 0; + while (rounds < n) : (rounds += 1) { + comptime var j: usize = 0; + inline while (j < 16) : (j += 1) { + s1 +%= input[i + n * j]; + s2 +%= s1; + } + } + } + + if (i < input.len) { + while (i + 16 <= input.len) : (i += 16) { + comptime var j: usize = 0; + inline while (j < 16) : (j += 1) { + s1 +%= input[i + j]; + s2 +%= s1; + } + } + while (i < input.len) : (i += 1) { + s1 +%= input[i]; + s2 +%= s1; + } + + s1 %= base; + s2 %= base; + } + } + + self.adler = s1 | (s2 << 16); + } + + pub fn final(self: &Adler32) u32 { + return self.adler; + } + + pub fn hash(input: []const u8) u32 { + var c = Adler32.init(); + c.update(input); + return c.final(); + } +}; + +test "adler32 sanity" { + debug.assert(Adler32.hash("a") == 0x620062); + debug.assert(Adler32.hash("example") == 0xbc002ed); +} + +test "adler32 long" { + const long1 = []u8 {1} ** 1024; + debug.assert(Adler32.hash(long1[0..]) == 0x06780401); + + const long2 = []u8 {1} ** 1025; + debug.assert(Adler32.hash(long2[0..]) == 0x0a7a0402); +} + +test "adler32 very long" { + const long = []u8 {1} ** 5553; + debug.assert(Adler32.hash(long[0..]) == 0x707f15b2); +} + diff --git a/std/hash/crc.zig b/std/hash/crc.zig new file mode 100644 index 0000000000..f88069ce3c --- /dev/null +++ b/std/hash/crc.zig @@ -0,0 +1,180 @@ +// There are two implementations of CRC32 implemented with the following key characteristics: +// +// - Crc32WithPoly uses 8Kb of tables but is ~10x faster than the small method. +// +// - Crc32SmallWithPoly uses only 64 bytes of memory but is slower. Be aware that this is +// still moderately fast just slow relative to the slicing approach. + +const std = @import("../index.zig"); +const debug = std.debug; + +pub const Polynomial = struct { + const IEEE = 0xedb88320; + const Castagnoli = 0x82f63b78; + const Koopman = 0xeb31d82e; +}; + +// IEEE is by far the most common CRC and so is aliased by default. +pub const Crc32 = Crc32WithPoly(Polynomial.IEEE); + +// slicing-by-8 crc32 implementation. +pub fn Crc32WithPoly(comptime poly: u32) type { + return struct { + const Self = this; + const lookup_tables = comptime block: { + @setEvalBranchQuota(20000); + var tables: [8][256]u32 = undefined; + + for (tables[0]) |*e, i| { + var crc = u32(i); + var j: usize = 0; while (j < 8) : (j += 1) { + if (crc & 1 == 1) { + crc = (crc >> 1) ^ poly; + } else { + crc = (crc >> 1); + } + } + *e = crc; + } + + var i: usize = 0; + while (i < 256) : (i += 1) { + var crc = tables[0][i]; + var j: usize = 1; while (j < 8) : (j += 1) { + const index = @truncate(u8, crc); + crc = tables[0][index] ^ (crc >> 8); + tables[j][i] = crc; + } + } + + break :block tables; + }; + + crc: u32, + + pub fn init() Self { + return Self { + .crc = 0xffffffff, + }; + } + + pub fn update(self: &Self, input: []const u8) void { + var i: usize = 0; + while (i + 8 <= input.len) : (i += 8) { + const p = input[i..i+8]; + + // Unrolling this way gives ~50Mb/s increase + self.crc ^= (u32(p[0]) << 0); + self.crc ^= (u32(p[1]) << 8); + self.crc ^= (u32(p[2]) << 16); + self.crc ^= (u32(p[3]) << 24); + + self.crc = + lookup_tables[0][p[7]] ^ + lookup_tables[1][p[6]] ^ + lookup_tables[2][p[5]] ^ + lookup_tables[3][p[4]] ^ + lookup_tables[4][@truncate(u8, self.crc >> 24)] ^ + lookup_tables[5][@truncate(u8, self.crc >> 16)] ^ + lookup_tables[6][@truncate(u8, self.crc >> 8)] ^ + lookup_tables[7][@truncate(u8, self.crc >> 0)]; + } + + while (i < input.len) : (i += 1) { + const index = @truncate(u8, self.crc) ^ input[i]; + self.crc = (self.crc >> 8) ^ lookup_tables[0][index]; + } + } + + pub fn final(self: &Self) u32 { + return ~self.crc; + } + + pub fn hash(input: []const u8) u32 { + var c = Self.init(); + c.update(input); + return c.final(); + } + }; +} + +test "crc32 ieee" { + const Crc32Ieee = Crc32WithPoly(Polynomial.IEEE); + + debug.assert(Crc32Ieee.hash("") == 0x00000000); + debug.assert(Crc32Ieee.hash("a") == 0xe8b7be43); + debug.assert(Crc32Ieee.hash("abc") == 0x352441c2); +} + +test "crc32 castagnoli" { + const Crc32Castagnoli = Crc32WithPoly(Polynomial.Castagnoli); + + debug.assert(Crc32Castagnoli.hash("") == 0x00000000); + debug.assert(Crc32Castagnoli.hash("a") == 0xc1d04330); + debug.assert(Crc32Castagnoli.hash("abc") == 0x364b3fb7); +} + +// half-byte lookup table implementation. +pub fn Crc32SmallWithPoly(comptime poly: u32) type { + return struct { + const Self = this; + const lookup_table = comptime block: { + var table: [16]u32 = undefined; + + for (table) |*e, i| { + var crc = u32(i * 16); + var j: usize = 0; while (j < 8) : (j += 1) { + if (crc & 1 == 1) { + crc = (crc >> 1) ^ poly; + } else { + crc = (crc >> 1); + } + } + *e = crc; + } + + break :block table; + }; + + crc: u32, + + pub fn init() Self { + return Self { + .crc = 0xffffffff, + }; + } + + pub fn update(self: &Self, input: []const u8) void { + for (input) |b| { + self.crc = lookup_table[@truncate(u4, self.crc ^ (b >> 0))] ^ (self.crc >> 4); + self.crc = lookup_table[@truncate(u4, self.crc ^ (b >> 4))] ^ (self.crc >> 4); + } + } + + pub fn final(self: &Self) u32 { + return ~self.crc; + } + + pub fn hash(input: []const u8) u32 { + var c = Self.init(); + c.update(input); + return c.final(); + } + }; +} + +test "small crc32 ieee" { + const Crc32Ieee = Crc32SmallWithPoly(Polynomial.IEEE); + + debug.assert(Crc32Ieee.hash("") == 0x00000000); + debug.assert(Crc32Ieee.hash("a") == 0xe8b7be43); + debug.assert(Crc32Ieee.hash("abc") == 0x352441c2); +} + +test "small crc32 castagnoli" { + const Crc32Castagnoli = Crc32SmallWithPoly(Polynomial.Castagnoli); + + debug.assert(Crc32Castagnoli.hash("") == 0x00000000); + debug.assert(Crc32Castagnoli.hash("a") == 0xc1d04330); + debug.assert(Crc32Castagnoli.hash("abc") == 0x364b3fb7); +} diff --git a/std/hash/fnv.zig b/std/hash/fnv.zig new file mode 100644 index 0000000000..88b965b76a --- /dev/null +++ b/std/hash/fnv.zig @@ -0,0 +1,60 @@ +// FNV1a - Fowler-Noll-Vo hash function +// +// FNV1a is a fast, non-cryptographic hash function with fairly good distribution properties. +// +// https://tools.ietf.org/html/draft-eastlake-fnv-14 + +const std = @import("../index.zig"); +const debug = std.debug; + +pub const Fnv1a_32 = Fnv1a(u32, 0x01000193 , 0x811c9dc5); +pub const Fnv1a_64 = Fnv1a(u64, 0x100000001b3, 0xcbf29ce484222325); +pub const Fnv1a_128 = Fnv1a(u128, 0x1000000000000000000013b, 0x6c62272e07bb014262b821756295c58d); + +fn Fnv1a(comptime T: type, comptime prime: T, comptime offset: T) type { + return struct { + const Self = this; + + value: T, + + pub fn init() Self { + return Self { + .value = offset, + }; + } + + pub fn update(self: &Self, input: []const u8) void { + for (input) |b| { + self.value ^= b; + self.value *%= prime; + } + } + + pub fn final(self: &Self) T { + return self.value; + } + + pub fn hash(input: []const u8) T { + var c = Self.init(); + c.update(input); + return c.final(); + } + }; +} + +test "fnv1a-32" { + debug.assert(Fnv1a_32.hash("") == 0x811c9dc5); + debug.assert(Fnv1a_32.hash("a") == 0xe40c292c); + debug.assert(Fnv1a_32.hash("foobar") == 0xbf9cf968); +} + +test "fnv1a-64" { + debug.assert(Fnv1a_64.hash("") == 0xcbf29ce484222325); + debug.assert(Fnv1a_64.hash("a") == 0xaf63dc4c8601ec8c); + debug.assert(Fnv1a_64.hash("foobar") == 0x85944171f73967e8); +} + +test "fnv1a-128" { + debug.assert(Fnv1a_128.hash("") == 0x6c62272e07bb014262b821756295c58d); + debug.assert(Fnv1a_128.hash("a") == 0xd228cb696f1a8caf78912b704e4a8964); +} diff --git a/std/hash/index.zig b/std/hash/index.zig new file mode 100644 index 0000000000..8cce35f3c5 --- /dev/null +++ b/std/hash/index.zig @@ -0,0 +1,22 @@ +const adler = @import("adler.zig"); +pub const Adler32 = adler.Adler32; + +// pub for polynomials + generic crc32 construction +pub const crc = @import("crc.zig"); +pub const Crc32 = crc.Crc32; + +const fnv = @import("fnv.zig"); +pub const Fnv1a_32 = fnv.Fnv1a_32; +pub const Fnv1a_64 = fnv.Fnv1a_64; +pub const Fnv1a_128 = fnv.Fnv1a_128; + +const siphash = @import("siphash.zig"); +pub const SipHash64 = siphash.SipHash64; +pub const SipHash128 = siphash.SipHash128; + +test "hash" { + _ = @import("adler.zig"); + _ = @import("crc.zig"); + _ = @import("fnv.zig"); + _ = @import("siphash.zig"); +} diff --git a/std/hash/siphash.zig b/std/hash/siphash.zig new file mode 100644 index 0000000000..301c35cf05 --- /dev/null +++ b/std/hash/siphash.zig @@ -0,0 +1,320 @@ +// Siphash +// +// SipHash is a moderately fast, non-cryptographic keyed hash function designed for resistance +// against hash flooding DoS attacks. +// +// https://131002.net/siphash/ + +const std = @import("../index.zig"); +const debug = std.debug; +const math = std.math; +const mem = std.mem; + +const Endian = @import("builtin").Endian; + +pub fn SipHash64(comptime c_rounds: usize, comptime d_rounds: usize) type { + return SipHash(u64, c_rounds, d_rounds); +} + +pub fn SipHash128(comptime c_rounds: usize, comptime d_rounds: usize) type { + return SipHash(u128, c_rounds, d_rounds); +} + +fn SipHash(comptime T: type, comptime c_rounds: usize, comptime d_rounds: usize) type { + debug.assert(T == u64 or T == u128); + debug.assert(c_rounds > 0 and d_rounds > 0); + + return struct { + const Self = this; + const digest_size = 64; + const block_size = 64; + + v0: u64, + v1: u64, + v2: u64, + v3: u64, + + // streaming cache + buf: [8]u8, + buf_len: usize, + msg_len: u8, + + pub fn init(key: []const u8) Self { + debug.assert(key.len >= 16); + + const k0 = mem.readInt(key[0..8], u64, Endian.Little); + const k1 = mem.readInt(key[8..16], u64, Endian.Little); + + var d = Self { + .v0 = k0 ^ 0x736f6d6570736575, + .v1 = k1 ^ 0x646f72616e646f6d, + .v2 = k0 ^ 0x6c7967656e657261, + .v3 = k1 ^ 0x7465646279746573, + + .buf = undefined, + .buf_len = 0, + .msg_len = 0, + }; + + if (T == u128) { + d.v1 ^= 0xee; + } + + return d; + } + + pub fn update(d: &Self, b: []const u8) void { + var off: usize = 0; + + // Partial from previous. + if (d.buf_len != 0 and d.buf_len + b.len > 8) { + off += 8 - d.buf_len; + mem.copy(u8, d.buf[d.buf_len..], b[0..off]); + d.round(d.buf[0..]); + d.buf_len = 0; + } + + // Full middle blocks. + while (off + 8 <= b.len) : (off += 8) { + d.round(b[off..off + 8]); + } + + // Remainder for next pass. + mem.copy(u8, d.buf[d.buf_len..], b[off..]); + d.buf_len += u8(b[off..].len); + d.msg_len +%= @truncate(u8, b.len); + } + + pub fn final(d: &Self) T { + // Padding + mem.set(u8, d.buf[d.buf_len..], 0); + d.buf[7] = d.msg_len; + d.round(d.buf[0..]); + + if (T == u128) { + d.v2 ^= 0xee; + } else { + d.v2 ^= 0xff; + } + + comptime var i: usize = 0; + inline while (i < d_rounds) : (i += 1) { + @inlineCall(sipRound, d); + } + + const b1 = d.v0 ^ d.v1 ^ d.v2 ^ d.v3; + if (T == u64) { + return b1; + } + + d.v1 ^= 0xdd; + + comptime var j: usize = 0; + inline while (j < d_rounds) : (j += 1) { + @inlineCall(sipRound, d); + } + + const b2 = d.v0 ^ d.v1 ^ d.v2 ^ d.v3; + return (u128(b2) << 64) | b1; + } + + fn round(d: &Self, b: []const u8) void { + debug.assert(b.len == 8); + + const m = mem.readInt(b[0..], u64, Endian.Little); + d.v3 ^= m; + + comptime var i: usize = 0; + inline while (i < c_rounds) : (i += 1) { + @inlineCall(sipRound, d); + } + + d.v0 ^= m; + } + + fn sipRound(d: &Self) void { + d.v0 +%= d.v1; + d.v1 = math.rotl(u64, d.v1, u64(13)); + d.v1 ^= d.v0; + d.v0 = math.rotl(u64, d.v0, u64(32)); + d.v2 +%= d.v3; + d.v3 = math.rotl(u64, d.v3, u64(16)); + d.v3 ^= d.v2; + d.v0 +%= d.v3; + d.v3 = math.rotl(u64, d.v3, u64(21)); + d.v3 ^= d.v0; + d.v2 +%= d.v1; + d.v1 = math.rotl(u64, d.v1, u64(17)); + d.v1 ^= d.v2; + d.v2 = math.rotl(u64, d.v2, u64(32)); + } + + pub fn hash(key: []const u8, input: []const u8) T { + var c = Self.init(key); + c.update(input); + return c.final(); + } + }; +} + +// Test vectors from reference implementation. +// https://github.com/veorq/SipHash/blob/master/vectors.h +const test_key = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"; + +test "siphash64-2-4 sanity" { + const vectors = [][]const u8 { + "\x31\x0e\x0e\xdd\x47\xdb\x6f\x72", // "" + "\xfd\x67\xdc\x93\xc5\x39\xf8\x74", // "\x00" + "\x5a\x4f\xa9\xd9\x09\x80\x6c\x0d", // "\x00\x01" ... etc + "\x2d\x7e\xfb\xd7\x96\x66\x67\x85", + "\xb7\x87\x71\x27\xe0\x94\x27\xcf", + "\x8d\xa6\x99\xcd\x64\x55\x76\x18", + "\xce\xe3\xfe\x58\x6e\x46\xc9\xcb", + "\x37\xd1\x01\x8b\xf5\x00\x02\xab", + "\x62\x24\x93\x9a\x79\xf5\xf5\x93", + "\xb0\xe4\xa9\x0b\xdf\x82\x00\x9e", + "\xf3\xb9\xdd\x94\xc5\xbb\x5d\x7a", + "\xa7\xad\x6b\x22\x46\x2f\xb3\xf4", + "\xfb\xe5\x0e\x86\xbc\x8f\x1e\x75", + "\x90\x3d\x84\xc0\x27\x56\xea\x14", + "\xee\xf2\x7a\x8e\x90\xca\x23\xf7", + "\xe5\x45\xbe\x49\x61\xca\x29\xa1", + "\xdb\x9b\xc2\x57\x7f\xcc\x2a\x3f", + "\x94\x47\xbe\x2c\xf5\xe9\x9a\x69", + "\x9c\xd3\x8d\x96\xf0\xb3\xc1\x4b", + "\xbd\x61\x79\xa7\x1d\xc9\x6d\xbb", + "\x98\xee\xa2\x1a\xf2\x5c\xd6\xbe", + "\xc7\x67\x3b\x2e\xb0\xcb\xf2\xd0", + "\x88\x3e\xa3\xe3\x95\x67\x53\x93", + "\xc8\xce\x5c\xcd\x8c\x03\x0c\xa8", + "\x94\xaf\x49\xf6\xc6\x50\xad\xb8", + "\xea\xb8\x85\x8a\xde\x92\xe1\xbc", + "\xf3\x15\xbb\x5b\xb8\x35\xd8\x17", + "\xad\xcf\x6b\x07\x63\x61\x2e\x2f", + "\xa5\xc9\x1d\xa7\xac\xaa\x4d\xde", + "\x71\x65\x95\x87\x66\x50\xa2\xa6", + "\x28\xef\x49\x5c\x53\xa3\x87\xad", + "\x42\xc3\x41\xd8\xfa\x92\xd8\x32", + "\xce\x7c\xf2\x72\x2f\x51\x27\x71", + "\xe3\x78\x59\xf9\x46\x23\xf3\xa7", + "\x38\x12\x05\xbb\x1a\xb0\xe0\x12", + "\xae\x97\xa1\x0f\xd4\x34\xe0\x15", + "\xb4\xa3\x15\x08\xbe\xff\x4d\x31", + "\x81\x39\x62\x29\xf0\x90\x79\x02", + "\x4d\x0c\xf4\x9e\xe5\xd4\xdc\xca", + "\x5c\x73\x33\x6a\x76\xd8\xbf\x9a", + "\xd0\xa7\x04\x53\x6b\xa9\x3e\x0e", + "\x92\x59\x58\xfc\xd6\x42\x0c\xad", + "\xa9\x15\xc2\x9b\xc8\x06\x73\x18", + "\x95\x2b\x79\xf3\xbc\x0a\xa6\xd4", + "\xf2\x1d\xf2\xe4\x1d\x45\x35\xf9", + "\x87\x57\x75\x19\x04\x8f\x53\xa9", + "\x10\xa5\x6c\xf5\xdf\xcd\x9a\xdb", + "\xeb\x75\x09\x5c\xcd\x98\x6c\xd0", + "\x51\xa9\xcb\x9e\xcb\xa3\x12\xe6", + "\x96\xaf\xad\xfc\x2c\xe6\x66\xc7", + "\x72\xfe\x52\x97\x5a\x43\x64\xee", + "\x5a\x16\x45\xb2\x76\xd5\x92\xa1", + "\xb2\x74\xcb\x8e\xbf\x87\x87\x0a", + "\x6f\x9b\xb4\x20\x3d\xe7\xb3\x81", + "\xea\xec\xb2\xa3\x0b\x22\xa8\x7f", + "\x99\x24\xa4\x3c\xc1\x31\x57\x24", + "\xbd\x83\x8d\x3a\xaf\xbf\x8d\xb7", + "\x0b\x1a\x2a\x32\x65\xd5\x1a\xea", + "\x13\x50\x79\xa3\x23\x1c\xe6\x60", + "\x93\x2b\x28\x46\xe4\xd7\x06\x66", + "\xe1\x91\x5f\x5c\xb1\xec\xa4\x6c", + "\xf3\x25\x96\x5c\xa1\x6d\x62\x9f", + "\x57\x5f\xf2\x8e\x60\x38\x1b\xe5", + "\x72\x45\x06\xeb\x4c\x32\x8a\x95", + }; + + const siphash = SipHash64(2, 4); + + var buffer: [64]u8 = undefined; + for (vectors) |vector, i| { + buffer[i] = u8(i); + + const expected = mem.readInt(vector, u64, Endian.Little); + debug.assert(siphash.hash(test_key, buffer[0..i]) == expected); + } +} + +test "siphash128-2-4 sanity" { + const vectors = [][]const u8 { + "\xa3\x81\x7f\x04\xba\x25\xa8\xe6\x6d\xf6\x72\x14\xc7\x55\x02\x93", + "\xda\x87\xc1\xd8\x6b\x99\xaf\x44\x34\x76\x59\x11\x9b\x22\xfc\x45", + "\x81\x77\x22\x8d\xa4\xa4\x5d\xc7\xfc\xa3\x8b\xde\xf6\x0a\xff\xe4", + "\x9c\x70\xb6\x0c\x52\x67\xa9\x4e\x5f\x33\xb6\xb0\x29\x85\xed\x51", + "\xf8\x81\x64\xc1\x2d\x9c\x8f\xaf\x7d\x0f\x6e\x7c\x7b\xcd\x55\x79", + "\x13\x68\x87\x59\x80\x77\x6f\x88\x54\x52\x7a\x07\x69\x0e\x96\x27", + "\x14\xee\xca\x33\x8b\x20\x86\x13\x48\x5e\xa0\x30\x8f\xd7\xa1\x5e", + "\xa1\xf1\xeb\xbe\xd8\xdb\xc1\x53\xc0\xb8\x4a\xa6\x1f\xf0\x82\x39", + "\x3b\x62\xa9\xba\x62\x58\xf5\x61\x0f\x83\xe2\x64\xf3\x14\x97\xb4", + "\x26\x44\x99\x06\x0a\xd9\xba\xab\xc4\x7f\x8b\x02\xbb\x6d\x71\xed", + "\x00\x11\x0d\xc3\x78\x14\x69\x56\xc9\x54\x47\xd3\xf3\xd0\xfb\xba", + "\x01\x51\xc5\x68\x38\x6b\x66\x77\xa2\xb4\xdc\x6f\x81\xe5\xdc\x18", + "\xd6\x26\xb2\x66\x90\x5e\xf3\x58\x82\x63\x4d\xf6\x85\x32\xc1\x25", + "\x98\x69\xe2\x47\xe9\xc0\x8b\x10\xd0\x29\x93\x4f\xc4\xb9\x52\xf7", + "\x31\xfc\xef\xac\x66\xd7\xde\x9c\x7e\xc7\x48\x5f\xe4\x49\x49\x02", + "\x54\x93\xe9\x99\x33\xb0\xa8\x11\x7e\x08\xec\x0f\x97\xcf\xc3\xd9", + "\x6e\xe2\xa4\xca\x67\xb0\x54\xbb\xfd\x33\x15\xbf\x85\x23\x05\x77", + "\x47\x3d\x06\xe8\x73\x8d\xb8\x98\x54\xc0\x66\xc4\x7a\xe4\x77\x40", + "\xa4\x26\xe5\xe4\x23\xbf\x48\x85\x29\x4d\xa4\x81\xfe\xae\xf7\x23", + "\x78\x01\x77\x31\xcf\x65\xfa\xb0\x74\xd5\x20\x89\x52\x51\x2e\xb1", + "\x9e\x25\xfc\x83\x3f\x22\x90\x73\x3e\x93\x44\xa5\xe8\x38\x39\xeb", + "\x56\x8e\x49\x5a\xbe\x52\x5a\x21\x8a\x22\x14\xcd\x3e\x07\x1d\x12", + "\x4a\x29\xb5\x45\x52\xd1\x6b\x9a\x46\x9c\x10\x52\x8e\xff\x0a\xae", + "\xc9\xd1\x84\xdd\xd5\xa9\xf5\xe0\xcf\x8c\xe2\x9a\x9a\xbf\x69\x1c", + "\x2d\xb4\x79\xae\x78\xbd\x50\xd8\x88\x2a\x8a\x17\x8a\x61\x32\xad", + "\x8e\xce\x5f\x04\x2d\x5e\x44\x7b\x50\x51\xb9\xea\xcb\x8d\x8f\x6f", + "\x9c\x0b\x53\xb4\xb3\xc3\x07\xe8\x7e\xae\xe0\x86\x78\x14\x1f\x66", + "\xab\xf2\x48\xaf\x69\xa6\xea\xe4\xbf\xd3\xeb\x2f\x12\x9e\xeb\x94", + "\x06\x64\xda\x16\x68\x57\x4b\x88\xb9\x35\xf3\x02\x73\x58\xae\xf4", + "\xaa\x4b\x9d\xc4\xbf\x33\x7d\xe9\x0c\xd4\xfd\x3c\x46\x7c\x6a\xb7", + "\xea\x5c\x7f\x47\x1f\xaf\x6b\xde\x2b\x1a\xd7\xd4\x68\x6d\x22\x87", + "\x29\x39\xb0\x18\x32\x23\xfa\xfc\x17\x23\xde\x4f\x52\xc4\x3d\x35", + "\x7c\x39\x56\xca\x5e\xea\xfc\x3e\x36\x3e\x9d\x55\x65\x46\xeb\x68", + "\x77\xc6\x07\x71\x46\xf0\x1c\x32\xb6\xb6\x9d\x5f\x4e\xa9\xff\xcf", + "\x37\xa6\x98\x6c\xb8\x84\x7e\xdf\x09\x25\xf0\xf1\x30\x9b\x54\xde", + "\xa7\x05\xf0\xe6\x9d\xa9\xa8\xf9\x07\x24\x1a\x2e\x92\x3c\x8c\xc8", + "\x3d\xc4\x7d\x1f\x29\xc4\x48\x46\x1e\x9e\x76\xed\x90\x4f\x67\x11", + "\x0d\x62\xbf\x01\xe6\xfc\x0e\x1a\x0d\x3c\x47\x51\xc5\xd3\x69\x2b", + "\x8c\x03\x46\x8b\xca\x7c\x66\x9e\xe4\xfd\x5e\x08\x4b\xbe\xe7\xb5", + "\x52\x8a\x5b\xb9\x3b\xaf\x2c\x9c\x44\x73\xcc\xe5\xd0\xd2\x2b\xd9", + "\xdf\x6a\x30\x1e\x95\xc9\x5d\xad\x97\xae\x0c\xc8\xc6\x91\x3b\xd8", + "\x80\x11\x89\x90\x2c\x85\x7f\x39\xe7\x35\x91\x28\x5e\x70\xb6\xdb", + "\xe6\x17\x34\x6a\xc9\xc2\x31\xbb\x36\x50\xae\x34\xcc\xca\x0c\x5b", + "\x27\xd9\x34\x37\xef\xb7\x21\xaa\x40\x18\x21\xdc\xec\x5a\xdf\x89", + "\x89\x23\x7d\x9d\xed\x9c\x5e\x78\xd8\xb1\xc9\xb1\x66\xcc\x73\x42", + "\x4a\x6d\x80\x91\xbf\x5e\x7d\x65\x11\x89\xfa\x94\xa2\x50\xb1\x4c", + "\x0e\x33\xf9\x60\x55\xe7\xae\x89\x3f\xfc\x0e\x3d\xcf\x49\x29\x02", + "\xe6\x1c\x43\x2b\x72\x0b\x19\xd1\x8e\xc8\xd8\x4b\xdc\x63\x15\x1b", + "\xf7\xe5\xae\xf5\x49\xf7\x82\xcf\x37\x90\x55\xa6\x08\x26\x9b\x16", + "\x43\x8d\x03\x0f\xd0\xb7\xa5\x4f\xa8\x37\xf2\xad\x20\x1a\x64\x03", + "\xa5\x90\xd3\xee\x4f\xbf\x04\xe3\x24\x7e\x0d\x27\xf2\x86\x42\x3f", + "\x5f\xe2\xc1\xa1\x72\xfe\x93\xc4\xb1\x5c\xd3\x7c\xae\xf9\xf5\x38", + "\x2c\x97\x32\x5c\xbd\x06\xb3\x6e\xb2\x13\x3d\xd0\x8b\x3a\x01\x7c", + "\x92\xc8\x14\x22\x7a\x6b\xca\x94\x9f\xf0\x65\x9f\x00\x2a\xd3\x9e", + "\xdc\xe8\x50\x11\x0b\xd8\x32\x8c\xfb\xd5\x08\x41\xd6\x91\x1d\x87", + "\x67\xf1\x49\x84\xc7\xda\x79\x12\x48\xe3\x2b\xb5\x92\x25\x83\xda", + "\x19\x38\xf2\xcf\x72\xd5\x4e\xe9\x7e\x94\x16\x6f\xa9\x1d\x2a\x36", + "\x74\x48\x1e\x96\x46\xed\x49\xfe\x0f\x62\x24\x30\x16\x04\x69\x8e", + "\x57\xfc\xa5\xde\x98\xa9\xd6\xd8\x00\x64\x38\xd0\x58\x3d\x8a\x1d", + "\x9f\xec\xde\x1c\xef\xdc\x1c\xbe\xd4\x76\x36\x74\xd9\x57\x53\x59", + "\xe3\x04\x0c\x00\xeb\x28\xf1\x53\x66\xca\x73\xcb\xd8\x72\xe7\x40", + "\x76\x97\x00\x9a\x6a\x83\x1d\xfe\xcc\xa9\x1c\x59\x93\x67\x0f\x7a", + "\x58\x53\x54\x23\x21\xf5\x67\xa0\x05\xd5\x47\xa4\xf0\x47\x59\xbd", + "\x51\x50\xd1\x77\x2f\x50\x83\x4a\x50\x3e\x06\x9a\x97\x3f\xbd\x7c", + }; + + const siphash = SipHash128(2, 4); + + var buffer: [64]u8 = undefined; + for (vectors) |vector, i| { + buffer[i] = u8(i); + + const expected = mem.readInt(vector, u128, Endian.Little); + debug.assert(siphash.hash(test_key, buffer[0..i]) == expected); + } +} diff --git a/std/index.zig b/std/index.zig index 4bc1444ac9..f2af70b28b 100644 --- a/std/index.zig +++ b/std/index.zig @@ -19,6 +19,7 @@ pub const elf = @import("elf.zig"); pub const empty_import = @import("empty.zig"); pub const endian = @import("endian.zig"); pub const fmt = @import("fmt/index.zig"); +pub const hash = @import("hash/index.zig"); pub const heap = @import("heap.zig"); pub const io = @import("io.zig"); pub const macho = @import("macho.zig"); @@ -51,6 +52,7 @@ test "std" { _ = @import("empty.zig"); _ = @import("endian.zig"); _ = @import("fmt/index.zig"); + _ = @import("hash/index.zig"); _ = @import("io.zig"); _ = @import("macho.zig"); _ = @import("math/index.zig"); From 820de1716b83ee99ed94461996ca592a302eccae Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Fri, 6 Apr 2018 15:37:49 +0200 Subject: [PATCH 31/87] std.zig.parser now parses labeled blocks. * There is also some code for switch range parsing --- std/zig/ast.zig | 88 +++++++++++++++++++++++-- std/zig/parser.zig | 161 ++++++++++++++++++++++++++++++++++++--------- 2 files changed, 215 insertions(+), 34 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 4aaeef8dab..bc066a9922 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -20,6 +20,8 @@ pub const Node = struct { FnProto, ParamDecl, Block, + Switch, + SwitchCase, InfixOp, PrefixOp, SuffixOp, @@ -55,6 +57,8 @@ pub const Node = struct { Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).iterate(index), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).iterate(index), Id.Block => @fieldParentPtr(NodeBlock, "base", base).iterate(index), + Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).iterate(index), + Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).iterate(index), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).iterate(index), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).iterate(index), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).iterate(index), @@ -91,6 +95,8 @@ pub const Node = struct { Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).firstToken(), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).firstToken(), Id.Block => @fieldParentPtr(NodeBlock, "base", base).firstToken(), + Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).firstToken(), + Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).firstToken(), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).firstToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).firstToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).firstToken(), @@ -127,6 +133,8 @@ pub const Node = struct { Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).lastToken(), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).lastToken(), Id.Block => @fieldParentPtr(NodeBlock, "base", base).lastToken(), + Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).lastToken(), + Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).lastToken(), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).lastToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).lastToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).lastToken(), @@ -506,9 +514,10 @@ pub const NodeParamDecl = struct { pub const NodeBlock = struct { base: Node, - begin_token: Token, - end_token: Token, + label: ?Token, + lbrace: Token, statements: ArrayList(&Node), + rbrace: Token, pub fn iterate(self: &NodeBlock, index: usize) ?&Node { var i = index; @@ -520,11 +529,80 @@ pub const NodeBlock = struct { } pub fn firstToken(self: &NodeBlock) Token { - return self.begin_token; + if (self.label) |label| { + return label; + } + + return self.lbrace; } pub fn lastToken(self: &NodeBlock) Token { - return self.end_token; + return self.rbrace; + } +}; + +pub const NodeSwitch = struct { + base: Node, + switch_token: Token, + expr: &Node, + cases: ArrayList(&NodeSwitchCase), + rbrace: Token, + + pub fn iterate(self: &NodeSwitch, index: usize) ?&Node { + var i = index; + + if (i < 1) return self.expr; + i -= 1; + + if (i < self.cases.len) return self.cases.at(i); + i -= self.cases.len; + + return null; + } + + pub fn firstToken(self: &NodeSwitch) Token { + return self.switch_token; + } + + pub fn lastToken(self: &NodeSwitch) Token { + return self.rbrace; + } +}; + +pub const NodeSwitchCase = struct { + base: Node, + items: ArrayList(&Node), + capture: ?Capture, + expr: &Node, + + const Capture = struct { + symbol: &NodeIdentifier, + is_ptr: bool, + }; + + pub fn iterate(self: &NodeSwitchCase, index: usize) ?&Node { + var i = index; + + if (i < self.items.len) return self.items.at(i); + i -= self.items.len; + + if (self.capture) |capture| { + if (i < 1) return &capture.base; + i -= 1; + } + + if (i < 1) return self.expr; + i -= 1; + + return null; + } + + pub fn firstToken(self: &NodeSwitchCase) Token { + return self.items.at(0).firstToken(); + } + + pub fn lastToken(self: &NodeSwitchCase) Token { + return self.expr.lastToken(); } }; @@ -575,6 +653,7 @@ pub const NodeInfixOp = struct { Mult, MultWrap, Period, + Range, Sub, SubWrap, UnwrapMaybe, @@ -625,6 +704,7 @@ pub const NodeInfixOp = struct { InfixOp.Mult, InfixOp.MultWrap, InfixOp.Period, + InfixOp.Range, InfixOp.Sub, InfixOp.SubWrap, InfixOp.UnwrapMaybe => {}, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index dfc3b2f82c..3337968d69 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -146,6 +146,8 @@ pub const Parser = struct { Required, Expression: DestPtr, + RangeExpressionBegin: DestPtr, + RangeExpressionEnd: DestPtr, AssignmentExpressionBegin: DestPtr, AssignmentExpressionEnd: DestPtr, UnwrapExpressionBegin: DestPtr, @@ -256,7 +258,7 @@ pub const Parser = struct { } const name = try self.createStringLiteral(arena, name_token); - const block = try self.createBlock(arena, token); + const block = try self.createBlock(arena, (?Token)(null), token); const test_decl = try self.createAttachTestDecl(arena, &root_node.decls, token, &name.base, block); stack.append(State { .Block = block }) catch unreachable; continue; @@ -643,6 +645,27 @@ pub const Parser = struct { } }, + State.RangeExpressionBegin => |dest_ptr| { + stack.append(State { .RangeExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .Expression = dest_ptr }); + continue; + }, + + State.RangeExpressionEnd => |dest_ptr| { + const token = self.getNextToken(); + if (token.id == Token.Id.Ellipsis3) { + const node = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.Range); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }) catch unreachable; + continue; + } else { + self.putBackToken(token); + continue; + } + }, + State.AssignmentExpressionBegin => |dest_ptr| { stack.append(State { .AssignmentExpressionEnd = dest_ptr }) catch unreachable; try stack.append(State { .UnwrapExpressionBegin = dest_ptr }); @@ -1205,10 +1228,6 @@ pub const Parser = struct { try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); continue; }, - Token.Id.Identifier => { - dest_ptr.store(&(try self.createIdentifier(arena, token)).base); - continue; - }, Token.Id.Builtin => { const node = try arena.create(ast.NodeBuiltinCall); *node = ast.NodeBuiltinCall { @@ -1348,8 +1367,32 @@ pub const Parser = struct { }, }) catch unreachable; }, + Token.Id.Identifier => { + const next = self.getNextToken(); + if (next.id != Token.Id.Colon) { + self.putBackToken(next); + dest_ptr.store(&(try self.createIdentifier(arena, token)).base); + continue; + } + + const block = try self.createBlock(arena, (?Token)(token), Token(undefined)); + dest_ptr.store(&block.base); + + stack.append(State { .Block = block }) catch unreachable; + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.LBrace, + .ptr = &block.lbrace, + } + }); + continue; + }, Token.Id.LBrace => { - @panic("TODO: Block expr"); + const block = try self.createBlock(arena, (?Token)(null), token); + dest_ptr.store(&block.base); + + stack.append(State { .Block = block }) catch unreachable; + continue; }, Token.Id.Keyword_fn => { @panic("TODO: fn proto"); @@ -1618,7 +1661,7 @@ pub const Parser = struct { const token = self.getNextToken(); switch(token.id) { Token.Id.LBrace => { - const block = try self.createBlock(arena, token); + const block = try self.createBlock(arena, (?Token)(null), token); fn_proto.body_node = &block.base; stack.append(State { .Block = block }) catch unreachable; continue; @@ -1635,7 +1678,7 @@ pub const Parser = struct { const token = self.getNextToken(); switch (token.id) { Token.Id.RBrace => { - block.end_token = token; + block.rbrace = token; continue; }, else => { @@ -1648,38 +1691,64 @@ pub const Parser = struct { }, State.Statement => |block| { - { - // Look for comptime var, comptime const - const comptime_token = self.getNextToken(); - if (comptime_token.id == Token.Id.Keyword_comptime) { + const next = self.getNextToken(); + switch (next.id) { + Token.Id.Keyword_comptime => { const mut_token = self.getNextToken(); if (mut_token.id == Token.Id.Keyword_var or mut_token.id == Token.Id.Keyword_const) { // TODO shouldn't need these casts const var_decl = try self.createAttachVarDecl(arena, &block.statements, (?Token)(null), - mut_token, (?Token)(comptime_token), (?Token)(null), null); + mut_token, (?Token)(next), (?Token)(null), null); stack.append(State { .VarDecl = var_decl }) catch unreachable; continue; + } else { + self.putBackToken(mut_token); + @panic("TODO: comptime block"); } - self.putBackToken(mut_token); - } - self.putBackToken(comptime_token); - } - { - // Look for const, var - const mut_token = self.getNextToken(); - if (mut_token.id == Token.Id.Keyword_var or mut_token.id == Token.Id.Keyword_const) { - // TODO shouldn't need these casts + }, + Token.Id.Keyword_var, Token.Id.Keyword_const => { const var_decl = try self.createAttachVarDecl(arena, &block.statements, (?Token)(null), - mut_token, (?Token)(null), (?Token)(null), null); + next, (?Token)(null), (?Token)(null), null); stack.append(State { .VarDecl = var_decl }) catch unreachable; continue; + }, + Token.Id.Identifier => { + const maybe_colon = self.getNextToken(); + if (maybe_colon.id != Token.Id.Colon) { + self.putBackToken(maybe_colon); + self.putBackToken(next); + stack.append(State { .ExpectToken = Token.Id.Semicolon }) catch unreachable; + try stack.append(State { .Expression = DestPtr{.Field = try block.statements.addOne() } }); + continue; + } + + const inner_block = try self.createBlock(arena, (?Token)(next), Token(undefined)); + try block.statements.append(&inner_block.base); + + stack.append(State { .Block = inner_block }) catch unreachable; + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.LBrace, + .ptr = &inner_block.lbrace, + } + }); + continue; + }, + Token.Id.LBrace => { + const inner_block = try self.createBlock(arena, (?Token)(null), next); + try block.statements.append(&inner_block.base); + + stack.append(State { .Block = inner_block }) catch unreachable; + continue; + }, + else => { + self.putBackToken(next); + stack.append(State { .ExpectToken = Token.Id.Semicolon }) catch unreachable; + try stack.append(State { .Expression = DestPtr{.Field = try block.statements.addOne() } }); + continue; } - self.putBackToken(mut_token); } - stack.append(State { .ExpectToken = Token.Id.Semicolon }) catch unreachable; - try stack.append(State { .Expression = DestPtr{.Field = try block.statements.addOne() } }); - continue; }, } } @@ -1905,14 +1974,15 @@ pub const Parser = struct { return node; } - fn createBlock(self: &Parser, arena: &mem.Allocator, begin_token: &const Token) !&ast.NodeBlock { + fn createBlock(self: &Parser, arena: &mem.Allocator, label: &const ?Token, lbrace: &const Token) !&ast.NodeBlock { const node = try arena.create(ast.NodeBlock); *node = ast.NodeBlock { .base = self.initNode(ast.Node.Id.Block), - .begin_token = *begin_token, - .end_token = undefined, + .label = *label, + .lbrace = *lbrace, .statements = ArrayList(&ast.Node).init(arena), + .rbrace = undefined, }; return node; } @@ -2340,6 +2410,10 @@ pub const Parser = struct { }, ast.Node.Id.Block => { const block = @fieldParentPtr(ast.NodeBlock, "base", base); + if (block.label) |label| { + try stream.print("{}: ", self.tokenizer.getTokenSlice(label)); + } + if (block.statements.len == 0) { try stream.write("{}"); } else { @@ -2747,6 +2821,8 @@ pub const Parser = struct { }, ast.Node.Id.FnProto => @panic("TODO fn proto in an expression"), ast.Node.Id.LineComment => @panic("TODO render line comment in an expression"), + ast.Node.Id.Switch => @panic("TODO switch"), + ast.Node.Id.SwitchCase => @panic("TODO switch case"), ast.Node.Id.StructField, ast.Node.Id.UnionTag, @@ -2791,6 +2867,9 @@ pub const Parser = struct { const var_decl = @fieldParentPtr(ast.NodeVarDecl, "base", base); try stack.append(RenderState { .VarDecl = var_decl}); }, + ast.Node.Id.Block => { + try stack.append(RenderState { .Expression = base}); + }, else => { try stack.append(RenderState { .Text = ";"}); try stack.append(RenderState { .Expression = base}); @@ -3323,6 +3402,28 @@ test "zig fmt: catch" { ); } +test "zig fmt: blocks" { + try testCanonical( + \\test "blocks" { + \\ { + \\ const a = 0; + \\ const b = 0; + \\ } + \\ + \\ blk: { + \\ const a = 0; + \\ const b = 0; + \\ } + \\ + \\ const r = blk: { + \\ const a = 0; + \\ const b = 0; + \\ }; + \\} + \\ + ); +} + test "zig fmt: switch" { try testCanonical( \\test "switch" { From bdff5bfa3e9f6ab490771b54109cb200b180b4da Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Sat, 7 Apr 2018 01:38:38 +0200 Subject: [PATCH 32/87] std.zig.parser now parses switch --- std/zig/ast.zig | 21 ++++ std/zig/parser.zig | 260 ++++++++++++++++++++++++++++++++++++------ std/zig/tokenizer.zig | 6 + 3 files changed, 251 insertions(+), 36 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index bc066a9922..ca384efedd 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -22,6 +22,7 @@ pub const Node = struct { Block, Switch, SwitchCase, + SwitchElse, InfixOp, PrefixOp, SuffixOp, @@ -59,6 +60,7 @@ pub const Node = struct { Id.Block => @fieldParentPtr(NodeBlock, "base", base).iterate(index), Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).iterate(index), Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).iterate(index), + Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).iterate(index), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).iterate(index), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).iterate(index), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).iterate(index), @@ -97,6 +99,7 @@ pub const Node = struct { Id.Block => @fieldParentPtr(NodeBlock, "base", base).firstToken(), Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).firstToken(), Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).firstToken(), + Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).firstToken(), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).firstToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).firstToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).firstToken(), @@ -135,6 +138,7 @@ pub const Node = struct { Id.Block => @fieldParentPtr(NodeBlock, "base", base).lastToken(), Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).lastToken(), Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).lastToken(), + Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).firstToken(), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).lastToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).lastToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).lastToken(), @@ -606,6 +610,23 @@ pub const NodeSwitchCase = struct { } }; +pub const NodeSwitchElse = struct { + base: Node, + token: Token, + + pub fn iterate(self: &NodeSwitchElse, index: usize) ?&Node { + return null; + } + + pub fn firstToken(self: &NodeSwitchElse) Token { + return self.token; + } + + pub fn lastToken(self: &NodeSwitchElse) Token { + return self.token; + } +}; + pub const NodeInfixOp = struct { base: Node, op_token: Token, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 3337968d69..09f7827505 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -98,10 +98,15 @@ pub const Parser = struct { ptr: &?&ast.Node, }; - fn ListState(comptime T: type) type { + const ExprListCtx = struct { + list: &ArrayList(&ast.Node), + end: Token.Id, + ptr: &Token, + }; + + fn ListSave(comptime T: type) type { return struct { list: &ArrayList(T), - end: Token.Id, ptr: &Token, }; } @@ -129,11 +134,16 @@ pub const Parser = struct { FnDef: &ast.NodeFnProto, Block: &ast.NodeBlock, Statement: &ast.NodeBlock, - ExprListItemOrEnd: ListState(&ast.Node), - ExprListCommaOrEnd: ListState(&ast.Node), - FieldInitListItemOrEnd: ListState(&ast.NodeFieldInitializer), - FieldInitListCommaOrEnd: ListState(&ast.NodeFieldInitializer), + ExprListItemOrEnd: ExprListCtx, + ExprListCommaOrEnd: ExprListCtx, + FieldInitListItemOrEnd: ListSave(&ast.NodeFieldInitializer), + FieldInitListCommaOrEnd: ListSave(&ast.NodeFieldInitializer), FieldListCommaOrEnd: &ast.NodeContainerDecl, + SwitchCaseOrEnd: ListSave(&ast.NodeSwitchCase), + SwitchCaseCapture: &?ast.NodeSwitchCase.Capture, + SwitchCaseCommaOrEnd: ListSave(&ast.NodeSwitchCase), + SwitchCaseItem: &ArrayList(&ast.Node), + SwitchCaseItemCommaOrEnd: &ArrayList(&ast.Node), /// A state that can be appended before any other State. If an error occures, /// the parser will first try looking for the closest optional state. If an @@ -245,17 +255,8 @@ pub const Parser = struct { Token.Id.Keyword_test => { stack.append(State.TopLevel) catch unreachable; - const name_token = self.getNextToken(); - if (name_token.id != Token.Id.StringLiteral) { - try self.parseError(&stack, token, "expected {}, found {}", @tagName(Token.Id.StringLiteral), @tagName(name_token.id)); - continue; - } - - const lbrace = self.getNextToken(); - if (lbrace.id != Token.Id.LBrace) { - try self.parseError(&stack, token, "expected {}, found {}", @tagName(Token.Id.LBrace), @tagName(name_token.id)); - continue; - } + const name_token = (try self.eatToken(&stack, Token.Id.StringLiteral)) ?? continue; + const lbrace = (try self.eatToken(&stack, Token.Id.LBrace)) ?? continue; const name = try self.createStringLiteral(arena, name_token); const block = try self.createBlock(arena, (?Token)(null), token); @@ -974,9 +975,8 @@ pub const Parser = struct { stack.append(State { .CurlySuffixExpressionEnd = dest_ptr }) catch unreachable; try stack.append(State { - .FieldInitListItemOrEnd = ListState(&ast.NodeFieldInitializer) { + .FieldInitListItemOrEnd = ListSave(&ast.NodeFieldInitializer) { .list = &node.op.StructInitializer, - .end = Token.Id.RBrace, .ptr = &node.rtoken, } }); @@ -992,7 +992,7 @@ pub const Parser = struct { stack.append(State { .CurlySuffixExpressionEnd = dest_ptr }) catch unreachable; try stack.append(State { - .ExprListItemOrEnd = ListState(&ast.Node) { + .ExprListItemOrEnd = ExprListCtx { .list = &node.op.ArrayInitializer, .end = Token.Id.RBrace, .ptr = &node.rtoken, @@ -1084,7 +1084,7 @@ pub const Parser = struct { stack.append(State { .SuffixOpExpressionEnd = dest_ptr }) catch unreachable; try stack.append(State { - .ExprListItemOrEnd = ListState(&ast.Node) { + .ExprListItemOrEnd = ExprListCtx { .list = &node.op.Call.params, .end = Token.Id.RParen, .ptr = &node.rtoken, @@ -1238,7 +1238,7 @@ pub const Parser = struct { }; dest_ptr.store(&node.base); stack.append(State { - .ExprListItemOrEnd = ListState(&ast.Node) { + .ExprListItemOrEnd = ExprListCtx { .list = &node.params, .end = Token.Id.RParen, .ptr = &node.rparen_token, @@ -1400,6 +1400,43 @@ pub const Parser = struct { Token.Id.Keyword_asm => { @panic("TODO: inline asm"); }, + Token.Id.Keyword_if => { + @panic("TODO: inline if"); + }, + Token.Id.Keyword_while => { + @panic("TODO: inline while"); + }, + Token.Id.Keyword_for => { + @panic("TODO: inline for"); + }, + Token.Id.Keyword_switch => { + const node = try arena.create(ast.NodeSwitch); + *node = ast.NodeSwitch { + .base = self.initNode(ast.Node.Id.Switch), + .switch_token = token, + .expr = undefined, + .cases = ArrayList(&ast.NodeSwitchCase).init(arena), + .rbrace = undefined, + }; + dest_ptr.store(&node.base); + + stack.append(State { + .SwitchCaseOrEnd = ListSave(&ast.NodeSwitchCase) { + .list = &node.cases, + .ptr = &node.rbrace, + }, + }) catch unreachable; + try stack.append(State { .ExpectToken = Token.Id.LBrace }); + try stack.append(State { .ExpectToken = Token.Id.RParen }); + try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); + try stack.append(State { .ExpectToken = Token.Id.LParen }); + }, + Token.Id.Keyword_comptime => { + @panic("TODO: inline comptime"); + }, + Token.Id.Keyword_suspend => { + @panic("TODO: inline suspend"); + }, else => { try self.parseError(&stack, token, "expected primary expression, found {}", @tagName(token.id)); continue; @@ -1463,8 +1500,7 @@ pub const Parser = struct { State.FieldInitListItemOrEnd => |list_state| { var token = self.getNextToken(); - const IdTag = @TagType(Token.Id); - if (IdTag(list_state.end) == token.id){ + if (token.id == Token.Id.RBrace){ *list_state.ptr = token; continue; } @@ -1497,13 +1533,82 @@ pub const Parser = struct { }); }, + State.SwitchCaseOrEnd => |list_state| { + var token = self.getNextToken(); + + if (token.id == Token.Id.RBrace){ + *list_state.ptr = token; + continue; + } + + self.putBackToken(token); + + const node = try arena.create(ast.NodeSwitchCase); + *node = ast.NodeSwitchCase { + .base = self.initNode(ast.Node.Id.SwitchCase), + .items = ArrayList(&ast.Node).init(arena), + .capture = null, + .expr = undefined, + }; + try list_state.list.append(node); + stack.append(State { .SwitchCaseCommaOrEnd = list_state }) catch unreachable; + try stack.append(State { .Expression = DestPtr{ .Field = &node.expr } }); + try stack.append(State { .SwitchCaseCapture = &node.capture }); + + const maybe_else = self.getNextToken(); + if (maybe_else.id == Token.Id.Keyword_else) { + const else_node = try arena.create(ast.NodeSwitchElse); + *else_node = ast.NodeSwitchElse { + .base = self.initNode(ast.Node.Id.SwitchElse), + .token = maybe_else, + }; + try node.items.append(&else_node.base); + try stack.append(State { .ExpectToken = Token.Id.EqualAngleBracketRight }); + continue; + } else { + self.putBackToken(maybe_else); + try stack.append(State { .SwitchCaseItem = &node.items }); + continue; + } + }, + + State.SwitchCaseCapture => |capture| { + const token = self.getNextToken(); + if (token.id != Token.Id.Pipe) { + self.putBackToken(token); + continue; + } + + const is_ptr = blk: { + const asterik = self.getNextToken(); + if (asterik.id == Token.Id.Asterisk) { + break :blk true; + } else { + self.putBackToken(asterik); + break :blk false; + } + }; + + const ident = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; + _ = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue; + *capture = ast.NodeSwitchCase.Capture { + .symbol = try self.createIdentifier(arena, ident), + .is_ptr = is_ptr + }; + }, + + State.SwitchCaseItem => |case_items| { + stack.append(State { .SwitchCaseItemCommaOrEnd = case_items }) catch unreachable; + try stack.append(State { .RangeExpressionBegin = DestPtr{ .Field = try case_items.addOne() } }); + }, + State.ExprListCommaOrEnd => |list_state| { try self.commaOrEnd(&stack, list_state.end, list_state.ptr, State { .ExprListItemOrEnd = list_state }); continue; }, State.FieldInitListCommaOrEnd => |list_state| { - try self.commaOrEnd(&stack, list_state.end, list_state.ptr, State { .FieldInitListItemOrEnd = list_state }); + try self.commaOrEnd(&stack, Token.Id.RBrace, list_state.ptr, State { .FieldInitListItemOrEnd = list_state }); continue; }, @@ -1513,6 +1618,16 @@ pub const Parser = struct { continue; }, + State.SwitchCaseCommaOrEnd => |list_state| { + try self.commaOrEnd(&stack, Token.Id.RBrace, list_state.ptr, State { .SwitchCaseOrEnd = list_state }); + continue; + }, + + State.SwitchCaseItemCommaOrEnd => |case_items| { + try self.commaOrEnd(&stack, Token.Id.EqualAngleBracketRight, null, State { .SwitchCaseItem = case_items }); + continue; + }, + State.AddrOfModifiers => |addr_of_info| { var token = self.getNextToken(); switch (token.id) { @@ -1741,6 +1856,11 @@ pub const Parser = struct { stack.append(State { .Block = inner_block }) catch unreachable; continue; }, + Token.Id.Keyword_switch => { + self.putBackToken(next); + stack.append(State { .Expression = DestPtr{.Field = try block.statements.addOne() } }) catch unreachable; + continue; + }, else => { self.putBackToken(next); stack.append(State { .ExpectToken = Token.Id.Semicolon }) catch unreachable; @@ -1754,7 +1874,7 @@ pub const Parser = struct { } } - fn commaOrEnd(self: &Parser, stack: &ArrayList(State), end: &const Token.Id, ptr: &Token, state_after_comma: &const State) !void { + fn commaOrEnd(self: &Parser, stack: &ArrayList(State), end: &const Token.Id, maybe_ptr: ?&Token, state_after_comma: &const State) !void { var token = self.getNextToken(); switch (token.id) { Token.Id.Comma => { @@ -1763,7 +1883,9 @@ pub const Parser = struct { else => { const IdTag = @TagType(Token.Id); if (IdTag(*end) == token.id) { - *ptr = token; + if (maybe_ptr) |ptr| { + *ptr = token; + } return; } @@ -2498,7 +2620,8 @@ pub const Parser = struct { ast.NodeInfixOp.InfixOp.Sub => " - ", ast.NodeInfixOp.InfixOp.SubWrap => " -% ", ast.NodeInfixOp.InfixOp.UnwrapMaybe => " ?? ", - else => unreachable, + ast.NodeInfixOp.InfixOp.Range => " ... ", + ast.NodeInfixOp.InfixOp.Catch => unreachable, }; try stack.append(RenderState { .Text = text }); @@ -2821,8 +2944,73 @@ pub const Parser = struct { }, ast.Node.Id.FnProto => @panic("TODO fn proto in an expression"), ast.Node.Id.LineComment => @panic("TODO render line comment in an expression"), - ast.Node.Id.Switch => @panic("TODO switch"), - ast.Node.Id.SwitchCase => @panic("TODO switch case"), + ast.Node.Id.Switch => { + const switch_node = @fieldParentPtr(ast.NodeSwitch, "base", base); + try stream.print("{} (", self.tokenizer.getTokenSlice(switch_node.switch_token)); + + try stack.append(RenderState { .Text = "}"}); + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { .Indent = indent }); + try stack.append(RenderState { .Text = "\n"}); + + const cases = switch_node.cases.toSliceConst(); + var i = cases.len; + while (i != 0) { + i -= 1; + const node = cases[i]; + try stack.append(RenderState { .Expression = &node.base}); + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { + .Text = blk: { + if (i != 0) { + const prev_node = cases[i - 1]; + const loc = self.tokenizer.getTokenLocation(prev_node.lastToken().end, node.firstToken()); + if (loc.line >= 2) { + break :blk "\n\n"; + } + } + break :blk "\n"; + }, + }); + + if (i != 0) { + try stack.append(RenderState { .Text = "," }); + } + } + try stack.append(RenderState { .Indent = indent + indent_delta}); + try stack.append(RenderState { .Text = ") {"}); + try stack.append(RenderState { .Expression = switch_node.expr }); + }, + ast.Node.Id.SwitchCase => { + const switch_case = @fieldParentPtr(ast.NodeSwitchCase, "base", base); + + try stack.append(RenderState { .Expression = switch_case.expr }); + if (switch_case.capture) |capture| { + try stack.append(RenderState { .Text = "| "}); + try stack.append(RenderState { .Expression = &capture.symbol.base }); + + if (capture.is_ptr) { + try stack.append(RenderState { .Text = "*"}); + } + try stack.append(RenderState { .Text = "|"}); + } + try stack.append(RenderState { .Text = " => "}); + + const items = switch_case.items.toSliceConst(); + var i = items.len; + while (i != 0) { + i -= 1; + try stack.append(RenderState { .Expression = items[i] }); + + if (i != 0) { + try stack.append(RenderState { .Text = ", " }); + } + } + }, + ast.Node.Id.SwitchElse => { + const switch_else = @fieldParentPtr(ast.NodeSwitchElse, "base", base); + try stream.print("{}", self.tokenizer.getTokenSlice(switch_else.token)); + }, ast.Node.Id.StructField, ast.Node.Id.UnionTag, @@ -2867,7 +3055,7 @@ pub const Parser = struct { const var_decl = @fieldParentPtr(ast.NodeVarDecl, "base", base); try stack.append(RenderState { .VarDecl = var_decl}); }, - ast.Node.Id.Block => { + ast.Node.Id.Block, ast.Node.Id.Switch => { try stack.append(RenderState { .Expression = base}); }, else => { @@ -3436,24 +3624,24 @@ test "zig fmt: switch" { \\ else => { \\ const a = 1; \\ const b = a; - \\ }, + \\ } \\ } \\ \\ const res = switch (0) { \\ 0 => 0, \\ 1 => 2, - \\ else => 4, + \\ else => 4 \\ }; \\ \\ const Union = union(enum) { \\ Int: i64, - \\ Float: f64, + \\ Float: f64 \\ }; \\ - \\ const u = Union { .Int = 0 }; + \\ const u = Union{ .Int = 0 }; \\ switch (u) { \\ Union.Int => |int| {}, - \\ Union.Float => |*float| unreachable, + \\ Union.Float => |*float| unreachable \\ } \\} \\ diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig index 87597adff1..422aa20629 100644 --- a/std/zig/tokenizer.zig +++ b/std/zig/tokenizer.zig @@ -86,6 +86,7 @@ pub const Token = struct { PipeEqual, Equal, EqualEqual, + EqualAngleBracketRight, BangEqual, LParen, RParen, @@ -688,6 +689,11 @@ pub const Tokenizer = struct { self.index += 1; break; }, + '>' => { + result.id = Token.Id.EqualAngleBracketRight; + self.index += 1; + break; + }, else => { result.id = Token.Id.Equal; break; From e4d0b46c0c07f3151b4959c5e2e0d4c304981ead Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Sun, 8 Apr 2018 17:05:08 +0200 Subject: [PATCH 33/87] std.zig.parser WIP generalizing parsing of payloads * Note, it doesn't work :) --- std/zig/ast.zig | 108 ++++++++++++++++++++++++++++++++++++++++----- std/zig/parser.zig | 104 ++++++++++++++++++++++++++----------------- 2 files changed, 161 insertions(+), 51 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index ca384efedd..0d060eb685 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -20,9 +20,11 @@ pub const Node = struct { FnProto, ParamDecl, Block, + Payload, Switch, SwitchCase, SwitchElse, + While, InfixOp, PrefixOp, SuffixOp, @@ -58,9 +60,11 @@ pub const Node = struct { Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).iterate(index), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).iterate(index), Id.Block => @fieldParentPtr(NodeBlock, "base", base).iterate(index), + Id.Payload => @fieldParentPtr(NodePayload, "base", base).iterate(index), Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).iterate(index), Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).iterate(index), Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).iterate(index), + Id.While => @fieldParentPtr(NodeWhile, "base", base).iterate(index), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).iterate(index), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).iterate(index), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).iterate(index), @@ -97,9 +101,11 @@ pub const Node = struct { Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).firstToken(), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).firstToken(), Id.Block => @fieldParentPtr(NodeBlock, "base", base).firstToken(), + Id.Payload => @fieldParentPtr(NodePayload, "base", base).firstToken(), Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).firstToken(), Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).firstToken(), Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).firstToken(), + Id.While => @fieldParentPtr(NodeWhile, "base", base).firstToken(), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).firstToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).firstToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).firstToken(), @@ -136,9 +142,11 @@ pub const Node = struct { Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).lastToken(), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).lastToken(), Id.Block => @fieldParentPtr(NodeBlock, "base", base).lastToken(), + Id.Payload => @fieldParentPtr(NodePayload, "base", base).lastToken(), Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).lastToken(), Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).lastToken(), - Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).firstToken(), + Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).lastToken(), + Id.While => @fieldParentPtr(NodeWhile, "base", base).lastToken(), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).lastToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).lastToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).lastToken(), @@ -545,6 +553,31 @@ pub const NodeBlock = struct { } }; +pub const NodePayload = struct { + base: Node, + lpipe: Token, + is_ptr: bool, + symbol: &NodeIdentifier, + rpipe: Token, + + pub fn iterate(self: &NodePayload, index: usize) ?&Node { + var i = index; + + if (i < 1) return &self.symbol.base; + i -= 1; + + return null; + } + + pub fn firstToken(self: &NodePayload) Token { + return self.lpipe; + } + + pub fn lastToken(self: &NodePayload) Token { + return self.rpipe; + } +}; + pub const NodeSwitch = struct { base: Node, switch_token: Token, @@ -576,22 +609,17 @@ pub const NodeSwitch = struct { pub const NodeSwitchCase = struct { base: Node, items: ArrayList(&Node), - capture: ?Capture, + payload: ?&Node, expr: &Node, - const Capture = struct { - symbol: &NodeIdentifier, - is_ptr: bool, - }; - pub fn iterate(self: &NodeSwitchCase, index: usize) ?&Node { var i = index; if (i < self.items.len) return self.items.at(i); i -= self.items.len; - if (self.capture) |capture| { - if (i < 1) return &capture.base; + if (self.payload) |payload| { + if (i < 1) return payload; i -= 1; } @@ -627,6 +655,66 @@ pub const NodeSwitchElse = struct { } }; +pub const NodeWhile = struct { + base: Node, + while_token: Token, + condition: &Node, + payload: ?&NodePayload, + continue_expr: ?&Node, + body: &Node, + @"else": ?Else, + + const Else = struct { + capture: ?&NodeIdentifier, + body: &Node, + }; + + + pub fn iterate(self: &NodeWhile, index: usize) ?&Node { + var i = index; + + if (i < 1) return self.condition; + i -= 1; + + if (self.payload) |payload| { + if (i < 1) return &payload.base; + i -= 1; + } + + if (self.continue_expr) |continue_expr| { + if (i < 1) return continue_expr; + i -= 1; + } + + if (i < 1) return self.body; + i -= 1; + + if (self.@"else") |@"else"| { + if (@"else".capture) |capture| { + if (i < 1) return &capture.base; + i -= 1; + } + + if (i < 1) return @"else".body; + i -= 1; + } + + return null; + } + + pub fn firstToken(self: &NodeWhile) Token { + return self.while_token; + } + + pub fn lastToken(self: &NodeWhile) Token { + if (self.@"else") |@"else"| { + return @"else".body.lastToken(); + } + + return self.body.lastToken(); + } +}; + pub const NodeInfixOp = struct { base: Node, op_token: Token, @@ -661,7 +749,7 @@ pub const NodeInfixOp = struct { BitXor, BoolAnd, BoolOr, - Catch: ?&NodeIdentifier, + Catch: ?&Node, Div, EqualEqual, ErrorUnion, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 09f7827505..0d8b8a3248 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -104,6 +104,11 @@ pub const Parser = struct { ptr: &Token, }; + const ElseCtx = struct { + payload: ?DestPtr, + body: DestPtr, + }; + fn ListSave(comptime T: type) type { return struct { list: &ArrayList(T), @@ -140,7 +145,7 @@ pub const Parser = struct { FieldInitListCommaOrEnd: ListSave(&ast.NodeFieldInitializer), FieldListCommaOrEnd: &ast.NodeContainerDecl, SwitchCaseOrEnd: ListSave(&ast.NodeSwitchCase), - SwitchCaseCapture: &?ast.NodeSwitchCase.Capture, + Payload: DestPtr, SwitchCaseCommaOrEnd: ListSave(&ast.NodeSwitchCase), SwitchCaseItem: &ArrayList(&ast.Node), SwitchCaseItemCommaOrEnd: &ArrayList(&ast.Node), @@ -635,9 +640,6 @@ pub const Parser = struct { Token.Id.Keyword_await => { @panic("TODO: await"); }, - Token.Id.Keyword_suspend => { - @panic("TODO: suspend"); - }, else => { self.putBackToken(token); stack.append(State { .AssignmentExpressionBegin = dest_ptr }) catch unreachable; @@ -705,21 +707,14 @@ pub const Parser = struct { stack.append(State { .UnwrapExpressionEnd = dest_ptr }) catch unreachable; try stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }); - - const next = self.getNextToken(); - if (next.id != Token.Id.Pipe) { - self.putBackToken(next); - continue; - } - - node.op.Catch = try self.createIdentifier(arena, Token(undefined)); - try stack.append(State { .ExpectToken = Token.Id.Pipe }); try stack.append(State { - .ExpectTokenSave = ExpectTokenSave { - .id = Token.Id.Identifier, - .ptr = &(??node.op.Catch).name_token + .Optional = RevertState { + .tokenizer = *self.tokenizer, + .parser = *self, + .ptr = &node.op.Catch, } }); + try stack.append(State { .Payload = DestPtr { .NullableField = &node.op.Catch } }); continue; }, Token.Id.QuestionMarkQuestionMark => { @@ -1404,12 +1399,25 @@ pub const Parser = struct { @panic("TODO: inline if"); }, Token.Id.Keyword_while => { + const node = try arena.create(ast.NodeWhile); + *node = ast.NodeWhile { + .base = self.initNode(ast.Node.Id.While), + .while_token = token, + .condition = undefined, + .payload = null, + .continue_expr = null, + .body = undefined, + .@"else" = null, + }; + dest_ptr.store(&node.base); + @panic("TODO: inline while"); }, Token.Id.Keyword_for => { @panic("TODO: inline for"); }, Token.Id.Keyword_switch => { + @breakpoint(); const node = try arena.create(ast.NodeSwitch); *node = ast.NodeSwitch { .base = self.initNode(ast.Node.Id.Switch), @@ -1434,9 +1442,6 @@ pub const Parser = struct { Token.Id.Keyword_comptime => { @panic("TODO: inline comptime"); }, - Token.Id.Keyword_suspend => { - @panic("TODO: inline suspend"); - }, else => { try self.parseError(&stack, token, "expected primary expression, found {}", @tagName(token.id)); continue; @@ -1547,13 +1552,21 @@ pub const Parser = struct { *node = ast.NodeSwitchCase { .base = self.initNode(ast.Node.Id.SwitchCase), .items = ArrayList(&ast.Node).init(arena), - .capture = null, + .payload = null, .expr = undefined, }; try list_state.list.append(node); stack.append(State { .SwitchCaseCommaOrEnd = list_state }) catch unreachable; try stack.append(State { .Expression = DestPtr{ .Field = &node.expr } }); - try stack.append(State { .SwitchCaseCapture = &node.capture }); + try stack.append(State { + .Optional = RevertState { + .tokenizer = *self.tokenizer, + .parser = *self, + .ptr = &node.payload, + } + }); + try stack.append(State { .Payload = DestPtr { .NullableField = &node.payload } }); + try stack.append(State.Required); const maybe_else = self.getNextToken(); if (maybe_else.id == Token.Id.Keyword_else) { @@ -1572,12 +1585,8 @@ pub const Parser = struct { } }, - State.SwitchCaseCapture => |capture| { - const token = self.getNextToken(); - if (token.id != Token.Id.Pipe) { - self.putBackToken(token); - continue; - } + State.Payload => |dest_ptr| { + const lpipe = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue; const is_ptr = blk: { const asterik = self.getNextToken(); @@ -1590,11 +1599,16 @@ pub const Parser = struct { }; const ident = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; - _ = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue; - *capture = ast.NodeSwitchCase.Capture { + const rpipe = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue; + const node = try arena.create(ast.NodePayload); + *node = ast.NodePayload { + .base = self.initNode(ast.Node.Id.Payload), + .lpipe = lpipe, + .is_ptr = is_ptr, .symbol = try self.createIdentifier(arena, ident), - .is_ptr = is_ptr + .rpipe = rpipe }; + dest_ptr.store(&node.base); }, State.SwitchCaseItem => |case_items| { @@ -1856,6 +1870,8 @@ pub const Parser = struct { stack.append(State { .Block = inner_block }) catch unreachable; continue; }, + Token.Id.Keyword_suspend, Token.Id.Keyword_if, + Token.Id.Keyword_while, Token.Id.Keyword_for, Token.Id.Keyword_switch => { self.putBackToken(next); stack.append(State { .Expression = DestPtr{.Field = try block.statements.addOne() } }) catch unreachable; @@ -2572,9 +2588,8 @@ pub const Parser = struct { if (prefix_op_node.op == ast.NodeInfixOp.InfixOp.Catch) { if (prefix_op_node.op.Catch) |payload| { - try stack.append(RenderState { .Text = "| " }); - try stack.append(RenderState { .Expression = &payload.base }); - try stack.append(RenderState { .Text = "|" }); + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Expression = payload }); } try stack.append(RenderState { .Text = " catch " }); } else { @@ -2764,6 +2779,17 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = rhs }); } }, + ast.Node.Id.Payload => { + const payload = @fieldParentPtr(ast.NodePayload, "base", base); + try stack.append(RenderState { .Text = "|"}); + try stack.append(RenderState { .Expression = &payload.symbol.base }); + + if (payload.is_ptr) { + try stack.append(RenderState { .Text = "*"}); + } + + try stack.append(RenderState { .Text = "|"}); + }, ast.Node.Id.GroupedExpression => { const grouped_expr = @fieldParentPtr(ast.NodeGroupedExpression, "base", base); try stack.append(RenderState { .Text = ")"}); @@ -2985,14 +3011,9 @@ pub const Parser = struct { const switch_case = @fieldParentPtr(ast.NodeSwitchCase, "base", base); try stack.append(RenderState { .Expression = switch_case.expr }); - if (switch_case.capture) |capture| { - try stack.append(RenderState { .Text = "| "}); - try stack.append(RenderState { .Expression = &capture.symbol.base }); - - if (capture.is_ptr) { - try stack.append(RenderState { .Text = "*"}); - } - try stack.append(RenderState { .Text = "|"}); + if (switch_case.payload) |payload| { + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Expression = payload }); } try stack.append(RenderState { .Text = " => "}); @@ -3011,6 +3032,7 @@ pub const Parser = struct { const switch_else = @fieldParentPtr(ast.NodeSwitchElse, "base", base); try stream.print("{}", self.tokenizer.getTokenSlice(switch_else.token)); }, + ast.Node.Id.While => @panic("TODO: Render while"), ast.Node.Id.StructField, ast.Node.Id.UnionTag, From d26905c102f45382a5aa4bf59deda0ccc8c6e50f Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 29 Mar 2018 00:24:04 -0400 Subject: [PATCH 34/87] error return traces for the early return case it would work but LLVM is not correctly spilling the addresses. See #821 --- src/all_types.hpp | 19 ++++ src/analyze.cpp | 22 +++- src/codegen.cpp | 272 +++++++++++++++++++++++++++++++++++++++------- src/ir.cpp | 146 ++++++++++++++++--------- src/ir_print.cpp | 24 +++- 5 files changed, 391 insertions(+), 92 deletions(-) diff --git a/src/all_types.hpp b/src/all_types.hpp index 6951230aa4..6893f60fb3 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -1656,6 +1656,8 @@ struct CodeGen { LLVMValueRef coro_save_fn_val; LLVMValueRef coro_promise_fn_val; LLVMValueRef coro_alloc_helper_fn_val; + LLVMValueRef merge_err_ret_traces_fn_val; + LLVMValueRef add_error_return_trace_addr_fn_val; bool error_during_imports; const char **clang_argv; @@ -2054,6 +2056,7 @@ enum IrInstructionId { IrInstructionIdAwaitBookkeeping, IrInstructionIdSaveErrRetAddr, IrInstructionIdAddImplicitReturnType, + IrInstructionIdMergeErrRetTraces, }; struct IrInstruction { @@ -2892,6 +2895,11 @@ struct IrInstructionExport { struct IrInstructionErrorReturnTrace { IrInstruction base; + + enum Nullable { + Null, + NonNull, + } nullable; }; struct IrInstructionErrorUnion { @@ -3024,6 +3032,13 @@ struct IrInstructionAddImplicitReturnType { IrInstruction *value; }; +struct IrInstructionMergeErrRetTraces { + IrInstruction base; + + IrInstruction *coro_promise_ptr; + TypeStructField *resolved_field; +}; + static const size_t slice_ptr_index = 0; static const size_t slice_len_index = 1; @@ -3033,11 +3048,15 @@ static const size_t maybe_null_index = 1; static const size_t err_union_err_index = 0; static const size_t err_union_payload_index = 1; +// TODO call graph analysis to find out what this number needs to be for every function +static const size_t stack_trace_ptr_count = 30; + #define ASYNC_ALLOC_FIELD_NAME "allocFn" #define ASYNC_FREE_FIELD_NAME "freeFn" #define AWAITER_HANDLE_FIELD_NAME "awaiter_handle" #define RESULT_FIELD_NAME "result" #define RESULT_PTR_FIELD_NAME "result_ptr" +#define ERR_RET_TRACE_PTR_FIELD_NAME "err_ret_trace_ptr" enum FloatMode { diff --git a/src/analyze.cpp b/src/analyze.cpp index 291e7e7644..16788c5e6c 100644 --- a/src/analyze.cpp +++ b/src/analyze.cpp @@ -468,10 +468,26 @@ TypeTableEntry *get_promise_frame_type(CodeGen *g, TypeTableEntry *return_type) TypeTableEntry *awaiter_handle_type = get_maybe_type(g, g->builtin_types.entry_promise); TypeTableEntry *result_ptr_type = get_pointer_to_type(g, return_type, false); - const char *field_names[] = {AWAITER_HANDLE_FIELD_NAME, RESULT_FIELD_NAME, RESULT_PTR_FIELD_NAME}; - TypeTableEntry *field_types[] = {awaiter_handle_type, return_type, result_ptr_type}; + + ZigList field_names = {}; + field_names.append(AWAITER_HANDLE_FIELD_NAME); + field_names.append(RESULT_FIELD_NAME); + field_names.append(RESULT_PTR_FIELD_NAME); + if (g->have_err_ret_tracing) { + field_names.append(ERR_RET_TRACE_PTR_FIELD_NAME); + } + + ZigList field_types = {}; + field_types.append(awaiter_handle_type); + field_types.append(return_type); + field_types.append(result_ptr_type); + if (g->have_err_ret_tracing) { + field_types.append(get_ptr_to_stack_trace_type(g)); + } + + assert(field_names.length == field_types.length); Buf *name = buf_sprintf("AsyncFramePromise(%s)", buf_ptr(&return_type->name)); - TypeTableEntry *entry = get_struct_type(g, buf_ptr(name), field_names, field_types, 3); + TypeTableEntry *entry = get_struct_type(g, buf_ptr(name), field_names.items, field_types.items, field_names.length); return_type->promise_frame_parent = entry; return entry; diff --git a/src/codegen.cpp b/src/codegen.cpp index 0dafcb9502..fa8e069e21 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -1114,6 +1114,207 @@ static LLVMValueRef get_return_address_fn_val(CodeGen *g) { return g->return_address_fn_val; } +static LLVMValueRef get_add_error_return_trace_addr_fn(CodeGen *g) { + if (g->add_error_return_trace_addr_fn_val != nullptr) + return g->add_error_return_trace_addr_fn_val; + + LLVMTypeRef arg_types[] = { + get_ptr_to_stack_trace_type(g)->type_ref, + g->builtin_types.entry_usize->type_ref, + }; + LLVMTypeRef fn_type_ref = LLVMFunctionType(LLVMVoidType(), arg_types, 2, false); + + Buf *fn_name = get_mangled_name(g, buf_create_from_str("__zig_add_err_ret_trace_addr"), false); + LLVMValueRef fn_val = LLVMAddFunction(g->module, buf_ptr(fn_name), fn_type_ref); + addLLVMFnAttr(fn_val, "alwaysinline"); + LLVMSetLinkage(fn_val, LLVMInternalLinkage); + LLVMSetFunctionCallConv(fn_val, get_llvm_cc(g, CallingConventionUnspecified)); + addLLVMFnAttr(fn_val, "nounwind"); + add_uwtable_attr(g, fn_val); + addLLVMArgAttr(fn_val, (unsigned)0, "nonnull"); + if (g->build_mode == BuildModeDebug) { + ZigLLVMAddFunctionAttr(fn_val, "no-frame-pointer-elim", "true"); + ZigLLVMAddFunctionAttr(fn_val, "no-frame-pointer-elim-non-leaf", nullptr); + } + + LLVMBasicBlockRef entry_block = LLVMAppendBasicBlock(fn_val, "Entry"); + LLVMBasicBlockRef prev_block = LLVMGetInsertBlock(g->builder); + LLVMValueRef prev_debug_location = LLVMGetCurrentDebugLocation(g->builder); + LLVMPositionBuilderAtEnd(g->builder, entry_block); + ZigLLVMClearCurrentDebugLocation(g->builder); + + LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->type_ref; + + // stack_trace.instruction_addresses[stack_trace.index % stack_trace.instruction_addresses.len] = return_address; + + LLVMValueRef err_ret_trace_ptr = LLVMGetParam(fn_val, 0); + LLVMValueRef address_value = LLVMGetParam(fn_val, 1); + + size_t index_field_index = g->stack_trace_type->data.structure.fields[0].gen_index; + LLVMValueRef index_field_ptr = LLVMBuildStructGEP(g->builder, err_ret_trace_ptr, (unsigned)index_field_index, ""); + size_t addresses_field_index = g->stack_trace_type->data.structure.fields[1].gen_index; + LLVMValueRef addresses_field_ptr = LLVMBuildStructGEP(g->builder, err_ret_trace_ptr, (unsigned)addresses_field_index, ""); + + TypeTableEntry *slice_type = g->stack_trace_type->data.structure.fields[1].type_entry; + size_t ptr_field_index = slice_type->data.structure.fields[slice_ptr_index].gen_index; + LLVMValueRef ptr_field_ptr = LLVMBuildStructGEP(g->builder, addresses_field_ptr, (unsigned)ptr_field_index, ""); + size_t len_field_index = slice_type->data.structure.fields[slice_len_index].gen_index; + LLVMValueRef len_field_ptr = LLVMBuildStructGEP(g->builder, addresses_field_ptr, (unsigned)len_field_index, ""); + + LLVMValueRef len_value = gen_load_untyped(g, len_field_ptr, 0, false, ""); + LLVMValueRef index_val = gen_load_untyped(g, index_field_ptr, 0, false, ""); + LLVMValueRef modded_val = LLVMBuildURem(g->builder, index_val, len_value, ""); + LLVMValueRef address_indices[] = { + modded_val, + }; + + LLVMValueRef ptr_value = gen_load_untyped(g, ptr_field_ptr, 0, false, ""); + LLVMValueRef address_slot = LLVMBuildInBoundsGEP(g->builder, ptr_value, address_indices, 1, ""); + + gen_store_untyped(g, address_value, address_slot, 0, false); + + // stack_trace.index += 1; + LLVMValueRef index_plus_one_val = LLVMBuildNUWAdd(g->builder, index_val, LLVMConstInt(usize_type_ref, 1, false), ""); + gen_store_untyped(g, index_plus_one_val, index_field_ptr, 0, false); + + // return; + LLVMBuildRetVoid(g->builder); + + LLVMPositionBuilderAtEnd(g->builder, prev_block); + LLVMSetCurrentDebugLocation(g->builder, prev_debug_location); + + g->add_error_return_trace_addr_fn_val = fn_val; + return fn_val; +} + +static LLVMValueRef get_merge_err_ret_traces_fn_val(CodeGen *g) { + if (g->merge_err_ret_traces_fn_val) + return g->merge_err_ret_traces_fn_val; + + assert(g->stack_trace_type != nullptr); + + LLVMTypeRef param_types[] = { + get_ptr_to_stack_trace_type(g)->type_ref, + get_ptr_to_stack_trace_type(g)->type_ref, + }; + LLVMTypeRef fn_type_ref = LLVMFunctionType(LLVMVoidType(), param_types, 2, false); + + Buf *fn_name = get_mangled_name(g, buf_create_from_str("__zig_merge_error_return_traces"), false); + LLVMValueRef fn_val = LLVMAddFunction(g->module, buf_ptr(fn_name), fn_type_ref); + LLVMSetLinkage(fn_val, LLVMInternalLinkage); + LLVMSetFunctionCallConv(fn_val, get_llvm_cc(g, CallingConventionUnspecified)); + addLLVMFnAttr(fn_val, "nounwind"); + add_uwtable_attr(g, fn_val); + addLLVMArgAttr(fn_val, (unsigned)0, "nonnull"); + addLLVMArgAttr(fn_val, (unsigned)0, "noalias"); + addLLVMArgAttr(fn_val, (unsigned)0, "writeonly"); + addLLVMArgAttr(fn_val, (unsigned)1, "nonnull"); + addLLVMArgAttr(fn_val, (unsigned)1, "noalias"); + addLLVMArgAttr(fn_val, (unsigned)1, "readonly"); + if (g->build_mode == BuildModeDebug) { + ZigLLVMAddFunctionAttr(fn_val, "no-frame-pointer-elim", "true"); + ZigLLVMAddFunctionAttr(fn_val, "no-frame-pointer-elim-non-leaf", nullptr); + } + + // this is above the ZigLLVMClearCurrentDebugLocation + LLVMValueRef add_error_return_trace_addr_fn_val = get_add_error_return_trace_addr_fn(g); + + LLVMBasicBlockRef entry_block = LLVMAppendBasicBlock(fn_val, "Entry"); + LLVMBasicBlockRef prev_block = LLVMGetInsertBlock(g->builder); + LLVMValueRef prev_debug_location = LLVMGetCurrentDebugLocation(g->builder); + LLVMPositionBuilderAtEnd(g->builder, entry_block); + ZigLLVMClearCurrentDebugLocation(g->builder); + + // var frame_index: usize = undefined; + // var frames_left: usize = undefined; + // if (src_stack_trace.index < src_stack_trace.instruction_addresses.len) { + // frame_index = 0; + // frames_left = src_stack_trace.index; + // if (frames_left == 0) return; + // } else { + // frame_index = (src_stack_trace.index + 1) % src_stack_trace.instruction_addresses.len; + // frames_left = src_stack_trace.instruction_addresses.len; + // } + // while (true) { + // __zig_add_err_ret_trace_addr(dest_stack_trace, src_stack_trace.instruction_addresses[frame_index]); + // frames_left -= 1; + // if (frames_left == 0) return; + // frame_index = (frame_index + 1) % src_stack_trace.instruction_addresses.len; + // } + LLVMBasicBlockRef return_block = LLVMAppendBasicBlock(fn_val, "Return"); + + LLVMValueRef frame_index_ptr = LLVMBuildAlloca(g->builder, g->builtin_types.entry_usize->type_ref, "frame_index"); + LLVMValueRef frames_left_ptr = LLVMBuildAlloca(g->builder, g->builtin_types.entry_usize->type_ref, "frames_left"); + + LLVMValueRef dest_stack_trace_ptr = LLVMGetParam(fn_val, 0); + LLVMValueRef src_stack_trace_ptr = LLVMGetParam(fn_val, 1); + + size_t src_index_field_index = g->stack_trace_type->data.structure.fields[0].gen_index; + size_t src_addresses_field_index = g->stack_trace_type->data.structure.fields[1].gen_index; + LLVMValueRef src_index_field_ptr = LLVMBuildStructGEP(g->builder, src_stack_trace_ptr, + (unsigned)src_index_field_index, ""); + LLVMValueRef src_addresses_field_ptr = LLVMBuildStructGEP(g->builder, src_stack_trace_ptr, + (unsigned)src_addresses_field_index, ""); + TypeTableEntry *slice_type = g->stack_trace_type->data.structure.fields[1].type_entry; + size_t ptr_field_index = slice_type->data.structure.fields[slice_ptr_index].gen_index; + LLVMValueRef src_ptr_field_ptr = LLVMBuildStructGEP(g->builder, src_addresses_field_ptr, (unsigned)ptr_field_index, ""); + size_t len_field_index = slice_type->data.structure.fields[slice_len_index].gen_index; + LLVMValueRef src_len_field_ptr = LLVMBuildStructGEP(g->builder, src_addresses_field_ptr, (unsigned)len_field_index, ""); + LLVMValueRef src_index_val = LLVMBuildLoad(g->builder, src_index_field_ptr, ""); + LLVMValueRef src_ptr_val = LLVMBuildLoad(g->builder, src_ptr_field_ptr, ""); + LLVMValueRef src_len_val = LLVMBuildLoad(g->builder, src_len_field_ptr, ""); + LLVMValueRef no_wrap_bit = LLVMBuildICmp(g->builder, LLVMIntULT, src_index_val, src_len_val, ""); + LLVMBasicBlockRef no_wrap_block = LLVMAppendBasicBlock(fn_val, "NoWrap"); + LLVMBasicBlockRef yes_wrap_block = LLVMAppendBasicBlock(fn_val, "YesWrap"); + LLVMBasicBlockRef loop_block = LLVMAppendBasicBlock(fn_val, "Loop"); + LLVMBuildCondBr(g->builder, no_wrap_bit, no_wrap_block, yes_wrap_block); + + LLVMPositionBuilderAtEnd(g->builder, no_wrap_block); + LLVMValueRef usize_zero = LLVMConstNull(g->builtin_types.entry_usize->type_ref); + LLVMBuildStore(g->builder, usize_zero, frame_index_ptr); + LLVMBuildStore(g->builder, src_index_val, frames_left_ptr); + LLVMValueRef frames_left_eq_zero_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, src_index_val, usize_zero, ""); + LLVMBuildCondBr(g->builder, frames_left_eq_zero_bit, return_block, loop_block); + + LLVMPositionBuilderAtEnd(g->builder, yes_wrap_block); + LLVMValueRef usize_one = LLVMConstInt(g->builtin_types.entry_usize->type_ref, 1, false); + LLVMValueRef plus_one = LLVMBuildNUWAdd(g->builder, src_index_val, usize_one, ""); + LLVMValueRef mod_len = LLVMBuildURem(g->builder, plus_one, src_len_val, ""); + LLVMBuildStore(g->builder, mod_len, frame_index_ptr); + LLVMBuildStore(g->builder, src_len_val, frames_left_ptr); + LLVMBuildBr(g->builder, loop_block); + + LLVMPositionBuilderAtEnd(g->builder, loop_block); + LLVMValueRef ptr_index = LLVMBuildLoad(g->builder, frame_index_ptr, ""); + LLVMValueRef addr_ptr = LLVMBuildInBoundsGEP(g->builder, src_ptr_val, &ptr_index, 1, ""); + LLVMValueRef this_addr_val = LLVMBuildLoad(g->builder, addr_ptr, ""); + LLVMValueRef args[] = {dest_stack_trace_ptr, this_addr_val}; + LLVMBuildCall(g->builder, add_error_return_trace_addr_fn_val, args, 2, ""); + LLVMValueRef prev_frames_left = LLVMBuildLoad(g->builder, frames_left_ptr, ""); + LLVMValueRef new_frames_left = LLVMBuildNUWSub(g->builder, prev_frames_left, usize_one, ""); + LLVMValueRef done_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, new_frames_left, usize_zero, ""); + LLVMBasicBlockRef continue_block = LLVMAppendBasicBlock(fn_val, "Continue"); + LLVMBuildCondBr(g->builder, done_bit, return_block, continue_block); + + LLVMPositionBuilderAtEnd(g->builder, return_block); + LLVMBuildRetVoid(g->builder); + + LLVMPositionBuilderAtEnd(g->builder, continue_block); + LLVMBuildStore(g->builder, new_frames_left, frames_left_ptr); + LLVMValueRef prev_index = LLVMBuildLoad(g->builder, frame_index_ptr, ""); + LLVMValueRef index_plus_one = LLVMBuildNUWAdd(g->builder, prev_index, usize_one, ""); + LLVMValueRef index_mod_len = LLVMBuildURem(g->builder, index_plus_one, src_len_val, ""); + LLVMBuildStore(g->builder, index_mod_len, frame_index_ptr); + LLVMBuildBr(g->builder, loop_block); + + LLVMPositionBuilderAtEnd(g->builder, prev_block); + LLVMSetCurrentDebugLocation(g->builder, prev_debug_location); + + g->merge_err_ret_traces_fn_val = fn_val; + return fn_val; + +} + static LLVMValueRef get_return_err_fn(CodeGen *g) { if (g->return_err_fn != nullptr) return g->return_err_fn; @@ -1140,50 +1341,24 @@ static LLVMValueRef get_return_err_fn(CodeGen *g) { ZigLLVMAddFunctionAttr(fn_val, "no-frame-pointer-elim-non-leaf", nullptr); } + // this is above the ZigLLVMClearCurrentDebugLocation + LLVMValueRef add_error_return_trace_addr_fn_val = get_add_error_return_trace_addr_fn(g); + LLVMBasicBlockRef entry_block = LLVMAppendBasicBlock(fn_val, "Entry"); LLVMBasicBlockRef prev_block = LLVMGetInsertBlock(g->builder); LLVMValueRef prev_debug_location = LLVMGetCurrentDebugLocation(g->builder); LLVMPositionBuilderAtEnd(g->builder, entry_block); ZigLLVMClearCurrentDebugLocation(g->builder); - LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->type_ref; - - // stack_trace.instruction_addresses[stack_trace.index % stack_trace.instruction_addresses.len] = return_address; - LLVMValueRef err_ret_trace_ptr = LLVMGetParam(fn_val, 0); - size_t index_field_index = g->stack_trace_type->data.structure.fields[0].gen_index; - LLVMValueRef index_field_ptr = LLVMBuildStructGEP(g->builder, err_ret_trace_ptr, (unsigned)index_field_index, ""); - size_t addresses_field_index = g->stack_trace_type->data.structure.fields[1].gen_index; - LLVMValueRef addresses_field_ptr = LLVMBuildStructGEP(g->builder, err_ret_trace_ptr, (unsigned)addresses_field_index, ""); - - TypeTableEntry *slice_type = g->stack_trace_type->data.structure.fields[1].type_entry; - size_t ptr_field_index = slice_type->data.structure.fields[slice_ptr_index].gen_index; - LLVMValueRef ptr_field_ptr = LLVMBuildStructGEP(g->builder, addresses_field_ptr, (unsigned)ptr_field_index, ""); - size_t len_field_index = slice_type->data.structure.fields[slice_len_index].gen_index; - LLVMValueRef len_field_ptr = LLVMBuildStructGEP(g->builder, addresses_field_ptr, (unsigned)len_field_index, ""); - - LLVMValueRef len_value = gen_load_untyped(g, len_field_ptr, 0, false, ""); - LLVMValueRef index_val = gen_load_untyped(g, index_field_ptr, 0, false, ""); - LLVMValueRef modded_val = LLVMBuildURem(g->builder, index_val, len_value, ""); - LLVMValueRef address_indices[] = { - modded_val, - }; - - LLVMValueRef ptr_value = gen_load_untyped(g, ptr_field_ptr, 0, false, ""); - LLVMValueRef address_slot = LLVMBuildInBoundsGEP(g->builder, ptr_value, address_indices, 1, ""); + LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->type_ref; LLVMValueRef zero = LLVMConstNull(g->builtin_types.entry_i32->type_ref); LLVMValueRef return_address_ptr = LLVMBuildCall(g->builder, get_return_address_fn_val(g), &zero, 1, ""); LLVMValueRef return_address = LLVMBuildPtrToInt(g->builder, return_address_ptr, usize_type_ref, ""); - LLVMValueRef address_value = LLVMBuildPtrToInt(g->builder, return_address, usize_type_ref, ""); - gen_store_untyped(g, address_value, address_slot, 0, false); - - // stack_trace.index += 1; - LLVMValueRef index_plus_one_val = LLVMBuildAdd(g->builder, index_val, LLVMConstInt(usize_type_ref, 1, false), ""); - gen_store_untyped(g, index_plus_one_val, index_field_ptr, 0, false); - - // return; + LLVMValueRef args[] = { err_ret_trace_ptr, return_address }; + LLVMBuildCall(g->builder, add_error_return_trace_addr_fn_val, args, 2, ""); LLVMBuildRetVoid(g->builder); LLVMPositionBuilderAtEnd(g->builder, prev_block); @@ -1641,7 +1816,6 @@ static LLVMValueRef ir_render_save_err_ret_addr(CodeGen *g, IrExecutable *execut }; LLVMValueRef call_instruction = ZigLLVMBuildCall(g->builder, return_err_fn, args, 1, get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAuto, ""); - LLVMSetTailCall(call_instruction, true); return call_instruction; } @@ -4204,6 +4378,22 @@ static LLVMValueRef ir_render_atomic_rmw(CodeGen *g, IrExecutable *executable, return LLVMBuildIntToPtr(g->builder, uncasted_result, operand_type->type_ref, ""); } +static LLVMValueRef ir_render_merge_err_ret_traces(CodeGen *g, IrExecutable *executable, + IrInstructionMergeErrRetTraces *instruction) +{ + assert(g->have_err_ret_tracing); + + LLVMValueRef coro_promise_ptr = ir_llvm_value(g, instruction->coro_promise_ptr); + TypeStructField *field = instruction->resolved_field; + LLVMValueRef ptr_field_ptr = LLVMBuildStructGEP(g->builder, coro_promise_ptr, field->gen_index, ""); + LLVMValueRef src_trace_ptr = LLVMBuildLoad(g->builder, ptr_field_ptr, ""); + LLVMValueRef dest_trace_ptr = get_cur_err_ret_trace_val(g, instruction->base.scope); + + LLVMValueRef args[] = { dest_trace_ptr, src_trace_ptr }; + LLVMBuildCall(g->builder, get_merge_err_ret_traces_fn_val(g), args, 2, ""); + return nullptr; +} + static void set_debug_location(CodeGen *g, IrInstruction *instruction) { AstNode *source_node = instruction->source_node; Scope *scope = instruction->scope; @@ -4421,6 +4611,8 @@ static LLVMValueRef ir_render_instruction(CodeGen *g, IrExecutable *executable, return ir_render_atomic_rmw(g, executable, (IrInstructionAtomicRmw *)instruction); case IrInstructionIdSaveErrRetAddr: return ir_render_save_err_ret_addr(g, executable, (IrInstructionSaveErrRetAddr *)instruction); + case IrInstructionIdMergeErrRetTraces: + return ir_render_merge_err_ret_traces(g, executable, (IrInstructionMergeErrRetTraces *)instruction); } zig_unreachable(); } @@ -5313,12 +5505,12 @@ static void do_code_gen(CodeGen *g) { bool is_async = fn_table_entry->type_entry->data.fn.fn_type_id.cc == CallingConventionAsync; bool have_err_ret_trace_stack = g->have_err_ret_tracing && fn_table_entry->calls_or_awaits_errorable_fn && (is_async || !have_err_ret_trace_arg); - if (have_err_ret_trace_stack) { - // TODO call graph analysis to find out what this number needs to be for every function - static const size_t stack_trace_ptr_count = 30; - + bool have_exactly_one_err_ret_value = !have_err_ret_trace_stack && g->have_err_ret_tracing && is_async && + type_can_fail(fn_table_entry->type_entry->data.fn.fn_type_id.return_type); + if (have_err_ret_trace_stack || have_exactly_one_err_ret_value) { TypeTableEntry *usize = g->builtin_types.entry_usize; - TypeTableEntry *array_type = get_array_type(g, usize, stack_trace_ptr_count); + uint32_t ret_addr_count = have_exactly_one_err_ret_value ? 1 : stack_trace_ptr_count; + TypeTableEntry *array_type = get_array_type(g, usize, ret_addr_count); LLVMValueRef err_ret_array_val = build_alloca(g, array_type, "error_return_trace_addresses", get_abi_alignment(g, array_type)); g->cur_err_ret_trace_val_stack = build_alloca(g, g->stack_trace_type, "error_return_trace", get_abi_alignment(g, g->stack_trace_type)); @@ -5341,7 +5533,7 @@ static void do_code_gen(CodeGen *g) { size_t len_field_index = slice_type->data.structure.fields[slice_len_index].gen_index; LLVMValueRef len_field_ptr = LLVMBuildStructGEP(g->builder, addresses_field_ptr, (unsigned)len_field_index, ""); - gen_store(g, LLVMConstInt(usize->type_ref, stack_trace_ptr_count, false), len_field_ptr, get_pointer_to_type(g, usize, false)); + gen_store(g, LLVMConstInt(usize->type_ref, ret_addr_count, false), len_field_ptr, get_pointer_to_type(g, usize, false)); } else { g->cur_err_ret_trace_val_stack = nullptr; } @@ -5943,6 +6135,8 @@ static void define_builtin_compile_vars(CodeGen *g) { os_path_join(g->cache_dir, buf_create_from_str(builtin_zig_basename), builtin_zig_path); Buf *contents = buf_alloc(); + // Modifications to this struct must be coordinated with code that does anything with + // g->stack_trace_type. There are hard-coded references to the field indexes. buf_append_str(contents, "pub const StackTrace = struct {\n" " index: usize,\n" diff --git a/src/ir.cpp b/src/ir.cpp index 4fe6769f78..c771adca44 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -725,6 +725,10 @@ static constexpr IrInstructionId ir_instruction_id(IrInstructionAddImplicitRetur return IrInstructionIdAddImplicitReturnType; } +static constexpr IrInstructionId ir_instruction_id(IrInstructionMergeErrRetTraces *) { + return IrInstructionIdMergeErrRetTraces; +} + template static T *ir_create_instruction(IrBuilder *irb, Scope *scope, AstNode *source_node) { T *special_instruction = allocate(1); @@ -972,6 +976,12 @@ static IrInstruction *ir_build_const_promise_init(IrBuilder *irb, Scope *scope, const_instruction->base.value.data.x_struct.fields[1].special = ConstValSpecialUndef; const_instruction->base.value.data.x_struct.fields[2].type = struct_type->data.structure.fields[2].type_entry; const_instruction->base.value.data.x_struct.fields[2].special = ConstValSpecialUndef; + if (irb->codegen->have_err_ret_tracing) { + assert(struct_type->data.structure.src_field_count == 4); + + const_instruction->base.value.data.x_struct.fields[3].type = struct_type->data.structure.fields[3].type_entry; + const_instruction->base.value.data.x_struct.fields[3].special = ConstValSpecialUndef; + } return &const_instruction->base; } @@ -2495,8 +2505,9 @@ static IrInstruction *ir_build_arg_type(IrBuilder *irb, Scope *scope, AstNode *s return &instruction->base; } -static IrInstruction *ir_build_error_return_trace(IrBuilder *irb, Scope *scope, AstNode *source_node) { +static IrInstruction *ir_build_error_return_trace(IrBuilder *irb, Scope *scope, AstNode *source_node, IrInstructionErrorReturnTrace::Nullable nullable) { IrInstructionErrorReturnTrace *instruction = ir_build_instruction(irb, scope, source_node); + instruction->nullable = nullable; return &instruction->base; } @@ -2717,6 +2728,18 @@ static IrInstruction *ir_build_add_implicit_return_type(IrBuilder *irb, Scope *s return &instruction->base; } +static IrInstruction *ir_build_merge_err_ret_traces(IrBuilder *irb, Scope *scope, AstNode *source_node, + IrInstruction *coro_promise_ptr, TypeStructField *resolved_field) +{ + IrInstructionMergeErrRetTraces *instruction = ir_build_instruction(irb, scope, source_node); + instruction->coro_promise_ptr = coro_promise_ptr; + instruction->resolved_field = resolved_field; + + ir_ref_instruction(coro_promise_ptr, irb->current_basic_block); + + return &instruction->base; +} + static void ir_count_defers(IrBuilder *irb, Scope *inner_scope, Scope *outer_scope, size_t *results) { results[ReturnKindUnconditional] = 0; results[ReturnKindError] = 0; @@ -2822,34 +2845,6 @@ static IrInstruction *ir_gen_async_return(IrBuilder *irb, Scope *scope, AstNode // the above blocks are rendered by ir_gen after the rest of codegen } -static bool exec_have_err_ret_trace(CodeGen *g, IrExecutable *exec) { - if (!g->have_err_ret_tracing) - return false; - FnTableEntry *fn_entry = exec_fn_entry(exec); - if (fn_entry == nullptr) - return false; - if (exec->is_inline) - return false; - return type_can_fail(fn_entry->type_entry->data.fn.fn_type_id.return_type); -} - -static void ir_gen_save_err_ret_addr(IrBuilder *irb, Scope *scope, AstNode *node) { - if (!exec_have_err_ret_trace(irb->codegen, irb->exec)) - return; - - bool is_async = exec_is_async(irb->exec); - - if (is_async) { - //IrInstruction *err_ret_addr_ptr = ir_build_load_ptr(irb, scope, node, irb->exec->coro_err_ret_addr_ptr); - //IrInstruction *return_address_ptr = ir_build_instr_addr(irb, scope, node); - //IrInstruction *return_address_usize = ir_build_ptr_to_int(irb, scope, node, return_address_ptr); - //ir_build_store_ptr(irb, scope, node, err_ret_addr_ptr, return_address_usize); - return; - } - - ir_build_save_err_ret_addr(irb, scope, node); -} - static IrInstruction *ir_gen_return(IrBuilder *irb, Scope *scope, AstNode *node, LVal lval) { assert(node->type == NodeTypeReturnExpr); @@ -2895,8 +2890,9 @@ static IrInstruction *ir_gen_return(IrBuilder *irb, Scope *scope, AstNode *node, IrInstruction *is_err = ir_build_test_err(irb, scope, node, return_value); + bool should_inline = ir_should_inline(irb->exec, scope); IrInstruction *is_comptime; - if (ir_should_inline(irb->exec, scope)) { + if (should_inline) { is_comptime = ir_build_const_bool(irb, scope, node, true); } else { is_comptime = ir_build_test_comptime(irb, scope, node, is_err); @@ -2909,7 +2905,9 @@ static IrInstruction *ir_gen_return(IrBuilder *irb, Scope *scope, AstNode *node, if (have_err_defers) { ir_gen_defers_for_block(irb, scope, outer_scope, true); } - ir_gen_save_err_ret_addr(irb, scope, node); + if (irb->codegen->have_err_ret_tracing && !should_inline) { + ir_build_save_err_ret_addr(irb, scope, node); + } ir_build_br(irb, scope, node, ret_stmt_block, is_comptime); ir_set_cursor_at_end_and_append_block(irb, ok_block); @@ -2938,7 +2936,8 @@ static IrInstruction *ir_gen_return(IrBuilder *irb, Scope *scope, AstNode *node, IrBasicBlock *return_block = ir_create_basic_block(irb, scope, "ErrRetReturn"); IrBasicBlock *continue_block = ir_create_basic_block(irb, scope, "ErrRetContinue"); IrInstruction *is_comptime; - if (ir_should_inline(irb->exec, scope)) { + bool should_inline = ir_should_inline(irb->exec, scope); + if (should_inline) { is_comptime = ir_build_const_bool(irb, scope, node, true); } else { is_comptime = ir_build_test_comptime(irb, scope, node, is_err_val); @@ -2948,7 +2947,9 @@ static IrInstruction *ir_gen_return(IrBuilder *irb, Scope *scope, AstNode *node, ir_set_cursor_at_end_and_append_block(irb, return_block); ir_gen_defers_for_block(irb, scope, outer_scope, true); IrInstruction *err_val = ir_build_unwrap_err_code(irb, scope, node, err_union_ptr); - ir_gen_save_err_ret_addr(irb, scope, node); + if (irb->codegen->have_err_ret_tracing && !should_inline) { + ir_build_save_err_ret_addr(irb, scope, node); + } ir_gen_async_return(irb, scope, node, err_val, false); ir_set_cursor_at_end_and_append_block(irb, continue_block); @@ -4242,7 +4243,7 @@ static IrInstruction *ir_gen_builtin_fn_call(IrBuilder *irb, Scope *scope, AstNo } case BuiltinFnIdErrorReturnTrace: { - return ir_build_error_return_trace(irb, scope, node); + return ir_build_error_return_trace(irb, scope, node, IrInstructionErrorReturnTrace::Null); } case BuiltinFnIdAtomicRmw: { @@ -6148,10 +6149,13 @@ static IrInstruction *ir_gen_await_expr(IrBuilder *irb, Scope *parent_scope, Ast IrInstruction *is_non_null = ir_build_test_nonnull(irb, parent_scope, node, maybe_await_handle); IrBasicBlock *yes_suspend_block = ir_create_basic_block(irb, parent_scope, "YesSuspend"); IrBasicBlock *no_suspend_block = ir_create_basic_block(irb, parent_scope, "NoSuspend"); - IrBasicBlock *merge_block = ir_create_basic_block(irb, parent_scope, "Merge"); + IrBasicBlock *merge_block = ir_create_basic_block(irb, parent_scope, "MergeSuspend"); ir_build_cond_br(irb, parent_scope, node, is_non_null, no_suspend_block, yes_suspend_block, const_bool_false); ir_set_cursor_at_end_and_append_block(irb, no_suspend_block); + if (irb->codegen->have_err_ret_tracing) { + ir_build_merge_err_ret_traces(irb, parent_scope, node, coro_promise_ptr, nullptr); + } Buf *result_field_name = buf_create_from_str(RESULT_FIELD_NAME); IrInstruction *promise_result_ptr = ir_build_field_ptr(irb, parent_scope, node, coro_promise_ptr, result_field_name); IrInstruction *no_suspend_result = ir_build_load_ptr(irb, parent_scope, node, promise_result_ptr); @@ -6460,13 +6464,19 @@ bool ir_gen(CodeGen *codegen, AstNode *node, Scope *scope, IrExecutable *ir_exec irb->exec->coro_handle = ir_build_coro_begin(irb, coro_scope, node, coro_id, coro_mem_ptr); Buf *awaiter_handle_field_name = buf_create_from_str(AWAITER_HANDLE_FIELD_NAME); - irb->exec->coro_awaiter_field_ptr = ir_build_field_ptr(irb, coro_scope, node, coro_promise_ptr, + irb->exec->coro_awaiter_field_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, awaiter_handle_field_name); Buf *result_field_name = buf_create_from_str(RESULT_FIELD_NAME); - irb->exec->coro_result_field_ptr = ir_build_field_ptr(irb, coro_scope, node, coro_promise_ptr, result_field_name); + irb->exec->coro_result_field_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, result_field_name); result_ptr_field_name = buf_create_from_str(RESULT_PTR_FIELD_NAME); - irb->exec->coro_result_ptr_field_ptr = ir_build_field_ptr(irb, coro_scope, node, coro_promise_ptr, result_ptr_field_name); - ir_build_store_ptr(irb, coro_scope, node, irb->exec->coro_result_ptr_field_ptr, irb->exec->coro_result_field_ptr); + irb->exec->coro_result_ptr_field_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, result_ptr_field_name); + ir_build_store_ptr(irb, scope, node, irb->exec->coro_result_ptr_field_ptr, irb->exec->coro_result_field_ptr); + if (irb->codegen->have_err_ret_tracing) { + IrInstruction *err_ret_trace_ptr = ir_build_error_return_trace(irb, scope, node, IrInstructionErrorReturnTrace::NonNull); + Buf *err_ret_trace_ptr_field_name = buf_create_from_str(ERR_RET_TRACE_PTR_FIELD_NAME); + IrInstruction *coro_err_ret_trace_ptr_field_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, err_ret_trace_ptr_field_name); + ir_build_store_ptr(irb, scope, node, coro_err_ret_trace_ptr_field_ptr, err_ret_trace_ptr); + } irb->exec->coro_early_final = ir_create_basic_block(irb, scope, "CoroEarlyFinal"); @@ -11579,18 +11589,25 @@ static bool exec_has_err_ret_trace(CodeGen *g, IrExecutable *exec) { static TypeTableEntry *ir_analyze_instruction_error_return_trace(IrAnalyze *ira, IrInstructionErrorReturnTrace *instruction) { - TypeTableEntry *ptr_to_stack_trace_type = get_ptr_to_stack_trace_type(ira->codegen); - TypeTableEntry *nullable_type = get_maybe_type(ira->codegen, ptr_to_stack_trace_type); - if (!exec_has_err_ret_trace(ira->codegen, ira->new_irb.exec)) { - ConstExprValue *out_val = ir_build_const_from(ira, &instruction->base); - out_val->data.x_maybe = nullptr; + if (instruction->nullable == IrInstructionErrorReturnTrace::Null) { + TypeTableEntry *ptr_to_stack_trace_type = get_ptr_to_stack_trace_type(ira->codegen); + TypeTableEntry *nullable_type = get_maybe_type(ira->codegen, ptr_to_stack_trace_type); + if (!exec_has_err_ret_trace(ira->codegen, ira->new_irb.exec)) { + ConstExprValue *out_val = ir_build_const_from(ira, &instruction->base); + out_val->data.x_maybe = nullptr; + return nullable_type; + } + IrInstruction *new_instruction = ir_build_error_return_trace(&ira->new_irb, instruction->base.scope, + instruction->base.source_node, instruction->nullable); + ir_link_new_instruction(new_instruction, &instruction->base); return nullable_type; + } else { + assert(ira->codegen->have_err_ret_tracing); + IrInstruction *new_instruction = ir_build_error_return_trace(&ira->new_irb, instruction->base.scope, + instruction->base.source_node, instruction->nullable); + ir_link_new_instruction(new_instruction, &instruction->base); + return get_ptr_to_stack_trace_type(ira->codegen); } - - IrInstruction *new_instruction = ir_build_error_return_trace(&ira->new_irb, instruction->base.scope, - instruction->base.source_node); - ir_link_new_instruction(new_instruction, &instruction->base); - return nullable_type; } static TypeTableEntry *ir_analyze_instruction_error_union(IrAnalyze *ira, @@ -17904,6 +17921,34 @@ static TypeTableEntry *ir_analyze_instruction_await_bookkeeping(IrAnalyze *ira, return out_val->type; } +static TypeTableEntry *ir_analyze_instruction_merge_err_ret_traces(IrAnalyze *ira, + IrInstructionMergeErrRetTraces *instruction) +{ + IrInstruction *coro_promise_ptr = instruction->coro_promise_ptr->other; + if (type_is_invalid(coro_promise_ptr->value.type)) + return ira->codegen->builtin_types.entry_invalid; + + assert(coro_promise_ptr->value.type->id == TypeTableEntryIdPointer); + TypeTableEntry *promise_frame_type = coro_promise_ptr->value.type->data.pointer.child_type; + assert(promise_frame_type->id == TypeTableEntryIdStruct); + TypeTableEntry *promise_result_type = promise_frame_type->data.structure.fields[1].type_entry; + + if (!type_can_fail(promise_result_type)) { + ConstExprValue *out_val = ir_build_const_from(ira, &instruction->base); + out_val->type = ira->codegen->builtin_types.entry_void; + return out_val->type; + } + + TypeStructField *field = find_struct_type_field(promise_frame_type, buf_create_from_str(ERR_RET_TRACE_PTR_FIELD_NAME)); + assert(field != nullptr); + + IrInstruction *result = ir_build_merge_err_ret_traces(&ira->new_irb, instruction->base.scope, + instruction->base.source_node, coro_promise_ptr, field); + ir_link_new_instruction(result, &instruction->base); + result->value.type = ira->codegen->builtin_types.entry_void; + return result->value.type; +} + static TypeTableEntry *ir_analyze_instruction_save_err_ret_addr(IrAnalyze *ira, IrInstructionSaveErrRetAddr *instruction) { IrInstruction *result = ir_build_save_err_ret_addr(&ira->new_irb, instruction->base.scope, instruction->base.source_node); @@ -18155,6 +18200,8 @@ static TypeTableEntry *ir_analyze_instruction_nocast(IrAnalyze *ira, IrInstructi return ir_analyze_instruction_save_err_ret_addr(ira, (IrInstructionSaveErrRetAddr *)instruction); case IrInstructionIdAddImplicitReturnType: return ir_analyze_instruction_add_implicit_return_type(ira, (IrInstructionAddImplicitReturnType *)instruction); + case IrInstructionIdMergeErrRetTraces: + return ir_analyze_instruction_merge_err_ret_traces(ira, (IrInstructionMergeErrRetTraces *)instruction); } zig_unreachable(); } @@ -18282,6 +18329,7 @@ bool ir_has_side_effects(IrInstruction *instruction) { case IrInstructionIdAwaitBookkeeping: case IrInstructionIdSaveErrRetAddr: case IrInstructionIdAddImplicitReturnType: + case IrInstructionIdMergeErrRetTraces: return true; case IrInstructionIdPhi: diff --git a/src/ir_print.cpp b/src/ir_print.cpp index b14d49a4ca..432d287bb8 100644 --- a/src/ir_print.cpp +++ b/src/ir_print.cpp @@ -1024,7 +1024,16 @@ static void ir_print_export(IrPrint *irp, IrInstructionExport *instruction) { } static void ir_print_error_return_trace(IrPrint *irp, IrInstructionErrorReturnTrace *instruction) { - fprintf(irp->f, "@errorReturnTrace()"); + fprintf(irp->f, "@errorReturnTrace("); + switch (instruction->nullable) { + case IrInstructionErrorReturnTrace::Null: + fprintf(irp->f, "Null"); + break; + case IrInstructionErrorReturnTrace::NonNull: + fprintf(irp->f, "NonNull"); + break; + } + fprintf(irp->f, ")"); } static void ir_print_error_union(IrPrint *irp, IrInstructionErrorUnion *instruction) { @@ -1179,6 +1188,16 @@ static void ir_print_add_implicit_return_type(IrPrint *irp, IrInstructionAddImpl fprintf(irp->f, ")"); } +static void ir_print_merge_err_ret_traces(IrPrint *irp, IrInstructionMergeErrRetTraces *instruction) { + fprintf(irp->f, "@mergeErrRetTraces("); + ir_print_other_instruction(irp, instruction->coro_promise_ptr); + fprintf(irp->f, ","); + if (instruction->resolved_field != nullptr) { + fprintf(irp->f, "field '%s'", buf_ptr(instruction->resolved_field->name)); + } + fprintf(irp->f, ")"); +} + static void ir_print_instruction(IrPrint *irp, IrInstruction *instruction) { ir_print_prefix(irp, instruction); switch (instruction->id) { @@ -1559,6 +1578,9 @@ static void ir_print_instruction(IrPrint *irp, IrInstruction *instruction) { case IrInstructionIdAddImplicitReturnType: ir_print_add_implicit_return_type(irp, (IrInstructionAddImplicitReturnType *)instruction); break; + case IrInstructionIdMergeErrRetTraces: + ir_print_merge_err_ret_traces(irp, (IrInstructionMergeErrRetTraces *)instruction); + break; } fprintf(irp->f, "\n"); } From e4083b7391fd829e3060d24e10d13c8d52d889b0 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 29 Mar 2018 01:24:07 -0400 Subject: [PATCH 35/87] codegen: fix not putting llvm allocas together --- src/codegen.cpp | 53 ++++++++++++++++++++++++++++--------------------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index fa8e069e21..88331e3027 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -5502,38 +5502,19 @@ static void do_code_gen(CodeGen *g) { g->cur_err_ret_trace_val_arg = nullptr; } + // error return tracing setup bool is_async = fn_table_entry->type_entry->data.fn.fn_type_id.cc == CallingConventionAsync; bool have_err_ret_trace_stack = g->have_err_ret_tracing && fn_table_entry->calls_or_awaits_errorable_fn && (is_async || !have_err_ret_trace_arg); bool have_exactly_one_err_ret_value = !have_err_ret_trace_stack && g->have_err_ret_tracing && is_async && type_can_fail(fn_table_entry->type_entry->data.fn.fn_type_id.return_type); + LLVMValueRef err_ret_array_val = nullptr; if (have_err_ret_trace_stack || have_exactly_one_err_ret_value) { - TypeTableEntry *usize = g->builtin_types.entry_usize; uint32_t ret_addr_count = have_exactly_one_err_ret_value ? 1 : stack_trace_ptr_count; - TypeTableEntry *array_type = get_array_type(g, usize, ret_addr_count); - LLVMValueRef err_ret_array_val = build_alloca(g, array_type, "error_return_trace_addresses", + TypeTableEntry *array_type = get_array_type(g, g->builtin_types.entry_usize, ret_addr_count); + err_ret_array_val = build_alloca(g, array_type, "error_return_trace_addresses", get_abi_alignment(g, array_type)); g->cur_err_ret_trace_val_stack = build_alloca(g, g->stack_trace_type, "error_return_trace", get_abi_alignment(g, g->stack_trace_type)); - size_t index_field_index = g->stack_trace_type->data.structure.fields[0].gen_index; - LLVMValueRef index_field_ptr = LLVMBuildStructGEP(g->builder, g->cur_err_ret_trace_val_stack, (unsigned)index_field_index, ""); - gen_store_untyped(g, LLVMConstNull(usize->type_ref), index_field_ptr, 0, false); - - size_t addresses_field_index = g->stack_trace_type->data.structure.fields[1].gen_index; - LLVMValueRef addresses_field_ptr = LLVMBuildStructGEP(g->builder, g->cur_err_ret_trace_val_stack, (unsigned)addresses_field_index, ""); - - TypeTableEntry *slice_type = g->stack_trace_type->data.structure.fields[1].type_entry; - size_t ptr_field_index = slice_type->data.structure.fields[slice_ptr_index].gen_index; - LLVMValueRef ptr_field_ptr = LLVMBuildStructGEP(g->builder, addresses_field_ptr, (unsigned)ptr_field_index, ""); - LLVMValueRef zero = LLVMConstNull(usize->type_ref); - LLVMValueRef indices[] = {zero, zero}; - LLVMValueRef err_ret_array_val_elem0_ptr = LLVMBuildInBoundsGEP(g->builder, err_ret_array_val, - indices, 2, ""); - gen_store(g, err_ret_array_val_elem0_ptr, ptr_field_ptr, - get_pointer_to_type(g, get_pointer_to_type(g, usize, false), false)); - - size_t len_field_index = slice_type->data.structure.fields[slice_len_index].gen_index; - LLVMValueRef len_field_ptr = LLVMBuildStructGEP(g->builder, addresses_field_ptr, (unsigned)len_field_index, ""); - gen_store(g, LLVMConstInt(usize->type_ref, ret_addr_count, false), len_field_ptr, get_pointer_to_type(g, usize, false)); } else { g->cur_err_ret_trace_val_stack = nullptr; } @@ -5628,6 +5609,32 @@ static void do_code_gen(CodeGen *g) { } } + // finishing error return trace setup. we have to do this after all the allocas. + if (have_err_ret_trace_stack || have_exactly_one_err_ret_value) { + uint32_t ret_addr_count = have_exactly_one_err_ret_value ? 1 : stack_trace_ptr_count; + TypeTableEntry *usize = g->builtin_types.entry_usize; + size_t index_field_index = g->stack_trace_type->data.structure.fields[0].gen_index; + LLVMValueRef index_field_ptr = LLVMBuildStructGEP(g->builder, g->cur_err_ret_trace_val_stack, (unsigned)index_field_index, ""); + gen_store_untyped(g, LLVMConstNull(usize->type_ref), index_field_ptr, 0, false); + + size_t addresses_field_index = g->stack_trace_type->data.structure.fields[1].gen_index; + LLVMValueRef addresses_field_ptr = LLVMBuildStructGEP(g->builder, g->cur_err_ret_trace_val_stack, (unsigned)addresses_field_index, ""); + + TypeTableEntry *slice_type = g->stack_trace_type->data.structure.fields[1].type_entry; + size_t ptr_field_index = slice_type->data.structure.fields[slice_ptr_index].gen_index; + LLVMValueRef ptr_field_ptr = LLVMBuildStructGEP(g->builder, addresses_field_ptr, (unsigned)ptr_field_index, ""); + LLVMValueRef zero = LLVMConstNull(usize->type_ref); + LLVMValueRef indices[] = {zero, zero}; + LLVMValueRef err_ret_array_val_elem0_ptr = LLVMBuildInBoundsGEP(g->builder, err_ret_array_val, + indices, 2, ""); + TypeTableEntry *ptr_ptr_usize_type = get_pointer_to_type(g, get_pointer_to_type(g, usize, false), false); + gen_store(g, err_ret_array_val_elem0_ptr, ptr_field_ptr, ptr_ptr_usize_type); + + size_t len_field_index = slice_type->data.structure.fields[slice_len_index].gen_index; + LLVMValueRef len_field_ptr = LLVMBuildStructGEP(g->builder, addresses_field_ptr, (unsigned)len_field_index, ""); + gen_store(g, LLVMConstInt(usize->type_ref, ret_addr_count, false), len_field_ptr, get_pointer_to_type(g, usize, false)); + } + FnTypeId *fn_type_id = &fn_table_entry->type_entry->data.fn.fn_type_id; // create debug variable declarations for parameters From ada441157f4a388950946e7f4db65c273f23c063 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 8 Apr 2018 16:04:21 -0400 Subject: [PATCH 36/87] put the error return addresses in the coro frame --- src/all_types.hpp | 12 +++++- src/analyze.cpp | 7 +++- src/codegen.cpp | 33 +++++++-------- src/ir.cpp | 101 +++++++++++++++++++++++++++++----------------- src/ir_print.cpp | 13 ++++-- 5 files changed, 105 insertions(+), 61 deletions(-) diff --git a/src/all_types.hpp b/src/all_types.hpp index 6893f60fb3..52b8ede82c 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -2057,6 +2057,7 @@ enum IrInstructionId { IrInstructionIdSaveErrRetAddr, IrInstructionIdAddImplicitReturnType, IrInstructionIdMergeErrRetTraces, + IrInstructionIdMarkErrRetTracePtr, }; struct IrInstruction { @@ -3036,7 +3037,13 @@ struct IrInstructionMergeErrRetTraces { IrInstruction base; IrInstruction *coro_promise_ptr; - TypeStructField *resolved_field; + IrInstruction *err_ret_trace_ptr; +}; + +struct IrInstructionMarkErrRetTracePtr { + IrInstruction base; + + IrInstruction *err_ret_trace_ptr; }; static const size_t slice_ptr_index = 0; @@ -3056,7 +3063,8 @@ static const size_t stack_trace_ptr_count = 30; #define AWAITER_HANDLE_FIELD_NAME "awaiter_handle" #define RESULT_FIELD_NAME "result" #define RESULT_PTR_FIELD_NAME "result_ptr" -#define ERR_RET_TRACE_PTR_FIELD_NAME "err_ret_trace_ptr" +#define RETURN_ADDRESSES_FIELD_NAME "return_addresses" +#define ERR_RET_TRACE_FIELD_NAME "err_ret_trace" enum FloatMode { diff --git a/src/analyze.cpp b/src/analyze.cpp index 16788c5e6c..ae2a1a1b1d 100644 --- a/src/analyze.cpp +++ b/src/analyze.cpp @@ -474,7 +474,8 @@ TypeTableEntry *get_promise_frame_type(CodeGen *g, TypeTableEntry *return_type) field_names.append(RESULT_FIELD_NAME); field_names.append(RESULT_PTR_FIELD_NAME); if (g->have_err_ret_tracing) { - field_names.append(ERR_RET_TRACE_PTR_FIELD_NAME); + field_names.append(ERR_RET_TRACE_FIELD_NAME); + field_names.append(RETURN_ADDRESSES_FIELD_NAME); } ZigList field_types = {}; @@ -482,7 +483,9 @@ TypeTableEntry *get_promise_frame_type(CodeGen *g, TypeTableEntry *return_type) field_types.append(return_type); field_types.append(result_ptr_type); if (g->have_err_ret_tracing) { - field_types.append(get_ptr_to_stack_trace_type(g)); + get_ptr_to_stack_trace_type(g); + field_types.append(g->stack_trace_type); + field_types.append(get_array_type(g, g->builtin_types.entry_usize, stack_trace_ptr_count)); } assert(field_names.length == field_types.length); diff --git a/src/codegen.cpp b/src/codegen.cpp index 88331e3027..914e9c7096 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -4383,10 +4383,7 @@ static LLVMValueRef ir_render_merge_err_ret_traces(CodeGen *g, IrExecutable *exe { assert(g->have_err_ret_tracing); - LLVMValueRef coro_promise_ptr = ir_llvm_value(g, instruction->coro_promise_ptr); - TypeStructField *field = instruction->resolved_field; - LLVMValueRef ptr_field_ptr = LLVMBuildStructGEP(g->builder, coro_promise_ptr, field->gen_index, ""); - LLVMValueRef src_trace_ptr = LLVMBuildLoad(g->builder, ptr_field_ptr, ""); + LLVMValueRef src_trace_ptr = ir_llvm_value(g, instruction->err_ret_trace_ptr); LLVMValueRef dest_trace_ptr = get_cur_err_ret_trace_val(g, instruction->base.scope); LLVMValueRef args[] = { dest_trace_ptr, src_trace_ptr }; @@ -4394,6 +4391,14 @@ static LLVMValueRef ir_render_merge_err_ret_traces(CodeGen *g, IrExecutable *exe return nullptr; } +static LLVMValueRef ir_render_mark_err_ret_trace_ptr(CodeGen *g, IrExecutable *executable, + IrInstructionMarkErrRetTracePtr *instruction) +{ + assert(g->have_err_ret_tracing); + g->cur_err_ret_trace_val_stack = ir_llvm_value(g, instruction->err_ret_trace_ptr); + return nullptr; +} + static void set_debug_location(CodeGen *g, IrInstruction *instruction) { AstNode *source_node = instruction->source_node; Scope *scope = instruction->scope; @@ -4613,6 +4618,8 @@ static LLVMValueRef ir_render_instruction(CodeGen *g, IrExecutable *executable, return ir_render_save_err_ret_addr(g, executable, (IrInstructionSaveErrRetAddr *)instruction); case IrInstructionIdMergeErrRetTraces: return ir_render_merge_err_ret_traces(g, executable, (IrInstructionMergeErrRetTraces *)instruction); + case IrInstructionIdMarkErrRetTracePtr: + return ir_render_mark_err_ret_trace_ptr(g, executable, (IrInstructionMarkErrRetTracePtr *)instruction); } zig_unreachable(); } @@ -5504,16 +5511,11 @@ static void do_code_gen(CodeGen *g) { // error return tracing setup bool is_async = fn_table_entry->type_entry->data.fn.fn_type_id.cc == CallingConventionAsync; - bool have_err_ret_trace_stack = g->have_err_ret_tracing && fn_table_entry->calls_or_awaits_errorable_fn && - (is_async || !have_err_ret_trace_arg); - bool have_exactly_one_err_ret_value = !have_err_ret_trace_stack && g->have_err_ret_tracing && is_async && - type_can_fail(fn_table_entry->type_entry->data.fn.fn_type_id.return_type); + bool have_err_ret_trace_stack = g->have_err_ret_tracing && fn_table_entry->calls_or_awaits_errorable_fn && !is_async && !have_err_ret_trace_arg; LLVMValueRef err_ret_array_val = nullptr; - if (have_err_ret_trace_stack || have_exactly_one_err_ret_value) { - uint32_t ret_addr_count = have_exactly_one_err_ret_value ? 1 : stack_trace_ptr_count; - TypeTableEntry *array_type = get_array_type(g, g->builtin_types.entry_usize, ret_addr_count); - err_ret_array_val = build_alloca(g, array_type, "error_return_trace_addresses", - get_abi_alignment(g, array_type)); + if (have_err_ret_trace_stack) { + TypeTableEntry *array_type = get_array_type(g, g->builtin_types.entry_usize, stack_trace_ptr_count); + err_ret_array_val = build_alloca(g, array_type, "error_return_trace_addresses", get_abi_alignment(g, array_type)); g->cur_err_ret_trace_val_stack = build_alloca(g, g->stack_trace_type, "error_return_trace", get_abi_alignment(g, g->stack_trace_type)); } else { g->cur_err_ret_trace_val_stack = nullptr; @@ -5610,8 +5612,7 @@ static void do_code_gen(CodeGen *g) { } // finishing error return trace setup. we have to do this after all the allocas. - if (have_err_ret_trace_stack || have_exactly_one_err_ret_value) { - uint32_t ret_addr_count = have_exactly_one_err_ret_value ? 1 : stack_trace_ptr_count; + if (have_err_ret_trace_stack) { TypeTableEntry *usize = g->builtin_types.entry_usize; size_t index_field_index = g->stack_trace_type->data.structure.fields[0].gen_index; LLVMValueRef index_field_ptr = LLVMBuildStructGEP(g->builder, g->cur_err_ret_trace_val_stack, (unsigned)index_field_index, ""); @@ -5632,7 +5633,7 @@ static void do_code_gen(CodeGen *g) { size_t len_field_index = slice_type->data.structure.fields[slice_len_index].gen_index; LLVMValueRef len_field_ptr = LLVMBuildStructGEP(g->builder, addresses_field_ptr, (unsigned)len_field_index, ""); - gen_store(g, LLVMConstInt(usize->type_ref, ret_addr_count, false), len_field_ptr, get_pointer_to_type(g, usize, false)); + gen_store(g, LLVMConstInt(usize->type_ref, stack_trace_ptr_count, false), len_field_ptr, get_pointer_to_type(g, usize, false)); } FnTypeId *fn_type_id = &fn_table_entry->type_entry->data.fn.fn_type_id; diff --git a/src/ir.cpp b/src/ir.cpp index c771adca44..3f8744ecb1 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -729,6 +729,10 @@ static constexpr IrInstructionId ir_instruction_id(IrInstructionMergeErrRetTrace return IrInstructionIdMergeErrRetTraces; } +static constexpr IrInstructionId ir_instruction_id(IrInstructionMarkErrRetTracePtr *) { + return IrInstructionIdMarkErrRetTracePtr; +} + template static T *ir_create_instruction(IrBuilder *irb, Scope *scope, AstNode *source_node) { T *special_instruction = allocate(1); @@ -960,31 +964,6 @@ static IrInstruction *ir_build_const_c_str_lit(IrBuilder *irb, Scope *scope, Ast return &const_instruction->base; } -static IrInstruction *ir_build_const_promise_init(IrBuilder *irb, Scope *scope, AstNode *source_node, - TypeTableEntry *return_type) -{ - TypeTableEntry *struct_type = get_promise_frame_type(irb->codegen, return_type); - - IrInstructionConst *const_instruction = ir_build_instruction(irb, scope, source_node); - const_instruction->base.value.type = struct_type; - const_instruction->base.value.special = ConstValSpecialStatic; - const_instruction->base.value.data.x_struct.fields = allocate(struct_type->data.structure.src_field_count); - const_instruction->base.value.data.x_struct.fields[0].type = struct_type->data.structure.fields[0].type_entry; - const_instruction->base.value.data.x_struct.fields[0].special = ConstValSpecialStatic; - const_instruction->base.value.data.x_struct.fields[0].data.x_maybe = nullptr; - const_instruction->base.value.data.x_struct.fields[1].type = return_type; - const_instruction->base.value.data.x_struct.fields[1].special = ConstValSpecialUndef; - const_instruction->base.value.data.x_struct.fields[2].type = struct_type->data.structure.fields[2].type_entry; - const_instruction->base.value.data.x_struct.fields[2].special = ConstValSpecialUndef; - if (irb->codegen->have_err_ret_tracing) { - assert(struct_type->data.structure.src_field_count == 4); - - const_instruction->base.value.data.x_struct.fields[3].type = struct_type->data.structure.fields[3].type_entry; - const_instruction->base.value.data.x_struct.fields[3].special = ConstValSpecialUndef; - } - return &const_instruction->base; -} - static IrInstruction *ir_build_bin_op(IrBuilder *irb, Scope *scope, AstNode *source_node, IrBinOp op_id, IrInstruction *op1, IrInstruction *op2, bool safety_check_on) { @@ -2729,13 +2708,23 @@ static IrInstruction *ir_build_add_implicit_return_type(IrBuilder *irb, Scope *s } static IrInstruction *ir_build_merge_err_ret_traces(IrBuilder *irb, Scope *scope, AstNode *source_node, - IrInstruction *coro_promise_ptr, TypeStructField *resolved_field) + IrInstruction *coro_promise_ptr, IrInstruction *err_ret_trace_ptr) { IrInstructionMergeErrRetTraces *instruction = ir_build_instruction(irb, scope, source_node); instruction->coro_promise_ptr = coro_promise_ptr; - instruction->resolved_field = resolved_field; + instruction->err_ret_trace_ptr = err_ret_trace_ptr; ir_ref_instruction(coro_promise_ptr, irb->current_basic_block); + ir_ref_instruction(err_ret_trace_ptr, irb->current_basic_block); + + return &instruction->base; +} + +static IrInstruction *ir_build_mark_err_ret_trace_ptr(IrBuilder *irb, Scope *scope, AstNode *source_node, IrInstruction *err_ret_trace_ptr) { + IrInstructionMarkErrRetTracePtr *instruction = ir_build_instruction(irb, scope, source_node); + instruction->err_ret_trace_ptr = err_ret_trace_ptr; + + ir_ref_instruction(err_ret_trace_ptr, irb->current_basic_block); return &instruction->base; } @@ -6154,7 +6143,9 @@ static IrInstruction *ir_gen_await_expr(IrBuilder *irb, Scope *parent_scope, Ast ir_set_cursor_at_end_and_append_block(irb, no_suspend_block); if (irb->codegen->have_err_ret_tracing) { - ir_build_merge_err_ret_traces(irb, parent_scope, node, coro_promise_ptr, nullptr); + Buf *err_ret_trace_field_name = buf_create_from_str(ERR_RET_TRACE_FIELD_NAME); + IrInstruction *err_ret_trace_ptr = ir_build_field_ptr(irb, parent_scope, node, coro_promise_ptr, err_ret_trace_field_name); + ir_build_merge_err_ret_traces(irb, parent_scope, node, coro_promise_ptr, err_ret_trace_ptr); } Buf *result_field_name = buf_create_from_str(RESULT_FIELD_NAME); IrInstruction *promise_result_ptr = ir_build_field_ptr(irb, parent_scope, node, coro_promise_ptr, result_field_name); @@ -6421,8 +6412,11 @@ bool ir_gen(CodeGen *codegen, AstNode *node, Scope *scope, IrExecutable *ir_exec VariableTableEntry *promise_var = ir_create_var(irb, node, coro_scope, nullptr, false, false, true, const_bool_false); return_type = fn_entry->type_entry->data.fn.fn_type_id.return_type; - IrInstruction *promise_init = ir_build_const_promise_init(irb, coro_scope, node, return_type); - ir_build_var_decl(irb, coro_scope, node, promise_var, nullptr, nullptr, promise_init); + IrInstruction *undef = ir_build_const_undefined(irb, coro_scope, node); + TypeTableEntry *coro_frame_type = get_promise_frame_type(irb->codegen, return_type); + IrInstruction *coro_frame_type_value = ir_build_const_type(irb, coro_scope, node, coro_frame_type); + // TODO mark this var decl as "no safety" e.g. disable initializing the undef value to 0xaa + ir_build_var_decl(irb, coro_scope, node, promise_var, coro_frame_type_value, nullptr, undef); IrInstruction *coro_promise_ptr = ir_build_var_ptr(irb, coro_scope, node, promise_var, false, false); VariableTableEntry *await_handle_var = ir_create_var(irb, node, coro_scope, nullptr, false, false, true, const_bool_false); @@ -6456,7 +6450,6 @@ bool ir_gen(CodeGen *codegen, AstNode *node, Scope *scope, IrExecutable *ir_exec ir_set_cursor_at_end_and_append_block(irb, alloc_err_block); // we can return undefined here, because the caller passes a pointer to the error struct field // in the error union result, and we populate it in case of allocation failure. - IrInstruction *undef = ir_build_const_undefined(irb, coro_scope, node); ir_build_return(irb, coro_scope, node, undef); ir_set_cursor_at_end_and_append_block(irb, alloc_ok_block); @@ -6466,16 +6459,32 @@ bool ir_gen(CodeGen *codegen, AstNode *node, Scope *scope, IrExecutable *ir_exec Buf *awaiter_handle_field_name = buf_create_from_str(AWAITER_HANDLE_FIELD_NAME); irb->exec->coro_awaiter_field_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, awaiter_handle_field_name); + ir_build_store_ptr(irb, scope, node, irb->exec->coro_awaiter_field_ptr, null_value); Buf *result_field_name = buf_create_from_str(RESULT_FIELD_NAME); irb->exec->coro_result_field_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, result_field_name); result_ptr_field_name = buf_create_from_str(RESULT_PTR_FIELD_NAME); irb->exec->coro_result_ptr_field_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, result_ptr_field_name); ir_build_store_ptr(irb, scope, node, irb->exec->coro_result_ptr_field_ptr, irb->exec->coro_result_field_ptr); if (irb->codegen->have_err_ret_tracing) { - IrInstruction *err_ret_trace_ptr = ir_build_error_return_trace(irb, scope, node, IrInstructionErrorReturnTrace::NonNull); - Buf *err_ret_trace_ptr_field_name = buf_create_from_str(ERR_RET_TRACE_PTR_FIELD_NAME); - IrInstruction *coro_err_ret_trace_ptr_field_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, err_ret_trace_ptr_field_name); - ir_build_store_ptr(irb, scope, node, coro_err_ret_trace_ptr_field_ptr, err_ret_trace_ptr); + // initialize the error return trace + Buf *return_addresses_field_name = buf_create_from_str(RETURN_ADDRESSES_FIELD_NAME); + IrInstruction *return_addresses_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, return_addresses_field_name); + + Buf *err_ret_trace_field_name = buf_create_from_str(ERR_RET_TRACE_FIELD_NAME); + IrInstruction *err_ret_trace_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, err_ret_trace_field_name); + ir_build_mark_err_ret_trace_ptr(irb, scope, node, err_ret_trace_ptr); + + // coordinate with builtin.zig + Buf *index_name = buf_create_from_str("index"); + IrInstruction *index_ptr = ir_build_field_ptr(irb, scope, node, err_ret_trace_ptr, index_name); + IrInstruction *zero = ir_build_const_usize(irb, scope, node, 0); + ir_build_store_ptr(irb, scope, node, index_ptr, zero); + + Buf *instruction_addresses_name = buf_create_from_str("instruction_addresses"); + IrInstruction *addrs_slice_ptr = ir_build_field_ptr(irb, scope, node, err_ret_trace_ptr, instruction_addresses_name); + + IrInstruction *slice_value = ir_build_slice(irb, scope, node, return_addresses_ptr, zero, nullptr, false); + ir_build_store_ptr(irb, scope, node, addrs_slice_ptr, slice_value); } @@ -17939,11 +17948,12 @@ static TypeTableEntry *ir_analyze_instruction_merge_err_ret_traces(IrAnalyze *ir return out_val->type; } - TypeStructField *field = find_struct_type_field(promise_frame_type, buf_create_from_str(ERR_RET_TRACE_PTR_FIELD_NAME)); - assert(field != nullptr); + IrInstruction *err_ret_trace_ptr = instruction->err_ret_trace_ptr->other; + if (type_is_invalid(err_ret_trace_ptr->value.type)) + return ira->codegen->builtin_types.entry_invalid; IrInstruction *result = ir_build_merge_err_ret_traces(&ira->new_irb, instruction->base.scope, - instruction->base.source_node, coro_promise_ptr, field); + instruction->base.source_node, coro_promise_ptr, err_ret_trace_ptr); ir_link_new_instruction(result, &instruction->base); result->value.type = ira->codegen->builtin_types.entry_void; return result->value.type; @@ -17957,6 +17967,18 @@ static TypeTableEntry *ir_analyze_instruction_save_err_ret_addr(IrAnalyze *ira, return result->value.type; } +static TypeTableEntry *ir_analyze_instruction_mark_err_ret_trace_ptr(IrAnalyze *ira, IrInstructionMarkErrRetTracePtr *instruction) { + IrInstruction *err_ret_trace_ptr = instruction->err_ret_trace_ptr->other; + if (type_is_invalid(err_ret_trace_ptr->value.type)) + return ira->codegen->builtin_types.entry_invalid; + + IrInstruction *result = ir_build_mark_err_ret_trace_ptr(&ira->new_irb, instruction->base.scope, + instruction->base.source_node, err_ret_trace_ptr); + ir_link_new_instruction(result, &instruction->base); + result->value.type = ira->codegen->builtin_types.entry_void; + return result->value.type; +} + static TypeTableEntry *ir_analyze_instruction_nocast(IrAnalyze *ira, IrInstruction *instruction) { switch (instruction->id) { case IrInstructionIdInvalid: @@ -18202,6 +18224,8 @@ static TypeTableEntry *ir_analyze_instruction_nocast(IrAnalyze *ira, IrInstructi return ir_analyze_instruction_add_implicit_return_type(ira, (IrInstructionAddImplicitReturnType *)instruction); case IrInstructionIdMergeErrRetTraces: return ir_analyze_instruction_merge_err_ret_traces(ira, (IrInstructionMergeErrRetTraces *)instruction); + case IrInstructionIdMarkErrRetTracePtr: + return ir_analyze_instruction_mark_err_ret_trace_ptr(ira, (IrInstructionMarkErrRetTracePtr *)instruction); } zig_unreachable(); } @@ -18330,6 +18354,7 @@ bool ir_has_side_effects(IrInstruction *instruction) { case IrInstructionIdSaveErrRetAddr: case IrInstructionIdAddImplicitReturnType: case IrInstructionIdMergeErrRetTraces: + case IrInstructionIdMarkErrRetTracePtr: return true; case IrInstructionIdPhi: diff --git a/src/ir_print.cpp b/src/ir_print.cpp index 432d287bb8..20dfb10b81 100644 --- a/src/ir_print.cpp +++ b/src/ir_print.cpp @@ -1192,9 +1192,13 @@ static void ir_print_merge_err_ret_traces(IrPrint *irp, IrInstructionMergeErrRet fprintf(irp->f, "@mergeErrRetTraces("); ir_print_other_instruction(irp, instruction->coro_promise_ptr); fprintf(irp->f, ","); - if (instruction->resolved_field != nullptr) { - fprintf(irp->f, "field '%s'", buf_ptr(instruction->resolved_field->name)); - } + ir_print_other_instruction(irp, instruction->err_ret_trace_ptr); + fprintf(irp->f, ")"); +} + +static void ir_print_mark_err_ret_trace_ptr(IrPrint *irp, IrInstructionMarkErrRetTracePtr *instruction) { + fprintf(irp->f, "@markErrRetTracePtr("); + ir_print_other_instruction(irp, instruction->err_ret_trace_ptr); fprintf(irp->f, ")"); } @@ -1581,6 +1585,9 @@ static void ir_print_instruction(IrPrint *irp, IrInstruction *instruction) { case IrInstructionIdMergeErrRetTraces: ir_print_merge_err_ret_traces(irp, (IrInstructionMergeErrRetTraces *)instruction); break; + case IrInstructionIdMarkErrRetTracePtr: + ir_print_mark_err_ret_trace_ptr(irp, (IrInstructionMarkErrRetTracePtr *)instruction); + break; } fprintf(irp->f, "\n"); } From 9e98ea552dcf03a4a05a920c8f027d09130dd688 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 8 Apr 2018 16:40:59 -0400 Subject: [PATCH 37/87] fix calling convention at callsite of zig-generated fns --- src/codegen.cpp | 6 +++--- test/cases/coroutines.zig | 28 ++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 914e9c7096..34eda6dd96 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -1289,7 +1289,7 @@ static LLVMValueRef get_merge_err_ret_traces_fn_val(CodeGen *g) { LLVMValueRef addr_ptr = LLVMBuildInBoundsGEP(g->builder, src_ptr_val, &ptr_index, 1, ""); LLVMValueRef this_addr_val = LLVMBuildLoad(g->builder, addr_ptr, ""); LLVMValueRef args[] = {dest_stack_trace_ptr, this_addr_val}; - LLVMBuildCall(g->builder, add_error_return_trace_addr_fn_val, args, 2, ""); + ZigLLVMBuildCall(g->builder, add_error_return_trace_addr_fn_val, args, 2, get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAlways, ""); LLVMValueRef prev_frames_left = LLVMBuildLoad(g->builder, frames_left_ptr, ""); LLVMValueRef new_frames_left = LLVMBuildNUWSub(g->builder, prev_frames_left, usize_one, ""); LLVMValueRef done_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, new_frames_left, usize_zero, ""); @@ -1358,7 +1358,7 @@ static LLVMValueRef get_return_err_fn(CodeGen *g) { LLVMValueRef return_address = LLVMBuildPtrToInt(g->builder, return_address_ptr, usize_type_ref, ""); LLVMValueRef args[] = { err_ret_trace_ptr, return_address }; - LLVMBuildCall(g->builder, add_error_return_trace_addr_fn_val, args, 2, ""); + ZigLLVMBuildCall(g->builder, add_error_return_trace_addr_fn_val, args, 2, get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAlways, ""); LLVMBuildRetVoid(g->builder); LLVMPositionBuilderAtEnd(g->builder, prev_block); @@ -4387,7 +4387,7 @@ static LLVMValueRef ir_render_merge_err_ret_traces(CodeGen *g, IrExecutable *exe LLVMValueRef dest_trace_ptr = get_cur_err_ret_trace_val(g, instruction->base.scope); LLVMValueRef args[] = { dest_trace_ptr, src_trace_ptr }; - LLVMBuildCall(g->builder, get_merge_err_ret_traces_fn_val(g), args, 2, ""); + ZigLLVMBuildCall(g->builder, get_merge_err_ret_traces_fn_val(g), args, 2, get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAuto, ""); return nullptr; } diff --git a/test/cases/coroutines.zig b/test/cases/coroutines.zig index 922c1a7e58..5537323734 100644 --- a/test/cases/coroutines.zig +++ b/test/cases/coroutines.zig @@ -1,4 +1,5 @@ const std = @import("std"); +const builtin = @import("builtin"); const assert = std.debug.assert; var x: i32 = 1; @@ -189,3 +190,30 @@ async fn failing() !void { suspend; return error.Fail; } + +test "error return trace across suspend points" { + const p = nonFailing(); + resume p; + const p2 = try async printTrace(p); + cancel p2; +} + +fn nonFailing() promise->error!void { + return async suspendThenFail() catch unreachable; +} + +async fn suspendThenFail() error!void { + suspend; + return error.Fail; +} + +async fn printTrace(p: promise->error!void) void { + (await p) catch |e| { + std.debug.assert(e == error.Fail); + if (@errorReturnTrace()) |trace| { + assert(trace.index == 1); + } else if (builtin.mode != builtin.Mode.ReleaseFast) { + @panic("expected return trace"); + } + }; +} From ee1a4f4c1d888d1485d8bb13ee0fa756bf729b08 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 8 Apr 2018 17:44:29 -0400 Subject: [PATCH 38/87] error return traces work with async return case --- src/all_types.hpp | 12 +++++++---- src/analyze.cpp | 3 ++- src/codegen.cpp | 4 ++-- src/ir.cpp | 43 ++++++++++++++++++++++++++++++--------- src/ir_print.cpp | 4 +++- test/cases/coroutines.zig | 9 +++++++- 6 files changed, 56 insertions(+), 19 deletions(-) diff --git a/src/all_types.hpp b/src/all_types.hpp index 52b8ede82c..25d4f70e2f 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -3037,7 +3037,8 @@ struct IrInstructionMergeErrRetTraces { IrInstruction base; IrInstruction *coro_promise_ptr; - IrInstruction *err_ret_trace_ptr; + IrInstruction *src_err_ret_trace_ptr; + IrInstruction *dest_err_ret_trace_ptr; }; struct IrInstructionMarkErrRetTracePtr { @@ -3058,13 +3059,16 @@ static const size_t err_union_payload_index = 1; // TODO call graph analysis to find out what this number needs to be for every function static const size_t stack_trace_ptr_count = 30; +// these belong to the async function +#define RETURN_ADDRESSES_FIELD_NAME "return_addresses" +#define ERR_RET_TRACE_FIELD_NAME "err_ret_trace" +#define RESULT_FIELD_NAME "result" #define ASYNC_ALLOC_FIELD_NAME "allocFn" #define ASYNC_FREE_FIELD_NAME "freeFn" #define AWAITER_HANDLE_FIELD_NAME "awaiter_handle" -#define RESULT_FIELD_NAME "result" +// these point to data belonging to the awaiter +#define ERR_RET_TRACE_PTR_FIELD_NAME "err_ret_trace_ptr" #define RESULT_PTR_FIELD_NAME "result_ptr" -#define RETURN_ADDRESSES_FIELD_NAME "return_addresses" -#define ERR_RET_TRACE_FIELD_NAME "err_ret_trace" enum FloatMode { diff --git a/src/analyze.cpp b/src/analyze.cpp index ae2a1a1b1d..3db49a11c9 100644 --- a/src/analyze.cpp +++ b/src/analyze.cpp @@ -474,6 +474,7 @@ TypeTableEntry *get_promise_frame_type(CodeGen *g, TypeTableEntry *return_type) field_names.append(RESULT_FIELD_NAME); field_names.append(RESULT_PTR_FIELD_NAME); if (g->have_err_ret_tracing) { + field_names.append(ERR_RET_TRACE_PTR_FIELD_NAME); field_names.append(ERR_RET_TRACE_FIELD_NAME); field_names.append(RETURN_ADDRESSES_FIELD_NAME); } @@ -483,7 +484,7 @@ TypeTableEntry *get_promise_frame_type(CodeGen *g, TypeTableEntry *return_type) field_types.append(return_type); field_types.append(result_ptr_type); if (g->have_err_ret_tracing) { - get_ptr_to_stack_trace_type(g); + field_types.append(get_ptr_to_stack_trace_type(g)); field_types.append(g->stack_trace_type); field_types.append(get_array_type(g, g->builtin_types.entry_usize, stack_trace_ptr_count)); } diff --git a/src/codegen.cpp b/src/codegen.cpp index 34eda6dd96..be83f68349 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -4383,8 +4383,8 @@ static LLVMValueRef ir_render_merge_err_ret_traces(CodeGen *g, IrExecutable *exe { assert(g->have_err_ret_tracing); - LLVMValueRef src_trace_ptr = ir_llvm_value(g, instruction->err_ret_trace_ptr); - LLVMValueRef dest_trace_ptr = get_cur_err_ret_trace_val(g, instruction->base.scope); + LLVMValueRef src_trace_ptr = ir_llvm_value(g, instruction->src_err_ret_trace_ptr); + LLVMValueRef dest_trace_ptr = ir_llvm_value(g, instruction->dest_err_ret_trace_ptr); LLVMValueRef args[] = { dest_trace_ptr, src_trace_ptr }; ZigLLVMBuildCall(g->builder, get_merge_err_ret_traces_fn_val(g), args, 2, get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAuto, ""); diff --git a/src/ir.cpp b/src/ir.cpp index 3f8744ecb1..4ab8b130c9 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -2708,14 +2708,16 @@ static IrInstruction *ir_build_add_implicit_return_type(IrBuilder *irb, Scope *s } static IrInstruction *ir_build_merge_err_ret_traces(IrBuilder *irb, Scope *scope, AstNode *source_node, - IrInstruction *coro_promise_ptr, IrInstruction *err_ret_trace_ptr) + IrInstruction *coro_promise_ptr, IrInstruction *src_err_ret_trace_ptr, IrInstruction *dest_err_ret_trace_ptr) { IrInstructionMergeErrRetTraces *instruction = ir_build_instruction(irb, scope, source_node); instruction->coro_promise_ptr = coro_promise_ptr; - instruction->err_ret_trace_ptr = err_ret_trace_ptr; + instruction->src_err_ret_trace_ptr = src_err_ret_trace_ptr; + instruction->dest_err_ret_trace_ptr = dest_err_ret_trace_ptr; ir_ref_instruction(coro_promise_ptr, irb->current_basic_block); - ir_ref_instruction(err_ret_trace_ptr, irb->current_basic_block); + ir_ref_instruction(src_err_ret_trace_ptr, irb->current_basic_block); + ir_ref_instruction(dest_err_ret_trace_ptr, irb->current_basic_block); return &instruction->base; } @@ -6115,6 +6117,13 @@ static IrInstruction *ir_gen_await_expr(IrBuilder *irb, Scope *parent_scope, Ast Buf *result_ptr_field_name = buf_create_from_str(RESULT_PTR_FIELD_NAME); IrInstruction *result_ptr_field_ptr = ir_build_field_ptr(irb, parent_scope, node, coro_promise_ptr, result_ptr_field_name); + if (irb->codegen->have_err_ret_tracing) { + IrInstruction *err_ret_trace_ptr = ir_build_error_return_trace(irb, parent_scope, node, IrInstructionErrorReturnTrace::NonNull); + Buf *err_ret_trace_ptr_field_name = buf_create_from_str(ERR_RET_TRACE_PTR_FIELD_NAME); + IrInstruction *err_ret_trace_ptr_field_ptr = ir_build_field_ptr(irb, parent_scope, node, coro_promise_ptr, err_ret_trace_ptr_field_name); + ir_build_store_ptr(irb, parent_scope, node, err_ret_trace_ptr_field_ptr, err_ret_trace_ptr); + } + Buf *awaiter_handle_field_name = buf_create_from_str(AWAITER_HANDLE_FIELD_NAME); IrInstruction *awaiter_field_ptr = ir_build_field_ptr(irb, parent_scope, node, coro_promise_ptr, awaiter_handle_field_name); @@ -6144,8 +6153,9 @@ static IrInstruction *ir_gen_await_expr(IrBuilder *irb, Scope *parent_scope, Ast ir_set_cursor_at_end_and_append_block(irb, no_suspend_block); if (irb->codegen->have_err_ret_tracing) { Buf *err_ret_trace_field_name = buf_create_from_str(ERR_RET_TRACE_FIELD_NAME); - IrInstruction *err_ret_trace_ptr = ir_build_field_ptr(irb, parent_scope, node, coro_promise_ptr, err_ret_trace_field_name); - ir_build_merge_err_ret_traces(irb, parent_scope, node, coro_promise_ptr, err_ret_trace_ptr); + IrInstruction *src_err_ret_trace_ptr = ir_build_field_ptr(irb, parent_scope, node, coro_promise_ptr, err_ret_trace_field_name); + IrInstruction *dest_err_ret_trace_ptr = ir_build_error_return_trace(irb, parent_scope, node, IrInstructionErrorReturnTrace::NonNull); + ir_build_merge_err_ret_traces(irb, parent_scope, node, coro_promise_ptr, src_err_ret_trace_ptr, dest_err_ret_trace_ptr); } Buf *result_field_name = buf_create_from_str(RESULT_FIELD_NAME); IrInstruction *promise_result_ptr = ir_build_field_ptr(irb, parent_scope, node, coro_promise_ptr, result_field_name); @@ -6402,6 +6412,8 @@ bool ir_gen(CodeGen *codegen, AstNode *node, Scope *scope, IrExecutable *ir_exec IrInstruction *coro_id; IrInstruction *u8_ptr_type; IrInstruction *const_bool_false; + IrInstruction *coro_promise_ptr; + IrInstruction *err_ret_trace_ptr; TypeTableEntry *return_type; Buf *result_ptr_field_name; VariableTableEntry *coro_size_var; @@ -6417,7 +6429,7 @@ bool ir_gen(CodeGen *codegen, AstNode *node, Scope *scope, IrExecutable *ir_exec IrInstruction *coro_frame_type_value = ir_build_const_type(irb, coro_scope, node, coro_frame_type); // TODO mark this var decl as "no safety" e.g. disable initializing the undef value to 0xaa ir_build_var_decl(irb, coro_scope, node, promise_var, coro_frame_type_value, nullptr, undef); - IrInstruction *coro_promise_ptr = ir_build_var_ptr(irb, coro_scope, node, promise_var, false, false); + coro_promise_ptr = ir_build_var_ptr(irb, coro_scope, node, promise_var, false, false); VariableTableEntry *await_handle_var = ir_create_var(irb, node, coro_scope, nullptr, false, false, true, const_bool_false); IrInstruction *null_value = ir_build_const_null(irb, coro_scope, node); @@ -6471,7 +6483,7 @@ bool ir_gen(CodeGen *codegen, AstNode *node, Scope *scope, IrExecutable *ir_exec IrInstruction *return_addresses_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, return_addresses_field_name); Buf *err_ret_trace_field_name = buf_create_from_str(ERR_RET_TRACE_FIELD_NAME); - IrInstruction *err_ret_trace_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, err_ret_trace_field_name); + err_ret_trace_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, err_ret_trace_field_name); ir_build_mark_err_ret_trace_ptr(irb, scope, node, err_ret_trace_ptr); // coordinate with builtin.zig @@ -6536,6 +6548,12 @@ bool ir_gen(CodeGen *codegen, AstNode *node, Scope *scope, IrExecutable *ir_exec IrInstruction *size_of_ret_val = ir_build_size_of(irb, scope, node, return_type_inst); ir_build_memcpy(irb, scope, node, result_ptr_as_u8_ptr, return_value_ptr_as_u8_ptr, size_of_ret_val); } + if (irb->codegen->have_err_ret_tracing) { + Buf *err_ret_trace_ptr_field_name = buf_create_from_str(ERR_RET_TRACE_PTR_FIELD_NAME); + IrInstruction *err_ret_trace_ptr_field_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, err_ret_trace_ptr_field_name); + IrInstruction *dest_err_ret_trace_ptr = ir_build_load_ptr(irb, scope, node, err_ret_trace_ptr_field_ptr); + ir_build_merge_err_ret_traces(irb, scope, node, coro_promise_ptr, err_ret_trace_ptr, dest_err_ret_trace_ptr); + } ir_build_br(irb, scope, node, check_free_block, const_bool_false); ir_set_cursor_at_end_and_append_block(irb, irb->exec->coro_final_cleanup_block); @@ -13098,6 +13116,7 @@ static IrInstruction *ir_analyze_container_member_access_inner(IrAnalyze *ira, { if (!is_slice(bare_struct_type)) { ScopeDecls *container_scope = get_container_scope(bare_struct_type); + assert(container_scope != nullptr); auto entry = container_scope->decl_table.maybe_get(field_name); Tld *tld = entry ? entry->value : nullptr; if (tld && tld->id == TldIdFn) { @@ -17948,12 +17967,16 @@ static TypeTableEntry *ir_analyze_instruction_merge_err_ret_traces(IrAnalyze *ir return out_val->type; } - IrInstruction *err_ret_trace_ptr = instruction->err_ret_trace_ptr->other; - if (type_is_invalid(err_ret_trace_ptr->value.type)) + IrInstruction *src_err_ret_trace_ptr = instruction->src_err_ret_trace_ptr->other; + if (type_is_invalid(src_err_ret_trace_ptr->value.type)) + return ira->codegen->builtin_types.entry_invalid; + + IrInstruction *dest_err_ret_trace_ptr = instruction->dest_err_ret_trace_ptr->other; + if (type_is_invalid(dest_err_ret_trace_ptr->value.type)) return ira->codegen->builtin_types.entry_invalid; IrInstruction *result = ir_build_merge_err_ret_traces(&ira->new_irb, instruction->base.scope, - instruction->base.source_node, coro_promise_ptr, err_ret_trace_ptr); + instruction->base.source_node, coro_promise_ptr, src_err_ret_trace_ptr, dest_err_ret_trace_ptr); ir_link_new_instruction(result, &instruction->base); result->value.type = ira->codegen->builtin_types.entry_void; return result->value.type; diff --git a/src/ir_print.cpp b/src/ir_print.cpp index 20dfb10b81..99f79ff75e 100644 --- a/src/ir_print.cpp +++ b/src/ir_print.cpp @@ -1192,7 +1192,9 @@ static void ir_print_merge_err_ret_traces(IrPrint *irp, IrInstructionMergeErrRet fprintf(irp->f, "@mergeErrRetTraces("); ir_print_other_instruction(irp, instruction->coro_promise_ptr); fprintf(irp->f, ","); - ir_print_other_instruction(irp, instruction->err_ret_trace_ptr); + ir_print_other_instruction(irp, instruction->src_err_ret_trace_ptr); + fprintf(irp->f, ","); + ir_print_other_instruction(irp, instruction->dest_err_ret_trace_ptr); fprintf(irp->f, ")"); } diff --git a/test/cases/coroutines.zig b/test/cases/coroutines.zig index 5537323734..6d28b98c9d 100644 --- a/test/cases/coroutines.zig +++ b/test/cases/coroutines.zig @@ -191,13 +191,20 @@ async fn failing() !void { return error.Fail; } -test "error return trace across suspend points" { +test "error return trace across suspend points - early return" { const p = nonFailing(); resume p; const p2 = try async printTrace(p); cancel p2; } +test "error return trace across suspend points - async return" { + const p = nonFailing(); + const p2 = try async printTrace(p); + resume p; + cancel p2; +} + fn nonFailing() promise->error!void { return async suspendThenFail() catch unreachable; } From eae355d77168d655273afa298dc921c40a36bd0b Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 6 Apr 2018 19:14:49 -0400 Subject: [PATCH 39/87] add docs for packed enum --- doc/langref.html.in | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/doc/langref.html.in b/doc/langref.html.in index 7f837186b5..ac597b3215 100644 --- a/doc/langref.html.in +++ b/doc/langref.html.in @@ -1947,8 +1947,24 @@ const Foo = extern enum { A, B, C }; export fn entry(foo: Foo) void { } {#code_end#} {#header_close#} -

TODO packed enum

- {#see_also|@memberName|@memberCount|@tagName#} + {#header_open|packed enum#} +

By default, the size of enums is not guaranteed.

+

packed enum causes the size of the enum to be the same as the size of the integer tag type + of the enum:

+ {#code_begin|test#} +const std = @import("std"); + +test "packed enum" { + const Number = packed enum(u8) { + One, + Two, + Three, + }; + std.debug.assert(@sizeOf(Number) == @sizeOf(u8)); +} + {#code_end#} + {#header_close#} + {#see_also|@memberName|@memberCount|@tagName|@sizeOf#} {#header_close#} {#header_open|union#} {#code_begin|test|union#} From 292d0cbdadd453874af6e2065638a88d4dda8a10 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 8 Apr 2018 18:03:02 -0400 Subject: [PATCH 40/87] add docs for union methods --- doc/langref.html.in | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/doc/langref.html.in b/doc/langref.html.in index ac597b3215..856d62f142 100644 --- a/doc/langref.html.in +++ b/doc/langref.html.in @@ -2033,7 +2033,27 @@ test "union variant switch" { assert(mem.eql(u8, what_is_it, "this is a number")); } -// TODO union methods +// Unions can have methods just like structs and enums: + +const Variant = union(enum) { + Int: i32, + Bool: bool, + + fn truthy(self: &const Variant) bool { + return switch (*self) { + Variant.Int => |x_int| x_int != 0, + Variant.Bool => |x_bool| x_bool, + }; + } +}; + +test "union method" { + var v1 = Variant { .Int = 1 }; + var v2 = Variant { .Bool = false }; + + assert(v1.truthy()); + assert(!v2.truthy()); +} const Small = union { From 0d22a00f6fde79f851a7d19c2096c07f541ed0be Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 7 Mar 2018 03:55:52 -0500 Subject: [PATCH 41/87] *WIP* async/await TCP server --- CMakeLists.txt | 2 - src/all_types.hpp | 12 +- src/analyze.cpp | 1 - src/ast_render.cpp | 3 - src/ir.cpp | 21 +- src/parser.cpp | 4 +- std/endian.zig | 25 -- std/event.zig | 202 +++++++++++++++ std/fmt/index.zig | 2 +- std/index.zig | 4 +- std/linked_list.zig | 1 + std/mem.zig | 26 ++ std/net.zig | 280 +++++++++------------ std/os/index.zig | 373 ++++++++++++++++++++++++++-- std/os/linux/i386.zig | 505 -------------------------------------- std/os/linux/index.zig | 216 +++++++++++----- test/cases/coroutines.zig | 15 ++ 17 files changed, 888 insertions(+), 804 deletions(-) delete mode 100644 std/endian.zig create mode 100644 std/event.zig delete mode 100644 std/os/linux/i386.zig diff --git a/CMakeLists.txt b/CMakeLists.txt index 2bb9bf517c..c6f169d635 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -432,7 +432,6 @@ set(ZIG_STD_FILES "dwarf.zig" "elf.zig" "empty.zig" - "endian.zig" "fmt/errol/enum3.zig" "fmt/errol/index.zig" "fmt/errol/lookup.zig" @@ -503,7 +502,6 @@ set(ZIG_STD_FILES "os/get_user_id.zig" "os/index.zig" "os/linux/errno.zig" - "os/linux/i386.zig" "os/linux/index.zig" "os/linux/x86_64.zig" "os/path.zig" diff --git a/src/all_types.hpp b/src/all_types.hpp index 25d4f70e2f..d434ea187e 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -359,7 +359,6 @@ enum NodeType { NodeTypeRoot, NodeTypeFnProto, NodeTypeFnDef, - NodeTypeFnDecl, NodeTypeParamDecl, NodeTypeBlock, NodeTypeGroupedExpr, @@ -453,10 +452,6 @@ struct AstNodeFnDef { AstNode *body; }; -struct AstNodeFnDecl { - AstNode *fn_proto; -}; - struct AstNodeParamDecl { Buf *name; AstNode *type; @@ -713,10 +708,6 @@ struct AstNodeSwitchRange { AstNode *end; }; -struct AstNodeLabel { - Buf *name; -}; - struct AstNodeCompTime { AstNode *expr; }; @@ -892,7 +883,6 @@ struct AstNode { union { AstNodeRoot root; AstNodeFnDef fn_def; - AstNodeFnDecl fn_decl; AstNodeFnProto fn_proto; AstNodeParamDecl param_decl; AstNodeBlock block; @@ -917,7 +907,6 @@ struct AstNode { AstNodeSwitchExpr switch_expr; AstNodeSwitchProng switch_prong; AstNodeSwitchRange switch_range; - AstNodeLabel label; AstNodeCompTime comptime_expr; AstNodeAsmExpr asm_expr; AstNodeFieldAccessExpr field_access_expr; @@ -2702,6 +2691,7 @@ struct IrInstructionFnProto { IrInstruction **param_types; IrInstruction *align_value; + IrInstruction *async_allocator_type_value; IrInstruction *return_type; IrInstruction *async_allocator_type_value; bool is_var_args; diff --git a/src/analyze.cpp b/src/analyze.cpp index 3db49a11c9..c73e6b39e3 100644 --- a/src/analyze.cpp +++ b/src/analyze.cpp @@ -3236,7 +3236,6 @@ void scan_decls(CodeGen *g, ScopeDecls *decls_scope, AstNode *node) { break; case NodeTypeContainerDecl: case NodeTypeParamDecl: - case NodeTypeFnDecl: case NodeTypeReturnExpr: case NodeTypeDefer: case NodeTypeBlock: diff --git a/src/ast_render.cpp b/src/ast_render.cpp index 7b5fc03ea8..2c3e1fc873 100644 --- a/src/ast_render.cpp +++ b/src/ast_render.cpp @@ -148,8 +148,6 @@ static const char *node_type_str(NodeType node_type) { return "Root"; case NodeTypeFnDef: return "FnDef"; - case NodeTypeFnDecl: - return "FnDecl"; case NodeTypeFnProto: return "FnProto"; case NodeTypeParamDecl: @@ -1098,7 +1096,6 @@ static void render_node_extra(AstRender *ar, AstNode *node, bool grouped) { } break; } - case NodeTypeFnDecl: case NodeTypeParamDecl: case NodeTypeTestDecl: case NodeTypeStructField: diff --git a/src/ir.cpp b/src/ir.cpp index 4ab8b130c9..a803183579 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -2153,12 +2153,12 @@ static IrInstruction *ir_build_unwrap_err_payload_from(IrBuilder *irb, IrInstruc } static IrInstruction *ir_build_fn_proto(IrBuilder *irb, Scope *scope, AstNode *source_node, - IrInstruction **param_types, IrInstruction *align_value, IrInstruction *return_type, - IrInstruction *async_allocator_type_value, bool is_var_args) + IrInstruction **param_types, IrInstruction *align_value, IrInstruction *return_type, IrInstruction *async_allocator_type_value, bool is_var_args) { IrInstructionFnProto *instruction = ir_build_instruction(irb, scope, source_node); instruction->param_types = param_types; instruction->align_value = align_value; + instruction->async_allocator_type_value = async_allocator_type_value; instruction->return_type = return_type; instruction->async_allocator_type_value = async_allocator_type_value; instruction->is_var_args = is_var_args; @@ -6041,6 +6041,13 @@ static IrInstruction *ir_gen_fn_proto(IrBuilder *irb, Scope *parent_scope, AstNo return irb->codegen->invalid_instruction; } + IrInstruction *async_allocator_type_value = nullptr; + if (node->data.fn_proto.async_allocator_type != nullptr) { + async_allocator_type_value = ir_gen_node(irb, node->data.fn_proto.async_allocator_type, parent_scope); + if (async_allocator_type_value == irb->codegen->invalid_instruction) + return irb->codegen->invalid_instruction; + } + IrInstruction *return_type; if (node->data.fn_proto.return_var_token == nullptr) { if (node->data.fn_proto.return_type == nullptr) { @@ -6061,8 +6068,7 @@ static IrInstruction *ir_gen_fn_proto(IrBuilder *irb, Scope *parent_scope, AstNo return irb->codegen->invalid_instruction; } - return ir_build_fn_proto(irb, parent_scope, node, param_types, align_value, return_type, - async_allocator_type_value, is_var_args); + return ir_build_fn_proto(irb, parent_scope, node, param_types, align_value, return_type, async_allocator_type_value, is_var_args); } static IrInstruction *ir_gen_cancel(IrBuilder *irb, Scope *parent_scope, AstNode *node) { @@ -6273,7 +6279,6 @@ static IrInstruction *ir_gen_node_raw(IrBuilder *irb, AstNode *node, Scope *scop case NodeTypeSwitchRange: case NodeTypeStructField: case NodeTypeFnDef: - case NodeTypeFnDecl: case NodeTypeTestDecl: zig_unreachable(); case NodeTypeBlock: @@ -16741,6 +16746,12 @@ static TypeTableEntry *ir_analyze_instruction_fn_proto(IrAnalyze *ira, IrInstruc return ira->codegen->builtin_types.entry_invalid; } + if (instruction->async_allocator_type_value != nullptr) { + fn_type_id.async_allocator_type = ir_resolve_type(ira, instruction->async_allocator_type_value->other); + if (type_is_invalid(fn_type_id.async_allocator_type)) + return ira->codegen->builtin_types.entry_invalid; + } + IrInstruction *return_type_value = instruction->return_type->other; fn_type_id.return_type = ir_resolve_type(ira, return_type_value); if (type_is_invalid(fn_type_id.return_type)) diff --git a/src/parser.cpp b/src/parser.cpp index d6faf4c984..b54a17362e 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -1037,6 +1037,7 @@ static AstNode *ast_parse_suffix_op_expr(ParseContext *pc, size_t *token_index, Token *async_token = &pc->tokens->at(*token_index); if (async_token->id == TokenIdKeywordAsync) { + size_t token_index_of_async = *token_index; *token_index += 1; AstNode *allocator_expr_node = nullptr; @@ -2923,9 +2924,6 @@ void ast_visit_node_children(AstNode *node, void (*visit)(AstNode **, void *cont visit_field(&node->data.fn_def.fn_proto, visit, context); visit_field(&node->data.fn_def.body, visit, context); break; - case NodeTypeFnDecl: - visit_field(&node->data.fn_decl.fn_proto, visit, context); - break; case NodeTypeParamDecl: visit_field(&node->data.param_decl.type, visit, context); break; diff --git a/std/endian.zig b/std/endian.zig deleted file mode 100644 index 121505d24d..0000000000 --- a/std/endian.zig +++ /dev/null @@ -1,25 +0,0 @@ -const mem = @import("mem.zig"); -const builtin = @import("builtin"); - -pub fn swapIfLe(comptime T: type, x: T) T { - return swapIf(builtin.Endian.Little, T, x); -} - -pub fn swapIfBe(comptime T: type, x: T) T { - return swapIf(builtin.Endian.Big, T, x); -} - -pub fn swapIf(endian: builtin.Endian, comptime T: type, x: T) T { - return if (builtin.endian == endian) swap(T, x) else x; -} - -pub fn swap(comptime T: type, x: T) T { - var buf: [@sizeOf(T)]u8 = undefined; - mem.writeInt(buf[0..], x, builtin.Endian.Little); - return mem.readInt(buf, T, builtin.Endian.Big); -} - -test "swap" { - const debug = @import("debug/index.zig"); - debug.assert(swap(u32, 0xDEADBEEF) == 0xEFBEADDE); -} diff --git a/std/event.zig b/std/event.zig new file mode 100644 index 0000000000..07fc64293c --- /dev/null +++ b/std/event.zig @@ -0,0 +1,202 @@ +const std = @import("index.zig"); +const assert = std.debug.assert; +const event = this; +const mem = std.mem; +const posix = std.os.posix; + +pub const TcpServer = struct { + handleRequestFn: async(&mem.Allocator) fn (&TcpServer, &const std.net.Address, &const std.os.File) void, + + loop: &Loop, + sockfd: i32, + accept_coro: ?promise, + + waiting_for_emfile_node: PromiseNode, + + const PromiseNode = std.LinkedList(promise).Node; + + pub fn init(loop: &Loop) !TcpServer { + const sockfd = try std.os.posixSocket(posix.AF_INET, + posix.SOCK_STREAM|posix.SOCK_CLOEXEC|posix.SOCK_NONBLOCK, + posix.PROTO_tcp); + errdefer std.os.close(sockfd); + + // TODO can't initialize handler coroutine here because we need well defined copy elision + return TcpServer { + .loop = loop, + .sockfd = sockfd, + .accept_coro = null, + .handleRequestFn = undefined, + .waiting_for_emfile_node = undefined, + }; + } + + pub fn listen(self: &TcpServer, address: &const std.net.Address, + handleRequestFn: async(&mem.Allocator) fn (&TcpServer, &const std.net.Address, &const std.os.File)void) !void + { + self.handleRequestFn = handleRequestFn; + + try std.os.posixBind(self.sockfd, &address.sockaddr); + try std.os.posixListen(self.sockfd, posix.SOMAXCONN); + + self.accept_coro = try async(self.loop.allocator) (TcpServer.handler)(self); // TODO #817 + errdefer cancel ??self.accept_coro; + + try self.loop.addFd(self.sockfd, ??self.accept_coro); + errdefer self.loop.removeFd(self.sockfd); + + } + + pub fn deinit(self: &TcpServer) void { + self.loop.removeFd(self.sockfd); + if (self.accept_coro) |accept_coro| cancel accept_coro; + std.os.close(self.sockfd); + } + + pub async fn handler(self: &TcpServer) void { + while (true) { + var accepted_addr: std.net.Address = undefined; + if (std.os.posixAccept(self.sockfd, &accepted_addr.sockaddr, + posix.SOCK_NONBLOCK | posix.SOCK_CLOEXEC)) |accepted_fd| + { + var socket = std.os.File.openHandle(accepted_fd); + // TODO #817 + _ = async(self.loop.allocator) (self.handleRequestFn)(self, accepted_addr, + socket) catch |err| switch (err) + { + error.OutOfMemory => { + socket.close(); + continue; + }, + }; + } else |err| switch (err) { + error.WouldBlock => { + suspend; // we will get resumed by epoll_wait in the event loop + continue; + }, + error.ProcessFdQuotaExceeded => { + errdefer std.os.emfile_promise_queue.remove(&self.waiting_for_emfile_node); + suspend |p| { + self.waiting_for_emfile_node = PromiseNode.init(p); + std.os.emfile_promise_queue.append(&self.waiting_for_emfile_node); + } + continue; + }, + error.ConnectionAborted, + error.FileDescriptorClosed => continue, + + error.PageFault => unreachable, + error.InvalidSyscall => unreachable, + error.FileDescriptorNotASocket => unreachable, + error.OperationNotSupported => unreachable, + + error.SystemFdQuotaExceeded, + error.SystemResources, + error.ProtocolFailure, + error.BlockedByFirewall, + error.Unexpected => { + @panic("TODO handle this error"); + }, + } + } + } +}; + +pub const Loop = struct { + allocator: &mem.Allocator, + epollfd: i32, + keep_running: bool, + + fn init(allocator: &mem.Allocator) !Loop { + const epollfd = try std.os.linuxEpollCreate(std.os.linux.EPOLL_CLOEXEC); + return Loop { + .keep_running = true, + .allocator = allocator, + .epollfd = epollfd, + }; + } + + pub fn addFd(self: &Loop, fd: i32, prom: promise) !void { + var ev = std.os.linux.epoll_event { + .events = std.os.linux.EPOLLIN|std.os.linux.EPOLLET, + .data = std.os.linux.epoll_data { + .ptr = @ptrToInt(prom), + }, + }; + try std.os.linuxEpollCtl(self.epollfd, std.os.linux.EPOLL_CTL_ADD, fd, &ev); + } + + pub fn removeFd(self: &Loop, fd: i32) void { + std.os.linuxEpollCtl(self.epollfd, std.os.linux.EPOLL_CTL_DEL, fd, undefined) catch {}; + } + + async fn waitFd(self: &Loop, fd: i32) !void { + defer self.removeFd(fd); + suspend |p| { + try self.addFd(fd, p); + } + } + + pub fn stop(self: &Loop) void { + // TODO make atomic + self.keep_running = false; + // TODO activate an fd in the epoll set + } + + pub fn run(self: &Loop) void { + while (self.keep_running) { + var events: [16]std.os.linux.epoll_event = undefined; + const count = std.os.linuxEpollWait(self.epollfd, events[0..], -1); + for (events[0..count]) |ev| { + const p = @intToPtr(promise, ev.data.ptr); + resume p; + } + } + } +}; + +test "listen on a port, send bytes, receive bytes" { + const MyServer = struct { + tcp_server: TcpServer, + + const Self = this; + + async(&mem.Allocator) fn handler(tcp_server: &TcpServer, _addr: &const std.net.Address, + _socket: &const std.os.File) void + { + const self = @fieldParentPtr(Self, "tcp_server", tcp_server); + var socket = *_socket; // TODO https://github.com/zig-lang/zig/issues/733 + defer socket.close(); + const next_handler = async errorableHandler(self, _addr, socket) catch |err| switch (err) { + error.OutOfMemory => return, + }; + (await next_handler) catch |err| switch (err) { + + }; + suspend |p| { cancel p; } + } + + async fn errorableHandler(self: &Self, _addr: &const std.net.Address, + _socket: &const std.os.File) !void + { + const addr = *_addr; // TODO https://github.com/zig-lang/zig/issues/733 + var socket = *_socket; // TODO https://github.com/zig-lang/zig/issues/733 + + var adapter = std.io.FileOutStream.init(&socket); + var stream = &adapter.stream; + try stream.print("hello from server\n") catch unreachable; + } + }; + + const ip4addr = std.net.parseIp4("127.0.0.1") catch unreachable; + const addr = std.net.Address.initIp4(ip4addr, 0); + + var loop = try Loop.init(std.debug.global_allocator); + var server = MyServer { + .tcp_server = try TcpServer.init(&loop), + }; + defer server.tcp_server.deinit(); + try server.tcp_server.listen(addr, MyServer.handler); + + loop.run(); +} diff --git a/std/fmt/index.zig b/std/fmt/index.zig index bd5b5710e0..cfdd70e95b 100644 --- a/std/fmt/index.zig +++ b/std/fmt/index.zig @@ -465,7 +465,7 @@ pub fn parseUnsigned(comptime T: type, buf: []const u8, radix: u8) ParseUnsigned return x; } -fn charToDigit(c: u8, radix: u8) (error{InvalidCharacter}!u8) { +pub fn charToDigit(c: u8, radix: u8) (error{InvalidCharacter}!u8) { const value = switch (c) { '0' ... '9' => c - '0', 'A' ... 'Z' => c - 'A' + 10, diff --git a/std/index.zig b/std/index.zig index f2af70b28b..f8ec787a01 100644 --- a/std/index.zig +++ b/std/index.zig @@ -17,7 +17,7 @@ pub const debug = @import("debug/index.zig"); pub const dwarf = @import("dwarf.zig"); pub const elf = @import("elf.zig"); pub const empty_import = @import("empty.zig"); -pub const endian = @import("endian.zig"); +pub const event = @import("event.zig"); pub const fmt = @import("fmt/index.zig"); pub const hash = @import("hash/index.zig"); pub const heap = @import("heap.zig"); @@ -50,13 +50,13 @@ test "std" { _ = @import("dwarf.zig"); _ = @import("elf.zig"); _ = @import("empty.zig"); - _ = @import("endian.zig"); _ = @import("fmt/index.zig"); _ = @import("hash/index.zig"); _ = @import("io.zig"); _ = @import("macho.zig"); _ = @import("math/index.zig"); _ = @import("mem.zig"); + _ = @import("net.zig"); _ = @import("heap.zig"); _ = @import("net.zig"); _ = @import("os/index.zig"); diff --git a/std/linked_list.zig b/std/linked_list.zig index c916a53133..45595f3efb 100644 --- a/std/linked_list.zig +++ b/std/linked_list.zig @@ -161,6 +161,7 @@ fn BaseLinkedList(comptime T: type, comptime ParentType: type, comptime field_na } list.len -= 1; + assert(list.len == 0 or (list.first != null and list.last != null)); } /// Remove and return the last node in the list. diff --git a/std/mem.zig b/std/mem.zig index 97cb35ae65..8a59d6251b 100644 --- a/std/mem.zig +++ b/std/mem.zig @@ -3,6 +3,7 @@ const debug = std.debug; const assert = debug.assert; const math = std.math; const builtin = @import("builtin"); +const mem = this; pub const Allocator = struct { const Error = error {OutOfMemory}; @@ -550,3 +551,28 @@ test "std.mem.rotate" { assert(eql(i32, arr, []i32{ 1, 2, 4, 5, 3 })); } + +// TODO: When https://github.com/zig-lang/zig/issues/649 is solved these can be done by +// endian-casting the pointer and then dereferencing + +pub fn endianSwapIfLe(comptime T: type, x: T) T { + return endianSwapIf(builtin.Endian.Little, T, x); +} + +pub fn endianSwapIfBe(comptime T: type, x: T) T { + return endianSwapIf(builtin.Endian.Big, T, x); +} + +pub fn endianSwapIf(endian: builtin.Endian, comptime T: type, x: T) T { + return if (builtin.endian == endian) endianSwap(T, x) else x; +} + +pub fn endianSwap(comptime T: type, x: T) T { + var buf: [@sizeOf(T)]u8 = undefined; + mem.writeInt(buf[0..], x, builtin.Endian.Little); + return mem.readInt(buf, T, builtin.Endian.Big); +} + +test "std.mem.endianSwap" { + assert(endianSwap(u32, 0xDEADBEEF) == 0xEFBEADDE); +} diff --git a/std/net.zig b/std/net.zig index 1140b6449b..595baae9dd 100644 --- a/std/net.zig +++ b/std/net.zig @@ -1,143 +1,103 @@ const std = @import("index.zig"); -const linux = std.os.linux; const assert = std.debug.assert; -const endian = std.endian; +const net = this; +const posix = std.os.posix; +const mem = std.mem; -// TODO don't trust this file, it bit rotted. start over +pub const Address = struct { + sockaddr: posix.sockaddr, -const Connection = struct { - socket_fd: i32, - - pub fn send(c: Connection, buf: []const u8) !usize { - const send_ret = linux.sendto(c.socket_fd, buf.ptr, buf.len, 0, null, 0); - const send_err = linux.getErrno(send_ret); - switch (send_err) { - 0 => return send_ret, - linux.EINVAL => unreachable, - linux.EFAULT => unreachable, - linux.ECONNRESET => return error.ConnectionReset, - linux.EINTR => return error.SigInterrupt, - // TODO there are more possible errors - else => return error.Unexpected, - } + pub fn initIp4(ip4: u32, port: u16) Address { + return Address { + .sockaddr = posix.sockaddr { + .in = posix.sockaddr_in { + .family = posix.AF_INET, + .port = std.mem.endianSwapIfLe(u16, port), + .addr = ip4, + .zero = []u8{0} ** 8, + }, + }, + }; } - pub fn recv(c: Connection, buf: []u8) ![]u8 { - const recv_ret = linux.recvfrom(c.socket_fd, buf.ptr, buf.len, 0, null, null); - const recv_err = linux.getErrno(recv_ret); - switch (recv_err) { - 0 => return buf[0..recv_ret], - linux.EINVAL => unreachable, - linux.EFAULT => unreachable, - linux.ENOTSOCK => return error.NotSocket, - linux.EINTR => return error.SigInterrupt, - linux.ENOMEM => return error.OutOfMemory, - linux.ECONNREFUSED => return error.ConnectionRefused, - linux.EBADF => return error.BadFd, - // TODO more error values - else => return error.Unexpected, - } + pub fn initIp6(ip6: &const Ip6Addr, port: u16) Address { + return Address { + .family = posix.AF_INET6, + .sockaddr = posix.sockaddr { + .in6 = posix.sockaddr_in6 { + .family = posix.AF_INET6, + .port = std.mem.endianSwapIfLe(u16, port), + .flowinfo = 0, + .addr = ip6.addr, + .scope_id = ip6.scope_id, + }, + }, + }; } - pub fn close(c: Connection) !void { - switch (linux.getErrno(linux.close(c.socket_fd))) { - 0 => return, - linux.EBADF => unreachable, - linux.EINTR => return error.SigInterrupt, - linux.EIO => return error.Io, - else => return error.Unexpected, + pub fn format(self: &const Address, out_stream: var) !void { + switch (self.sockaddr.in.family) { + posix.AF_INET => { + const native_endian_port = std.mem.endianSwapIfLe(u16, self.sockaddr.in.port); + const bytes = ([]const u8)((&self.sockaddr.in.addr)[0..1]); + try out_stream.print("{}.{}.{}.{}:{}", bytes[0], bytes[1], bytes[2], bytes[3], native_endian_port); + }, + posix.AF_INET6 => { + const native_endian_port = std.mem.endianSwapIfLe(u16, self.sockaddr.in6.port); + try out_stream.print("[TODO render ip6 address]:{}", native_endian_port); + }, + else => try out_stream.write("(unrecognized address family)"), } } }; -const Address = struct { - family: u16, +pub fn parseIp4(buf: []const u8) !u32 { + var result: u32 = undefined; + const out_ptr = ([]u8)((&result)[0..1]); + + var x: u8 = 0; + var index: u8 = 0; + var saw_any_digits = false; + for (buf) |c| { + if (c == '.') { + if (!saw_any_digits) { + return error.InvalidCharacter; + } + if (index == 3) { + return error.InvalidEnd; + } + out_ptr[index] = x; + index += 1; + x = 0; + saw_any_digits = false; + } else if (c >= '0' and c <= '9') { + saw_any_digits = true; + const digit = c - '0'; + if (@mulWithOverflow(u8, x, 10, &x)) { + return error.Overflow; + } + if (@addWithOverflow(u8, x, digit, &x)) { + return error.Overflow; + } + } else { + return error.InvalidCharacter; + } + } + if (index == 3 and saw_any_digits) { + out_ptr[index] = x; + return result; + } + + return error.Incomplete; +} + +pub const Ip6Addr = struct { scope_id: u32, addr: [16]u8, - sort_key: i32, }; -pub fn lookup(hostname: []const u8, out_addrs: []Address) ![]Address { - if (hostname.len == 0) { - - unreachable; // TODO - } - - unreachable; // TODO -} - -pub fn connectAddr(addr: &Address, port: u16) !Connection { - const socket_ret = linux.socket(addr.family, linux.SOCK_STREAM, linux.PROTO_tcp); - const socket_err = linux.getErrno(socket_ret); - if (socket_err > 0) { - // TODO figure out possible errors from socket() - return error.Unexpected; - } - const socket_fd = i32(socket_ret); - - const connect_ret = if (addr.family == linux.AF_INET) x: { - var os_addr: linux.sockaddr_in = undefined; - os_addr.family = addr.family; - os_addr.port = endian.swapIfLe(u16, port); - @memcpy((&u8)(&os_addr.addr), &addr.addr[0], 4); - @memset(&os_addr.zero[0], 0, @sizeOf(@typeOf(os_addr.zero))); - break :x linux.connect(socket_fd, (&linux.sockaddr)(&os_addr), @sizeOf(linux.sockaddr_in)); - } else if (addr.family == linux.AF_INET6) x: { - var os_addr: linux.sockaddr_in6 = undefined; - os_addr.family = addr.family; - os_addr.port = endian.swapIfLe(u16, port); - os_addr.flowinfo = 0; - os_addr.scope_id = addr.scope_id; - @memcpy(&os_addr.addr[0], &addr.addr[0], 16); - break :x linux.connect(socket_fd, (&linux.sockaddr)(&os_addr), @sizeOf(linux.sockaddr_in6)); - } else { - unreachable; - }; - const connect_err = linux.getErrno(connect_ret); - if (connect_err > 0) { - switch (connect_err) { - linux.ETIMEDOUT => return error.TimedOut, - else => { - // TODO figure out possible errors from connect() - return error.Unexpected; - }, - } - } - - return Connection { - .socket_fd = socket_fd, - }; -} - -pub fn connect(hostname: []const u8, port: u16) !Connection { - var addrs_buf: [1]Address = undefined; - const addrs_slice = try lookup(hostname, addrs_buf[0..]); - const main_addr = &addrs_slice[0]; - - return connectAddr(main_addr, port); -} - -pub fn parseIpLiteral(buf: []const u8) !Address { - - return error.InvalidIpLiteral; -} - -fn hexDigit(c: u8) u8 { - // TODO use switch with range - if ('0' <= c and c <= '9') { - return c - '0'; - } else if ('A' <= c and c <= 'Z') { - return c - 'A' + 10; - } else if ('a' <= c and c <= 'z') { - return c - 'a' + 10; - } else { - return @maxValue(u8); - } -} - -fn parseIp6(buf: []const u8) !Address { - var result: Address = undefined; - result.family = linux.AF_INET6; +pub fn parseIp6(buf: []const u8) !Ip6Addr { + var result: Ip6Addr = undefined; result.scope_id = 0; const ip_slice = result.addr[0..]; @@ -156,14 +116,14 @@ fn parseIp6(buf: []const u8) !Address { return error.Overflow; } } else { - return error.InvalidChar; + return error.InvalidCharacter; } } else if (c == ':') { if (!saw_any_digits) { - return error.InvalidChar; + return error.InvalidCharacter; } if (index == 14) { - return error.JunkAtEnd; + return error.InvalidEnd; } ip_slice[index] = @truncate(u8, x >> 8); index += 1; @@ -174,7 +134,7 @@ fn parseIp6(buf: []const u8) !Address { saw_any_digits = false; } else if (c == '%') { if (!saw_any_digits) { - return error.InvalidChar; + return error.InvalidCharacter; } if (index == 14) { ip_slice[index] = @truncate(u8, x >> 8); @@ -185,10 +145,7 @@ fn parseIp6(buf: []const u8) !Address { scope_id = true; saw_any_digits = false; } else { - const digit = hexDigit(c); - if (digit == @maxValue(u8)) { - return error.InvalidChar; - } + const digit = try std.fmt.charToDigit(c, 16); if (@mulWithOverflow(u16, x, 16, &x)) { return error.Overflow; } @@ -216,42 +173,27 @@ fn parseIp6(buf: []const u8) !Address { return error.Incomplete; } -fn parseIp4(buf: []const u8) !u32 { - var result: u32 = undefined; - const out_ptr = ([]u8)((&result)[0..1]); +test "std.net.parseIp4" { + assert((try parseIp4("127.0.0.1")) == std.mem.endianSwapIfLe(u32, 0x7f000001)); - var x: u8 = 0; - var index: u8 = 0; - var saw_any_digits = false; - for (buf) |c| { - if (c == '.') { - if (!saw_any_digits) { - return error.InvalidChar; - } - if (index == 3) { - return error.JunkAtEnd; - } - out_ptr[index] = x; - index += 1; - x = 0; - saw_any_digits = false; - } else if (c >= '0' and c <= '9') { - saw_any_digits = true; - const digit = c - '0'; - if (@mulWithOverflow(u8, x, 10, &x)) { - return error.Overflow; - } - if (@addWithOverflow(u8, x, digit, &x)) { - return error.Overflow; - } - } else { - return error.InvalidChar; - } - } - if (index == 3 and saw_any_digits) { - out_ptr[index] = x; - return result; - } - - return error.Incomplete; + testParseIp4Fail("256.0.0.1", error.Overflow); + testParseIp4Fail("x.0.0.1", error.InvalidCharacter); + testParseIp4Fail("127.0.0.1.1", error.InvalidEnd); + testParseIp4Fail("127.0.0.", error.Incomplete); + testParseIp4Fail("100..0.1", error.InvalidCharacter); +} + +fn testParseIp4Fail(buf: []const u8, expected_err: error) void { + if (parseIp4(buf)) |_| { + @panic("expected error"); + } else |e| { + assert(e == expected_err); + } +} + +test "std.net.parseIp6" { + const addr = try parseIp6("FF01:0:0:0:0:0:0:FB"); + assert(addr.addr[0] == 0xff); + assert(addr.addr[1] == 0x01); + assert(addr.addr[2] == 0x00); } diff --git a/std/os/index.zig b/std/os/index.zig index 4b74af035e..6f9db7edcd 100644 --- a/std/os/index.zig +++ b/std/os/index.zig @@ -4,6 +4,19 @@ const Os = builtin.Os; const is_windows = builtin.os == Os.windows; const os = this; +test "std.os" { + _ = @import("child_process.zig"); + _ = @import("darwin.zig"); + _ = @import("darwin_errno.zig"); + _ = @import("get_user_id.zig"); + _ = @import("linux/errno.zig"); + _ = @import("linux/index.zig"); + _ = @import("linux/x86_64.zig"); + _ = @import("path.zig"); + _ = @import("test.zig"); + _ = @import("windows/index.zig"); +} + pub const windows = @import("windows/index.zig"); pub const darwin = @import("darwin.zig"); pub const linux = @import("linux/index.zig"); @@ -14,6 +27,7 @@ pub const posix = switch(builtin.os) { Os.zen => zen, else => @compileError("Unsupported OS"), }; +pub const net = @import("net.zig"); pub const ChildProcess = @import("child_process.zig").ChildProcess; pub const path = @import("path.zig"); @@ -173,6 +187,13 @@ pub fn exit(status: u8) noreturn { } } +/// When a file descriptor is closed on linux, it pops the first +/// node from this queue and resumes it. +/// Async functions which get the EMFILE error code can suspend, +/// putting their coroutine handle into this list. +/// TODO make this an atomic linked list +pub var emfile_promise_queue = std.LinkedList(promise).init(); + /// Closes the file handle. Keeps trying if it gets interrupted by a signal. pub fn close(handle: FileHandle) void { if (is_windows) { @@ -180,10 +201,12 @@ pub fn close(handle: FileHandle) void { } else { while (true) { const err = posix.getErrno(posix.close(handle)); - if (err == posix.EINTR) { - continue; - } else { - return; + switch (err) { + posix.EINTR => continue, + else => { + if (emfile_promise_queue.popFirst()) |p| resume p.data; + return; + }, } } } @@ -1753,27 +1776,16 @@ fn testWindowsCmdLine(input_cmd_line: &const u8, expected_args: []const []const assert(it.next(debug.global_allocator) == null); } -test "std.os" { - _ = @import("child_process.zig"); - _ = @import("darwin_errno.zig"); - _ = @import("darwin.zig"); - _ = @import("get_user_id.zig"); - _ = @import("linux/errno.zig"); - //_ = @import("linux_i386.zig"); - _ = @import("linux/x86_64.zig"); - _ = @import("linux/index.zig"); - _ = @import("path.zig"); - _ = @import("windows/index.zig"); - _ = @import("test.zig"); -} - - // TODO make this a build variable that you can set const unexpected_error_tracing = false; +const UnexpectedError = error { + /// The Operating System returned an undocumented error code. + Unexpected, +}; /// Call this when you made a syscall or something that sets errno /// and you get an unexpected error. -pub fn unexpectedErrorPosix(errno: usize) (error{Unexpected}) { +pub fn unexpectedErrorPosix(errno: usize) UnexpectedError { if (unexpected_error_tracing) { debug.warn("unexpected errno: {}\n", errno); debug.dumpCurrentStackTrace(null); @@ -1783,7 +1795,7 @@ pub fn unexpectedErrorPosix(errno: usize) (error{Unexpected}) { /// Call this when you made a windows DLL call or something that does SetLastError /// and you get an unexpected error. -pub fn unexpectedErrorWindows(err: windows.DWORD) (error{Unexpected}) { +pub fn unexpectedErrorWindows(err: windows.DWORD) UnexpectedError { if (unexpected_error_tracing) { debug.warn("unexpected GetLastError(): {}\n", err); debug.dumpCurrentStackTrace(null); @@ -1898,3 +1910,322 @@ pub fn isTty(handle: FileHandle) bool { } } } + +pub const PosixSocketError = error { + /// Permission to create a socket of the specified type and/or + /// pro‐tocol is denied. + PermissionDenied, + + /// The implementation does not support the specified address family. + AddressFamilyNotSupported, + + /// Unknown protocol, or protocol family not available. + ProtocolFamilyNotAvailable, + + /// The per-process limit on the number of open file descriptors has been reached. + ProcessFdQuotaExceeded, + + /// The system-wide limit on the total number of open files has been reached. + SystemFdQuotaExceeded, + + /// Insufficient memory is available. The socket cannot be created until sufficient + /// resources are freed. + SystemResources, + + /// The protocol type or the specified protocol is not supported within this domain. + ProtocolNotSupported, +}; + +pub fn posixSocket(domain: u32, socket_type: u32, protocol: u32) !i32 { + const rc = posix.socket(domain, socket_type, protocol); + const err = posix.getErrno(rc); + switch (err) { + 0 => return i32(rc), + posix.EACCES => return PosixSocketError.PermissionDenied, + posix.EAFNOSUPPORT => return PosixSocketError.AddressFamilyNotSupported, + posix.EINVAL => return PosixSocketError.ProtocolFamilyNotAvailable, + posix.EMFILE => return PosixSocketError.ProcessFdQuotaExceeded, + posix.ENFILE => return PosixSocketError.SystemFdQuotaExceeded, + posix.ENOBUFS, posix.ENOMEM => return PosixSocketError.SystemResources, + posix.EPROTONOSUPPORT => return PosixSocketError.ProtocolNotSupported, + else => return unexpectedErrorPosix(err), + } +} + +pub const PosixBindError = error { + /// The address is protected, and the user is not the superuser. + /// For UNIX domain sockets: Search permission is denied on a component + /// of the path prefix. + AccessDenied, + + /// The given address is already in use, or in the case of Internet domain sockets, + /// The port number was specified as zero in the socket + /// address structure, but, upon attempting to bind to an ephemeral port, it was + /// determined that all port numbers in the ephemeral port range are currently in + /// use. See the discussion of /proc/sys/net/ipv4/ip_local_port_range ip(7). + AddressInUse, + + /// sockfd is not a valid file descriptor. + InvalidFileDescriptor, + + /// The socket is already bound to an address, or addrlen is wrong, or addr is not + /// a valid address for this socket's domain. + InvalidSocketOrAddress, + + /// The file descriptor sockfd does not refer to a socket. + FileDescriptorNotASocket, + + /// A nonexistent interface was requested or the requested address was not local. + AddressNotAvailable, + + /// addr points outside the user's accessible address space. + PageFault, + + /// Too many symbolic links were encountered in resolving addr. + SymLinkLoop, + + /// addr is too long. + NameTooLong, + + /// A component in the directory prefix of the socket pathname does not exist. + FileNotFound, + + /// Insufficient kernel memory was available. + SystemResources, + + /// A component of the path prefix is not a directory. + NotDir, + + /// The socket inode would reside on a read-only filesystem. + ReadOnlyFileSystem, + + Unexpected, +}; + +/// addr is `&const T` where T is one of the sockaddr +pub fn posixBind(fd: i32, addr: &const posix.sockaddr) PosixBindError!void { + const rc = posix.bind(fd, addr, @sizeOf(posix.sockaddr)); + const err = posix.getErrno(rc); + switch (err) { + 0 => return, + posix.EACCES => return PosixBindError.AccessDenied, + posix.EADDRINUSE => return PosixBindError.AddressInUse, + posix.EBADF => return PosixBindError.InvalidFileDescriptor, + posix.EINVAL => return PosixBindError.InvalidSocketOrAddress, + posix.ENOTSOCK => return PosixBindError.FileDescriptorNotASocket, + posix.EADDRNOTAVAIL => return PosixBindError.AddressNotAvailable, + posix.EFAULT => return PosixBindError.PageFault, + posix.ELOOP => return PosixBindError.SymLinkLoop, + posix.ENAMETOOLONG => return PosixBindError.NameTooLong, + posix.ENOENT => return PosixBindError.FileNotFound, + posix.ENOMEM => return PosixBindError.SystemResources, + posix.ENOTDIR => return PosixBindError.NotDir, + posix.EROFS => return PosixBindError.ReadOnlyFileSystem, + else => return unexpectedErrorPosix(err), + } +} + +const PosixListenError = error { + /// Another socket is already listening on the same port. + /// For Internet domain sockets, the socket referred to by sockfd had not previously + /// been bound to an address and, upon attempting to bind it to an ephemeral port, it + /// was determined that all port numbers in the ephemeral port range are currently in + /// use. See the discussion of /proc/sys/net/ipv4/ip_local_port_range in ip(7). + AddressInUse, + + /// The argument sockfd is not a valid file descriptor. + InvalidFileDescriptor, + + /// The file descriptor sockfd does not refer to a socket. + FileDescriptorNotASocket, + + /// The socket is not of a type that supports the listen() operation. + OperationNotSupported, + + Unexpected, +}; + +pub fn posixListen(sockfd: i32, backlog: u32) PosixListenError!void { + const rc = posix.listen(sockfd, backlog); + const err = posix.getErrno(rc); + switch (err) { + 0 => return, + posix.EADDRINUSE => return PosixListenError.AddressInUse, + posix.EBADF => return PosixListenError.InvalidFileDescriptor, + posix.ENOTSOCK => return PosixListenError.FileDescriptorNotASocket, + posix.EOPNOTSUPP => return PosixListenError.OperationNotSupported, + else => return unexpectedErrorPosix(err), + } +} + +pub const PosixAcceptError = error { + /// The socket is marked nonblocking and no connections are present to be accepted. + WouldBlock, + + /// sockfd is not an open file descriptor. + FileDescriptorClosed, + + ConnectionAborted, + + /// The addr argument is not in a writable part of the user address space. + PageFault, + + /// Socket is not listening for connections, or addrlen is invalid (e.g., is negative), + /// or invalid value in flags. + InvalidSyscall, + + /// The per-process limit on the number of open file descriptors has been reached. + ProcessFdQuotaExceeded, + + /// The system-wide limit on the total number of open files has been reached. + SystemFdQuotaExceeded, + + /// Not enough free memory. This often means that the memory allocation is limited + /// by the socket buffer limits, not by the system memory. + SystemResources, + + /// The file descriptor sockfd does not refer to a socket. + FileDescriptorNotASocket, + + /// The referenced socket is not of type SOCK_STREAM. + OperationNotSupported, + + ProtocolFailure, + + /// Firewall rules forbid connection. + BlockedByFirewall, + + Unexpected, +}; + +pub fn posixAccept(fd: i32, addr: &posix.sockaddr, flags: u32) PosixAcceptError!i32 { + while (true) { + var sockaddr_size = u32(@sizeOf(posix.sockaddr)); + const rc = posix.accept4(fd, addr, &sockaddr_size, flags); + const err = posix.getErrno(rc); + switch (err) { + 0 => return i32(rc), + posix.EINTR => continue, + else => return unexpectedErrorPosix(err), + + posix.EAGAIN => return PosixAcceptError.WouldBlock, + posix.EBADF => return PosixAcceptError.FileDescriptorClosed, + posix.ECONNABORTED => return PosixAcceptError.ConnectionAborted, + posix.EFAULT => return PosixAcceptError.PageFault, + posix.EINVAL => return PosixAcceptError.InvalidSyscall, + posix.EMFILE => return PosixAcceptError.ProcessFdQuotaExceeded, + posix.ENFILE => return PosixAcceptError.SystemFdQuotaExceeded, + posix.ENOBUFS, posix.ENOMEM => return PosixAcceptError.SystemResources, + posix.ENOTSOCK => return PosixAcceptError.FileDescriptorNotASocket, + posix.EOPNOTSUPP => return PosixAcceptError.OperationNotSupported, + posix.EPROTO => return PosixAcceptError.ProtocolFailure, + posix.EPERM => return PosixAcceptError.BlockedByFirewall, + } + } +} + +pub const LinuxEpollCreateError = error { + /// Invalid value specified in flags. + InvalidSyscall, + + /// The per-user limit on the number of epoll instances imposed by + /// /proc/sys/fs/epoll/max_user_instances was encountered. See epoll(7) for further + /// details. + /// Or, The per-process limit on the number of open file descriptors has been reached. + ProcessFdQuotaExceeded, + + /// The system-wide limit on the total number of open files has been reached. + SystemFdQuotaExceeded, + + /// There was insufficient memory to create the kernel object. + SystemResources, + + Unexpected, +}; + +pub fn linuxEpollCreate(flags: u32) LinuxEpollCreateError!i32 { + const rc = posix.epoll_create1(flags); + const err = posix.getErrno(rc); + switch (err) { + 0 => return i32(rc), + else => return unexpectedErrorPosix(err), + + posix.EINVAL => return LinuxEpollCreateError.InvalidSyscall, + posix.EMFILE => return LinuxEpollCreateError.ProcessFdQuotaExceeded, + posix.ENFILE => return LinuxEpollCreateError.SystemFdQuotaExceeded, + posix.ENOMEM => return LinuxEpollCreateError.SystemResources, + } +} + +pub const LinuxEpollCtlError = error { + /// epfd or fd is not a valid file descriptor. + InvalidFileDescriptor, + + /// op was EPOLL_CTL_ADD, and the supplied file descriptor fd is already registered + /// with this epoll instance. + FileDescriptorAlreadyPresentInSet, + + /// epfd is not an epoll file descriptor, or fd is the same as epfd, or the requested + /// operation op is not supported by this interface, or + /// An invalid event type was specified along with EPOLLEXCLUSIVE in events, or + /// op was EPOLL_CTL_MOD and events included EPOLLEXCLUSIVE, or + /// op was EPOLL_CTL_MOD and the EPOLLEXCLUSIVE flag has previously been applied to + /// this epfd, fd pair, or + /// EPOLLEXCLUSIVE was specified in event and fd refers to an epoll instance. + InvalidSyscall, + + /// fd refers to an epoll instance and this EPOLL_CTL_ADD operation would result in a + /// circular loop of epoll instances monitoring one another. + OperationCausesCircularLoop, + + /// op was EPOLL_CTL_MOD or EPOLL_CTL_DEL, and fd is not registered with this epoll + /// instance. + FileDescriptorNotRegistered, + + /// There was insufficient memory to handle the requested op control operation. + SystemResources, + + /// The limit imposed by /proc/sys/fs/epoll/max_user_watches was encountered while + /// trying to register (EPOLL_CTL_ADD) a new file descriptor on an epoll instance. + /// See epoll(7) for further details. + UserResourceLimitReached, + + /// The target file fd does not support epoll. This error can occur if fd refers to, + /// for example, a regular file or a directory. + FileDescriptorIncompatibleWithEpoll, + + Unexpected, +}; + +pub fn linuxEpollCtl(epfd: i32, op: u32, fd: i32, event: &linux.epoll_event) LinuxEpollCtlError!void { + const rc = posix.epoll_ctl(epfd, op, fd, event); + const err = posix.getErrno(rc); + switch (err) { + 0 => return, + else => return unexpectedErrorPosix(err), + + posix.EBADF => return LinuxEpollCtlError.InvalidFileDescriptor, + posix.EEXIST => return LinuxEpollCtlError.FileDescriptorAlreadyPresentInSet, + posix.EINVAL => return LinuxEpollCtlError.InvalidSyscall, + posix.ELOOP => return LinuxEpollCtlError.OperationCausesCircularLoop, + posix.ENOENT => return LinuxEpollCtlError.FileDescriptorNotRegistered, + posix.ENOMEM => return LinuxEpollCtlError.SystemResources, + posix.ENOSPC => return LinuxEpollCtlError.UserResourceLimitReached, + posix.EPERM => return LinuxEpollCtlError.FileDescriptorIncompatibleWithEpoll, + } +} + +pub fn linuxEpollWait(epfd: i32, events: []linux.epoll_event, timeout: i32) usize { + while (true) { + const rc = posix.epoll_wait(epfd, &events[0], u32(events.len), timeout); + const err = posix.getErrno(rc); + switch (err) { + 0 => return rc, + posix.EINTR => continue, + posix.EBADF => unreachable, + posix.EFAULT => unreachable, + posix.EINVAL => unreachable, + else => unreachable, + } + } +} diff --git a/std/os/linux/i386.zig b/std/os/linux/i386.zig deleted file mode 100644 index 7450ad34fa..0000000000 --- a/std/os/linux/i386.zig +++ /dev/null @@ -1,505 +0,0 @@ -const std = @import("../../index.zig"); -const linux = std.os.linux; -const socklen_t = linux.socklen_t; -const iovec = linux.iovec; - -pub const SYS_restart_syscall = 0; -pub const SYS_exit = 1; -pub const SYS_fork = 2; -pub const SYS_read = 3; -pub const SYS_write = 4; -pub const SYS_open = 5; -pub const SYS_close = 6; -pub const SYS_waitpid = 7; -pub const SYS_creat = 8; -pub const SYS_link = 9; -pub const SYS_unlink = 10; -pub const SYS_execve = 11; -pub const SYS_chdir = 12; -pub const SYS_time = 13; -pub const SYS_mknod = 14; -pub const SYS_chmod = 15; -pub const SYS_lchown = 16; -pub const SYS_break = 17; -pub const SYS_oldstat = 18; -pub const SYS_lseek = 19; -pub const SYS_getpid = 20; -pub const SYS_mount = 21; -pub const SYS_umount = 22; -pub const SYS_setuid = 23; -pub const SYS_getuid = 24; -pub const SYS_stime = 25; -pub const SYS_ptrace = 26; -pub const SYS_alarm = 27; -pub const SYS_oldfstat = 28; -pub const SYS_pause = 29; -pub const SYS_utime = 30; -pub const SYS_stty = 31; -pub const SYS_gtty = 32; -pub const SYS_access = 33; -pub const SYS_nice = 34; -pub const SYS_ftime = 35; -pub const SYS_sync = 36; -pub const SYS_kill = 37; -pub const SYS_rename = 38; -pub const SYS_mkdir = 39; -pub const SYS_rmdir = 40; -pub const SYS_dup = 41; -pub const SYS_pipe = 42; -pub const SYS_times = 43; -pub const SYS_prof = 44; -pub const SYS_brk = 45; -pub const SYS_setgid = 46; -pub const SYS_getgid = 47; -pub const SYS_signal = 48; -pub const SYS_geteuid = 49; -pub const SYS_getegid = 50; -pub const SYS_acct = 51; -pub const SYS_umount2 = 52; -pub const SYS_lock = 53; -pub const SYS_ioctl = 54; -pub const SYS_fcntl = 55; -pub const SYS_mpx = 56; -pub const SYS_setpgid = 57; -pub const SYS_ulimit = 58; -pub const SYS_oldolduname = 59; -pub const SYS_umask = 60; -pub const SYS_chroot = 61; -pub const SYS_ustat = 62; -pub const SYS_dup2 = 63; -pub const SYS_getppid = 64; -pub const SYS_getpgrp = 65; -pub const SYS_setsid = 66; -pub const SYS_sigaction = 67; -pub const SYS_sgetmask = 68; -pub const SYS_ssetmask = 69; -pub const SYS_setreuid = 70; -pub const SYS_setregid = 71; -pub const SYS_sigsuspend = 72; -pub const SYS_sigpending = 73; -pub const SYS_sethostname = 74; -pub const SYS_setrlimit = 75; -pub const SYS_getrlimit = 76; -pub const SYS_getrusage = 77; -pub const SYS_gettimeofday = 78; -pub const SYS_settimeofday = 79; -pub const SYS_getgroups = 80; -pub const SYS_setgroups = 81; -pub const SYS_select = 82; -pub const SYS_symlink = 83; -pub const SYS_oldlstat = 84; -pub const SYS_readlink = 85; -pub const SYS_uselib = 86; -pub const SYS_swapon = 87; -pub const SYS_reboot = 88; -pub const SYS_readdir = 89; -pub const SYS_mmap = 90; -pub const SYS_munmap = 91; -pub const SYS_truncate = 92; -pub const SYS_ftruncate = 93; -pub const SYS_fchmod = 94; -pub const SYS_fchown = 95; -pub const SYS_getpriority = 96; -pub const SYS_setpriority = 97; -pub const SYS_profil = 98; -pub const SYS_statfs = 99; -pub const SYS_fstatfs = 100; -pub const SYS_ioperm = 101; -pub const SYS_socketcall = 102; -pub const SYS_syslog = 103; -pub const SYS_setitimer = 104; -pub const SYS_getitimer = 105; -pub const SYS_stat = 106; -pub const SYS_lstat = 107; -pub const SYS_fstat = 108; -pub const SYS_olduname = 109; -pub const SYS_iopl = 110; -pub const SYS_vhangup = 111; -pub const SYS_idle = 112; -pub const SYS_vm86old = 113; -pub const SYS_wait4 = 114; -pub const SYS_swapoff = 115; -pub const SYS_sysinfo = 116; -pub const SYS_ipc = 117; -pub const SYS_fsync = 118; -pub const SYS_sigreturn = 119; -pub const SYS_clone = 120; -pub const SYS_setdomainname = 121; -pub const SYS_uname = 122; -pub const SYS_modify_ldt = 123; -pub const SYS_adjtimex = 124; -pub const SYS_mprotect = 125; -pub const SYS_sigprocmask = 126; -pub const SYS_create_module = 127; -pub const SYS_init_module = 128; -pub const SYS_delete_module = 129; -pub const SYS_get_kernel_syms = 130; -pub const SYS_quotactl = 131; -pub const SYS_getpgid = 132; -pub const SYS_fchdir = 133; -pub const SYS_bdflush = 134; -pub const SYS_sysfs = 135; -pub const SYS_personality = 136; -pub const SYS_afs_syscall = 137; -pub const SYS_setfsuid = 138; -pub const SYS_setfsgid = 139; -pub const SYS__llseek = 140; -pub const SYS_getdents = 141; -pub const SYS__newselect = 142; -pub const SYS_flock = 143; -pub const SYS_msync = 144; -pub const SYS_readv = 145; -pub const SYS_writev = 146; -pub const SYS_getsid = 147; -pub const SYS_fdatasync = 148; -pub const SYS__sysctl = 149; -pub const SYS_mlock = 150; -pub const SYS_munlock = 151; -pub const SYS_mlockall = 152; -pub const SYS_munlockall = 153; -pub const SYS_sched_setparam = 154; -pub const SYS_sched_getparam = 155; -pub const SYS_sched_setscheduler = 156; -pub const SYS_sched_getscheduler = 157; -pub const SYS_sched_yield = 158; -pub const SYS_sched_get_priority_max = 159; -pub const SYS_sched_get_priority_min = 160; -pub const SYS_sched_rr_get_interval = 161; -pub const SYS_nanosleep = 162; -pub const SYS_mremap = 163; -pub const SYS_setresuid = 164; -pub const SYS_getresuid = 165; -pub const SYS_vm86 = 166; -pub const SYS_query_module = 167; -pub const SYS_poll = 168; -pub const SYS_nfsservctl = 169; -pub const SYS_setresgid = 170; -pub const SYS_getresgid = 171; -pub const SYS_prctl = 172; -pub const SYS_rt_sigreturn = 173; -pub const SYS_rt_sigaction = 174; -pub const SYS_rt_sigprocmask = 175; -pub const SYS_rt_sigpending = 176; -pub const SYS_rt_sigtimedwait = 177; -pub const SYS_rt_sigqueueinfo = 178; -pub const SYS_rt_sigsuspend = 179; -pub const SYS_pread64 = 180; -pub const SYS_pwrite64 = 181; -pub const SYS_chown = 182; -pub const SYS_getcwd = 183; -pub const SYS_capget = 184; -pub const SYS_capset = 185; -pub const SYS_sigaltstack = 186; -pub const SYS_sendfile = 187; -pub const SYS_getpmsg = 188; -pub const SYS_putpmsg = 189; -pub const SYS_vfork = 190; -pub const SYS_ugetrlimit = 191; -pub const SYS_mmap2 = 192; -pub const SYS_truncate64 = 193; -pub const SYS_ftruncate64 = 194; -pub const SYS_stat64 = 195; -pub const SYS_lstat64 = 196; -pub const SYS_fstat64 = 197; -pub const SYS_lchown32 = 198; -pub const SYS_getuid32 = 199; -pub const SYS_getgid32 = 200; -pub const SYS_geteuid32 = 201; -pub const SYS_getegid32 = 202; -pub const SYS_setreuid32 = 203; -pub const SYS_setregid32 = 204; -pub const SYS_getgroups32 = 205; -pub const SYS_setgroups32 = 206; -pub const SYS_fchown32 = 207; -pub const SYS_setresuid32 = 208; -pub const SYS_getresuid32 = 209; -pub const SYS_setresgid32 = 210; -pub const SYS_getresgid32 = 211; -pub const SYS_chown32 = 212; -pub const SYS_setuid32 = 213; -pub const SYS_setgid32 = 214; -pub const SYS_setfsuid32 = 215; -pub const SYS_setfsgid32 = 216; -pub const SYS_pivot_root = 217; -pub const SYS_mincore = 218; -pub const SYS_madvise = 219; -pub const SYS_madvise1 = 219; -pub const SYS_getdents64 = 220; -pub const SYS_fcntl64 = 221; -pub const SYS_gettid = 224; -pub const SYS_readahead = 225; -pub const SYS_setxattr = 226; -pub const SYS_lsetxattr = 227; -pub const SYS_fsetxattr = 228; -pub const SYS_getxattr = 229; -pub const SYS_lgetxattr = 230; -pub const SYS_fgetxattr = 231; -pub const SYS_listxattr = 232; -pub const SYS_llistxattr = 233; -pub const SYS_flistxattr = 234; -pub const SYS_removexattr = 235; -pub const SYS_lremovexattr = 236; -pub const SYS_fremovexattr = 237; -pub const SYS_tkill = 238; -pub const SYS_sendfile64 = 239; -pub const SYS_futex = 240; -pub const SYS_sched_setaffinity = 241; -pub const SYS_sched_getaffinity = 242; -pub const SYS_set_thread_area = 243; -pub const SYS_get_thread_area = 244; -pub const SYS_io_setup = 245; -pub const SYS_io_destroy = 246; -pub const SYS_io_getevents = 247; -pub const SYS_io_submit = 248; -pub const SYS_io_cancel = 249; -pub const SYS_fadvise64 = 250; -pub const SYS_exit_group = 252; -pub const SYS_lookup_dcookie = 253; -pub const SYS_epoll_create = 254; -pub const SYS_epoll_ctl = 255; -pub const SYS_epoll_wait = 256; -pub const SYS_remap_file_pages = 257; -pub const SYS_set_tid_address = 258; -pub const SYS_timer_create = 259; -pub const SYS_timer_settime = SYS_timer_create+1; -pub const SYS_timer_gettime = SYS_timer_create+2; -pub const SYS_timer_getoverrun = SYS_timer_create+3; -pub const SYS_timer_delete = SYS_timer_create+4; -pub const SYS_clock_settime = SYS_timer_create+5; -pub const SYS_clock_gettime = SYS_timer_create+6; -pub const SYS_clock_getres = SYS_timer_create+7; -pub const SYS_clock_nanosleep = SYS_timer_create+8; -pub const SYS_statfs64 = 268; -pub const SYS_fstatfs64 = 269; -pub const SYS_tgkill = 270; -pub const SYS_utimes = 271; -pub const SYS_fadvise64_64 = 272; -pub const SYS_vserver = 273; -pub const SYS_mbind = 274; -pub const SYS_get_mempolicy = 275; -pub const SYS_set_mempolicy = 276; -pub const SYS_mq_open = 277; -pub const SYS_mq_unlink = SYS_mq_open+1; -pub const SYS_mq_timedsend = SYS_mq_open+2; -pub const SYS_mq_timedreceive = SYS_mq_open+3; -pub const SYS_mq_notify = SYS_mq_open+4; -pub const SYS_mq_getsetattr = SYS_mq_open+5; -pub const SYS_kexec_load = 283; -pub const SYS_waitid = 284; -pub const SYS_add_key = 286; -pub const SYS_request_key = 287; -pub const SYS_keyctl = 288; -pub const SYS_ioprio_set = 289; -pub const SYS_ioprio_get = 290; -pub const SYS_inotify_init = 291; -pub const SYS_inotify_add_watch = 292; -pub const SYS_inotify_rm_watch = 293; -pub const SYS_migrate_pages = 294; -pub const SYS_openat = 295; -pub const SYS_mkdirat = 296; -pub const SYS_mknodat = 297; -pub const SYS_fchownat = 298; -pub const SYS_futimesat = 299; -pub const SYS_fstatat64 = 300; -pub const SYS_unlinkat = 301; -pub const SYS_renameat = 302; -pub const SYS_linkat = 303; -pub const SYS_symlinkat = 304; -pub const SYS_readlinkat = 305; -pub const SYS_fchmodat = 306; -pub const SYS_faccessat = 307; -pub const SYS_pselect6 = 308; -pub const SYS_ppoll = 309; -pub const SYS_unshare = 310; -pub const SYS_set_robust_list = 311; -pub const SYS_get_robust_list = 312; -pub const SYS_splice = 313; -pub const SYS_sync_file_range = 314; -pub const SYS_tee = 315; -pub const SYS_vmsplice = 316; -pub const SYS_move_pages = 317; -pub const SYS_getcpu = 318; -pub const SYS_epoll_pwait = 319; -pub const SYS_utimensat = 320; -pub const SYS_signalfd = 321; -pub const SYS_timerfd_create = 322; -pub const SYS_eventfd = 323; -pub const SYS_fallocate = 324; -pub const SYS_timerfd_settime = 325; -pub const SYS_timerfd_gettime = 326; -pub const SYS_signalfd4 = 327; -pub const SYS_eventfd2 = 328; -pub const SYS_epoll_create1 = 329; -pub const SYS_dup3 = 330; -pub const SYS_pipe2 = 331; -pub const SYS_inotify_init1 = 332; -pub const SYS_preadv = 333; -pub const SYS_pwritev = 334; -pub const SYS_rt_tgsigqueueinfo = 335; -pub const SYS_perf_event_open = 336; -pub const SYS_recvmmsg = 337; -pub const SYS_fanotify_init = 338; -pub const SYS_fanotify_mark = 339; -pub const SYS_prlimit64 = 340; -pub const SYS_name_to_handle_at = 341; -pub const SYS_open_by_handle_at = 342; -pub const SYS_clock_adjtime = 343; -pub const SYS_syncfs = 344; -pub const SYS_sendmmsg = 345; -pub const SYS_setns = 346; -pub const SYS_process_vm_readv = 347; -pub const SYS_process_vm_writev = 348; -pub const SYS_kcmp = 349; -pub const SYS_finit_module = 350; -pub const SYS_sched_setattr = 351; -pub const SYS_sched_getattr = 352; -pub const SYS_renameat2 = 353; -pub const SYS_seccomp = 354; -pub const SYS_getrandom = 355; -pub const SYS_memfd_create = 356; -pub const SYS_bpf = 357; -pub const SYS_execveat = 358; -pub const SYS_socket = 359; -pub const SYS_socketpair = 360; -pub const SYS_bind = 361; -pub const SYS_connect = 362; -pub const SYS_listen = 363; -pub const SYS_accept4 = 364; -pub const SYS_getsockopt = 365; -pub const SYS_setsockopt = 366; -pub const SYS_getsockname = 367; -pub const SYS_getpeername = 368; -pub const SYS_sendto = 369; -pub const SYS_sendmsg = 370; -pub const SYS_recvfrom = 371; -pub const SYS_recvmsg = 372; -pub const SYS_shutdown = 373; -pub const SYS_userfaultfd = 374; -pub const SYS_membarrier = 375; -pub const SYS_mlock2 = 376; - - -pub const O_CREAT = 0o100; -pub const O_EXCL = 0o200; -pub const O_NOCTTY = 0o400; -pub const O_TRUNC = 0o1000; -pub const O_APPEND = 0o2000; -pub const O_NONBLOCK = 0o4000; -pub const O_DSYNC = 0o10000; -pub const O_SYNC = 0o4010000; -pub const O_RSYNC = 0o4010000; -pub const O_DIRECTORY = 0o200000; -pub const O_NOFOLLOW = 0o400000; -pub const O_CLOEXEC = 0o2000000; - -pub const O_ASYNC = 0o20000; -pub const O_DIRECT = 0o40000; -pub const O_LARGEFILE = 0o100000; -pub const O_NOATIME = 0o1000000; -pub const O_PATH = 0o10000000; -pub const O_TMPFILE = 0o20200000; -pub const O_NDELAY = O_NONBLOCK; - -pub const F_DUPFD = 0; -pub const F_GETFD = 1; -pub const F_SETFD = 2; -pub const F_GETFL = 3; -pub const F_SETFL = 4; - -pub const F_SETOWN = 8; -pub const F_GETOWN = 9; -pub const F_SETSIG = 10; -pub const F_GETSIG = 11; - -pub const F_GETLK = 12; -pub const F_SETLK = 13; -pub const F_SETLKW = 14; - -pub const F_SETOWN_EX = 15; -pub const F_GETOWN_EX = 16; - -pub const F_GETOWNER_UIDS = 17; - -pub inline fn syscall0(number: usize) usize { - return asm volatile ("int $0x80" - : [ret] "={eax}" (-> usize) - : [number] "{eax}" (number)); -} - -pub inline fn syscall1(number: usize, arg1: usize) usize { - return asm volatile ("int $0x80" - : [ret] "={eax}" (-> usize) - : [number] "{eax}" (number), - [arg1] "{ebx}" (arg1)); -} - -pub inline fn syscall2(number: usize, arg1: usize, arg2: usize) usize { - return asm volatile ("int $0x80" - : [ret] "={eax}" (-> usize) - : [number] "{eax}" (number), - [arg1] "{ebx}" (arg1), - [arg2] "{ecx}" (arg2)); -} - -pub inline fn syscall3(number: usize, arg1: usize, arg2: usize, arg3: usize) usize { - return asm volatile ("int $0x80" - : [ret] "={eax}" (-> usize) - : [number] "{eax}" (number), - [arg1] "{ebx}" (arg1), - [arg2] "{ecx}" (arg2), - [arg3] "{edx}" (arg3)); -} - -pub inline fn syscall4(number: usize, arg1: usize, arg2: usize, arg3: usize, arg4: usize) usize { - return asm volatile ("int $0x80" - : [ret] "={eax}" (-> usize) - : [number] "{eax}" (number), - [arg1] "{ebx}" (arg1), - [arg2] "{ecx}" (arg2), - [arg3] "{edx}" (arg3), - [arg4] "{esi}" (arg4)); -} - -pub inline fn syscall5(number: usize, arg1: usize, arg2: usize, arg3: usize, - arg4: usize, arg5: usize) usize -{ - return asm volatile ("int $0x80" - : [ret] "={eax}" (-> usize) - : [number] "{eax}" (number), - [arg1] "{ebx}" (arg1), - [arg2] "{ecx}" (arg2), - [arg3] "{edx}" (arg3), - [arg4] "{esi}" (arg4), - [arg5] "{edi}" (arg5)); -} - -pub inline fn syscall6(number: usize, arg1: usize, arg2: usize, arg3: usize, - arg4: usize, arg5: usize, arg6: usize) usize -{ - return asm volatile ("int $0x80" - : [ret] "={eax}" (-> usize) - : [number] "{eax}" (number), - [arg1] "{ebx}" (arg1), - [arg2] "{ecx}" (arg2), - [arg3] "{edx}" (arg3), - [arg4] "{esi}" (arg4), - [arg5] "{edi}" (arg5), - [arg6] "{ebp}" (arg6)); -} - -pub nakedcc fn restore() void { - asm volatile ( - \\popl %%eax - \\movl $119, %%eax - \\int $0x80 - : - : - : "rcx", "r11"); -} - -pub nakedcc fn restore_rt() void { - asm volatile ("int $0x80" - : - : [number] "{eax}" (usize(SYS_rt_sigreturn)) - : "rcx", "r11"); -} diff --git a/std/os/linux/index.zig b/std/os/linux/index.zig index 8fd8bcbe78..602ff66e74 100644 --- a/std/os/linux/index.zig +++ b/std/os/linux/index.zig @@ -101,17 +101,6 @@ pub const SIG_BLOCK = 0; pub const SIG_UNBLOCK = 1; pub const SIG_SETMASK = 2; -pub const SOCK_STREAM = 1; -pub const SOCK_DGRAM = 2; -pub const SOCK_RAW = 3; -pub const SOCK_RDM = 4; -pub const SOCK_SEQPACKET = 5; -pub const SOCK_DCCP = 6; -pub const SOCK_PACKET = 10; -pub const SOCK_CLOEXEC = 0o2000000; -pub const SOCK_NONBLOCK = 0o4000; - - pub const PROTO_ip = 0o000; pub const PROTO_icmp = 0o001; pub const PROTO_igmp = 0o002; @@ -149,6 +138,20 @@ pub const PROTO_encap = 0o142; pub const PROTO_pim = 0o147; pub const PROTO_raw = 0o377; +pub const SHUT_RD = 0; +pub const SHUT_WR = 1; +pub const SHUT_RDWR = 2; + +pub const SOCK_STREAM = 1; +pub const SOCK_DGRAM = 2; +pub const SOCK_RAW = 3; +pub const SOCK_RDM = 4; +pub const SOCK_SEQPACKET = 5; +pub const SOCK_DCCP = 6; +pub const SOCK_PACKET = 10; +pub const SOCK_CLOEXEC = 0o2000000; +pub const SOCK_NONBLOCK = 0o4000; + pub const PF_UNSPEC = 0; pub const PF_LOCAL = 1; pub const PF_UNIX = PF_LOCAL; @@ -193,7 +196,10 @@ pub const PF_CAIF = 37; pub const PF_ALG = 38; pub const PF_NFC = 39; pub const PF_VSOCK = 40; -pub const PF_MAX = 41; +pub const PF_KCM = 41; +pub const PF_QIPCRTR = 42; +pub const PF_SMC = 43; +pub const PF_MAX = 44; pub const AF_UNSPEC = PF_UNSPEC; pub const AF_LOCAL = PF_LOCAL; @@ -239,8 +245,137 @@ pub const AF_CAIF = PF_CAIF; pub const AF_ALG = PF_ALG; pub const AF_NFC = PF_NFC; pub const AF_VSOCK = PF_VSOCK; +pub const AF_KCM = PF_KCM; +pub const AF_QIPCRTR = PF_QIPCRTR; +pub const AF_SMC = PF_SMC; pub const AF_MAX = PF_MAX; +pub const SO_DEBUG = 1; +pub const SO_REUSEADDR = 2; +pub const SO_TYPE = 3; +pub const SO_ERROR = 4; +pub const SO_DONTROUTE = 5; +pub const SO_BROADCAST = 6; +pub const SO_SNDBUF = 7; +pub const SO_RCVBUF = 8; +pub const SO_KEEPALIVE = 9; +pub const SO_OOBINLINE = 10; +pub const SO_NO_CHECK = 11; +pub const SO_PRIORITY = 12; +pub const SO_LINGER = 13; +pub const SO_BSDCOMPAT = 14; +pub const SO_REUSEPORT = 15; +pub const SO_PASSCRED = 16; +pub const SO_PEERCRED = 17; +pub const SO_RCVLOWAT = 18; +pub const SO_SNDLOWAT = 19; +pub const SO_RCVTIMEO = 20; +pub const SO_SNDTIMEO = 21; +pub const SO_ACCEPTCONN = 30; +pub const SO_SNDBUFFORCE = 32; +pub const SO_RCVBUFFORCE = 33; +pub const SO_PROTOCOL = 38; +pub const SO_DOMAIN = 39; + +pub const SO_SECURITY_AUTHENTICATION = 22; +pub const SO_SECURITY_ENCRYPTION_TRANSPORT = 23; +pub const SO_SECURITY_ENCRYPTION_NETWORK = 24; + +pub const SO_BINDTODEVICE = 25; + +pub const SO_ATTACH_FILTER = 26; +pub const SO_DETACH_FILTER = 27; +pub const SO_GET_FILTER = SO_ATTACH_FILTER; + +pub const SO_PEERNAME = 28; +pub const SO_TIMESTAMP = 29; +pub const SCM_TIMESTAMP = SO_TIMESTAMP; + +pub const SO_PEERSEC = 31; +pub const SO_PASSSEC = 34; +pub const SO_TIMESTAMPNS = 35; +pub const SCM_TIMESTAMPNS = SO_TIMESTAMPNS; +pub const SO_MARK = 36; +pub const SO_TIMESTAMPING = 37; +pub const SCM_TIMESTAMPING = SO_TIMESTAMPING; +pub const SO_RXQ_OVFL = 40; +pub const SO_WIFI_STATUS = 41; +pub const SCM_WIFI_STATUS = SO_WIFI_STATUS; +pub const SO_PEEK_OFF = 42; +pub const SO_NOFCS = 43; +pub const SO_LOCK_FILTER = 44; +pub const SO_SELECT_ERR_QUEUE = 45; +pub const SO_BUSY_POLL = 46; +pub const SO_MAX_PACING_RATE = 47; +pub const SO_BPF_EXTENSIONS = 48; +pub const SO_INCOMING_CPU = 49; +pub const SO_ATTACH_BPF = 50; +pub const SO_DETACH_BPF = SO_DETACH_FILTER; +pub const SO_ATTACH_REUSEPORT_CBPF = 51; +pub const SO_ATTACH_REUSEPORT_EBPF = 52; +pub const SO_CNX_ADVICE = 53; +pub const SCM_TIMESTAMPING_OPT_STATS = 54; +pub const SO_MEMINFO = 55; +pub const SO_INCOMING_NAPI_ID = 56; +pub const SO_COOKIE = 57; +pub const SCM_TIMESTAMPING_PKTINFO = 58; +pub const SO_PEERGROUPS = 59; +pub const SO_ZEROCOPY = 60; + +pub const SOL_SOCKET = 1; + +pub const SOL_IP = 0; +pub const SOL_IPV6 = 41; +pub const SOL_ICMPV6 = 58; + +pub const SOL_RAW = 255; +pub const SOL_DECNET = 261; +pub const SOL_X25 = 262; +pub const SOL_PACKET = 263; +pub const SOL_ATM = 264; +pub const SOL_AAL = 265; +pub const SOL_IRDA = 266; +pub const SOL_NETBEUI = 267; +pub const SOL_LLC = 268; +pub const SOL_DCCP = 269; +pub const SOL_NETLINK = 270; +pub const SOL_TIPC = 271; +pub const SOL_RXRPC = 272; +pub const SOL_PPPOL2TP = 273; +pub const SOL_BLUETOOTH = 274; +pub const SOL_PNPIPE = 275; +pub const SOL_RDS = 276; +pub const SOL_IUCV = 277; +pub const SOL_CAIF = 278; +pub const SOL_ALG = 279; +pub const SOL_NFC = 280; +pub const SOL_KCM = 281; +pub const SOL_TLS = 282; + +pub const SOMAXCONN = 128; + +pub const MSG_OOB = 0x0001; +pub const MSG_PEEK = 0x0002; +pub const MSG_DONTROUTE = 0x0004; +pub const MSG_CTRUNC = 0x0008; +pub const MSG_PROXY = 0x0010; +pub const MSG_TRUNC = 0x0020; +pub const MSG_DONTWAIT = 0x0040; +pub const MSG_EOR = 0x0080; +pub const MSG_WAITALL = 0x0100; +pub const MSG_FIN = 0x0200; +pub const MSG_SYN = 0x0400; +pub const MSG_CONFIRM = 0x0800; +pub const MSG_RST = 0x1000; +pub const MSG_ERRQUEUE = 0x2000; +pub const MSG_NOSIGNAL = 0x4000; +pub const MSG_MORE = 0x8000; +pub const MSG_WAITFORONE = 0x10000; +pub const MSG_BATCH = 0x40000; +pub const MSG_ZEROCOPY = 0x4000000; +pub const MSG_FASTOPEN = 0x20000000; +pub const MSG_CMSG_CLOEXEC = 0x40000000; + pub const DT_UNKNOWN = 0; pub const DT_FIFO = 1; pub const DT_CHR = 2; @@ -599,30 +734,27 @@ pub fn sigismember(set: &const sigset_t, sig: u6) bool { return ((*set)[usize(s) / usize.bit_count] & (usize(1) << (s & (usize.bit_count - 1)))) != 0; } - +pub const in_port_t = u16; pub const sa_family_t = u16; pub const socklen_t = u32; -pub const in_addr = u32; -pub const in6_addr = [16]u8; -pub const sockaddr = extern struct { - family: sa_family_t, - port: u16, - data: [12]u8, +pub const sockaddr = extern union { + in: sockaddr_in, + in6: sockaddr_in6, }; pub const sockaddr_in = extern struct { family: sa_family_t, - port: u16, - addr: in_addr, + port: in_port_t, + addr: u32, zero: [8]u8, }; pub const sockaddr_in6 = extern struct { family: sa_family_t, - port: u16, + port: in_port_t, flowinfo: u32, - addr: in6_addr, + addr: [16]u8, scope_id: u32, }; @@ -639,8 +771,8 @@ pub fn getpeername(fd: i32, noalias addr: &sockaddr, noalias len: &socklen_t) us return syscall3(SYS_getpeername, usize(fd), @ptrToInt(addr), @ptrToInt(len)); } -pub fn socket(domain: i32, socket_type: i32, protocol: i32) usize { - return syscall3(SYS_socket, usize(domain), usize(socket_type), usize(protocol)); +pub fn socket(domain: u32, socket_type: u32, protocol: u32) usize { + return syscall3(SYS_socket, domain, socket_type, protocol); } pub fn setsockopt(fd: i32, level: i32, optname: i32, optval: &const u8, optlen: socklen_t) usize { @@ -677,8 +809,8 @@ pub fn bind(fd: i32, addr: &const sockaddr, len: socklen_t) usize { return syscall3(SYS_bind, usize(fd), @ptrToInt(addr), usize(len)); } -pub fn listen(fd: i32, backlog: i32) usize { - return syscall2(SYS_listen, usize(fd), usize(backlog)); +pub fn listen(fd: i32, backlog: u32) usize { + return syscall2(SYS_listen, usize(fd), backlog); } pub fn sendto(fd: i32, buf: &const u8, len: usize, flags: u32, addr: ?&const sockaddr, alen: socklen_t) usize { @@ -697,34 +829,6 @@ pub fn accept4(fd: i32, noalias addr: &sockaddr, noalias len: &socklen_t, flags: return syscall4(SYS_accept4, usize(fd), @ptrToInt(addr), @ptrToInt(len), flags); } -// error NameTooLong; -// error SystemResources; -// error Io; -// -// pub fn if_nametoindex(name: []u8) !u32 { -// var ifr: ifreq = undefined; -// -// if (name.len >= ifr.ifr_name.len) { -// return error.NameTooLong; -// } -// -// const socket_ret = socket(AF_UNIX, SOCK_DGRAM|SOCK_CLOEXEC, 0); -// const socket_err = getErrno(socket_ret); -// if (socket_err > 0) { -// return error.SystemResources; -// } -// const socket_fd = i32(socket_ret); -// @memcpy(&ifr.ifr_name[0], &name[0], name.len); -// ifr.ifr_name[name.len] = 0; -// const ioctl_ret = ioctl(socket_fd, SIOCGIFINDEX, &ifr); -// close(socket_fd); -// const ioctl_err = getErrno(ioctl_ret); -// if (ioctl_err > 0) { -// return error.Io; -// } -// return ifr.ifr_ifindex; -// } - pub fn fstat(fd: i32, stat_buf: &Stat) usize { return syscall2(SYS_fstat, usize(fd), @ptrToInt(stat_buf)); } @@ -749,7 +853,7 @@ pub fn epoll_create1(flags: usize) usize { return syscall1(SYS_epoll_create1, flags); } -pub fn epoll_ctl(epoll_fd: i32, op: i32, fd: i32, ev: &epoll_event) usize { +pub fn epoll_ctl(epoll_fd: i32, op: u32, fd: i32, ev: &epoll_event) usize { return syscall4(SYS_epoll_ctl, usize(epoll_fd), usize(op), usize(fd), @ptrToInt(ev)); } diff --git a/test/cases/coroutines.zig b/test/cases/coroutines.zig index 6d28b98c9d..c4149d2f5f 100644 --- a/test/cases/coroutines.zig +++ b/test/cases/coroutines.zig @@ -133,6 +133,7 @@ fn early_seq(c: u8) void { early_points[early_seq_index] = c; early_seq_index += 1; } +<<<<<<< HEAD test "coro allocation failure" { var failing_allocator = std.debug.FailingAllocator.init(std.debug.global_allocator, 0); @@ -224,3 +225,17 @@ async fn printTrace(p: promise->error!void) void { } }; } + +test "coroutine in a struct field" { + const Foo = struct { + bar: async fn() void, + }; + var foo = Foo { + .bar = simpleAsyncFn2, + }; + cancel try async foo.bar(); +} + +async fn simpleAsyncFn2() void { + suspend; +} From b85ef656caa4d6845996d60a091332e9113d17e2 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 8 Mar 2018 10:07:21 -0500 Subject: [PATCH 42/87] running into the llvm corosplit error again --- src/ir.cpp | 11 +++++++++++ src/ir_print.cpp | 6 ++++++ std/event.zig | 8 ++++---- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/ir.cpp b/src/ir.cpp index a803183579..5fb4a61ef9 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -2647,6 +2647,17 @@ static IrInstruction *ir_build_coro_alloc_helper(IrBuilder *irb, Scope *scope, A return &instruction->base; } +static IrInstruction *ir_build_add_implicit_return_type(IrBuilder *irb, Scope *scope, AstNode *source_node, + IrInstruction *value) +{ + IrInstructionAddImplicitReturnType *instruction = ir_build_instruction(irb, scope, source_node); + instruction->value = value; + + ir_ref_instruction(value, irb->current_basic_block); + + return &instruction->base; +} + static IrInstruction *ir_build_atomic_rmw(IrBuilder *irb, Scope *scope, AstNode *source_node, IrInstruction *operand_type, IrInstruction *ptr, IrInstruction *op, IrInstruction *operand, IrInstruction *ordering, AtomicRmwOp resolved_op, AtomicOrder resolved_ordering) diff --git a/src/ir_print.cpp b/src/ir_print.cpp index 99f79ff75e..33fa3ce138 100644 --- a/src/ir_print.cpp +++ b/src/ir_print.cpp @@ -1146,6 +1146,12 @@ static void ir_print_coro_alloc_helper(IrPrint *irp, IrInstructionCoroAllocHelpe fprintf(irp->f, ")"); } +static void ir_print_add_implicit_return_type(IrPrint *irp, IrInstructionAddImplicitReturnType *instruction) { + fprintf(irp->f, "@addImplicitReturnType("); + ir_print_other_instruction(irp, instruction->value); + fprintf(irp->f, ")"); +} + static void ir_print_atomic_rmw(IrPrint *irp, IrInstructionAtomicRmw *instruction) { fprintf(irp->f, "@atomicRmw("); if (instruction->operand_type != nullptr) { diff --git a/std/event.zig b/std/event.zig index 07fc64293c..7ede8e20d8 100644 --- a/std/event.zig +++ b/std/event.zig @@ -168,10 +168,10 @@ test "listen on a port, send bytes, receive bytes" { var socket = *_socket; // TODO https://github.com/zig-lang/zig/issues/733 defer socket.close(); const next_handler = async errorableHandler(self, _addr, socket) catch |err| switch (err) { - error.OutOfMemory => return, + error.OutOfMemory => @panic("unable to handle connection: out of memory"), }; - (await next_handler) catch |err| switch (err) { - + (await next_handler) catch |err| { + std.debug.panic("unable to handle connection: {}\n", err); }; suspend |p| { cancel p; } } @@ -184,7 +184,7 @@ test "listen on a port, send bytes, receive bytes" { var adapter = std.io.FileOutStream.init(&socket); var stream = &adapter.stream; - try stream.print("hello from server\n") catch unreachable; + try stream.print("hello from server\n"); } }; From 8f4ad95777bb0fd14c095bf626dfc2928a7dddca Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 8 Mar 2018 22:57:12 -0500 Subject: [PATCH 43/87] update what std tests to run --- std/index.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/std/index.zig b/std/index.zig index f8ec787a01..7084c55189 100644 --- a/std/index.zig +++ b/std/index.zig @@ -50,6 +50,7 @@ test "std" { _ = @import("dwarf.zig"); _ = @import("elf.zig"); _ = @import("empty.zig"); + //TODO_ = @import("event.zig"); _ = @import("fmt/index.zig"); _ = @import("hash/index.zig"); _ = @import("io.zig"); @@ -58,7 +59,6 @@ test "std" { _ = @import("mem.zig"); _ = @import("net.zig"); _ = @import("heap.zig"); - _ = @import("net.zig"); _ = @import("os/index.zig"); _ = @import("rand/index.zig"); _ = @import("sort.zig"); From acd8f6ef184c031f672fa6d6a690d831344de4b4 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 8 Apr 2018 18:49:20 -0400 Subject: [PATCH 44/87] fixups from rebase --- src/all_types.hpp | 1 - src/ir.cpp | 31 ++++--------------------------- src/ir_print.cpp | 6 ------ src/parser.cpp | 1 - test/cases/coroutines.zig | 1 - 5 files changed, 4 insertions(+), 36 deletions(-) diff --git a/src/all_types.hpp b/src/all_types.hpp index d434ea187e..d27a5c7a1c 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -2691,7 +2691,6 @@ struct IrInstructionFnProto { IrInstruction **param_types; IrInstruction *align_value; - IrInstruction *async_allocator_type_value; IrInstruction *return_type; IrInstruction *async_allocator_type_value; bool is_var_args; diff --git a/src/ir.cpp b/src/ir.cpp index 5fb4a61ef9..5348e8ba13 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -2153,12 +2153,12 @@ static IrInstruction *ir_build_unwrap_err_payload_from(IrBuilder *irb, IrInstruc } static IrInstruction *ir_build_fn_proto(IrBuilder *irb, Scope *scope, AstNode *source_node, - IrInstruction **param_types, IrInstruction *align_value, IrInstruction *return_type, IrInstruction *async_allocator_type_value, bool is_var_args) + IrInstruction **param_types, IrInstruction *align_value, IrInstruction *return_type, + IrInstruction *async_allocator_type_value, bool is_var_args) { IrInstructionFnProto *instruction = ir_build_instruction(irb, scope, source_node); instruction->param_types = param_types; instruction->align_value = align_value; - instruction->async_allocator_type_value = async_allocator_type_value; instruction->return_type = return_type; instruction->async_allocator_type_value = async_allocator_type_value; instruction->is_var_args = is_var_args; @@ -2647,17 +2647,6 @@ static IrInstruction *ir_build_coro_alloc_helper(IrBuilder *irb, Scope *scope, A return &instruction->base; } -static IrInstruction *ir_build_add_implicit_return_type(IrBuilder *irb, Scope *scope, AstNode *source_node, - IrInstruction *value) -{ - IrInstructionAddImplicitReturnType *instruction = ir_build_instruction(irb, scope, source_node); - instruction->value = value; - - ir_ref_instruction(value, irb->current_basic_block); - - return &instruction->base; -} - static IrInstruction *ir_build_atomic_rmw(IrBuilder *irb, Scope *scope, AstNode *source_node, IrInstruction *operand_type, IrInstruction *ptr, IrInstruction *op, IrInstruction *operand, IrInstruction *ordering, AtomicRmwOp resolved_op, AtomicOrder resolved_ordering) @@ -6052,13 +6041,6 @@ static IrInstruction *ir_gen_fn_proto(IrBuilder *irb, Scope *parent_scope, AstNo return irb->codegen->invalid_instruction; } - IrInstruction *async_allocator_type_value = nullptr; - if (node->data.fn_proto.async_allocator_type != nullptr) { - async_allocator_type_value = ir_gen_node(irb, node->data.fn_proto.async_allocator_type, parent_scope); - if (async_allocator_type_value == irb->codegen->invalid_instruction) - return irb->codegen->invalid_instruction; - } - IrInstruction *return_type; if (node->data.fn_proto.return_var_token == nullptr) { if (node->data.fn_proto.return_type == nullptr) { @@ -6079,7 +6061,8 @@ static IrInstruction *ir_gen_fn_proto(IrBuilder *irb, Scope *parent_scope, AstNo return irb->codegen->invalid_instruction; } - return ir_build_fn_proto(irb, parent_scope, node, param_types, align_value, return_type, async_allocator_type_value, is_var_args); + return ir_build_fn_proto(irb, parent_scope, node, param_types, align_value, return_type, + async_allocator_type_value, is_var_args); } static IrInstruction *ir_gen_cancel(IrBuilder *irb, Scope *parent_scope, AstNode *node) { @@ -16757,12 +16740,6 @@ static TypeTableEntry *ir_analyze_instruction_fn_proto(IrAnalyze *ira, IrInstruc return ira->codegen->builtin_types.entry_invalid; } - if (instruction->async_allocator_type_value != nullptr) { - fn_type_id.async_allocator_type = ir_resolve_type(ira, instruction->async_allocator_type_value->other); - if (type_is_invalid(fn_type_id.async_allocator_type)) - return ira->codegen->builtin_types.entry_invalid; - } - IrInstruction *return_type_value = instruction->return_type->other; fn_type_id.return_type = ir_resolve_type(ira, return_type_value); if (type_is_invalid(fn_type_id.return_type)) diff --git a/src/ir_print.cpp b/src/ir_print.cpp index 33fa3ce138..99f79ff75e 100644 --- a/src/ir_print.cpp +++ b/src/ir_print.cpp @@ -1146,12 +1146,6 @@ static void ir_print_coro_alloc_helper(IrPrint *irp, IrInstructionCoroAllocHelpe fprintf(irp->f, ")"); } -static void ir_print_add_implicit_return_type(IrPrint *irp, IrInstructionAddImplicitReturnType *instruction) { - fprintf(irp->f, "@addImplicitReturnType("); - ir_print_other_instruction(irp, instruction->value); - fprintf(irp->f, ")"); -} - static void ir_print_atomic_rmw(IrPrint *irp, IrInstructionAtomicRmw *instruction) { fprintf(irp->f, "@atomicRmw("); if (instruction->operand_type != nullptr) { diff --git a/src/parser.cpp b/src/parser.cpp index b54a17362e..2bd94033cc 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -1037,7 +1037,6 @@ static AstNode *ast_parse_suffix_op_expr(ParseContext *pc, size_t *token_index, Token *async_token = &pc->tokens->at(*token_index); if (async_token->id == TokenIdKeywordAsync) { - size_t token_index_of_async = *token_index; *token_index += 1; AstNode *allocator_expr_node = nullptr; diff --git a/test/cases/coroutines.zig b/test/cases/coroutines.zig index c4149d2f5f..fbd8f08607 100644 --- a/test/cases/coroutines.zig +++ b/test/cases/coroutines.zig @@ -133,7 +133,6 @@ fn early_seq(c: u8) void { early_points[early_seq_index] = c; early_seq_index += 1; } -<<<<<<< HEAD test "coro allocation failure" { var failing_allocator = std.debug.FailingAllocator.init(std.debug.global_allocator, 0); From cbda0fa78c37dd84821061309469c85a2281174c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 8 Apr 2018 20:08:40 -0400 Subject: [PATCH 45/87] basic tcp server working when used with netcat --- std/event.zig | 22 ++++++++++++++-------- std/net.zig | 6 ++++++ std/os/index.zig | 24 ++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 8 deletions(-) diff --git a/std/event.zig b/std/event.zig index 7ede8e20d8..2ea5d03865 100644 --- a/std/event.zig +++ b/std/event.zig @@ -5,11 +5,12 @@ const mem = std.mem; const posix = std.os.posix; pub const TcpServer = struct { - handleRequestFn: async(&mem.Allocator) fn (&TcpServer, &const std.net.Address, &const std.os.File) void, + handleRequestFn: async<&mem.Allocator> fn (&TcpServer, &const std.net.Address, &const std.os.File) void, loop: &Loop, sockfd: i32, accept_coro: ?promise, + listen_address: std.net.Address, waiting_for_emfile_node: PromiseNode, @@ -28,18 +29,20 @@ pub const TcpServer = struct { .accept_coro = null, .handleRequestFn = undefined, .waiting_for_emfile_node = undefined, + .listen_address = undefined, }; } pub fn listen(self: &TcpServer, address: &const std.net.Address, - handleRequestFn: async(&mem.Allocator) fn (&TcpServer, &const std.net.Address, &const std.os.File)void) !void + handleRequestFn: async<&mem.Allocator> fn (&TcpServer, &const std.net.Address, &const std.os.File)void) !void { self.handleRequestFn = handleRequestFn; try std.os.posixBind(self.sockfd, &address.sockaddr); try std.os.posixListen(self.sockfd, posix.SOMAXCONN); + self.listen_address = std.net.Address.initPosix(try std.os.posixGetSockName(self.sockfd)); - self.accept_coro = try async(self.loop.allocator) (TcpServer.handler)(self); // TODO #817 + self.accept_coro = try async TcpServer.handler(self); errdefer cancel ??self.accept_coro; try self.loop.addFd(self.sockfd, ??self.accept_coro); @@ -60,10 +63,7 @@ pub const TcpServer = struct { posix.SOCK_NONBLOCK | posix.SOCK_CLOEXEC)) |accepted_fd| { var socket = std.os.File.openHandle(accepted_fd); - // TODO #817 - _ = async(self.loop.allocator) (self.handleRequestFn)(self, accepted_addr, - socket) catch |err| switch (err) - { + _ = async self.handleRequestFn(self, accepted_addr, socket) catch |err| switch (err) { error.OutOfMemory => { socket.close(); continue; @@ -161,7 +161,7 @@ test "listen on a port, send bytes, receive bytes" { const Self = this; - async(&mem.Allocator) fn handler(tcp_server: &TcpServer, _addr: &const std.net.Address, + async<&mem.Allocator> fn handler(tcp_server: &TcpServer, _addr: &const std.net.Address, _socket: &const std.os.File) void { const self = @fieldParentPtr(Self, "tcp_server", tcp_server); @@ -198,5 +198,11 @@ test "listen on a port, send bytes, receive bytes" { defer server.tcp_server.deinit(); try server.tcp_server.listen(addr, MyServer.handler); + var stderr_file = try std.io.getStdErr(); + var stderr_stream = &std.io.FileOutStream.init(&stderr_file).stream; + try stderr_stream.print("\nlistening at "); + try server.tcp_server.listen_address.format(stderr_stream); + try stderr_stream.print("\n"); + loop.run(); } diff --git a/std/net.zig b/std/net.zig index 595baae9dd..3dddffda90 100644 --- a/std/net.zig +++ b/std/net.zig @@ -35,6 +35,12 @@ pub const Address = struct { }; } + pub fn initPosix(addr: &const posix.sockaddr) Address { + return Address { + .sockaddr = *addr, + }; + } + pub fn format(self: &const Address, out_stream: var) !void { switch (self.sockaddr.in.family) { posix.AF_INET => { diff --git a/std/os/index.zig b/std/os/index.zig index 6f9db7edcd..e3ab34b355 100644 --- a/std/os/index.zig +++ b/std/os/index.zig @@ -2229,3 +2229,27 @@ pub fn linuxEpollWait(epfd: i32, events: []linux.epoll_event, timeout: i32) usiz } } } + +pub const PosixGetSockNameError = error { + /// Insufficient resources were available in the system to perform the operation. + SystemResources, + + Unexpected, +}; + +pub fn posixGetSockName(sockfd: i32) PosixGetSockNameError!posix.sockaddr { + var addr: posix.sockaddr = undefined; + var addrlen: posix.socklen_t = @sizeOf(posix.sockaddr); + const rc = posix.getsockname(sockfd, &addr, &addrlen); + const err = posix.getErrno(rc); + switch (err) { + 0 => return addr, + else => return unexpectedErrorPosix(err), + + posix.EBADF => unreachable, + posix.EFAULT => unreachable, + posix.EINVAL => unreachable, + posix.ENOTSOCK => unreachable, + posix.ENOBUFS => return PosixGetSockNameError.SystemResources, + } +} From e85a10e9f5f6b17736babd321da8dceb72ea17af Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 9 Apr 2018 00:52:45 -0400 Subject: [PATCH 46/87] async tcp server proof of concept --- CMakeLists.txt | 1 + src/codegen.cpp | 3 + src/ir.cpp | 33 ++++++---- std/c/darwin.zig | 8 +++ std/event.zig | 45 ++++++++++--- std/index.zig | 2 +- std/net.zig | 27 +++++--- std/os/darwin.zig | 3 + std/os/index.zig | 133 +++++++++++++++++++++++++++++++++++++- std/os/linux/index.zig | 12 ++-- test/cases/coroutines.zig | 14 ---- 11 files changed, 231 insertions(+), 50 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c6f169d635..b5171d7266 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -432,6 +432,7 @@ set(ZIG_STD_FILES "dwarf.zig" "elf.zig" "empty.zig" + "event.zig" "fmt/errol/enum3.zig" "fmt/errol/index.zig" "fmt/errol/lookup.zig" diff --git a/src/codegen.cpp b/src/codegen.cpp index be83f68349..2aca143524 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -408,6 +408,9 @@ static uint32_t get_err_ret_trace_arg_index(CodeGen *g, FnTableEntry *fn_table_e if (!g->have_err_ret_tracing) { return UINT32_MAX; } + if (fn_table_entry->type_entry->data.fn.fn_type_id.cc == CallingConventionAsync) { + return 0; + } TypeTableEntry *fn_type = fn_table_entry->type_entry; if (!fn_type_can_fail(&fn_type->data.fn.fn_type_id)) { return UINT32_MAX; diff --git a/src/ir.cpp b/src/ir.cpp index 5348e8ba13..0b072cc696 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -2755,9 +2755,10 @@ static IrInstruction *ir_mark_gen(IrInstruction *instruction) { static bool ir_gen_defers_for_block(IrBuilder *irb, Scope *inner_scope, Scope *outer_scope, bool gen_error_defers) { Scope *scope = inner_scope; + bool is_noreturn = false; while (scope != outer_scope) { if (!scope) - return false; + return is_noreturn; if (scope->id == ScopeIdDefer) { AstNode *defer_node = scope->source_node; @@ -2770,14 +2771,18 @@ static bool ir_gen_defers_for_block(IrBuilder *irb, Scope *inner_scope, Scope *o Scope *defer_expr_scope = defer_node->data.defer.expr_scope; IrInstruction *defer_expr_value = ir_gen_node(irb, defer_expr_node, defer_expr_scope); if (defer_expr_value != irb->codegen->invalid_instruction) { - ir_mark_gen(ir_build_check_statement_is_void(irb, defer_expr_scope, defer_expr_node, defer_expr_value)); + if (defer_expr_value->value.type != nullptr && defer_expr_value->value.type->id == TypeTableEntryIdUnreachable) { + is_noreturn = true; + } else { + ir_mark_gen(ir_build_check_statement_is_void(irb, defer_expr_scope, defer_expr_node, defer_expr_value)); + } } } } scope = scope->parent; } - return true; + return is_noreturn; } static void ir_set_cursor_at_end(IrBuilder *irb, IrBasicBlock *basic_block) { @@ -2936,12 +2941,13 @@ static IrInstruction *ir_gen_return(IrBuilder *irb, Scope *scope, AstNode *node, ir_mark_gen(ir_build_cond_br(irb, scope, node, is_err_val, return_block, continue_block, is_comptime)); ir_set_cursor_at_end_and_append_block(irb, return_block); - ir_gen_defers_for_block(irb, scope, outer_scope, true); - IrInstruction *err_val = ir_build_unwrap_err_code(irb, scope, node, err_union_ptr); - if (irb->codegen->have_err_ret_tracing && !should_inline) { - ir_build_save_err_ret_addr(irb, scope, node); + if (!ir_gen_defers_for_block(irb, scope, outer_scope, true)) { + IrInstruction *err_val = ir_build_unwrap_err_code(irb, scope, node, err_union_ptr); + if (irb->codegen->have_err_ret_tracing && !should_inline) { + ir_build_save_err_ret_addr(irb, scope, node); + } + ir_gen_async_return(irb, scope, node, err_val, false); } - ir_gen_async_return(irb, scope, node, err_val, false); ir_set_cursor_at_end_and_append_block(irb, continue_block); IrInstruction *unwrapped_ptr = ir_build_unwrap_err_payload(irb, scope, node, err_union_ptr, false); @@ -5695,7 +5701,7 @@ static IrInstruction *ir_gen_continue(IrBuilder *irb, Scope *continue_scope, Ast IrBasicBlock *dest_block = loop_scope->continue_block; ir_gen_defers_for_block(irb, continue_scope, dest_block->scope, false); - return ir_build_br(irb, continue_scope, node, dest_block, is_comptime); + return ir_mark_gen(ir_build_br(irb, continue_scope, node, dest_block, is_comptime)); } static IrInstruction *ir_gen_error_type(IrBuilder *irb, Scope *scope, AstNode *node) { @@ -6178,7 +6184,7 @@ static IrInstruction *ir_gen_await_expr(IrBuilder *irb, Scope *parent_scope, Ast ir_set_cursor_at_end_and_append_block(irb, cleanup_block); ir_gen_defers_for_block(irb, parent_scope, outer_scope, true); - ir_build_br(irb, parent_scope, node, irb->exec->coro_final_cleanup_block, const_bool_false); + ir_mark_gen(ir_build_br(irb, parent_scope, node, irb->exec->coro_final_cleanup_block, const_bool_false)); ir_set_cursor_at_end_and_append_block(irb, resume_block); IrInstruction *yes_suspend_result = ir_build_load_ptr(irb, parent_scope, node, my_result_var_ptr); @@ -6254,7 +6260,7 @@ static IrInstruction *ir_gen_suspend(IrBuilder *irb, Scope *parent_scope, AstNod ir_set_cursor_at_end_and_append_block(irb, cleanup_block); ir_gen_defers_for_block(irb, parent_scope, outer_scope, true); - ir_build_br(irb, parent_scope, node, irb->exec->coro_final_cleanup_block, const_bool_false); + ir_mark_gen(ir_build_br(irb, parent_scope, node, irb->exec->coro_final_cleanup_block, const_bool_false)); ir_set_cursor_at_end_and_append_block(irb, resume_block); return ir_build_const_void(irb, parent_scope, node); @@ -16746,6 +16752,11 @@ static TypeTableEntry *ir_analyze_instruction_fn_proto(IrAnalyze *ira, IrInstruc return ira->codegen->builtin_types.entry_invalid; if (fn_type_id.cc == CallingConventionAsync) { + if (instruction->async_allocator_type_value == nullptr) { + ir_add_error(ira, &instruction->base, + buf_sprintf("async fn proto missing allocator type")); + return ira->codegen->builtin_types.entry_invalid; + } IrInstruction *async_allocator_type_value = instruction->async_allocator_type_value->other; fn_type_id.async_allocator_type = ir_resolve_type(ira, async_allocator_type_value); if (type_is_invalid(fn_type_id.async_allocator_type)) diff --git a/std/c/darwin.zig b/std/c/darwin.zig index aa49dfa3df..feb689cdc5 100644 --- a/std/c/darwin.zig +++ b/std/c/darwin.zig @@ -55,3 +55,11 @@ pub const dirent = extern struct { d_type: u8, d_name: u8, // field address is address of first byte of name }; + +pub const sockaddr = extern struct { + sa_len: u8, + sa_family: sa_family_t, + sa_data: [14]u8, +}; + +pub const sa_family_t = u8; diff --git a/std/event.zig b/std/event.zig index 2ea5d03865..bdad7fcc18 100644 --- a/std/event.zig +++ b/std/event.zig @@ -1,4 +1,5 @@ const std = @import("index.zig"); +const builtin = @import("builtin"); const assert = std.debug.assert; const event = this; const mem = std.mem; @@ -38,7 +39,7 @@ pub const TcpServer = struct { { self.handleRequestFn = handleRequestFn; - try std.os.posixBind(self.sockfd, &address.sockaddr); + try std.os.posixBind(self.sockfd, &address.os_addr); try std.os.posixListen(self.sockfd, posix.SOMAXCONN); self.listen_address = std.net.Address.initPosix(try std.os.posixGetSockName(self.sockfd)); @@ -59,7 +60,7 @@ pub const TcpServer = struct { pub async fn handler(self: &TcpServer) void { while (true) { var accepted_addr: std.net.Address = undefined; - if (std.os.posixAccept(self.sockfd, &accepted_addr.sockaddr, + if (std.os.posixAccept(self.sockfd, &accepted_addr.os_addr, posix.SOCK_NONBLOCK | posix.SOCK_CLOEXEC)) |accepted_fd| { var socket = std.os.File.openHandle(accepted_fd); @@ -118,7 +119,7 @@ pub const Loop = struct { pub fn addFd(self: &Loop, fd: i32, prom: promise) !void { var ev = std.os.linux.epoll_event { - .events = std.os.linux.EPOLLIN|std.os.linux.EPOLLET, + .events = std.os.linux.EPOLLIN|std.os.linux.EPOLLOUT|std.os.linux.EPOLLET, .data = std.os.linux.epoll_data { .ptr = @ptrToInt(prom), }, @@ -155,7 +156,24 @@ pub const Loop = struct { } }; +pub async fn connect(loop: &Loop, _address: &const std.net.Address) !std.os.File { + var address = *_address; // TODO https://github.com/zig-lang/zig/issues/733 + + const sockfd = try std.os.posixSocket(posix.AF_INET, posix.SOCK_STREAM|posix.SOCK_CLOEXEC|posix.SOCK_NONBLOCK, posix.PROTO_tcp); + errdefer std.os.close(sockfd); + + try std.os.posixConnectAsync(sockfd, &address.os_addr); + try await try async loop.waitFd(sockfd); + try std.os.posixGetSockOptConnectError(sockfd); + + return std.os.File.openHandle(sockfd); +} + test "listen on a port, send bytes, receive bytes" { + if (builtin.os != builtin.Os.linux) { + // TODO build abstractions for other operating systems + return; + } const MyServer = struct { tcp_server: TcpServer, @@ -198,11 +216,20 @@ test "listen on a port, send bytes, receive bytes" { defer server.tcp_server.deinit(); try server.tcp_server.listen(addr, MyServer.handler); - var stderr_file = try std.io.getStdErr(); - var stderr_stream = &std.io.FileOutStream.init(&stderr_file).stream; - try stderr_stream.print("\nlistening at "); - try server.tcp_server.listen_address.format(stderr_stream); - try stderr_stream.print("\n"); - + const p = try async doAsyncTest(&loop, server.tcp_server.listen_address); + defer cancel p; loop.run(); } + +async fn doAsyncTest(loop: &Loop, address: &const std.net.Address) void { + errdefer @panic("test failure"); + + var socket_file = try await try async event.connect(loop, address); + defer socket_file.close(); + + var buf: [512]u8 = undefined; + const amt_read = try socket_file.read(buf[0..]); + const msg = buf[0..amt_read]; + assert(mem.eql(u8, msg, "hello from server\n")); + loop.stop(); +} diff --git a/std/index.zig b/std/index.zig index 7084c55189..07c4360aab 100644 --- a/std/index.zig +++ b/std/index.zig @@ -50,7 +50,7 @@ test "std" { _ = @import("dwarf.zig"); _ = @import("elf.zig"); _ = @import("empty.zig"); - //TODO_ = @import("event.zig"); + _ = @import("event.zig"); _ = @import("fmt/index.zig"); _ = @import("hash/index.zig"); _ = @import("io.zig"); diff --git a/std/net.zig b/std/net.zig index 3dddffda90..8e1b8d97b2 100644 --- a/std/net.zig +++ b/std/net.zig @@ -1,15 +1,26 @@ const std = @import("index.zig"); +const builtin = @import("builtin"); const assert = std.debug.assert; const net = this; const posix = std.os.posix; const mem = std.mem; +pub const TmpWinAddr = struct { + family: u8, + data: [14]u8, +}; + +pub const OsAddress = switch (builtin.os) { + builtin.Os.windows => TmpWinAddr, + else => posix.sockaddr, +}; + pub const Address = struct { - sockaddr: posix.sockaddr, + os_addr: OsAddress, pub fn initIp4(ip4: u32, port: u16) Address { return Address { - .sockaddr = posix.sockaddr { + .os_addr = posix.sockaddr { .in = posix.sockaddr_in { .family = posix.AF_INET, .port = std.mem.endianSwapIfLe(u16, port), @@ -23,7 +34,7 @@ pub const Address = struct { pub fn initIp6(ip6: &const Ip6Addr, port: u16) Address { return Address { .family = posix.AF_INET6, - .sockaddr = posix.sockaddr { + .os_addr = posix.sockaddr { .in6 = posix.sockaddr_in6 { .family = posix.AF_INET6, .port = std.mem.endianSwapIfLe(u16, port), @@ -37,19 +48,19 @@ pub const Address = struct { pub fn initPosix(addr: &const posix.sockaddr) Address { return Address { - .sockaddr = *addr, + .os_addr = *addr, }; } pub fn format(self: &const Address, out_stream: var) !void { - switch (self.sockaddr.in.family) { + switch (self.os_addr.in.family) { posix.AF_INET => { - const native_endian_port = std.mem.endianSwapIfLe(u16, self.sockaddr.in.port); - const bytes = ([]const u8)((&self.sockaddr.in.addr)[0..1]); + const native_endian_port = std.mem.endianSwapIfLe(u16, self.os_addr.in.port); + const bytes = ([]const u8)((&self.os_addr.in.addr)[0..1]); try out_stream.print("{}.{}.{}.{}:{}", bytes[0], bytes[1], bytes[2], bytes[3], native_endian_port); }, posix.AF_INET6 => { - const native_endian_port = std.mem.endianSwapIfLe(u16, self.sockaddr.in6.port); + const native_endian_port = std.mem.endianSwapIfLe(u16, self.os_addr.in6.port); try out_stream.print("[TODO render ip6 address]:{}", native_endian_port); }, else => try out_stream.write("(unrecognized address family)"), diff --git a/std/os/darwin.zig b/std/os/darwin.zig index f8b1fbed3b..40da55315c 100644 --- a/std/os/darwin.zig +++ b/std/os/darwin.zig @@ -301,6 +301,9 @@ pub const timespec = c.timespec; pub const Stat = c.Stat; pub const dirent = c.dirent; +pub const sa_family_t = c.sa_family_t; +pub const sockaddr = c.sockaddr; + /// Renamed from `sigaction` to `Sigaction` to avoid conflict with the syscall. pub const Sigaction = struct { handler: extern fn(i32)void, diff --git a/std/os/index.zig b/std/os/index.zig index e3ab34b355..b6caed6f53 100644 --- a/std/os/index.zig +++ b/std/os/index.zig @@ -2217,7 +2217,7 @@ pub fn linuxEpollCtl(epfd: i32, op: u32, fd: i32, event: &linux.epoll_event) Lin pub fn linuxEpollWait(epfd: i32, events: []linux.epoll_event, timeout: i32) usize { while (true) { - const rc = posix.epoll_wait(epfd, &events[0], u32(events.len), timeout); + const rc = posix.epoll_wait(epfd, events.ptr, u32(events.len), timeout); const err = posix.getErrno(rc); switch (err) { 0 => return rc, @@ -2253,3 +2253,134 @@ pub fn posixGetSockName(sockfd: i32) PosixGetSockNameError!posix.sockaddr { posix.ENOBUFS => return PosixGetSockNameError.SystemResources, } } + +pub const PosixConnectError = error { + /// For UNIX domain sockets, which are identified by pathname: Write permission is denied on the socket + /// file, or search permission is denied for one of the directories in the path prefix. + /// or + /// The user tried to connect to a broadcast address without having the socket broadcast flag enabled or + /// the connection request failed because of a local firewall rule. + PermissionDenied, + + /// Local address is already in use. + AddressInUse, + + /// (Internet domain sockets) The socket referred to by sockfd had not previously been bound to an + /// address and, upon attempting to bind it to an ephemeral port, it was determined that all port numbers + /// in the ephemeral port range are currently in use. See the discussion of + /// /proc/sys/net/ipv4/ip_local_port_range in ip(7). + AddressNotAvailable, + + /// The passed address didn't have the correct address family in its sa_family field. + AddressFamilyNotSupported, + + /// Insufficient entries in the routing cache. + SystemResources, + + /// A connect() on a stream socket found no one listening on the remote address. + ConnectionRefused, + + /// Network is unreachable. + NetworkUnreachable, + + /// Timeout while attempting connection. The server may be too busy to accept new connections. Note + /// that for IP sockets the timeout may be very long when syncookies are enabled on the server. + ConnectionTimedOut, + + Unexpected, +}; + +pub fn posixConnect(sockfd: i32, sockaddr: &const posix.sockaddr) PosixConnectError!void { + while (true) { + const rc = posix.connect(sockfd, sockaddr, @sizeOf(posix.sockaddr)); + const err = posix.getErrno(rc); + switch (err) { + 0 => return, + else => return unexpectedErrorPosix(err), + + posix.EACCES => return PosixConnectError.PermissionDenied, + posix.EPERM => return PosixConnectError.PermissionDenied, + posix.EADDRINUSE => return PosixConnectError.AddressInUse, + posix.EADDRNOTAVAIL => return PosixConnectError.AddressNotAvailable, + posix.EAFNOSUPPORT => return PosixConnectError.AddressFamilyNotSupported, + posix.EAGAIN => return PosixConnectError.SystemResources, + posix.EALREADY => unreachable, // The socket is nonblocking and a previous connection attempt has not yet been completed. + posix.EBADF => unreachable, // sockfd is not a valid open file descriptor. + posix.ECONNREFUSED => return PosixConnectError.ConnectionRefused, + posix.EFAULT => unreachable, // The socket structure address is outside the user's address space. + posix.EINPROGRESS => unreachable, // The socket is nonblocking and the connection cannot be completed immediately. + posix.EINTR => continue, + posix.EISCONN => unreachable, // The socket is already connected. + posix.ENETUNREACH => return PosixConnectError.NetworkUnreachable, + posix.ENOTSOCK => unreachable, // The file descriptor sockfd does not refer to a socket. + posix.EPROTOTYPE => unreachable, // The socket type does not support the requested communications protocol. + posix.ETIMEDOUT => return PosixConnectError.ConnectionTimedOut, + } + } +} + +/// Same as posixConnect except it is for blocking socket file descriptors. +/// It expects to receive EINPROGRESS. +pub fn posixConnectAsync(sockfd: i32, sockaddr: &const posix.sockaddr) PosixConnectError!void { + while (true) { + const rc = posix.connect(sockfd, sockaddr, @sizeOf(posix.sockaddr)); + const err = posix.getErrno(rc); + switch (err) { + 0, posix.EINPROGRESS => return, + else => return unexpectedErrorPosix(err), + + posix.EACCES => return PosixConnectError.PermissionDenied, + posix.EPERM => return PosixConnectError.PermissionDenied, + posix.EADDRINUSE => return PosixConnectError.AddressInUse, + posix.EADDRNOTAVAIL => return PosixConnectError.AddressNotAvailable, + posix.EAFNOSUPPORT => return PosixConnectError.AddressFamilyNotSupported, + posix.EAGAIN => return PosixConnectError.SystemResources, + posix.EALREADY => unreachable, // The socket is nonblocking and a previous connection attempt has not yet been completed. + posix.EBADF => unreachable, // sockfd is not a valid open file descriptor. + posix.ECONNREFUSED => return PosixConnectError.ConnectionRefused, + posix.EFAULT => unreachable, // The socket structure address is outside the user's address space. + posix.EINTR => continue, + posix.EISCONN => unreachable, // The socket is already connected. + posix.ENETUNREACH => return PosixConnectError.NetworkUnreachable, + posix.ENOTSOCK => unreachable, // The file descriptor sockfd does not refer to a socket. + posix.EPROTOTYPE => unreachable, // The socket type does not support the requested communications protocol. + posix.ETIMEDOUT => return PosixConnectError.ConnectionTimedOut, + } + } +} + +pub fn posixGetSockOptConnectError(sockfd: i32) PosixConnectError!void { + var err_code: i32 = undefined; + var size: u32 = @sizeOf(i32); + const rc = posix.getsockopt(sockfd, posix.SOL_SOCKET, posix.SO_ERROR, @ptrCast(&u8, &err_code), &size); + assert(size == 4); + const err = posix.getErrno(rc); + switch (err) { + 0 => switch (err_code) { + 0 => return, + else => return unexpectedErrorPosix(err), + + posix.EACCES => return PosixConnectError.PermissionDenied, + posix.EPERM => return PosixConnectError.PermissionDenied, + posix.EADDRINUSE => return PosixConnectError.AddressInUse, + posix.EADDRNOTAVAIL => return PosixConnectError.AddressNotAvailable, + posix.EAFNOSUPPORT => return PosixConnectError.AddressFamilyNotSupported, + posix.EAGAIN => return PosixConnectError.SystemResources, + posix.EALREADY => unreachable, // The socket is nonblocking and a previous connection attempt has not yet been completed. + posix.EBADF => unreachable, // sockfd is not a valid open file descriptor. + posix.ECONNREFUSED => return PosixConnectError.ConnectionRefused, + posix.EFAULT => unreachable, // The socket structure address is outside the user's address space. + posix.EISCONN => unreachable, // The socket is already connected. + posix.ENETUNREACH => return PosixConnectError.NetworkUnreachable, + posix.ENOTSOCK => unreachable, // The file descriptor sockfd does not refer to a socket. + posix.EPROTOTYPE => unreachable, // The socket type does not support the requested communications protocol. + posix.ETIMEDOUT => return PosixConnectError.ConnectionTimedOut, + }, + else => return unexpectedErrorPosix(err), + posix.EBADF => unreachable, // The argument sockfd is not a valid file descriptor. + posix.EFAULT => unreachable, // The address pointed to by optval or optlen is not in a valid part of the process address space. + posix.EINVAL => unreachable, + posix.ENOPROTOOPT => unreachable, // The option is unknown at the level indicated. + posix.ENOTSOCK => unreachable, // The file descriptor sockfd does not refer to a socket. + } +} diff --git a/std/os/linux/index.zig b/std/os/linux/index.zig index 602ff66e74..7f27fc83d9 100644 --- a/std/os/linux/index.zig +++ b/std/os/linux/index.zig @@ -775,12 +775,12 @@ pub fn socket(domain: u32, socket_type: u32, protocol: u32) usize { return syscall3(SYS_socket, domain, socket_type, protocol); } -pub fn setsockopt(fd: i32, level: i32, optname: i32, optval: &const u8, optlen: socklen_t) usize { - return syscall5(SYS_setsockopt, usize(fd), usize(level), usize(optname), usize(optval), @ptrToInt(optlen)); +pub fn setsockopt(fd: i32, level: u32, optname: u32, optval: &const u8, optlen: socklen_t) usize { + return syscall5(SYS_setsockopt, usize(fd), level, optname, usize(optval), @ptrToInt(optlen)); } -pub fn getsockopt(fd: i32, level: i32, optname: i32, noalias optval: &u8, noalias optlen: &socklen_t) usize { - return syscall5(SYS_getsockopt, usize(fd), usize(level), usize(optname), @ptrToInt(optval), @ptrToInt(optlen)); +pub fn getsockopt(fd: i32, level: u32, optname: u32, noalias optval: &u8, noalias optlen: &socklen_t) usize { + return syscall5(SYS_getsockopt, usize(fd), level, optname, @ptrToInt(optval), @ptrToInt(optlen)); } pub fn sendmsg(fd: i32, msg: &const msghdr, flags: u32) usize { @@ -833,14 +833,14 @@ pub fn fstat(fd: i32, stat_buf: &Stat) usize { return syscall2(SYS_fstat, usize(fd), @ptrToInt(stat_buf)); } -pub const epoll_data = extern union { +pub const epoll_data = packed union { ptr: usize, fd: i32, @"u32": u32, @"u64": u64, }; -pub const epoll_event = extern struct { +pub const epoll_event = packed struct { events: u32, data: epoll_data, }; diff --git a/test/cases/coroutines.zig b/test/cases/coroutines.zig index fbd8f08607..6d28b98c9d 100644 --- a/test/cases/coroutines.zig +++ b/test/cases/coroutines.zig @@ -224,17 +224,3 @@ async fn printTrace(p: promise->error!void) void { } }; } - -test "coroutine in a struct field" { - const Foo = struct { - bar: async fn() void, - }; - var foo = Foo { - .bar = simpleAsyncFn2, - }; - cancel try async foo.bar(); -} - -async fn simpleAsyncFn2() void { - suspend; -} From e260c8ca632fb2569f99d182dd6d1daea2b6df63 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Mon, 9 Apr 2018 11:11:18 +0200 Subject: [PATCH 47/87] std.zig.parser now parses while loops and labeled break and continue --- std/zig/ast.zig | 62 ++++-- std/zig/parser.zig | 541 +++++++++++++++++++++++++++++++++++---------- 2 files changed, 468 insertions(+), 135 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 0d060eb685..22b14fe363 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -21,6 +21,7 @@ pub const Node = struct { ParamDecl, Block, Payload, + Else, Switch, SwitchCase, SwitchElse, @@ -61,6 +62,7 @@ pub const Node = struct { Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).iterate(index), Id.Block => @fieldParentPtr(NodeBlock, "base", base).iterate(index), Id.Payload => @fieldParentPtr(NodePayload, "base", base).iterate(index), + Id.Else => @fieldParentPtr(NodeSwitch, "base", base).iterate(index), Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).iterate(index), Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).iterate(index), Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).iterate(index), @@ -102,6 +104,7 @@ pub const Node = struct { Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).firstToken(), Id.Block => @fieldParentPtr(NodeBlock, "base", base).firstToken(), Id.Payload => @fieldParentPtr(NodePayload, "base", base).firstToken(), + Id.Else => @fieldParentPtr(NodeSwitch, "base", base).firstToken(), Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).firstToken(), Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).firstToken(), Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).firstToken(), @@ -143,6 +146,7 @@ pub const Node = struct { Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).lastToken(), Id.Block => @fieldParentPtr(NodeBlock, "base", base).lastToken(), Id.Payload => @fieldParentPtr(NodePayload, "base", base).lastToken(), + Id.Else => @fieldParentPtr(NodeElse, "base", base).lastToken(), Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).lastToken(), Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).lastToken(), Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).lastToken(), @@ -578,6 +582,35 @@ pub const NodePayload = struct { } }; +pub const NodeElse = struct { + base: Node, + else_token: Token, + payload: ?&NodePayload, + body: &Node, + + pub fn iterate(self: &NodeElse, index: usize) ?&Node { + var i = index; + + if (self.payload) |payload| { + if (i < 1) return &payload.base; + i -= 1; + } + + if (i < 1) return self.body; + i -= 1; + + return null; + } + + pub fn firstToken(self: &NodeElse) Token { + return self.else_token; + } + + pub fn lastToken(self: &NodeElse) Token { + return self.body.lastToken(); + } +}; + pub const NodeSwitch = struct { base: Node, switch_token: Token, @@ -609,7 +642,7 @@ pub const NodeSwitch = struct { pub const NodeSwitchCase = struct { base: Node, items: ArrayList(&Node), - payload: ?&Node, + payload: ?&NodePayload, expr: &Node, pub fn iterate(self: &NodeSwitchCase, index: usize) ?&Node { @@ -657,18 +690,14 @@ pub const NodeSwitchElse = struct { pub const NodeWhile = struct { base: Node, + label: ?Token, + inline_token: ?Token, while_token: Token, condition: &Node, payload: ?&NodePayload, continue_expr: ?&Node, body: &Node, - @"else": ?Else, - - const Else = struct { - capture: ?&NodeIdentifier, - body: &Node, - }; - + @"else": ?&NodeElse, pub fn iterate(self: &NodeWhile, index: usize) ?&Node { var i = index; @@ -690,12 +719,7 @@ pub const NodeWhile = struct { i -= 1; if (self.@"else") |@"else"| { - if (@"else".capture) |capture| { - if (i < 1) return &capture.base; - i -= 1; - } - - if (i < 1) return @"else".body; + if (i < 1) return &@"else".base; i -= 1; } @@ -703,6 +727,14 @@ pub const NodeWhile = struct { } pub fn firstToken(self: &NodeWhile) Token { + if (self.label) |label| { + return label; + } + + if (self.inline_token) |inline_token| { + return inline_token; + } + return self.while_token; } @@ -749,7 +781,7 @@ pub const NodeInfixOp = struct { BitXor, BoolAnd, BoolOr, - Catch: ?&Node, + Catch: ?&NodePayload, Div, EqualEqual, ErrorUnion, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 0d8b8a3248..7c8fbb4e52 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -116,6 +116,24 @@ pub const Parser = struct { }; } + const LabelCtx = struct { + label: ?Token, + dest_ptr: DestPtr, + }; + + const InlineCtx = struct { + label: ?Token, + inline_token: ?Token, + dest_ptr: DestPtr, + }; + + const LoopCtx = struct { + label: ?Token, + inline_token: ?Token, + loop_token: Token, + dest_ptr: DestPtr, + }; + const State = union(enum) { TopLevel, TopLevelExtern: TopLevelDeclCtx, @@ -137,15 +155,22 @@ pub const Parser = struct { ParamDecl: &ast.NodeFnProto, ParamDeclComma, FnDef: &ast.NodeFnProto, + LabeledExpression: LabelCtx, + Inline: InlineCtx, + While: LoopCtx, + For: LoopCtx, Block: &ast.NodeBlock, + Else: &?&ast.NodeElse, + WhileContinueExpr: &?&ast.Node, Statement: &ast.NodeBlock, + Semicolon: &const &const ast.Node, ExprListItemOrEnd: ExprListCtx, ExprListCommaOrEnd: ExprListCtx, FieldInitListItemOrEnd: ListSave(&ast.NodeFieldInitializer), FieldInitListCommaOrEnd: ListSave(&ast.NodeFieldInitializer), FieldListCommaOrEnd: &ast.NodeContainerDecl, SwitchCaseOrEnd: ListSave(&ast.NodeSwitchCase), - Payload: DestPtr, + Payload: &?&ast.NodePayload, SwitchCaseCommaOrEnd: ListSave(&ast.NodeSwitchCase), SwitchCaseItem: &ArrayList(&ast.Node), SwitchCaseItemCommaOrEnd: &ArrayList(&ast.Node), @@ -157,9 +182,6 @@ pub const Parser = struct { /// "leaked" nodes. TODO: Figure out if it's nessesary to handle leaked nodes. Optional: RevertState, - /// Optional can be reverted by adding the Required state to the stack. - Required, - Expression: DestPtr, RangeExpressionBegin: DestPtr, RangeExpressionEnd: DestPtr, @@ -598,8 +620,7 @@ pub const Parser = struct { continue; }, - State.Optional, - State.Required => { }, + State.Optional => { }, State.Expression => |dest_ptr| { const token = self.getNextToken(); @@ -626,10 +647,51 @@ pub const Parser = struct { continue; }, Token.Id.Keyword_break => { - @panic("TODO: break"); + const label = blk: { + const colon = self.getNextToken(); + if (colon.id != Token.Id.Colon) { + self.putBackToken(colon); + break :blk null; + } + + break :blk (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; + }; + + const node = try self.createControlFlowExpr(arena, token, + ast.NodeControlFlowExpression.Kind { + .Break = label, + } + ); + dest_ptr.store(&node.base); + + stack.append(State { + .Optional = RevertState { + .parser = *self, + .tokenizer = *self.tokenizer, + .ptr = &node.rhs, + } + }) catch unreachable; + try stack.append(State { .Expression = DestPtr { .NullableField = &node.rhs } }); + continue; }, Token.Id.Keyword_continue => { - @panic("TODO: break"); + const label = blk: { + const colon = self.getNextToken(); + if (colon.id != Token.Id.Colon) { + self.putBackToken(colon); + break :blk null; + } + + break :blk (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; + }; + + const node = try self.createControlFlowExpr(arena, token, + ast.NodeControlFlowExpression.Kind { + .Continue = label, + } + ); + dest_ptr.store(&node.base); + continue; }, Token.Id.Keyword_cancel => { @panic("TODO: cancel"); @@ -707,14 +769,7 @@ pub const Parser = struct { stack.append(State { .UnwrapExpressionEnd = dest_ptr }) catch unreachable; try stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }); - try stack.append(State { - .Optional = RevertState { - .tokenizer = *self.tokenizer, - .parser = *self, - .ptr = &node.op.Catch, - } - }); - try stack.append(State { .Payload = DestPtr { .NullableField = &node.op.Catch } }); + try stack.append(State { .Payload = &node.op.Catch }); continue; }, Token.Id.QuestionMarkQuestionMark => { @@ -1370,16 +1425,12 @@ pub const Parser = struct { continue; } - const block = try self.createBlock(arena, (?Token)(token), Token(undefined)); - dest_ptr.store(&block.base); - - stack.append(State { .Block = block }) catch unreachable; - try stack.append(State { - .ExpectTokenSave = ExpectTokenSave { - .id = Token.Id.LBrace, - .ptr = &block.lbrace, + stack.append(State { + .LabeledExpression = LabelCtx { + .label = token, + .dest_ptr = dest_ptr } - }); + }) catch unreachable; continue; }, Token.Id.LBrace => { @@ -1398,26 +1449,31 @@ pub const Parser = struct { Token.Id.Keyword_if => { @panic("TODO: inline if"); }, + Token.Id.Keyword_inline => { + stack.append(State { + .Inline = InlineCtx { + .label = null, + .inline_token = token, + .dest_ptr = dest_ptr, + } + }) catch unreachable; + continue; + }, Token.Id.Keyword_while => { - const node = try arena.create(ast.NodeWhile); - *node = ast.NodeWhile { - .base = self.initNode(ast.Node.Id.While), - .while_token = token, - .condition = undefined, - .payload = null, - .continue_expr = null, - .body = undefined, - .@"else" = null, - }; - dest_ptr.store(&node.base); - - @panic("TODO: inline while"); + stack.append(State { + .While = LoopCtx { + .label = null, + .inline_token = null, + .loop_token = token, + .dest_ptr = dest_ptr, + } + }) catch unreachable; + continue; }, Token.Id.Keyword_for => { @panic("TODO: inline for"); }, Token.Id.Keyword_switch => { - @breakpoint(); const node = try arena.create(ast.NodeSwitch); *node = ast.NodeSwitch { .base = self.initNode(ast.Node.Id.Switch), @@ -1558,15 +1614,7 @@ pub const Parser = struct { try list_state.list.append(node); stack.append(State { .SwitchCaseCommaOrEnd = list_state }) catch unreachable; try stack.append(State { .Expression = DestPtr{ .Field = &node.expr } }); - try stack.append(State { - .Optional = RevertState { - .tokenizer = *self.tokenizer, - .parser = *self, - .ptr = &node.payload, - } - }); - try stack.append(State { .Payload = DestPtr { .NullableField = &node.payload } }); - try stack.append(State.Required); + try stack.append(State { .Payload = &node.payload }); const maybe_else = self.getNextToken(); if (maybe_else.id == Token.Id.Keyword_else) { @@ -1585,32 +1633,6 @@ pub const Parser = struct { } }, - State.Payload => |dest_ptr| { - const lpipe = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue; - - const is_ptr = blk: { - const asterik = self.getNextToken(); - if (asterik.id == Token.Id.Asterisk) { - break :blk true; - } else { - self.putBackToken(asterik); - break :blk false; - } - }; - - const ident = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; - const rpipe = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue; - const node = try arena.create(ast.NodePayload); - *node = ast.NodePayload { - .base = self.initNode(ast.Node.Id.Payload), - .lpipe = lpipe, - .is_ptr = is_ptr, - .symbol = try self.createIdentifier(arena, ident), - .rpipe = rpipe - }; - dest_ptr.store(&node.base); - }, - State.SwitchCaseItem => |case_items| { stack.append(State { .SwitchCaseItemCommaOrEnd = case_items }) catch unreachable; try stack.append(State { .RangeExpressionBegin = DestPtr{ .Field = try case_items.addOne() } }); @@ -1642,6 +1664,68 @@ pub const Parser = struct { continue; }, + State.Else => |dest| { + const else_token = self.getNextToken(); + if (else_token.id != Token.Id.Keyword_else) { + self.putBackToken(else_token); + continue; + } + + const node = try arena.create(ast.NodeElse); + *node = ast.NodeElse { + .base = self.initNode(ast.Node.Id.Else), + .else_token = else_token, + .payload = null, + .body = undefined, + }; + *dest = node; + + stack.append(State { .Expression = DestPtr { .Field = &node.body } }) catch unreachable; + try stack.append(State { .Payload = &node.payload }); + }, + + State.WhileContinueExpr => |dest| { + const colon = self.getNextToken(); + if (colon.id != Token.Id.Colon) { + self.putBackToken(colon); + continue; + } + + _ = (try self.eatToken(&stack, Token.Id.LParen)) ?? continue; + stack.append(State { .ExpectToken = Token.Id.RParen }) catch unreachable; + try stack.append(State { .Expression = DestPtr { .NullableField = dest } }); + }, + + State.Payload => |dest| { + const lpipe = self.getNextToken(); + if (lpipe.id != Token.Id.Pipe) { + self.putBackToken(lpipe); + continue; + } + + const is_ptr = blk: { + const asterik = self.getNextToken(); + if (asterik.id == Token.Id.Asterisk) { + break :blk true; + } else { + self.putBackToken(asterik); + break :blk false; + } + }; + + const ident = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; + const rpipe = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue; + const node = try arena.create(ast.NodePayload); + *node = ast.NodePayload { + .base = self.initNode(ast.Node.Id.Payload), + .lpipe = lpipe, + .is_ptr = is_ptr, + .symbol = try self.createIdentifier(arena, ident), + .rpipe = rpipe + }; + *dest = node; + }, + State.AddrOfModifiers => |addr_of_info| { var token = self.getNextToken(); switch (token.id) { @@ -1803,6 +1887,114 @@ pub const Parser = struct { } }, + State.LabeledExpression => |ctx| { + const token = self.getNextToken(); + switch (token.id) { + Token.Id.LBrace => { + const block = try self.createBlock(arena, (?Token)(ctx.label), token); + ctx.dest_ptr.store(&block.base); + + stack.append(State { .Block = block }) catch unreachable; + continue; + }, + Token.Id.Keyword_while => { + stack.append(State { + .While = LoopCtx { + .label = ctx.label, + .inline_token = null, + .loop_token = token, + .dest_ptr = ctx.dest_ptr, + } + }) catch unreachable; + continue; + }, + Token.Id.Keyword_for => { + stack.append(State { + .For = LoopCtx { + .label = ctx.label, + .inline_token = null, + .loop_token = token, + .dest_ptr = ctx.dest_ptr, + } + }) catch unreachable; + continue; + }, + Token.Id.Keyword_inline => { + stack.append(State { + .Inline = InlineCtx { + .label = ctx.label, + .inline_token = token, + .dest_ptr = ctx.dest_ptr, + } + }) catch unreachable; + continue; + }, + else => { + try self.parseError(&stack, token, "expected 'while', 'for', 'inline' or '{{', found {}", @tagName(token.id)); + continue; + }, + } + }, + + State.Inline => |ctx| { + const token = self.getNextToken(); + switch (token.id) { + Token.Id.Keyword_while => { + stack.append(State { + .While = LoopCtx { + .inline_token = ctx.inline_token, + .label = ctx.label, + .loop_token = token, + .dest_ptr = ctx.dest_ptr, + } + }) catch unreachable; + continue; + }, + Token.Id.Keyword_for => { + stack.append(State { + .For = LoopCtx { + .inline_token = ctx.inline_token, + .label = ctx.label, + .loop_token = token, + .dest_ptr = ctx.dest_ptr, + } + }) catch unreachable; + continue; + }, + else => { + try self.parseError(&stack, token, "expected 'while' or 'for', found {}", @tagName(token.id)); + continue; + }, + } + }, + + State.While => |ctx| { + const node = try arena.create(ast.NodeWhile); + *node = ast.NodeWhile { + .base = self.initNode(ast.Node.Id.While), + .label = ctx.label, + .inline_token = ctx.inline_token, + .while_token = ctx.loop_token, + .condition = undefined, + .payload = null, + .continue_expr = null, + .body = undefined, + .@"else" = null, + }; + ctx.dest_ptr.store(&node.base); + + stack.append(State { .Else = &node.@"else" }) catch unreachable; + try stack.append(State { .Expression = DestPtr { .Field = &node.body } }); + try stack.append(State { .WhileContinueExpr = &node.continue_expr }); + try stack.append(State { .Payload = &node.payload }); + try stack.append(State { .ExpectToken = Token.Id.RParen }); + try stack.append(State { .Expression = DestPtr { .Field = &node.condition } }); + try stack.append(State { .ExpectToken = Token.Id.LParen }); + }, + + State.For => |ctx| { + }, + State.Block => |block| { const token = self.getNextToken(); switch (token.id) { @@ -1841,28 +2033,6 @@ pub const Parser = struct { stack.append(State { .VarDecl = var_decl }) catch unreachable; continue; }, - Token.Id.Identifier => { - const maybe_colon = self.getNextToken(); - if (maybe_colon.id != Token.Id.Colon) { - self.putBackToken(maybe_colon); - self.putBackToken(next); - stack.append(State { .ExpectToken = Token.Id.Semicolon }) catch unreachable; - try stack.append(State { .Expression = DestPtr{.Field = try block.statements.addOne() } }); - continue; - } - - const inner_block = try self.createBlock(arena, (?Token)(next), Token(undefined)); - try block.statements.append(&inner_block.base); - - stack.append(State { .Block = inner_block }) catch unreachable; - try stack.append(State { - .ExpectTokenSave = ExpectTokenSave { - .id = Token.Id.LBrace, - .ptr = &inner_block.lbrace, - } - }); - continue; - }, Token.Id.LBrace => { const inner_block = try self.createBlock(arena, (?Token)(null), next); try block.statements.append(&inner_block.base); @@ -1870,22 +2040,53 @@ pub const Parser = struct { stack.append(State { .Block = inner_block }) catch unreachable; continue; }, - Token.Id.Keyword_suspend, Token.Id.Keyword_if, - Token.Id.Keyword_while, Token.Id.Keyword_for, - Token.Id.Keyword_switch => { - self.putBackToken(next); - stack.append(State { .Expression = DestPtr{.Field = try block.statements.addOne() } }) catch unreachable; - continue; - }, else => { self.putBackToken(next); - stack.append(State { .ExpectToken = Token.Id.Semicolon }) catch unreachable; - try stack.append(State { .Expression = DestPtr{.Field = try block.statements.addOne() } }); + const statememt = try block.statements.addOne(); + stack.append(State { .Semicolon = statememt }) catch unreachable; + try stack.append(State { .Expression = DestPtr{.Field = statememt } }); continue; } } }, + + State.Semicolon => |node_ptr| { + const node = *node_ptr; + switch (node.id) { + ast.Node.Id.Root, + ast.Node.Id.StructField, + ast.Node.Id.UnionTag, + ast.Node.Id.EnumTag, + ast.Node.Id.ParamDecl, + ast.Node.Id.Block, + ast.Node.Id.Payload, + ast.Node.Id.Switch, + ast.Node.Id.SwitchCase, + ast.Node.Id.SwitchElse, + ast.Node.Id.FieldInitializer, + ast.Node.Id.LineComment, + ast.Node.Id.TestDecl => continue, + ast.Node.Id.While => { + const while_node = @fieldParentPtr(ast.NodeWhile, "base", node); + if (while_node.@"else") |@"else"| { + stack.append(State { .Semicolon = &@"else".base }) catch unreachable; + continue; + } + + stack.append(State { .Semicolon = &while_node.body }) catch unreachable; + continue; + }, + ast.Node.Id.Else => { + const else_node = @fieldParentPtr(ast.NodeElse, "base", node); + stack.append(State { .Semicolon = &else_node.body }) catch unreachable; + continue; + }, + else => { + _ = (try self.eatToken(&stack, Token.Id.Semicolon)) ?? continue; + } + } + } } } } @@ -2295,9 +2496,6 @@ pub const Parser = struct { *revert.ptr = null; return; }, - State.Required => { - return error.StateRequired; - }, else => { } } } @@ -2361,6 +2559,7 @@ pub const Parser = struct { Expression: &ast.Node, VarDecl: &ast.NodeVarDecl, Statement: &ast.Node, + Semicolon: &ast.Node, FieldInitializer: &ast.NodeFieldInitializer, PrintIndent, Indent: usize, @@ -2589,7 +2788,7 @@ pub const Parser = struct { if (prefix_op_node.op == ast.NodeInfixOp.InfixOp.Catch) { if (prefix_op_node.op.Catch) |payload| { try stack.append(RenderState { .Text = " " }); - try stack.append(RenderState { .Expression = payload }); + try stack.append(RenderState { .Expression = &payload.base }); } try stack.append(RenderState { .Text = " catch " }); } else { @@ -3013,7 +3212,7 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = switch_case.expr }); if (switch_case.payload) |payload| { try stack.append(RenderState { .Text = " " }); - try stack.append(RenderState { .Expression = payload }); + try stack.append(RenderState { .Expression = &payload.base }); } try stack.append(RenderState { .Text = " => "}); @@ -3032,7 +3231,74 @@ pub const Parser = struct { const switch_else = @fieldParentPtr(ast.NodeSwitchElse, "base", base); try stream.print("{}", self.tokenizer.getTokenSlice(switch_else.token)); }, - ast.Node.Id.While => @panic("TODO: Render while"), + ast.Node.Id.Else => { + const else_node = @fieldParentPtr(ast.NodeElse, "base", base); + try stream.print("{} ", self.tokenizer.getTokenSlice(else_node.else_token)); + + if (else_node.body.id == ast.Node.Id.Block) { + try stack.append(RenderState { .Expression = else_node.body }); + } else { + try stack.append(RenderState { .Indent = indent }); + try stack.append(RenderState { .Expression = else_node.body }); + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { .Indent = indent + indent_delta }); + try stack.append(RenderState { .Text = "\n" }); + } + + if (else_node.payload) |payload| { + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Expression = &payload.base }); + } + }, + ast.Node.Id.While => { + const while_node = @fieldParentPtr(ast.NodeWhile, "base", base); + if (while_node.label) |label| { + try stream.print("{}: ", self.tokenizer.getTokenSlice(label)); + } + + if (while_node.inline_token) |inline_token| { + try stream.print("{} ", self.tokenizer.getTokenSlice(inline_token)); + } + + try stream.print("{} ", self.tokenizer.getTokenSlice(while_node.while_token)); + + if (while_node.@"else") |@"else"| { + try stack.append(RenderState { .Expression = &@"else".base }); + + if (while_node.body.id == ast.Node.Id.Block) { + try stack.append(RenderState { .Text = " " }); + } else { + try stack.append(RenderState { .Text = "\n" }); + } + } + + if (while_node.body.id == ast.Node.Id.Block) { + try stack.append(RenderState { .Expression = while_node.body }); + try stack.append(RenderState { .Text = " " }); + } else { + try stack.append(RenderState { .Indent = indent }); + try stack.append(RenderState { .Expression = while_node.body }); + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { .Indent = indent + indent_delta }); + try stack.append(RenderState { .Text = "\n" }); + } + + if (while_node.continue_expr) |continue_expr| { + try stack.append(RenderState { .Text = ")" }); + try stack.append(RenderState { .Expression = continue_expr }); + try stack.append(RenderState { .Text = ": (" }); + try stack.append(RenderState { .Text = " " }); + } + + if (while_node.payload) |payload| { + try stack.append(RenderState { .Expression = &payload.base }); + try stack.append(RenderState { .Text = " " }); + } + + try stack.append(RenderState { .Text = ")" }); + try stack.append(RenderState { .Expression = while_node.condition }); + try stack.append(RenderState { .Text = "(" }); + }, ast.Node.Id.StructField, ast.Node.Id.UnionTag, @@ -3077,13 +3343,45 @@ pub const Parser = struct { const var_decl = @fieldParentPtr(ast.NodeVarDecl, "base", base); try stack.append(RenderState { .VarDecl = var_decl}); }, - ast.Node.Id.Block, ast.Node.Id.Switch => { - try stack.append(RenderState { .Expression = base}); + else => { + try stack.append(RenderState { .Semicolon = base }); + try stack.append(RenderState { .Expression = base }); + }, + } + }, + RenderState.Semicolon => |base| { + switch (base.id) { + ast.Node.Id.Root, + ast.Node.Id.StructField, + ast.Node.Id.UnionTag, + ast.Node.Id.EnumTag, + ast.Node.Id.ParamDecl, + ast.Node.Id.Block, + ast.Node.Id.Payload, + ast.Node.Id.Switch, + ast.Node.Id.SwitchCase, + ast.Node.Id.SwitchElse, + ast.Node.Id.FieldInitializer, + ast.Node.Id.LineComment, + ast.Node.Id.TestDecl => {}, + ast.Node.Id.While => { + const while_node = @fieldParentPtr(ast.NodeWhile, "base", base); + if (while_node.@"else") |@"else"| { + stack.append(RenderState { .Semicolon = &@"else".base }) catch unreachable; + continue; + } + + stack.append(RenderState { .Semicolon = while_node.body }) catch unreachable; + continue; + }, + ast.Node.Id.Else => { + const else_node = @fieldParentPtr(ast.NodeElse, "base", base); + stack.append(RenderState { .Semicolon = else_node.body }) catch unreachable; + continue; }, else => { - try stack.append(RenderState { .Text = ";"}); - try stack.append(RenderState { .Expression = base}); - }, + try stack.append(RenderState { .Text = ";" }); + } } }, RenderState.Indent => |new_indent| indent = new_indent, @@ -3690,8 +3988,11 @@ test "zig fmt: while" { \\ continue; \\ \\ i = 0; - \\ var j usize = 0; - \\ while (i < 10) : ({ i += 1; j += 1; }) { + \\ var j: usize = 0; + \\ while (i < 10) : ({ + \\ i += 1; + \\ j += 1; + \\ }) { \\ continue; \\ } \\ @@ -3711,7 +4012,7 @@ test "zig fmt: while" { \\ break 7; \\ } else { \\ unreachable; - \\ } + \\ }; \\ \\ var a: error!u8 = 0; \\ while (a) |v| { @@ -3721,7 +4022,7 @@ test "zig fmt: while" { \\ } \\ \\ comptime var k: usize = 0; - \\ inline while (i < 10) (i += 1) + \\ inline while (i < 10) : (i += 1) \\ j += 2; \\} \\ From e24409ebe0f50be9e01810a5f61bb4c09db57d28 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Mon, 9 Apr 2018 11:17:57 +0200 Subject: [PATCH 48/87] std.zig.parser unified code for rendering and parsing semicolon in statements --- std/zig/parser.zig | 109 +++++++++++++++++---------------------------- 1 file changed, 40 insertions(+), 69 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 7c8fbb4e52..0fd757a01f 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -2053,44 +2053,49 @@ pub const Parser = struct { State.Semicolon => |node_ptr| { const node = *node_ptr; - switch (node.id) { - ast.Node.Id.Root, - ast.Node.Id.StructField, - ast.Node.Id.UnionTag, - ast.Node.Id.EnumTag, - ast.Node.Id.ParamDecl, - ast.Node.Id.Block, - ast.Node.Id.Payload, - ast.Node.Id.Switch, - ast.Node.Id.SwitchCase, - ast.Node.Id.SwitchElse, - ast.Node.Id.FieldInitializer, - ast.Node.Id.LineComment, - ast.Node.Id.TestDecl => continue, - ast.Node.Id.While => { - const while_node = @fieldParentPtr(ast.NodeWhile, "base", node); - if (while_node.@"else") |@"else"| { - stack.append(State { .Semicolon = &@"else".base }) catch unreachable; - continue; - } - - stack.append(State { .Semicolon = &while_node.body }) catch unreachable; - continue; - }, - ast.Node.Id.Else => { - const else_node = @fieldParentPtr(ast.NodeElse, "base", node); - stack.append(State { .Semicolon = &else_node.body }) catch unreachable; - continue; - }, - else => { - _ = (try self.eatToken(&stack, Token.Id.Semicolon)) ?? continue; - } + if (requireSemiColon(node)) { + _ = (try self.eatToken(&stack, Token.Id.Semicolon)) ?? continue; } } } } } + fn requireSemiColon(node: &const ast.Node) bool { + var n = node; + while (true) { + switch (n.id) { + ast.Node.Id.Root, + ast.Node.Id.StructField, + ast.Node.Id.UnionTag, + ast.Node.Id.EnumTag, + ast.Node.Id.ParamDecl, + ast.Node.Id.Block, + ast.Node.Id.Payload, + ast.Node.Id.Switch, + ast.Node.Id.SwitchCase, + ast.Node.Id.SwitchElse, + ast.Node.Id.FieldInitializer, + ast.Node.Id.LineComment, + ast.Node.Id.TestDecl => return false, + ast.Node.Id.While => { + const while_node = @fieldParentPtr(ast.NodeWhile, "base", n); + if (while_node.@"else") |@"else"| { + n = @"else".base; + continue; + } + + n = while_node.body; + }, + ast.Node.Id.Else => { + const else_node = @fieldParentPtr(ast.NodeElse, "base", n); + n = else_node.body; + }, + else => return true, + } + } + } + fn commaOrEnd(self: &Parser, stack: &ArrayList(State), end: &const Token.Id, maybe_ptr: ?&Token, state_after_comma: &const State) !void { var token = self.getNextToken(); switch (token.id) { @@ -2559,7 +2564,6 @@ pub const Parser = struct { Expression: &ast.Node, VarDecl: &ast.NodeVarDecl, Statement: &ast.Node, - Semicolon: &ast.Node, FieldInitializer: &ast.NodeFieldInitializer, PrintIndent, Indent: usize, @@ -3344,46 +3348,13 @@ pub const Parser = struct { try stack.append(RenderState { .VarDecl = var_decl}); }, else => { - try stack.append(RenderState { .Semicolon = base }); + if (requireSemiColon(base)) { + try stack.append(RenderState { .Text = ";" }); + } try stack.append(RenderState { .Expression = base }); }, } }, - RenderState.Semicolon => |base| { - switch (base.id) { - ast.Node.Id.Root, - ast.Node.Id.StructField, - ast.Node.Id.UnionTag, - ast.Node.Id.EnumTag, - ast.Node.Id.ParamDecl, - ast.Node.Id.Block, - ast.Node.Id.Payload, - ast.Node.Id.Switch, - ast.Node.Id.SwitchCase, - ast.Node.Id.SwitchElse, - ast.Node.Id.FieldInitializer, - ast.Node.Id.LineComment, - ast.Node.Id.TestDecl => {}, - ast.Node.Id.While => { - const while_node = @fieldParentPtr(ast.NodeWhile, "base", base); - if (while_node.@"else") |@"else"| { - stack.append(RenderState { .Semicolon = &@"else".base }) catch unreachable; - continue; - } - - stack.append(RenderState { .Semicolon = while_node.body }) catch unreachable; - continue; - }, - ast.Node.Id.Else => { - const else_node = @fieldParentPtr(ast.NodeElse, "base", base); - stack.append(RenderState { .Semicolon = else_node.body }) catch unreachable; - continue; - }, - else => { - try stack.append(RenderState { .Text = ";" }); - } - } - }, RenderState.Indent => |new_indent| indent = new_indent, RenderState.PrintIndent => try stream.writeByteNTimes(' ', indent), } From 7dd55a8007c540415d3e704e490609dd86bea924 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Mon, 9 Apr 2018 11:48:25 +0200 Subject: [PATCH 49/87] std.zig.parser now parses for loops --- std/zig/ast.zig | 150 +++++++++++++++++++++++++++++---- std/zig/parser.zig | 202 +++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 319 insertions(+), 33 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 22b14fe363..c97e33ba10 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -20,12 +20,15 @@ pub const Node = struct { FnProto, ParamDecl, Block, - Payload, + ErrorPayload, + ValuePayload, + ValueIndexPayload, Else, Switch, SwitchCase, SwitchElse, While, + For, InfixOp, PrefixOp, SuffixOp, @@ -61,12 +64,15 @@ pub const Node = struct { Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).iterate(index), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).iterate(index), Id.Block => @fieldParentPtr(NodeBlock, "base", base).iterate(index), - Id.Payload => @fieldParentPtr(NodePayload, "base", base).iterate(index), + Id.ErrorPayload => @fieldParentPtr(NodeErrorPayload, "base", base).iterate(index), + Id.ValuePayload => @fieldParentPtr(NodeValuePayload, "base", base).iterate(index), + Id.ValueIndexPayload => @fieldParentPtr(NodeValueIndexPayload, "base", base).iterate(index), Id.Else => @fieldParentPtr(NodeSwitch, "base", base).iterate(index), Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).iterate(index), Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).iterate(index), Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).iterate(index), Id.While => @fieldParentPtr(NodeWhile, "base", base).iterate(index), + Id.For => @fieldParentPtr(NodeFor, "base", base).iterate(index), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).iterate(index), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).iterate(index), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).iterate(index), @@ -103,12 +109,15 @@ pub const Node = struct { Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).firstToken(), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).firstToken(), Id.Block => @fieldParentPtr(NodeBlock, "base", base).firstToken(), - Id.Payload => @fieldParentPtr(NodePayload, "base", base).firstToken(), + Id.ErrorPayload => @fieldParentPtr(NodeErrorPayload, "base", base).firstToken(), + Id.ValuePayload => @fieldParentPtr(NodeValuePayload, "base", base).firstToken(), + Id.ValueIndexPayload => @fieldParentPtr(NodeValueIndexPayload, "base", base).firstToken(), Id.Else => @fieldParentPtr(NodeSwitch, "base", base).firstToken(), Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).firstToken(), Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).firstToken(), Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).firstToken(), Id.While => @fieldParentPtr(NodeWhile, "base", base).firstToken(), + Id.For => @fieldParentPtr(NodeFor, "base", base).firstToken(), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).firstToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).firstToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).firstToken(), @@ -145,12 +154,15 @@ pub const Node = struct { Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).lastToken(), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).lastToken(), Id.Block => @fieldParentPtr(NodeBlock, "base", base).lastToken(), - Id.Payload => @fieldParentPtr(NodePayload, "base", base).lastToken(), + Id.ErrorPayload => @fieldParentPtr(NodeErrorPayload, "base", base).lastToken(), + Id.ValuePayload => @fieldParentPtr(NodeValuePayload, "base", base).lastToken(), + Id.ValueIndexPayload => @fieldParentPtr(NodeValueIndexPayload, "base", base).lastToken(), Id.Else => @fieldParentPtr(NodeElse, "base", base).lastToken(), Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).lastToken(), Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).lastToken(), Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).lastToken(), Id.While => @fieldParentPtr(NodeWhile, "base", base).lastToken(), + Id.For => @fieldParentPtr(NodeFor, "base", base).lastToken(), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).lastToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).lastToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).lastToken(), @@ -557,27 +569,82 @@ pub const NodeBlock = struct { } }; -pub const NodePayload = struct { +pub const NodeErrorPayload = struct { base: Node, lpipe: Token, - is_ptr: bool, - symbol: &NodeIdentifier, + error_symbol: &NodeIdentifier, rpipe: Token, - pub fn iterate(self: &NodePayload, index: usize) ?&Node { + pub fn iterate(self: &NodeErrorPayload, index: usize) ?&Node { var i = index; - if (i < 1) return &self.symbol.base; + if (i < 1) return &self.error_symbol.base; i -= 1; return null; } - pub fn firstToken(self: &NodePayload) Token { + pub fn firstToken(self: &NodeErrorPayload) Token { return self.lpipe; } - pub fn lastToken(self: &NodePayload) Token { + pub fn lastToken(self: &NodeErrorPayload) Token { + return self.rpipe; + } +}; + +pub const NodeValuePayload = struct { + base: Node, + lpipe: Token, + is_ptr: bool, + value_symbol: &NodeIdentifier, + rpipe: Token, + + pub fn iterate(self: &NodeValuePayload, index: usize) ?&Node { + var i = index; + + if (i < 1) return &self.value_symbol.base; + i -= 1; + + return null; + } + + pub fn firstToken(self: &NodeValuePayload) Token { + return self.lpipe; + } + + pub fn lastToken(self: &NodeValuePayload) Token { + return self.rpipe; + } +}; + +pub const NodeValueIndexPayload = struct { + base: Node, + lpipe: Token, + is_ptr: bool, + value_symbol: &NodeIdentifier, + index_symbol: ?&NodeIdentifier, + rpipe: Token, + + pub fn iterate(self: &NodeValueIndexPayload, index: usize) ?&Node { + var i = index; + + if (i < 1) return &self.value_symbol.base; + i -= 1; + + if (self.index_symbol) |index_symbol| { + if (i < 1) return &index_symbol.base; + i -= 1; + } + + return null; + } + + pub fn firstToken(self: &NodeValueIndexPayload) Token { + return self.lpipe; + } + + pub fn lastToken(self: &NodeValueIndexPayload) Token { return self.rpipe; } }; @@ -585,7 +652,7 @@ pub const NodePayload = struct { pub const NodeElse = struct { base: Node, else_token: Token, - payload: ?&NodePayload, + payload: ?&NodeErrorPayload, body: &Node, pub fn iterate(self: &NodeElse, index: usize) ?&Node { @@ -642,7 +709,7 @@ pub const NodeSwitch = struct { pub const NodeSwitchCase = struct { base: Node, items: ArrayList(&Node), - payload: ?&NodePayload, + payload: ?&NodeValuePayload, expr: &Node, pub fn iterate(self: &NodeSwitchCase, index: usize) ?&Node { @@ -694,7 +761,7 @@ pub const NodeWhile = struct { inline_token: ?Token, while_token: Token, condition: &Node, - payload: ?&NodePayload, + payload: ?&NodeValuePayload, continue_expr: ?&Node, body: &Node, @"else": ?&NodeElse, @@ -747,6 +814,59 @@ pub const NodeWhile = struct { } }; +pub const NodeFor = struct { + base: Node, + label: ?Token, + inline_token: ?Token, + for_token: Token, + array_expr: &Node, + payload: ?&NodeValueIndexPayload, + body: &Node, + @"else": ?&NodeElse, + + pub fn iterate(self: &NodeFor, index: usize) ?&Node { + var i = index; + + if (i < 1) return self.array_expr; + i -= 1; + + if (self.payload) |payload| { + if (i < 1) return &payload.base; + i -= 1; + } + + if (i < 1) return self.body; + i -= 1; + + if (self.@"else") |@"else"| { + if (i < 1) return &@"else".base; + i -= 1; + } + + return null; + } + + pub fn firstToken(self: &NodeFor) Token { + if (self.label) |label| { + return label; + } + + if (self.inline_token) |inline_token| { + return inline_token; + } + + return self.for_token; + } + + pub fn lastToken(self: &NodeFor) Token { + if (self.@"else") |@"else"| { + return @"else".body.lastToken(); + } + + return self.body.lastToken(); + } +}; + pub const NodeInfixOp = struct { base: Node, op_token: Token, @@ -781,7 +901,7 @@ pub const NodeInfixOp = struct { BitXor, BoolAnd, BoolOr, - Catch: ?&NodePayload, + Catch: ?&NodeErrorPayload, Div, EqualEqual, ErrorUnion, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 0fd757a01f..2b2afaeabb 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -170,7 +170,9 @@ pub const Parser = struct { FieldInitListCommaOrEnd: ListSave(&ast.NodeFieldInitializer), FieldListCommaOrEnd: &ast.NodeContainerDecl, SwitchCaseOrEnd: ListSave(&ast.NodeSwitchCase), - Payload: &?&ast.NodePayload, + ErrorPayload: &?&ast.NodeErrorPayload, + ValuePayload: &?&ast.NodeValuePayload, + ValueIndexPayload: &?&ast.NodeValueIndexPayload, SwitchCaseCommaOrEnd: ListSave(&ast.NodeSwitchCase), SwitchCaseItem: &ArrayList(&ast.Node), SwitchCaseItemCommaOrEnd: &ArrayList(&ast.Node), @@ -769,7 +771,7 @@ pub const Parser = struct { stack.append(State { .UnwrapExpressionEnd = dest_ptr }) catch unreachable; try stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }); - try stack.append(State { .Payload = &node.op.Catch }); + try stack.append(State { .ErrorPayload = &node.op.Catch }); continue; }, Token.Id.QuestionMarkQuestionMark => { @@ -1471,7 +1473,15 @@ pub const Parser = struct { continue; }, Token.Id.Keyword_for => { - @panic("TODO: inline for"); + stack.append(State { + .For = LoopCtx { + .label = null, + .inline_token = null, + .loop_token = token, + .dest_ptr = dest_ptr, + } + }) catch unreachable; + continue; }, Token.Id.Keyword_switch => { const node = try arena.create(ast.NodeSwitch); @@ -1614,7 +1624,7 @@ pub const Parser = struct { try list_state.list.append(node); stack.append(State { .SwitchCaseCommaOrEnd = list_state }) catch unreachable; try stack.append(State { .Expression = DestPtr{ .Field = &node.expr } }); - try stack.append(State { .Payload = &node.payload }); + try stack.append(State { .ValuePayload = &node.payload }); const maybe_else = self.getNextToken(); if (maybe_else.id == Token.Id.Keyword_else) { @@ -1681,7 +1691,7 @@ pub const Parser = struct { *dest = node; stack.append(State { .Expression = DestPtr { .Field = &node.body } }) catch unreachable; - try stack.append(State { .Payload = &node.payload }); + try stack.append(State { .ErrorPayload = &node.payload }); }, State.WhileContinueExpr => |dest| { @@ -1696,7 +1706,26 @@ pub const Parser = struct { try stack.append(State { .Expression = DestPtr { .NullableField = dest } }); }, - State.Payload => |dest| { + State.ErrorPayload => |dest| { + const lpipe = self.getNextToken(); + if (lpipe.id != Token.Id.Pipe) { + self.putBackToken(lpipe); + continue; + } + + const error_symbol = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; + const rpipe = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue; + const node = try arena.create(ast.NodeErrorPayload); + *node = ast.NodeErrorPayload { + .base = self.initNode(ast.Node.Id.ErrorPayload), + .lpipe = lpipe, + .error_symbol = try self.createIdentifier(arena, error_symbol), + .rpipe = rpipe + }; + *dest = node; + }, + + State.ValuePayload => |dest| { const lpipe = self.getNextToken(); if (lpipe.id != Token.Id.Pipe) { self.putBackToken(lpipe); @@ -1713,14 +1742,56 @@ pub const Parser = struct { } }; - const ident = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; + const value_symbol = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; const rpipe = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue; - const node = try arena.create(ast.NodePayload); - *node = ast.NodePayload { - .base = self.initNode(ast.Node.Id.Payload), + const node = try arena.create(ast.NodeValuePayload); + *node = ast.NodeValuePayload { + .base = self.initNode(ast.Node.Id.ValuePayload), .lpipe = lpipe, .is_ptr = is_ptr, - .symbol = try self.createIdentifier(arena, ident), + .value_symbol = try self.createIdentifier(arena, value_symbol), + .rpipe = rpipe + }; + *dest = node; + }, + + State.ValueIndexPayload => |dest| { + const lpipe = self.getNextToken(); + if (lpipe.id != Token.Id.Pipe) { + self.putBackToken(lpipe); + continue; + } + + const is_ptr = blk: { + const asterik = self.getNextToken(); + if (asterik.id == Token.Id.Asterisk) { + break :blk true; + } else { + self.putBackToken(asterik); + break :blk false; + } + }; + + const value_symbol = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; + const index_symbol = blk: { + const comma = self.getNextToken(); + if (comma.id != Token.Id.Comma) { + self.putBackToken(comma); + break :blk null; + } + + const symbol = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; + break :blk try self.createIdentifier(arena, symbol); + }; + + const rpipe = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue; + const node = try arena.create(ast.NodeValueIndexPayload); + *node = ast.NodeValueIndexPayload { + .base = self.initNode(ast.Node.Id.ValueIndexPayload), + .lpipe = lpipe, + .is_ptr = is_ptr, + .value_symbol = try self.createIdentifier(arena, value_symbol), + .index_symbol = index_symbol, .rpipe = rpipe }; *dest = node; @@ -1986,13 +2057,32 @@ pub const Parser = struct { stack.append(State { .Else = &node.@"else" }) catch unreachable; try stack.append(State { .Expression = DestPtr { .Field = &node.body } }); try stack.append(State { .WhileContinueExpr = &node.continue_expr }); - try stack.append(State { .Payload = &node.payload }); + try stack.append(State { .ValuePayload = &node.payload }); try stack.append(State { .ExpectToken = Token.Id.RParen }); try stack.append(State { .Expression = DestPtr { .Field = &node.condition } }); try stack.append(State { .ExpectToken = Token.Id.LParen }); }, State.For => |ctx| { + const node = try arena.create(ast.NodeFor); + *node = ast.NodeFor { + .base = self.initNode(ast.Node.Id.For), + .label = ctx.label, + .inline_token = ctx.inline_token, + .for_token = ctx.loop_token, + .array_expr = undefined, + .payload = null, + .body = undefined, + .@"else" = null, + }; + ctx.dest_ptr.store(&node.base); + + stack.append(State { .Else = &node.@"else" }) catch unreachable; + try stack.append(State { .Expression = DestPtr { .Field = &node.body } }); + try stack.append(State { .ValueIndexPayload = &node.payload }); + try stack.append(State { .ExpectToken = Token.Id.RParen }); + try stack.append(State { .Expression = DestPtr { .Field = &node.array_expr } }); + try stack.append(State { .ExpectToken = Token.Id.LParen }); }, State.Block => |block| { @@ -2071,7 +2161,9 @@ pub const Parser = struct { ast.Node.Id.EnumTag, ast.Node.Id.ParamDecl, ast.Node.Id.Block, - ast.Node.Id.Payload, + ast.Node.Id.ErrorPayload, + ast.Node.Id.ValuePayload, + ast.Node.Id.ValueIndexPayload, ast.Node.Id.Switch, ast.Node.Id.SwitchCase, ast.Node.Id.SwitchElse, @@ -2087,6 +2179,15 @@ pub const Parser = struct { n = while_node.body; }, + ast.Node.Id.For => { + const for_node = @fieldParentPtr(ast.NodeFor, "base", n); + if (for_node.@"else") |@"else"| { + n = @"else".base; + continue; + } + + n = for_node.body; + }, ast.Node.Id.Else => { const else_node = @fieldParentPtr(ast.NodeElse, "base", n); n = else_node.body; @@ -2982,10 +3083,33 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = rhs }); } }, - ast.Node.Id.Payload => { - const payload = @fieldParentPtr(ast.NodePayload, "base", base); + ast.Node.Id.ErrorPayload => { + const payload = @fieldParentPtr(ast.NodeErrorPayload, "base", base); try stack.append(RenderState { .Text = "|"}); - try stack.append(RenderState { .Expression = &payload.symbol.base }); + try stack.append(RenderState { .Expression = &payload.error_symbol.base }); + try stack.append(RenderState { .Text = "|"}); + }, + ast.Node.Id.ValuePayload => { + const payload = @fieldParentPtr(ast.NodeValuePayload, "base", base); + try stack.append(RenderState { .Text = "|"}); + try stack.append(RenderState { .Expression = &payload.value_symbol.base }); + + if (payload.is_ptr) { + try stack.append(RenderState { .Text = "*"}); + } + + try stack.append(RenderState { .Text = "|"}); + }, + ast.Node.Id.ValueIndexPayload => { + const payload = @fieldParentPtr(ast.NodeValueIndexPayload, "base", base); + try stack.append(RenderState { .Text = "|"}); + + if (payload.index_symbol) |index_symbol| { + try stack.append(RenderState { .Expression = &index_symbol.base }); + try stack.append(RenderState { .Text = ", "}); + } + + try stack.append(RenderState { .Expression = &payload.value_symbol.base }); if (payload.is_ptr) { try stack.append(RenderState { .Text = "*"}); @@ -3303,6 +3427,48 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = while_node.condition }); try stack.append(RenderState { .Text = "(" }); }, + ast.Node.Id.For => { + const for_node = @fieldParentPtr(ast.NodeFor, "base", base); + if (for_node.label) |label| { + try stream.print("{}: ", self.tokenizer.getTokenSlice(label)); + } + + if (for_node.inline_token) |inline_token| { + try stream.print("{} ", self.tokenizer.getTokenSlice(inline_token)); + } + + try stream.print("{} ", self.tokenizer.getTokenSlice(for_node.for_token)); + + if (for_node.@"else") |@"else"| { + try stack.append(RenderState { .Expression = &@"else".base }); + + if (for_node.body.id == ast.Node.Id.Block) { + try stack.append(RenderState { .Text = " " }); + } else { + try stack.append(RenderState { .Text = "\n" }); + } + } + + if (for_node.body.id == ast.Node.Id.Block) { + try stack.append(RenderState { .Expression = for_node.body }); + try stack.append(RenderState { .Text = " " }); + } else { + try stack.append(RenderState { .Indent = indent }); + try stack.append(RenderState { .Expression = for_node.body }); + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { .Indent = indent + indent_delta }); + try stack.append(RenderState { .Text = "\n" }); + } + + if (for_node.payload) |payload| { + try stack.append(RenderState { .Expression = &payload.base }); + try stack.append(RenderState { .Text = " " }); + } + + try stack.append(RenderState { .Text = ")" }); + try stack.append(RenderState { .Expression = for_node.array_expr }); + try stack.append(RenderState { .Text = "(" }); + }, ast.Node.Id.StructField, ast.Node.Id.UnionTag, @@ -4022,10 +4188,10 @@ test "zig fmt: for" { \\ continue; \\ \\ const res = for (a) |v, i| { - \\ breal v; + \\ break v; \\ } else { \\ unreachable; - \\ } + \\ }; \\ \\ var num: usize = 0; \\ inline for (a) |v, i| { From c19f5a2356b441e3dde95e85e029fec6ef4f836d Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Mon, 9 Apr 2018 12:51:18 +0200 Subject: [PATCH 50/87] std.zig.parser now parses if statements --- std/zig/ast.zig | 47 +++++++++++++++++++++ std/zig/parser.zig | 101 ++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 137 insertions(+), 11 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index c97e33ba10..9e077a3567 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -29,6 +29,7 @@ pub const Node = struct { SwitchElse, While, For, + If, InfixOp, PrefixOp, SuffixOp, @@ -73,6 +74,7 @@ pub const Node = struct { Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).iterate(index), Id.While => @fieldParentPtr(NodeWhile, "base", base).iterate(index), Id.For => @fieldParentPtr(NodeFor, "base", base).iterate(index), + Id.If => @fieldParentPtr(NodeIf, "base", base).iterate(index), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).iterate(index), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).iterate(index), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).iterate(index), @@ -118,6 +120,7 @@ pub const Node = struct { Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).firstToken(), Id.While => @fieldParentPtr(NodeWhile, "base", base).firstToken(), Id.For => @fieldParentPtr(NodeFor, "base", base).firstToken(), + Id.If => @fieldParentPtr(NodeIf, "base", base).firstToken(), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).firstToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).firstToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).firstToken(), @@ -163,6 +166,7 @@ pub const Node = struct { Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).lastToken(), Id.While => @fieldParentPtr(NodeWhile, "base", base).lastToken(), Id.For => @fieldParentPtr(NodeFor, "base", base).lastToken(), + Id.If => @fieldParentPtr(NodeIf, "base", base).lastToken(), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).lastToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).lastToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).lastToken(), @@ -867,6 +871,49 @@ pub const NodeFor = struct { } }; +pub const NodeIf = struct { + base: Node, + if_token: Token, + condition: &Node, + payload: ?&NodeValuePayload, + body: &Node, + @"else": ?&NodeElse, + + pub fn iterate(self: &NodeIf, index: usize) ?&Node { + var i = index; + + if (i < 1) return self.condition; + i -= 1; + + if (self.payload) |payload| { + if (i < 1) return &payload.base; + i -= 1; + } + + if (i < 1) return self.body; + i -= 1; + + if (self.@"else") |@"else"| { + if (i < 1) return &@"else".base; + i -= 1; + } + + return null; + } + + pub fn firstToken(self: &NodeIf) Token { + return self.if_token; + } + + pub fn lastToken(self: &NodeIf) Token { + if (self.@"else") |@"else"| { + return @"else".body.lastToken(); + } + + return self.body.lastToken(); + } +}; + pub const NodeInfixOp = struct { base: Node, op_token: Token, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 2b2afaeabb..09383a313b 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -1449,7 +1449,24 @@ pub const Parser = struct { @panic("TODO: inline asm"); }, Token.Id.Keyword_if => { - @panic("TODO: inline if"); + const node = try arena.create(ast.NodeIf); + *node = ast.NodeIf { + .base = self.initNode(ast.Node.Id.If), + .if_token = token, + .condition = undefined, + .payload = null, + .body = undefined, + .@"else" = null, + }; + dest_ptr.store(&node.base); + + stack.append(State { .Else = &node.@"else" }) catch unreachable; + try stack.append(State { .Expression = DestPtr { .Field = &node.body } }); + try stack.append(State { .ValuePayload = &node.payload }); + try stack.append(State { .ExpectToken = Token.Id.RParen }); + try stack.append(State { .Expression = DestPtr { .Field = &node.condition } }); + try stack.append(State { .ExpectToken = Token.Id.LParen }); + continue; }, Token.Id.Keyword_inline => { stack.append(State { @@ -2188,6 +2205,15 @@ pub const Parser = struct { n = for_node.body; }, + ast.Node.Id.If => { + const if_node = @fieldParentPtr(ast.NodeIf, "base", n); + if (if_node.@"else") |@"else"| { + n = @"else".base; + continue; + } + + n = if_node.body; + }, ast.Node.Id.Else => { const else_node = @fieldParentPtr(ast.NodeElse, "base", n); n = else_node.body; @@ -3363,14 +3389,19 @@ pub const Parser = struct { const else_node = @fieldParentPtr(ast.NodeElse, "base", base); try stream.print("{} ", self.tokenizer.getTokenSlice(else_node.else_token)); - if (else_node.body.id == ast.Node.Id.Block) { - try stack.append(RenderState { .Expression = else_node.body }); - } else { - try stack.append(RenderState { .Indent = indent }); - try stack.append(RenderState { .Expression = else_node.body }); - try stack.append(RenderState.PrintIndent); - try stack.append(RenderState { .Indent = indent + indent_delta }); - try stack.append(RenderState { .Text = "\n" }); + switch (else_node.body.id) { + ast.Node.Id.Block, ast.Node.Id.If, + ast.Node.Id.For, ast.Node.Id.While, + ast.Node.Id.Switch => { + try stack.append(RenderState { .Expression = else_node.body }); + }, + else => { + try stack.append(RenderState { .Indent = indent }); + try stack.append(RenderState { .Expression = else_node.body }); + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { .Indent = indent + indent_delta }); + try stack.append(RenderState { .Text = "\n" }); + } } if (else_node.payload) |payload| { @@ -3396,6 +3427,7 @@ pub const Parser = struct { if (while_node.body.id == ast.Node.Id.Block) { try stack.append(RenderState { .Text = " " }); } else { + try stack.append(RenderState.PrintIndent); try stack.append(RenderState { .Text = "\n" }); } } @@ -3445,6 +3477,7 @@ pub const Parser = struct { if (for_node.body.id == ast.Node.Id.Block) { try stack.append(RenderState { .Text = " " }); } else { + try stack.append(RenderState.PrintIndent); try stack.append(RenderState { .Text = "\n" }); } } @@ -3469,6 +3502,53 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = for_node.array_expr }); try stack.append(RenderState { .Text = "(" }); }, + ast.Node.Id.If => { + const if_node = @fieldParentPtr(ast.NodeIf, "base", base); + try stream.print("{} ", self.tokenizer.getTokenSlice(if_node.if_token)); + + switch (if_node.body.id) { + ast.Node.Id.Block, ast.Node.Id.If, + ast.Node.Id.For, ast.Node.Id.While, + ast.Node.Id.Switch => { + if (if_node.@"else") |@"else"| { + try stack.append(RenderState { .Expression = &@"else".base }); + + if (if_node.body.id == ast.Node.Id.Block) { + try stack.append(RenderState { .Text = " " }); + } else { + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { .Text = "\n" }); + } + } + }, + else => { + if (if_node.@"else") |@"else"| { + try stack.append(RenderState { .Expression = @"else".body }); + + if (@"else".payload) |payload| { + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Expression = &payload.base }); + } + + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(@"else".else_token) }); + try stack.append(RenderState { .Text = " " }); + } + } + } + + try stack.append(RenderState { .Expression = if_node.body }); + try stack.append(RenderState { .Text = " " }); + + if (if_node.payload) |payload| { + try stack.append(RenderState { .Expression = &payload.base }); + try stack.append(RenderState { .Text = " " }); + } + + try stack.append(RenderState { .Text = ")" }); + try stack.append(RenderState { .Expression = if_node.condition }); + try stack.append(RenderState { .Text = "(" }); + }, ast.Node.Id.StructField, ast.Node.Id.UnionTag, @@ -4210,8 +4290,7 @@ test "zig fmt: if" { \\ unreachable; \\ } \\ - \\ if (10 < 0) - \\ unreachable; + \\ if (10 < 0) unreachable; \\ \\ if (10 < 0) { \\ unreachable; From d04346d2ac04c079c1722db691d1536c3d717735 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Mon, 9 Apr 2018 13:07:46 +0200 Subject: [PATCH 51/87] ast.zig.parser now parses defer statements --- std/zig/ast.zig | 33 +++++++++++++++++++++++++++++++++ std/zig/parser.zig | 27 +++++++++++++++++++++++++++ std/zig/tokenizer.zig | 2 ++ 3 files changed, 62 insertions(+) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 9e077a3567..29eaa56be4 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -20,6 +20,7 @@ pub const Node = struct { FnProto, ParamDecl, Block, + Defer, ErrorPayload, ValuePayload, ValueIndexPayload, @@ -65,6 +66,7 @@ pub const Node = struct { Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).iterate(index), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).iterate(index), Id.Block => @fieldParentPtr(NodeBlock, "base", base).iterate(index), + Id.Defer => @fieldParentPtr(NodeDefer, "base", base).iterate(index), Id.ErrorPayload => @fieldParentPtr(NodeErrorPayload, "base", base).iterate(index), Id.ValuePayload => @fieldParentPtr(NodeValuePayload, "base", base).iterate(index), Id.ValueIndexPayload => @fieldParentPtr(NodeValueIndexPayload, "base", base).iterate(index), @@ -111,6 +113,7 @@ pub const Node = struct { Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).firstToken(), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).firstToken(), Id.Block => @fieldParentPtr(NodeBlock, "base", base).firstToken(), + Id.Defer => @fieldParentPtr(NodeDefer, "base", base).firstToken(), Id.ErrorPayload => @fieldParentPtr(NodeErrorPayload, "base", base).firstToken(), Id.ValuePayload => @fieldParentPtr(NodeValuePayload, "base", base).firstToken(), Id.ValueIndexPayload => @fieldParentPtr(NodeValueIndexPayload, "base", base).firstToken(), @@ -157,6 +160,7 @@ pub const Node = struct { Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).lastToken(), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).lastToken(), Id.Block => @fieldParentPtr(NodeBlock, "base", base).lastToken(), + Id.Defer => @fieldParentPtr(NodeDefer, "base", base).lastToken(), Id.ErrorPayload => @fieldParentPtr(NodeErrorPayload, "base", base).lastToken(), Id.ValuePayload => @fieldParentPtr(NodeValuePayload, "base", base).lastToken(), Id.ValueIndexPayload => @fieldParentPtr(NodeValueIndexPayload, "base", base).lastToken(), @@ -573,6 +577,35 @@ pub const NodeBlock = struct { } }; +pub const NodeDefer = struct { + base: Node, + defer_token: Token, + kind: Kind, + expr: &Node, + + const Kind = enum { + Error, + Unconditional, + }; + + pub fn iterate(self: &NodeDefer, index: usize) ?&Node { + var i = index; + + if (i < 1) return self.expr; + i -= 1; + + return null; + } + + pub fn firstToken(self: &NodeDefer) Token { + return self.defer_token; + } + + pub fn lastToken(self: &NodeDefer) Token { + return self.expr.lastToken(); + } +}; + pub const NodeErrorPayload = struct { base: Node, lpipe: Token, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 09383a313b..b783af3ea7 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -2140,6 +2140,24 @@ pub const Parser = struct { stack.append(State { .VarDecl = var_decl }) catch unreachable; continue; }, + Token.Id.Keyword_defer, Token.Id.Keyword_errdefer => { + const node = try arena.create(ast.NodeDefer); + *node = ast.NodeDefer { + .base = self.initNode(ast.Node.Id.Defer), + .defer_token = next, + .kind = switch (next.id) { + Token.Id.Keyword_defer => ast.NodeDefer.Kind.Unconditional, + Token.Id.Keyword_errdefer => ast.NodeDefer.Kind.Error, + else => unreachable, + }, + .expr = undefined, + }; + try block.statements.append(&node.base); + + stack.append(State { .Semicolon = &node.base }) catch unreachable; + try stack.append(State { .Expression = DestPtr{.Field = &node.expr } }); + continue; + }, Token.Id.LBrace => { const inner_block = try self.createBlock(arena, (?Token)(null), next); try block.statements.append(&inner_block.base); @@ -2218,6 +2236,10 @@ pub const Parser = struct { const else_node = @fieldParentPtr(ast.NodeElse, "base", n); n = else_node.body; }, + ast.Node.Id.Defer => { + const defer_node = @fieldParentPtr(ast.NodeDefer, "base", n); + n = defer_node.expr; + }, else => return true, } } @@ -2912,6 +2934,11 @@ pub const Parser = struct { } } }, + ast.Node.Id.Defer => { + const defer_node = @fieldParentPtr(ast.NodeDefer, "base", base); + try stream.print("{} ", self.tokenizer.getTokenSlice(defer_node.defer_token)); + try stack.append(RenderState { .Expression = defer_node.expr }); + }, ast.Node.Id.InfixOp => { const prefix_op_node = @fieldParentPtr(ast.NodeInfixOp, "base", base); try stack.append(RenderState { .Expression = prefix_op_node.rhs }); diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig index 422aa20629..4cce31baeb 100644 --- a/std/zig/tokenizer.zig +++ b/std/zig/tokenizer.zig @@ -26,6 +26,7 @@ pub const Token = struct { KeywordId{.bytes="defer", .id = Id.Keyword_defer}, KeywordId{.bytes="else", .id = Id.Keyword_else}, KeywordId{.bytes="enum", .id = Id.Keyword_enum}, + KeywordId{.bytes="errdefer", .id = Id.Keyword_errdefer}, KeywordId{.bytes="error", .id = Id.Keyword_error}, KeywordId{.bytes="export", .id = Id.Keyword_export}, KeywordId{.bytes="extern", .id = Id.Keyword_extern}, @@ -151,6 +152,7 @@ pub const Token = struct { Keyword_defer, Keyword_else, Keyword_enum, + Keyword_errdefer, Keyword_error, Keyword_export, Keyword_extern, From 7d32c9521fff3d2a18293d38562636c5a0e3408e Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Mon, 9 Apr 2018 13:24:47 +0200 Subject: [PATCH 52/87] std.zig.parser now parses comptime --- std/zig/ast.zig | 27 +++++++++++++++++++++++++++ std/zig/parser.zig | 42 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 29eaa56be4..2da819556c 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -21,6 +21,7 @@ pub const Node = struct { ParamDecl, Block, Defer, + Comptime, ErrorPayload, ValuePayload, ValueIndexPayload, @@ -67,6 +68,7 @@ pub const Node = struct { Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).iterate(index), Id.Block => @fieldParentPtr(NodeBlock, "base", base).iterate(index), Id.Defer => @fieldParentPtr(NodeDefer, "base", base).iterate(index), + Id.Comptime => @fieldParentPtr(NodeComptime, "base", base).iterate(index), Id.ErrorPayload => @fieldParentPtr(NodeErrorPayload, "base", base).iterate(index), Id.ValuePayload => @fieldParentPtr(NodeValuePayload, "base", base).iterate(index), Id.ValueIndexPayload => @fieldParentPtr(NodeValueIndexPayload, "base", base).iterate(index), @@ -114,6 +116,7 @@ pub const Node = struct { Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).firstToken(), Id.Block => @fieldParentPtr(NodeBlock, "base", base).firstToken(), Id.Defer => @fieldParentPtr(NodeDefer, "base", base).firstToken(), + Id.Comptime => @fieldParentPtr(NodeComptime, "base", base).firstToken(), Id.ErrorPayload => @fieldParentPtr(NodeErrorPayload, "base", base).firstToken(), Id.ValuePayload => @fieldParentPtr(NodeValuePayload, "base", base).firstToken(), Id.ValueIndexPayload => @fieldParentPtr(NodeValueIndexPayload, "base", base).firstToken(), @@ -161,6 +164,7 @@ pub const Node = struct { Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).lastToken(), Id.Block => @fieldParentPtr(NodeBlock, "base", base).lastToken(), Id.Defer => @fieldParentPtr(NodeDefer, "base", base).lastToken(), + Id.Comptime => @fieldParentPtr(NodeComptime, "base", base).lastToken(), Id.ErrorPayload => @fieldParentPtr(NodeErrorPayload, "base", base).lastToken(), Id.ValuePayload => @fieldParentPtr(NodeValuePayload, "base", base).lastToken(), Id.ValueIndexPayload => @fieldParentPtr(NodeValueIndexPayload, "base", base).lastToken(), @@ -606,6 +610,29 @@ pub const NodeDefer = struct { } }; +pub const NodeComptime = struct { + base: Node, + comptime_token: Token, + expr: &Node, + + pub fn iterate(self: &NodeComptime, index: usize) ?&Node { + var i = index; + + if (i < 1) return self.expr; + i -= 1; + + return null; + } + + pub fn firstToken(self: &NodeComptime) Token { + return self.comptime_token; + } + + pub fn lastToken(self: &NodeComptime) Token { + return self.expr.lastToken(); + } +}; + pub const NodeErrorPayload = struct { base: Node, lpipe: Token, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index b783af3ea7..aa04ad68d3 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -309,6 +309,18 @@ pub const Parser = struct { }); continue; }, + Token.Id.Keyword_comptime => { + const node = try arena.create(ast.NodeComptime); + *node = ast.NodeComptime { + .base = self.initNode(ast.Node.Id.Comptime), + .comptime_token = token, + .expr = undefined, + }; + try root_node.decls.append(&node.base); + stack.append(State.TopLevel) catch unreachable; + try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); + continue; + }, else => { self.putBackToken(token); stack.append(State.TopLevel) catch unreachable; @@ -1523,7 +1535,15 @@ pub const Parser = struct { try stack.append(State { .ExpectToken = Token.Id.LParen }); }, Token.Id.Keyword_comptime => { - @panic("TODO: inline comptime"); + const node = try arena.create(ast.NodeComptime); + *node = ast.NodeComptime { + .base = self.initNode(ast.Node.Id.Comptime), + .comptime_token = token, + .expr = undefined, + }; + dest_ptr.store(&node.base); + try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); + continue; }, else => { try self.parseError(&stack, token, "expected primary expression, found {}", @tagName(token.id)); @@ -2131,7 +2151,10 @@ pub const Parser = struct { continue; } else { self.putBackToken(mut_token); - @panic("TODO: comptime block"); + self.putBackToken(next); + const statememt = try block.statements.addOne(); + stack.append(State { .Semicolon = statememt }) catch unreachable; + try stack.append(State { .Expression = DestPtr{.Field = statememt } }); } }, Token.Id.Keyword_var, Token.Id.Keyword_const => { @@ -2240,6 +2263,10 @@ pub const Parser = struct { const defer_node = @fieldParentPtr(ast.NodeDefer, "base", n); n = defer_node.expr; }, + ast.Node.Id.Comptime => { + const comptime_node = @fieldParentPtr(ast.NodeComptime, "base", n); + n = comptime_node.expr; + }, else => return true, } } @@ -2824,6 +2851,12 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = value}); } }, + ast.Node.Id.Comptime => { + if (requireSemiColon(decl)) { + try stack.append(RenderState { .Text = ";" }); + } + try stack.append(RenderState { .Expression = decl }); + }, else => unreachable, } }, @@ -2939,6 +2972,11 @@ pub const Parser = struct { try stream.print("{} ", self.tokenizer.getTokenSlice(defer_node.defer_token)); try stack.append(RenderState { .Expression = defer_node.expr }); }, + ast.Node.Id.Comptime => { + const comptime_node = @fieldParentPtr(ast.NodeComptime, "base", base); + try stream.print("{} ", self.tokenizer.getTokenSlice(comptime_node.comptime_token)); + try stack.append(RenderState { .Expression = comptime_node.expr }); + }, ast.Node.Id.InfixOp => { const prefix_op_node = @fieldParentPtr(ast.NodeInfixOp, "base", base); try stack.append(RenderState { .Expression = prefix_op_node.rhs }); From aa552633cc0bc66d46f61ea2105f7a4392df37be Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Mon, 9 Apr 2018 14:02:03 +0200 Subject: [PATCH 53/87] std.zig.parser now parses fn types --- std/zig/parser.zig | 134 +++++++++++++++++++++++++++++++++------------ 1 file changed, 99 insertions(+), 35 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index aa04ad68d3..7bdff56b85 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -1413,6 +1413,17 @@ pub const Parser = struct { }) catch unreachable; }, Token.Id.Keyword_extern => { + const next = self.getNextToken(); + if (next.id == Token.Id.Keyword_fn) { + // TODO shouldn't need this cast + const fn_proto = try self.createFnProto(arena, next, + (?Token)(token), (?&ast.Node)(null), (?Token)(null), (?Token)(null), (?Token)(null)); + dest_ptr.store(&fn_proto.base); + stack.append(State { .FnProto = fn_proto }) catch unreachable; + continue; + } + + self.putBackToken(next); stack.append(State { .ContainerExtern = ContainerExternCtx { .dest_ptr = dest_ptr, @@ -1455,7 +1466,26 @@ pub const Parser = struct { continue; }, Token.Id.Keyword_fn => { - @panic("TODO: fn proto"); + // TODO shouldn't need these casts + const fn_proto = try self.createFnProto(arena, token, + (?Token)(null), (?&ast.Node)(null), (?Token)(null), (?Token)(null), (?Token)(null)); + dest_ptr.store(&fn_proto.base); + stack.append(State { .FnProto = fn_proto }) catch unreachable; + continue; + }, + Token.Id.Keyword_nakedcc, Token.Id.Keyword_stdcallcc => { + // TODO shouldn't need this cast + const fn_proto = try self.createFnProto(arena, undefined, + (?Token)(null), (?&ast.Node)(null), (?Token)(token), (?Token)(null), (?Token)(null)); + dest_ptr.store(&fn_proto.base); + stack.append(State { .FnProto = fn_proto }) catch unreachable; + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.Keyword_fn, + .ptr = &fn_proto.fn_token, + } + }); + continue; }, Token.Id.Keyword_asm => { @panic("TODO: inline asm"); @@ -2781,41 +2811,14 @@ pub const Parser = struct { ast.Node.Id.FnProto => { const fn_proto = @fieldParentPtr(ast.NodeFnProto, "base", decl); - if (fn_proto.body_node == null) { - try stack.append(RenderState { .Text = ";" }); + if (fn_proto.body_node) |body_node| { + stack.append(RenderState { .Expression = body_node}) catch unreachable; + try stack.append(RenderState { .Text = " "}); + } else { + stack.append(RenderState { .Text = ";" }) catch unreachable; } - try stack.append(RenderState { .FnProtoRParen = fn_proto}); - var i = fn_proto.params.len; - while (i != 0) { - i -= 1; - const param_decl_node = fn_proto.params.items[i]; - try stack.append(RenderState { .ParamDecl = param_decl_node}); - if (i != 0) { - try stack.append(RenderState { .Text = ", " }); - } - } - - try stack.append(RenderState { .Text = "(" }); - if (fn_proto.name_token) |name_token| { - try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(name_token) }); - } - - try stack.append(RenderState { .Text = "fn " }); - if (fn_proto.lib_name) |lib_name| { - try stack.append(RenderState { .Text = " " }); - try stack.append(RenderState { .Expression = lib_name }); - } - if (fn_proto.extern_token) |extern_token| { - try stack.append(RenderState { .Text = " " }); - try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(extern_token) }); - } - - if (fn_proto.visib_token) |visib_token| { - assert(visib_token.id == Token.Id.Keyword_pub or visib_token.id == Token.Id.Keyword_export); - try stack.append(RenderState { .Text = " " }); - try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(visib_token) }); - } + try stack.append(RenderState { .Expression = decl }); }, ast.Node.Id.VarDecl => { const var_decl = @fieldParentPtr(ast.NodeVarDecl, "base", decl); @@ -3386,7 +3389,65 @@ pub const Parser = struct { } } }, - ast.Node.Id.FnProto => @panic("TODO fn proto in an expression"), + ast.Node.Id.FnProto => { + const fn_proto = @fieldParentPtr(ast.NodeFnProto, "base", base); + + switch (fn_proto.return_type) { + ast.NodeFnProto.ReturnType.Explicit => |node| { + try stack.append(RenderState { .Expression = node}); + }, + ast.NodeFnProto.ReturnType.Infer => { + try stack.append(RenderState { .Text = "var"}); + }, + ast.NodeFnProto.ReturnType.InferErrorSet => |node| { + try stack.append(RenderState { .Expression = node}); + try stack.append(RenderState { .Text = "!"}); + }, + } + + if (fn_proto.align_expr != null) { + @panic("TODO"); + } + + try stack.append(RenderState { .Text = ") " }); + var i = fn_proto.params.len; + while (i != 0) { + i -= 1; + const param_decl_node = fn_proto.params.items[i]; + try stack.append(RenderState { .ParamDecl = param_decl_node}); + if (i != 0) { + try stack.append(RenderState { .Text = ", " }); + } + } + + try stack.append(RenderState { .Text = "(" }); + if (fn_proto.name_token) |name_token| { + try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(name_token) }); + try stack.append(RenderState { .Text = " " }); + } + + try stack.append(RenderState { .Text = "fn" }); + + if (fn_proto.cc_token) |cc_token| { + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(cc_token) }); + } + + if (fn_proto.lib_name) |lib_name| { + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Expression = lib_name }); + } + if (fn_proto.extern_token) |extern_token| { + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(extern_token) }); + } + + if (fn_proto.visib_token) |visib_token| { + assert(visib_token.id == Token.Id.Keyword_pub or visib_token.id == Token.Id.Keyword_export); + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(visib_token) }); + } + }, ast.Node.Id.LineComment => @panic("TODO render line comment in an expression"), ast.Node.Id.Switch => { const switch_node = @fieldParentPtr(ast.NodeSwitch, "base", base); @@ -4461,6 +4522,9 @@ test "zig fmt: fn type" { \\ return i + 1; \\} \\ + \\const a: fn(u8) u8 = undefined; + \\const b: extern fn(u8) u8 = undefined; + \\const c: nakedcc fn(u8) u8 = undefined; \\const ap: fn(u8) u8 = a; \\ ); From a09bb408a20fcfc0575a9298b318803b07517b0a Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Mon, 9 Apr 2018 15:40:16 +0200 Subject: [PATCH 54/87] std.zig.parser now parses asm expressions * We cannot render asm expressions yet --- std/zig/ast.zig | 41 ++++++++++++++ std/zig/parser.zig | 138 ++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 178 insertions(+), 1 deletion(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 2da819556c..d048f4ed43 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -47,6 +47,7 @@ pub const Node = struct { NullLiteral, UndefinedLiteral, ThisLiteral, + Asm, Unreachable, ErrorType, BuiltinCall, @@ -94,6 +95,7 @@ pub const Node = struct { Id.NullLiteral => @fieldParentPtr(NodeNullLiteral, "base", base).iterate(index), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).iterate(index), Id.ThisLiteral => @fieldParentPtr(NodeThisLiteral, "base", base).iterate(index), + Id.Asm => @fieldParentPtr(NodeAsm, "base", base).iterate(index), Id.Unreachable => @fieldParentPtr(NodeUnreachable, "base", base).iterate(index), Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).iterate(index), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index), @@ -143,6 +145,7 @@ pub const Node = struct { Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).firstToken(), Id.Unreachable => @fieldParentPtr(NodeUnreachable, "base", base).firstToken(), Id.ThisLiteral => @fieldParentPtr(NodeThisLiteral, "base", base).firstToken(), + Id.Asm => @fieldParentPtr(NodeAsm, "base", base).firstToken(), Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).firstToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).firstToken(), @@ -190,6 +193,7 @@ pub const Node = struct { Id.NullLiteral => @fieldParentPtr(NodeNullLiteral, "base", base).lastToken(), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).lastToken(), Id.ThisLiteral => @fieldParentPtr(NodeThisLiteral, "base", base).lastToken(), + Id.Asm => @fieldParentPtr(NodeAsm, "base", base).lastToken(), Id.Unreachable => @fieldParentPtr(NodeUnreachable, "base", base).lastToken(), Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).lastToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(), @@ -1512,6 +1516,43 @@ pub const NodeThisLiteral = struct { } }; +pub const NodeAsm = struct { + base: Node, + asm_token: Token, + is_volatile: bool, + template: Token, + //tokens: ArrayList(AsmToken), + outputs: ArrayList(AsmOutput), + inputs: ArrayList(AsmInput), + cloppers: ArrayList(&NodeStringLiteral), + rparen: Token, + + const AsmOutput = struct { + symbolic_name: Token, + constraint: Token, + variable_name: ?Token, + return_type: ?&Node, + }; + + const AsmInput = struct { + symbolic_name: Token, + constraint: Token, + expr: &Node, + }; + + pub fn iterate(self: &NodeAsm, index: usize) ?&Node { + return null; + } + + pub fn firstToken(self: &NodeAsm) Token { + return self.asm_token; + } + + pub fn lastToken(self: &NodeAsm) Token { + return self.rparen; + } +}; + pub const NodeUnreachable = struct { base: Node, token: Token, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 7bdff56b85..7f5427146a 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -164,6 +164,9 @@ pub const Parser = struct { WhileContinueExpr: &?&ast.Node, Statement: &ast.NodeBlock, Semicolon: &const &const ast.Node, + AsmOutputItems: &ArrayList(ast.NodeAsm.AsmOutput), + AsmInputItems: &ArrayList(ast.NodeAsm.AsmInput), + AsmClopperItems: &ArrayList(&ast.NodeStringLiteral), ExprListItemOrEnd: ExprListCtx, ExprListCommaOrEnd: ExprListCtx, FieldInitListItemOrEnd: ListSave(&ast.NodeFieldInitializer), @@ -1488,7 +1491,44 @@ pub const Parser = struct { continue; }, Token.Id.Keyword_asm => { - @panic("TODO: inline asm"); + const is_volatile = blk: { + const volatile_token = self.getNextToken(); + if (volatile_token.id != Token.Id.Keyword_volatile) { + self.putBackToken(volatile_token); + break :blk false; + } + break :blk true; + }; + _ = (try self.eatToken(&stack, Token.Id.LParen)) ?? continue; + const template = (try self.eatToken(&stack, Token.Id.StringLiteral)) ?? continue; + // TODO parse template + + const node = try arena.create(ast.NodeAsm); + *node = ast.NodeAsm { + .base = self.initNode(ast.Node.Id.Asm), + .asm_token = token, + .is_volatile = is_volatile, + .template = template, + //.tokens = ArrayList(ast.NodeAsm.AsmToken).init(arena), + .outputs = ArrayList(ast.NodeAsm.AsmOutput).init(arena), + .inputs = ArrayList(ast.NodeAsm.AsmInput).init(arena), + .cloppers = ArrayList(&ast.NodeStringLiteral).init(arena), + .rparen = undefined, + }; + dest_ptr.store(&node.base); + + stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.RParen, + .ptr = &node.rparen, + } + }) catch unreachable; + try stack.append(State { .AsmClopperItems = &node.cloppers }); + try stack.append(State { .IfToken = Token.Id.Colon }); + try stack.append(State { .AsmInputItems = &node.inputs }); + try stack.append(State { .IfToken = Token.Id.Colon }); + try stack.append(State { .AsmOutputItems = &node.outputs }); + try stack.append(State { .IfToken = Token.Id.Colon }); }, Token.Id.Keyword_if => { const node = try arena.create(ast.NodeIf); @@ -1621,6 +1661,84 @@ pub const Parser = struct { } }, + + State.AsmOutputItems => |items| { + const lbracket = self.getNextToken(); + if (lbracket.id != Token.Id.LBracket) { + self.putBackToken(lbracket); + continue; + } + + stack.append(State { .AsmOutputItems = items }) catch unreachable; + try stack.append(State { .IfToken = Token.Id.Comma }); + + const symbolic_name = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; + _ = (try self.eatToken(&stack, Token.Id.RBracket)) ?? continue; + const constraint = (try self.eatToken(&stack, Token.Id.StringLiteral)) ?? continue; + + _ = (try self.eatToken(&stack, Token.Id.LParen)) ?? continue; + try stack.append(State { .ExpectToken = Token.Id.RParen }); + + const res = try items.addOne(); + *res = ast.NodeAsm.AsmOutput { + .symbolic_name = symbolic_name, + .constraint = constraint, + .variable_name = null, + .return_type = null, + }; + const symbol_or_arrow = self.getNextToken(); + switch (symbol_or_arrow.id) { + Token.Id.Identifier => res.variable_name = symbol_or_arrow, + Token.Id.Arrow => { + try stack.append(State { .TypeExprBegin = DestPtr { .NullableField = &res.return_type } }); + }, + else => { + try self.parseError(&stack, symbol_or_arrow, "expected '->' or {}, found {}", + @tagName(Token.Id.Identifier), + @tagName(symbol_or_arrow.id)); + continue; + }, + } + }, + + State.AsmInputItems => |items| { + const lbracket = self.getNextToken(); + if (lbracket.id != Token.Id.LBracket) { + self.putBackToken(lbracket); + continue; + } + + stack.append(State { .AsmInputItems = items }) catch unreachable; + try stack.append(State { .IfToken = Token.Id.Comma }); + + const symbolic_name = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; + _ = (try self.eatToken(&stack, Token.Id.RBracket)) ?? continue; + const constraint = (try self.eatToken(&stack, Token.Id.StringLiteral)) ?? continue; + + _ = (try self.eatToken(&stack, Token.Id.LParen)) ?? continue; + try stack.append(State { .ExpectToken = Token.Id.RParen }); + + const res = try items.addOne(); + *res = ast.NodeAsm.AsmInput { + .symbolic_name = symbolic_name, + .constraint = constraint, + .expr = undefined, + }; + try stack.append(State { .Expression = DestPtr { .Field = &res.expr } }); + }, + + State.AsmClopperItems => |items| { + const string = self.getNextToken(); + if (string.id != Token.Id.StringLiteral) { + self.putBackToken(string); + continue; + } + + try items.append(try self.createStringLiteral(arena, string)); + stack.append(State { .AsmClopperItems = items }) catch unreachable; + try stack.append(State { .IfToken = Token.Id.Comma }); + }, + State.ExprListItemOrEnd => |list_state| { var token = self.getNextToken(); @@ -3675,6 +3793,24 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = if_node.condition }); try stack.append(RenderState { .Text = "(" }); }, + ast.Node.Id.Asm => { + const asm_node = @fieldParentPtr(ast.NodeAsm, "base", base); + try stream.print("{} ", self.tokenizer.getTokenSlice(asm_node.asm_token)); + + if (asm_node.is_volatile) { + try stream.write("volatile "); + } + + try stream.print("({}", self.tokenizer.getTokenSlice(asm_node.template)); + + try stack.append(RenderState { .Text = ")" }); + @panic("TODO: Render asm"); + //\\ return asm volatile ("syscall" + //\\ : [ret] "={rax}" (-> usize) + //\\ : [number] "{rax}" (number), + //\\ [arg1] "{rdi}" (arg1) + //\\ : "rcx", "r11"); + },, ast.Node.Id.StructField, ast.Node.Id.UnionTag, From 4545be360a723a4c5b141cbefcc07ac7f17f0757 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 9 Apr 2018 21:14:55 -0400 Subject: [PATCH 55/87] fix std.io.readline to work on windows closes #882 --- std/io.zig | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/std/io.zig b/std/io.zig index 93d50e6709..7b72af15e4 100644 --- a/std/io.zig +++ b/std/io.zig @@ -486,6 +486,11 @@ pub fn readLine(buf: []u8) !usize { while (true) { const byte = stream.readByte() catch return error.EndOfFile; switch (byte) { + '\r' => { + // trash the following \n + _ = stream.readByte() catch return error.EndOfFile; + return index; + }, '\n' => return index, else => { if (index == buf.len) return error.InputTooLong; From 2c7996f4006b94f81fc3b1c1e3f73a4a7a291782 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 10 Apr 2018 09:27:11 +0200 Subject: [PATCH 56/87] std.zig.parser can now render asm expressions --- std/zig/ast.zig | 112 +++++++++++++++++++++++++++----- std/zig/parser.zig | 156 +++++++++++++++++++++++++++++++++++++-------- 2 files changed, 228 insertions(+), 40 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index d048f4ed43..86a706f1c7 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -48,6 +48,8 @@ pub const Node = struct { UndefinedLiteral, ThisLiteral, Asm, + AsmInput, + AsmOutput, Unreachable, ErrorType, BuiltinCall, @@ -96,6 +98,8 @@ pub const Node = struct { Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).iterate(index), Id.ThisLiteral => @fieldParentPtr(NodeThisLiteral, "base", base).iterate(index), Id.Asm => @fieldParentPtr(NodeAsm, "base", base).iterate(index), + Id.AsmInput => @fieldParentPtr(NodeAsmInput, "base", base).iterate(index), + Id.AsmOutput => @fieldParentPtr(NodeAsmOutput, "base", base).iterate(index), Id.Unreachable => @fieldParentPtr(NodeUnreachable, "base", base).iterate(index), Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).iterate(index), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index), @@ -146,6 +150,8 @@ pub const Node = struct { Id.Unreachable => @fieldParentPtr(NodeUnreachable, "base", base).firstToken(), Id.ThisLiteral => @fieldParentPtr(NodeThisLiteral, "base", base).firstToken(), Id.Asm => @fieldParentPtr(NodeAsm, "base", base).firstToken(), + Id.AsmInput => @fieldParentPtr(NodeAsmInput, "base", base).firstToken(), + Id.AsmOutput => @fieldParentPtr(NodeAsmOutput, "base", base).firstToken(), Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).firstToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).firstToken(), @@ -194,6 +200,8 @@ pub const Node = struct { Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).lastToken(), Id.ThisLiteral => @fieldParentPtr(NodeThisLiteral, "base", base).lastToken(), Id.Asm => @fieldParentPtr(NodeAsm, "base", base).lastToken(), + Id.AsmInput => @fieldParentPtr(NodeAsmInput, "base", base).lastToken(), + Id.AsmOutput => @fieldParentPtr(NodeAsmOutput, "base", base).lastToken(), Id.Unreachable => @fieldParentPtr(NodeUnreachable, "base", base).lastToken(), Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).lastToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(), @@ -1516,31 +1524,105 @@ pub const NodeThisLiteral = struct { } }; +pub const NodeAsmOutput = struct { + base: Node, + symbolic_name: &NodeIdentifier, + constraint: &NodeStringLiteral, + kind: Kind, + + const Kind = union(enum) { + Variable: &NodeIdentifier, + Return: &Node + }; + + pub fn iterate(self: &NodeAsmOutput, index: usize) ?&Node { + var i = index; + + if (i < 1) return &self.symbolic_name.base; + i -= 1; + + if (i < 1) return &self.constraint.base; + i -= 1; + + switch (self.kind) { + Kind.Variable => |variable_name| { + if (i < 1) return &variable_name.base; + i -= 1; + }, + Kind.Return => |return_type| { + if (i < 1) return return_type; + i -= 1; + } + } + + return null; + } + + pub fn firstToken(self: &NodeAsmOutput) Token { + return self.symbolic_name.firstToken(); + } + + pub fn lastToken(self: &NodeAsmOutput) Token { + return switch (self.kind) { + Kind.Variable => |variable_name| variable_name.lastToken(), + Kind.Return => |return_type| return_type.lastToken(), + }; + } +}; + +pub const NodeAsmInput = struct { + base: Node, + symbolic_name: &NodeIdentifier, + constraint: &NodeStringLiteral, + expr: &Node, + + pub fn iterate(self: &NodeAsmInput, index: usize) ?&Node { + var i = index; + + if (i < 1) return &self.symbolic_name.base; + i -= 1; + + if (i < 1) return &self.constraint.base; + i -= 1; + + if (i < 1) return self.expr; + i -= 1; + + return null; + } + + pub fn firstToken(self: &NodeAsmInput) Token { + return self.symbolic_name.firstToken(); + } + + pub fn lastToken(self: &NodeAsmInput) Token { + return self.expr.lastToken(); + } +}; + pub const NodeAsm = struct { base: Node, asm_token: Token, is_volatile: bool, template: Token, //tokens: ArrayList(AsmToken), - outputs: ArrayList(AsmOutput), - inputs: ArrayList(AsmInput), + outputs: ArrayList(&NodeAsmOutput), + inputs: ArrayList(&NodeAsmInput), cloppers: ArrayList(&NodeStringLiteral), rparen: Token, - const AsmOutput = struct { - symbolic_name: Token, - constraint: Token, - variable_name: ?Token, - return_type: ?&Node, - }; - - const AsmInput = struct { - symbolic_name: Token, - constraint: Token, - expr: &Node, - }; - pub fn iterate(self: &NodeAsm, index: usize) ?&Node { + var i = index; + + if (i < self.outputs.len) return &self.outputs.at(index).base; + i -= self.outputs.len; + + if (i < self.inputs.len) return &self.inputs.at(index).base; + i -= self.inputs.len; + + if (i < self.cloppers.len) return &self.cloppers.at(index).base; + i -= self.cloppers.len; + return null; } diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 7f5427146a..9851f6cc30 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -164,8 +164,8 @@ pub const Parser = struct { WhileContinueExpr: &?&ast.Node, Statement: &ast.NodeBlock, Semicolon: &const &const ast.Node, - AsmOutputItems: &ArrayList(ast.NodeAsm.AsmOutput), - AsmInputItems: &ArrayList(ast.NodeAsm.AsmInput), + AsmOutputItems: &ArrayList(&ast.NodeAsmOutput), + AsmInputItems: &ArrayList(&ast.NodeAsmInput), AsmClopperItems: &ArrayList(&ast.NodeStringLiteral), ExprListItemOrEnd: ExprListCtx, ExprListCommaOrEnd: ExprListCtx, @@ -1510,8 +1510,8 @@ pub const Parser = struct { .is_volatile = is_volatile, .template = template, //.tokens = ArrayList(ast.NodeAsm.AsmToken).init(arena), - .outputs = ArrayList(ast.NodeAsm.AsmOutput).init(arena), - .inputs = ArrayList(ast.NodeAsm.AsmInput).init(arena), + .outputs = ArrayList(&ast.NodeAsmOutput).init(arena), + .inputs = ArrayList(&ast.NodeAsmInput).init(arena), .cloppers = ArrayList(&ast.NodeStringLiteral).init(arena), .rparen = undefined, }; @@ -1679,18 +1679,23 @@ pub const Parser = struct { _ = (try self.eatToken(&stack, Token.Id.LParen)) ?? continue; try stack.append(State { .ExpectToken = Token.Id.RParen }); - const res = try items.addOne(); - *res = ast.NodeAsm.AsmOutput { - .symbolic_name = symbolic_name, - .constraint = constraint, - .variable_name = null, - .return_type = null, + const node = try arena.create(ast.NodeAsmOutput); + *node = ast.NodeAsmOutput { + .base = self.initNode(ast.Node.Id.AsmOutput), + .symbolic_name = try self.createIdentifier(arena, symbolic_name), + .constraint = try self.createStringLiteral(arena, constraint), + .kind = undefined, }; + try items.append(node); + const symbol_or_arrow = self.getNextToken(); switch (symbol_or_arrow.id) { - Token.Id.Identifier => res.variable_name = symbol_or_arrow, + Token.Id.Identifier => { + node.kind = ast.NodeAsmOutput.Kind { .Variable = try self.createIdentifier(arena, symbol_or_arrow) }; + }, Token.Id.Arrow => { - try stack.append(State { .TypeExprBegin = DestPtr { .NullableField = &res.return_type } }); + node.kind = ast.NodeAsmOutput.Kind { .Return = undefined }; + try stack.append(State { .TypeExprBegin = DestPtr { .Field = &node.kind.Return } }); }, else => { try self.parseError(&stack, symbol_or_arrow, "expected '->' or {}, found {}", @@ -1718,13 +1723,15 @@ pub const Parser = struct { _ = (try self.eatToken(&stack, Token.Id.LParen)) ?? continue; try stack.append(State { .ExpectToken = Token.Id.RParen }); - const res = try items.addOne(); - *res = ast.NodeAsm.AsmInput { - .symbolic_name = symbolic_name, - .constraint = constraint, + const node = try arena.create(ast.NodeAsmInput); + *node = ast.NodeAsmInput { + .base = self.initNode(ast.Node.Id.AsmInput), + .symbolic_name = try self.createIdentifier(arena, symbolic_name), + .constraint = try self.createStringLiteral(arena, constraint), .expr = undefined, }; - try stack.append(State { .Expression = DestPtr { .Field = &res.expr } }); + try items.append(node); + try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); }, State.AsmClopperItems => |items| { @@ -3803,14 +3810,113 @@ pub const Parser = struct { try stream.print("({}", self.tokenizer.getTokenSlice(asm_node.template)); + try stack.append(RenderState { .Indent = indent }); try stack.append(RenderState { .Text = ")" }); - @panic("TODO: Render asm"); - //\\ return asm volatile ("syscall" - //\\ : [ret] "={rax}" (-> usize) - //\\ : [number] "{rax}" (number), - //\\ [arg1] "{rdi}" (arg1) - //\\ : "rcx", "r11"); - },, + { + const cloppers = asm_node.cloppers.toSliceConst(); + var i = cloppers.len; + while (i != 0) { + i -= 1; + try stack.append(RenderState { .Expression = &cloppers[i].base }); + + if (i != 0) { + try stack.append(RenderState { .Text = ", " }); + } + } + } + try stack.append(RenderState { .Text = ": " }); + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { .Indent = indent + indent_delta }); + try stack.append(RenderState { .Text = "\n" }); + { + const inputs = asm_node.inputs.toSliceConst(); + var i = inputs.len; + while (i != 0) { + i -= 1; + const node = inputs[i]; + try stack.append(RenderState { .Expression = &node.base}); + + if (i != 0) { + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { + .Text = blk: { + const prev_node = inputs[i - 1]; + const loc = self.tokenizer.getTokenLocation(prev_node.lastToken().end, node.firstToken()); + if (loc.line >= 2) { + break :blk "\n\n"; + } + break :blk "\n"; + }, + }); + try stack.append(RenderState { .Text = "," }); + } + } + } + try stack.append(RenderState { .Indent = indent + indent_delta + 2}); + try stack.append(RenderState { .Text = ": "}); + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { .Indent = indent + indent_delta}); + try stack.append(RenderState { .Text = "\n" }); + { + const outputs = asm_node.outputs.toSliceConst(); + var i = outputs.len; + while (i != 0) { + i -= 1; + const node = outputs[i]; + try stack.append(RenderState { .Expression = &node.base}); + + if (i != 0) { + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { + .Text = blk: { + const prev_node = outputs[i - 1]; + const loc = self.tokenizer.getTokenLocation(prev_node.lastToken().end, node.firstToken()); + if (loc.line >= 2) { + break :blk "\n\n"; + } + break :blk "\n"; + }, + }); + try stack.append(RenderState { .Text = "," }); + } + } + } + try stack.append(RenderState { .Indent = indent + indent_delta + 2}); + try stack.append(RenderState { .Text = ": "}); + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { .Indent = indent + indent_delta}); + try stack.append(RenderState { .Text = "\n" }); + }, + ast.Node.Id.AsmInput => { + const asm_input = @fieldParentPtr(ast.NodeAsmInput, "base", base); + + try stack.append(RenderState { .Text = ")"}); + try stack.append(RenderState { .Expression = asm_input.expr}); + try stack.append(RenderState { .Text = " ("}); + try stack.append(RenderState { .Expression = &asm_input.constraint.base}); + try stack.append(RenderState { .Text = "] "}); + try stack.append(RenderState { .Expression = &asm_input.symbolic_name.base}); + try stack.append(RenderState { .Text = "["}); + }, + ast.Node.Id.AsmOutput => { + const asm_output = @fieldParentPtr(ast.NodeAsmOutput, "base", base); + + try stack.append(RenderState { .Text = ")"}); + switch (asm_output.kind) { + ast.NodeAsmOutput.Kind.Variable => |variable_name| { + try stack.append(RenderState { .Expression = &variable_name.base}); + }, + ast.NodeAsmOutput.Kind.Return => |return_type| { + try stack.append(RenderState { .Expression = return_type}); + try stack.append(RenderState { .Text = "-> "}); + }, + } + try stack.append(RenderState { .Text = " ("}); + try stack.append(RenderState { .Expression = &asm_output.constraint.base}); + try stack.append(RenderState { .Text = "] "}); + try stack.append(RenderState { .Expression = &asm_output.symbolic_name.base}); + try stack.append(RenderState { .Text = "["}); + }, ast.Node.Id.StructField, ast.Node.Id.UnionTag, @@ -4672,7 +4778,7 @@ test "zig fmt: inline asm" { \\ return asm volatile ("syscall" \\ : [ret] "={rax}" (-> usize) \\ : [number] "{rax}" (number), - \\ [arg1] "{rdi}" (arg1) + \\ [arg1] "{rdi}" (arg1) \\ : "rcx", "r11"); \\} \\ From 5cd69ee6a4b7d02dbc48d93e8f8f95bd608d9d7c Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 10 Apr 2018 09:37:29 +0200 Subject: [PATCH 57/87] std.zig.parser changed assign expr to only be allowed in some contexts * Only allowed in while continue expr and statement expr --- std/zig/parser.zig | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 9851f6cc30..ff3b35c193 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -721,7 +721,7 @@ pub const Parser = struct { }, else => { self.putBackToken(token); - stack.append(State { .AssignmentExpressionBegin = dest_ptr }) catch unreachable; + stack.append(State { .UnwrapExpressionBegin = dest_ptr }) catch unreachable; continue; } } @@ -750,7 +750,7 @@ pub const Parser = struct { State.AssignmentExpressionBegin => |dest_ptr| { stack.append(State { .AssignmentExpressionEnd = dest_ptr }) catch unreachable; - try stack.append(State { .UnwrapExpressionBegin = dest_ptr }); + try stack.append(State { .Expression = dest_ptr }); continue; }, @@ -762,7 +762,7 @@ pub const Parser = struct { dest_ptr.store(&node.base); stack.append(State { .AssignmentExpressionEnd = dest_ptr }) catch unreachable; - try stack.append(State { .UnwrapExpressionBegin = DestPtr { .Field = &node.rhs } }); + try stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }); continue; } else { self.putBackToken(token); @@ -1895,7 +1895,7 @@ pub const Parser = struct { _ = (try self.eatToken(&stack, Token.Id.LParen)) ?? continue; stack.append(State { .ExpectToken = Token.Id.RParen }) catch unreachable; - try stack.append(State { .Expression = DestPtr { .NullableField = dest } }); + try stack.append(State { .AssignmentExpressionBegin = DestPtr { .NullableField = dest } }); }, State.ErrorPayload => |dest| { @@ -2333,7 +2333,7 @@ pub const Parser = struct { try block.statements.append(&node.base); stack.append(State { .Semicolon = &node.base }) catch unreachable; - try stack.append(State { .Expression = DestPtr{.Field = &node.expr } }); + try stack.append(State { .AssignmentExpressionBegin = DestPtr{.Field = &node.expr } }); continue; }, Token.Id.LBrace => { @@ -2347,7 +2347,7 @@ pub const Parser = struct { self.putBackToken(next); const statememt = try block.statements.addOne(); stack.append(State { .Semicolon = statememt }) catch unreachable; - try stack.append(State { .Expression = DestPtr{.Field = statememt } }); + try stack.append(State { .AssignmentExpressionBegin = DestPtr{.Field = statememt } }); continue; } } @@ -4260,8 +4260,6 @@ test "zig fmt: precedence" { \\ a or b and c; \\ (a or b) and c; \\ (a or b) and c; - \\ a = b or c; - \\ (a = b) or c; \\} \\ ); From f85b9f2bf3ac0c5af7804921d2003c2992558cb7 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 10 Apr 2018 11:25:58 +0200 Subject: [PATCH 58/87] std.zig.parser now parses coroutine code --- std/zig/ast.zig | 157 +++++++++++++++++++------ std/zig/parser.zig | 277 ++++++++++++++++++++++++++++++++++++--------- 2 files changed, 345 insertions(+), 89 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 86a706f1c7..9043a66ac3 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -17,14 +17,15 @@ pub const Node = struct { UnionTag, EnumTag, Identifier, + AsyncAttribute, FnProto, ParamDecl, Block, Defer, Comptime, - ErrorPayload, - ValuePayload, - ValueIndexPayload, + Payload, + PointerPayload, + PointerIndexPayload, Else, Switch, SwitchCase, @@ -37,6 +38,7 @@ pub const Node = struct { SuffixOp, GroupedExpression, ControlFlowExpression, + Suspend, FieldInitializer, IntegerLiteral, FloatLiteral, @@ -67,14 +69,15 @@ pub const Node = struct { Id.UnionTag => @fieldParentPtr(NodeUnionTag, "base", base).iterate(index), Id.EnumTag => @fieldParentPtr(NodeEnumTag, "base", base).iterate(index), Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).iterate(index), + Id.AsyncAttribute => @fieldParentPtr(NodeAsyncAttribute, "base", base).iterate(index), Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).iterate(index), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).iterate(index), Id.Block => @fieldParentPtr(NodeBlock, "base", base).iterate(index), Id.Defer => @fieldParentPtr(NodeDefer, "base", base).iterate(index), Id.Comptime => @fieldParentPtr(NodeComptime, "base", base).iterate(index), - Id.ErrorPayload => @fieldParentPtr(NodeErrorPayload, "base", base).iterate(index), - Id.ValuePayload => @fieldParentPtr(NodeValuePayload, "base", base).iterate(index), - Id.ValueIndexPayload => @fieldParentPtr(NodeValueIndexPayload, "base", base).iterate(index), + Id.Payload => @fieldParentPtr(NodePayload, "base", base).iterate(index), + Id.PointerPayload => @fieldParentPtr(NodePointerPayload, "base", base).iterate(index), + Id.PointerIndexPayload => @fieldParentPtr(NodePointerIndexPayload, "base", base).iterate(index), Id.Else => @fieldParentPtr(NodeSwitch, "base", base).iterate(index), Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).iterate(index), Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).iterate(index), @@ -87,6 +90,7 @@ pub const Node = struct { Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).iterate(index), Id.GroupedExpression => @fieldParentPtr(NodeGroupedExpression, "base", base).iterate(index), Id.ControlFlowExpression => @fieldParentPtr(NodeControlFlowExpression, "base", base).iterate(index), + Id.Suspend => @fieldParentPtr(NodeSuspend, "base", base).iterate(index), Id.FieldInitializer => @fieldParentPtr(NodeFieldInitializer, "base", base).iterate(index), Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).iterate(index), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index), @@ -118,14 +122,15 @@ pub const Node = struct { Id.UnionTag => @fieldParentPtr(NodeUnionTag, "base", base).firstToken(), Id.EnumTag => @fieldParentPtr(NodeEnumTag, "base", base).firstToken(), Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).firstToken(), + Id.AsyncAttribute => @fieldParentPtr(NodeAsyncAttribute, "base", base).firstToken(), Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).firstToken(), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).firstToken(), Id.Block => @fieldParentPtr(NodeBlock, "base", base).firstToken(), Id.Defer => @fieldParentPtr(NodeDefer, "base", base).firstToken(), Id.Comptime => @fieldParentPtr(NodeComptime, "base", base).firstToken(), - Id.ErrorPayload => @fieldParentPtr(NodeErrorPayload, "base", base).firstToken(), - Id.ValuePayload => @fieldParentPtr(NodeValuePayload, "base", base).firstToken(), - Id.ValueIndexPayload => @fieldParentPtr(NodeValueIndexPayload, "base", base).firstToken(), + Id.Payload => @fieldParentPtr(NodePayload, "base", base).firstToken(), + Id.PointerPayload => @fieldParentPtr(NodePointerPayload, "base", base).firstToken(), + Id.PointerIndexPayload => @fieldParentPtr(NodePointerIndexPayload, "base", base).firstToken(), Id.Else => @fieldParentPtr(NodeSwitch, "base", base).firstToken(), Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).firstToken(), Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).firstToken(), @@ -138,6 +143,7 @@ pub const Node = struct { Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).firstToken(), Id.GroupedExpression => @fieldParentPtr(NodeGroupedExpression, "base", base).firstToken(), Id.ControlFlowExpression => @fieldParentPtr(NodeControlFlowExpression, "base", base).firstToken(), + Id.Suspend => @fieldParentPtr(NodeSuspend, "base", base).firstToken(), Id.FieldInitializer => @fieldParentPtr(NodeFieldInitializer, "base", base).firstToken(), Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).firstToken(), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(), @@ -169,14 +175,15 @@ pub const Node = struct { Id.UnionTag => @fieldParentPtr(NodeUnionTag, "base", base).lastToken(), Id.EnumTag => @fieldParentPtr(NodeEnumTag, "base", base).lastToken(), Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).lastToken(), + Id.AsyncAttribute => @fieldParentPtr(NodeAsyncAttribute, "base", base).lastToken(), Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).lastToken(), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).lastToken(), Id.Block => @fieldParentPtr(NodeBlock, "base", base).lastToken(), Id.Defer => @fieldParentPtr(NodeDefer, "base", base).lastToken(), Id.Comptime => @fieldParentPtr(NodeComptime, "base", base).lastToken(), - Id.ErrorPayload => @fieldParentPtr(NodeErrorPayload, "base", base).lastToken(), - Id.ValuePayload => @fieldParentPtr(NodeValuePayload, "base", base).lastToken(), - Id.ValueIndexPayload => @fieldParentPtr(NodeValueIndexPayload, "base", base).lastToken(), + Id.Payload => @fieldParentPtr(NodePayload, "base", base).lastToken(), + Id.PointerPayload => @fieldParentPtr(NodePointerPayload, "base", base).lastToken(), + Id.PointerIndexPayload => @fieldParentPtr(NodePointerIndexPayload, "base", base).lastToken(), Id.Else => @fieldParentPtr(NodeElse, "base", base).lastToken(), Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).lastToken(), Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).lastToken(), @@ -189,6 +196,7 @@ pub const Node = struct { Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).lastToken(), Id.GroupedExpression => @fieldParentPtr(NodeGroupedExpression, "base", base).lastToken(), Id.ControlFlowExpression => @fieldParentPtr(NodeControlFlowExpression, "base", base).lastToken(), + Id.Suspend => @fieldParentPtr(NodeSuspend, "base", base).lastToken(), Id.FieldInitializer => @fieldParentPtr(NodeFieldInitializer, "base", base).lastToken(), Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).lastToken(), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(), @@ -456,6 +464,36 @@ pub const NodeIdentifier = struct { } }; +pub const NodeAsyncAttribute = struct { + base: Node, + async_token: Token, + allocator_type: ?&Node, + rangle_bracket: ?Token, + + pub fn iterate(self: &NodeAsyncAttribute, index: usize) ?&Node { + var i = index; + + if (self.allocator_type) |allocator_type| { + if (i < 1) return allocator_type; + i -= 1; + } + + return null; + } + + pub fn firstToken(self: &NodeAsyncAttribute) Token { + return self.async_token; + } + + pub fn lastToken(self: &NodeAsyncAttribute) Token { + if (self.rangle_bracket) |rangle_bracket| { + return rangle_bracket; + } + + return self.async_token; + } +}; + pub const NodeFnProto = struct { base: Node, visib_token: ?Token, @@ -467,6 +505,7 @@ pub const NodeFnProto = struct { extern_token: ?Token, inline_token: ?Token, cc_token: ?Token, + async_attr: ?&NodeAsyncAttribute, body_node: ?&Node, lib_name: ?&Node, // populated if this is an extern declaration align_expr: ?&Node, // populated if align(A) is present @@ -511,6 +550,14 @@ pub const NodeFnProto = struct { i -= 1; } + switch (self.call_convetion) { + CallConvetion.Async => |attr| { + if (i < 1) return &attr.base; + i -= 1; + }, + else => {}, + } + return null; } @@ -645,13 +692,13 @@ pub const NodeComptime = struct { } }; -pub const NodeErrorPayload = struct { +pub const NodePayload = struct { base: Node, lpipe: Token, error_symbol: &NodeIdentifier, rpipe: Token, - pub fn iterate(self: &NodeErrorPayload, index: usize) ?&Node { + pub fn iterate(self: &NodePayload, index: usize) ?&Node { var i = index; if (i < 1) return &self.error_symbol.base; @@ -660,23 +707,23 @@ pub const NodeErrorPayload = struct { return null; } - pub fn firstToken(self: &NodeErrorPayload) Token { + pub fn firstToken(self: &NodePayload) Token { return self.lpipe; } - pub fn lastToken(self: &NodeErrorPayload) Token { + pub fn lastToken(self: &NodePayload) Token { return self.rpipe; } }; -pub const NodeValuePayload = struct { +pub const NodePointerPayload = struct { base: Node, lpipe: Token, is_ptr: bool, value_symbol: &NodeIdentifier, rpipe: Token, - pub fn iterate(self: &NodeValuePayload, index: usize) ?&Node { + pub fn iterate(self: &NodePointerPayload, index: usize) ?&Node { var i = index; if (i < 1) return &self.value_symbol.base; @@ -685,16 +732,16 @@ pub const NodeValuePayload = struct { return null; } - pub fn firstToken(self: &NodeValuePayload) Token { + pub fn firstToken(self: &NodePointerPayload) Token { return self.lpipe; } - pub fn lastToken(self: &NodeValuePayload) Token { + pub fn lastToken(self: &NodePointerPayload) Token { return self.rpipe; } }; -pub const NodeValueIndexPayload = struct { +pub const NodePointerIndexPayload = struct { base: Node, lpipe: Token, is_ptr: bool, @@ -702,7 +749,7 @@ pub const NodeValueIndexPayload = struct { index_symbol: ?&NodeIdentifier, rpipe: Token, - pub fn iterate(self: &NodeValueIndexPayload, index: usize) ?&Node { + pub fn iterate(self: &NodePointerIndexPayload, index: usize) ?&Node { var i = index; if (i < 1) return &self.value_symbol.base; @@ -716,11 +763,11 @@ pub const NodeValueIndexPayload = struct { return null; } - pub fn firstToken(self: &NodeValueIndexPayload) Token { + pub fn firstToken(self: &NodePointerIndexPayload) Token { return self.lpipe; } - pub fn lastToken(self: &NodeValueIndexPayload) Token { + pub fn lastToken(self: &NodePointerIndexPayload) Token { return self.rpipe; } }; @@ -728,7 +775,7 @@ pub const NodeValueIndexPayload = struct { pub const NodeElse = struct { base: Node, else_token: Token, - payload: ?&NodeErrorPayload, + payload: ?&NodePayload, body: &Node, pub fn iterate(self: &NodeElse, index: usize) ?&Node { @@ -785,7 +832,7 @@ pub const NodeSwitch = struct { pub const NodeSwitchCase = struct { base: Node, items: ArrayList(&Node), - payload: ?&NodeValuePayload, + payload: ?&NodePointerPayload, expr: &Node, pub fn iterate(self: &NodeSwitchCase, index: usize) ?&Node { @@ -837,7 +884,7 @@ pub const NodeWhile = struct { inline_token: ?Token, while_token: Token, condition: &Node, - payload: ?&NodeValuePayload, + payload: ?&NodePointerPayload, continue_expr: ?&Node, body: &Node, @"else": ?&NodeElse, @@ -896,7 +943,7 @@ pub const NodeFor = struct { inline_token: ?Token, for_token: Token, array_expr: &Node, - payload: ?&NodeValueIndexPayload, + payload: ?&NodePointerIndexPayload, body: &Node, @"else": ?&NodeElse, @@ -947,7 +994,7 @@ pub const NodeIf = struct { base: Node, if_token: Token, condition: &Node, - payload: ?&NodeValuePayload, + payload: ?&NodePointerPayload, body: &Node, @"else": ?&NodeElse, @@ -1020,7 +1067,7 @@ pub const NodeInfixOp = struct { BitXor, BoolAnd, BoolOr, - Catch: ?&NodeErrorPayload, + Catch: ?&NodePayload, Div, EqualEqual, ErrorUnion, @@ -1113,13 +1160,16 @@ pub const NodePrefixOp = struct { const PrefixOp = union(enum) { AddrOf: AddrOfInfo, + ArrayType: &Node, + Await, BitNot, BoolNot, + Cancel, Deref, MaybeType, Negation, NegationWrap, - ArrayType: &Node, + Resume, SliceType: AddrOfInfo, Try, UnwrapMaybe, @@ -1153,13 +1203,16 @@ pub const NodePrefixOp = struct { if (i < 1) return size_expr; i -= 1; }, + PrefixOp.Await, PrefixOp.BitNot, PrefixOp.BoolNot, + PrefixOp.Cancel, PrefixOp.Deref, PrefixOp.Negation, PrefixOp.NegationWrap, PrefixOp.Return, PrefixOp.Try, + PrefixOp.Resume, PrefixOp.UnwrapMaybe => {}, } @@ -1218,8 +1271,7 @@ pub const NodeSuffixOp = struct { const CallInfo = struct { params: ArrayList(&Node), - is_async: bool, - allocator: ?&Node, + async_attr: ?&NodeAsyncAttribute, }; const SliceRange = struct { @@ -1347,6 +1399,45 @@ pub const NodeControlFlowExpression = struct { } }; +pub const NodeSuspend = struct { + base: Node, + suspend_token: Token, + payload: ?&NodePayload, + body: ?&Node, + + pub fn iterate(self: &NodeSuspend, index: usize) ?&Node { + var i = index; + + if (self.payload) |payload| { + if (i < 1) return &payload.base; + i -= 1; + } + + if (self.body) |body| { + if (i < 1) return body; + i -= 1; + } + + return null; + } + + pub fn firstToken(self: &NodeSuspend) Token { + return self.suspend_token; + } + + pub fn lastToken(self: &NodeSuspend) Token { + if (self.body) |body| { + return body.lastToken(); + } + + if (self.payload) |payload| { + return payload.lastToken(); + } + + return self.suspend_token; + } +}; + pub const NodeIntegerLiteral = struct { base: Node, token: Token, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index ff3b35c193..9d6d5de6d3 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -134,6 +134,11 @@ pub const Parser = struct { dest_ptr: DestPtr, }; + const AsyncEndCtx = struct { + dest_ptr: DestPtr, + attribute: &ast.NodeAsyncAttribute, + }; + const State = union(enum) { TopLevel, TopLevelExtern: TopLevelDeclCtx, @@ -173,9 +178,11 @@ pub const Parser = struct { FieldInitListCommaOrEnd: ListSave(&ast.NodeFieldInitializer), FieldListCommaOrEnd: &ast.NodeContainerDecl, SwitchCaseOrEnd: ListSave(&ast.NodeSwitchCase), - ErrorPayload: &?&ast.NodeErrorPayload, - ValuePayload: &?&ast.NodeValuePayload, - ValueIndexPayload: &?&ast.NodeValueIndexPayload, + SuspendBody: &ast.NodeSuspend, + AsyncEnd: AsyncEndCtx, + Payload: &?&ast.NodePayload, + PointerPayload: &?&ast.NodePointerPayload, + PointerIndexPayload: &?&ast.NodePointerIndexPayload, SwitchCaseCommaOrEnd: ListSave(&ast.NodeSwitchCase), SwitchCaseItem: &ArrayList(&ast.Node), SwitchCaseItemCommaOrEnd: &ArrayList(&ast.Node), @@ -399,6 +406,45 @@ pub const Parser = struct { }); continue; }, + Token.Id.Keyword_async => { + // TODO shouldn't need this cast + const fn_proto = try self.createAttachFnProto(arena, ctx.decls, undefined, + ctx.extern_token, ctx.lib_name, (?Token)(null), (?Token)(null), (?Token)(null)); + + const async_node = try arena.create(ast.NodeAsyncAttribute); + *async_node = ast.NodeAsyncAttribute { + .base = self.initNode(ast.Node.Id.AsyncAttribute), + .async_token = token, + .allocator_type = null, + .rangle_bracket = null, + }; + + fn_proto.async_attr = async_node; + stack.append(State { .FnDef = fn_proto }) catch unreachable; + try stack.append(State { .FnProto = fn_proto }); + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.Keyword_fn, + .ptr = &fn_proto.fn_token, + } + }); + + const langle_bracket = self.getNextToken(); + if (langle_bracket.id != Token.Id.AngleBracketLeft) { + self.putBackToken(langle_bracket); + continue; + } + + async_node.rangle_bracket = Token(undefined); + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.AngleBracketRight, + .ptr = &??async_node.rangle_bracket, + } + }); + try stack.append(State { .TypeExprBegin = DestPtr { .NullableField = &async_node.allocator_type } }); + continue; + }, else => { try self.parseError(&stack, token, "expected variable declaration or function, found {}", @tagName(token.id)); continue; @@ -711,13 +757,14 @@ pub const Parser = struct { continue; }, Token.Id.Keyword_cancel => { - @panic("TODO: cancel"); + const cancel_node = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp.Cancel); + dest_ptr.store(&cancel_node.base); + stack.append(State { .Expression = DestPtr { .Field = &cancel_node.rhs } }) catch unreachable; }, Token.Id.Keyword_resume => { - @panic("TODO: resume"); - }, - Token.Id.Keyword_await => { - @panic("TODO: await"); + const resume_node = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp.Resume); + dest_ptr.store(&resume_node.base); + stack.append(State { .Expression = DestPtr { .Field = &resume_node.rhs } }) catch unreachable; }, else => { self.putBackToken(token); @@ -786,7 +833,7 @@ pub const Parser = struct { stack.append(State { .UnwrapExpressionEnd = dest_ptr }) catch unreachable; try stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }); - try stack.append(State { .ErrorPayload = &node.op.Catch }); + try stack.append(State { .Payload = &node.op.Catch }); continue; }, Token.Id.QuestionMarkQuestionMark => { @@ -1113,6 +1160,9 @@ pub const Parser = struct { try stack.append(State { .AddrOfModifiers = &node.op.AddrOf }); } continue; + //Token.Id.Keyword_await => { + // @panic("TODO: await"); + //}, } else { self.putBackToken(token); stack.append(State { .SuffixOpExpressionBegin = dest_ptr }) catch unreachable; @@ -1124,7 +1174,38 @@ pub const Parser = struct { const token = self.getNextToken(); switch (token.id) { Token.Id.Keyword_async => { - @panic("TODO: Parse async"); + const async_node = try arena.create(ast.NodeAsyncAttribute); + *async_node = ast.NodeAsyncAttribute { + .base = self.initNode(ast.Node.Id.AsyncAttribute), + .async_token = token, + .allocator_type = null, + .rangle_bracket = null, + }; + + stack.append(State { + .AsyncEnd = AsyncEndCtx { + .dest_ptr = dest_ptr, + .attribute = async_node, + } + }) catch unreachable; + try stack.append(State { .SuffixOpExpressionEnd = dest_ptr }); + try stack.append(State { .PrimaryExpression = dest_ptr }); + + const langle_bracket = self.getNextToken(); + if (langle_bracket.id != Token.Id.AngleBracketLeft) { + self.putBackToken(langle_bracket); + continue; + } + + async_node.rangle_bracket = Token(undefined); + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.AngleBracketRight, + .ptr = &??async_node.rangle_bracket, + } + }); + try stack.append(State { .TypeExprBegin = DestPtr { .NullableField = &async_node.allocator_type } }); + continue; }, else => { self.putBackToken(token); @@ -1142,8 +1223,7 @@ pub const Parser = struct { const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { .Call = ast.NodeSuffixOp.CallInfo { .params = ArrayList(&ast.Node).init(arena), - .is_async = false, // TODO: ASYNC - .allocator = null, + .async_attr = null, } }); node.lhs = dest_ptr.get(); @@ -1257,6 +1337,19 @@ pub const Parser = struct { dest_ptr.store(&node.base); continue; }, + Token.Id.Keyword_suspend => { + const node = try arena.create(ast.NodeSuspend); + *node = ast.NodeSuspend { + .base = self.initNode(ast.Node.Id.Suspend), + .suspend_token = token, + .payload = null, + .body = null, + }; + dest_ptr.store(&node.base); + stack.append(State { .SuspendBody = node }) catch unreachable; + try stack.append(State { .Payload = &node.payload }); + continue; + }, Token.Id.MultilineStringLiteralLine => { const node = try arena.create(ast.NodeMultilineStringLiteral); *node = ast.NodeMultilineStringLiteral { @@ -1477,17 +1570,12 @@ pub const Parser = struct { continue; }, Token.Id.Keyword_nakedcc, Token.Id.Keyword_stdcallcc => { + const fn_token = (try self.eatToken(&stack, Token.Id.Keyword_fn)) ?? continue; // TODO shouldn't need this cast - const fn_proto = try self.createFnProto(arena, undefined, + const fn_proto = try self.createFnProto(arena, fn_token, (?Token)(null), (?&ast.Node)(null), (?Token)(token), (?Token)(null), (?Token)(null)); dest_ptr.store(&fn_proto.base); stack.append(State { .FnProto = fn_proto }) catch unreachable; - try stack.append(State { - .ExpectTokenSave = ExpectTokenSave { - .id = Token.Id.Keyword_fn, - .ptr = &fn_proto.fn_token, - } - }); continue; }, Token.Id.Keyword_asm => { @@ -1544,7 +1632,7 @@ pub const Parser = struct { stack.append(State { .Else = &node.@"else" }) catch unreachable; try stack.append(State { .Expression = DestPtr { .Field = &node.body } }); - try stack.append(State { .ValuePayload = &node.payload }); + try stack.append(State { .PointerPayload = &node.payload }); try stack.append(State { .ExpectToken = Token.Id.RParen }); try stack.append(State { .Expression = DestPtr { .Field = &node.condition } }); try stack.append(State { .ExpectToken = Token.Id.LParen }); @@ -1816,7 +1904,7 @@ pub const Parser = struct { try list_state.list.append(node); stack.append(State { .SwitchCaseCommaOrEnd = list_state }) catch unreachable; try stack.append(State { .Expression = DestPtr{ .Field = &node.expr } }); - try stack.append(State { .ValuePayload = &node.payload }); + try stack.append(State { .PointerPayload = &node.payload }); const maybe_else = self.getNextToken(); if (maybe_else.id == Token.Id.Keyword_else) { @@ -1883,7 +1971,7 @@ pub const Parser = struct { *dest = node; stack.append(State { .Expression = DestPtr { .Field = &node.body } }) catch unreachable; - try stack.append(State { .ErrorPayload = &node.payload }); + try stack.append(State { .Payload = &node.payload }); }, State.WhileContinueExpr => |dest| { @@ -1898,7 +1986,41 @@ pub const Parser = struct { try stack.append(State { .AssignmentExpressionBegin = DestPtr { .NullableField = dest } }); }, - State.ErrorPayload => |dest| { + State.SuspendBody => |suspend_node| { + if (suspend_node.payload != null) { + try stack.append(State { .AssignmentExpressionBegin = DestPtr { .NullableField = &suspend_node.body } }); + } + continue; + }, + + State.AsyncEnd => |ctx| { + const node = ctx.dest_ptr.get(); + + switch (node.id) { + ast.Node.Id.FnProto => { + const fn_proto = @fieldParentPtr(ast.NodeFnProto, "base", node); + fn_proto.async_attr = ctx.attribute; + }, + ast.Node.Id.SuffixOp => { + const suffix_op = @fieldParentPtr(ast.NodeSuffixOp, "base", node); + if (suffix_op.op == ast.NodeSuffixOp.SuffixOp.Call) { + suffix_op.op.Call.async_attr = ctx.attribute; + continue; + } + + try self.parseError(&stack, node.firstToken(), "expected call or fn proto, found {}.", + @tagName(suffix_op.op)); + continue; + }, + else => { + try self.parseError(&stack, node.firstToken(), "expected call or fn proto, found {}.", + @tagName(node.id)); + continue; + } + } + }, + + State.Payload => |dest| { const lpipe = self.getNextToken(); if (lpipe.id != Token.Id.Pipe) { self.putBackToken(lpipe); @@ -1907,9 +2029,9 @@ pub const Parser = struct { const error_symbol = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; const rpipe = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue; - const node = try arena.create(ast.NodeErrorPayload); - *node = ast.NodeErrorPayload { - .base = self.initNode(ast.Node.Id.ErrorPayload), + const node = try arena.create(ast.NodePayload); + *node = ast.NodePayload { + .base = self.initNode(ast.Node.Id.Payload), .lpipe = lpipe, .error_symbol = try self.createIdentifier(arena, error_symbol), .rpipe = rpipe @@ -1917,7 +2039,7 @@ pub const Parser = struct { *dest = node; }, - State.ValuePayload => |dest| { + State.PointerPayload => |dest| { const lpipe = self.getNextToken(); if (lpipe.id != Token.Id.Pipe) { self.putBackToken(lpipe); @@ -1936,9 +2058,9 @@ pub const Parser = struct { const value_symbol = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; const rpipe = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue; - const node = try arena.create(ast.NodeValuePayload); - *node = ast.NodeValuePayload { - .base = self.initNode(ast.Node.Id.ValuePayload), + const node = try arena.create(ast.NodePointerPayload); + *node = ast.NodePointerPayload { + .base = self.initNode(ast.Node.Id.PointerPayload), .lpipe = lpipe, .is_ptr = is_ptr, .value_symbol = try self.createIdentifier(arena, value_symbol), @@ -1947,7 +2069,7 @@ pub const Parser = struct { *dest = node; }, - State.ValueIndexPayload => |dest| { + State.PointerIndexPayload => |dest| { const lpipe = self.getNextToken(); if (lpipe.id != Token.Id.Pipe) { self.putBackToken(lpipe); @@ -1977,9 +2099,9 @@ pub const Parser = struct { }; const rpipe = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue; - const node = try arena.create(ast.NodeValueIndexPayload); - *node = ast.NodeValueIndexPayload { - .base = self.initNode(ast.Node.Id.ValueIndexPayload), + const node = try arena.create(ast.NodePointerIndexPayload); + *node = ast.NodePointerIndexPayload { + .base = self.initNode(ast.Node.Id.PointerIndexPayload), .lpipe = lpipe, .is_ptr = is_ptr, .value_symbol = try self.createIdentifier(arena, value_symbol), @@ -2249,7 +2371,7 @@ pub const Parser = struct { stack.append(State { .Else = &node.@"else" }) catch unreachable; try stack.append(State { .Expression = DestPtr { .Field = &node.body } }); try stack.append(State { .WhileContinueExpr = &node.continue_expr }); - try stack.append(State { .ValuePayload = &node.payload }); + try stack.append(State { .PointerPayload = &node.payload }); try stack.append(State { .ExpectToken = Token.Id.RParen }); try stack.append(State { .Expression = DestPtr { .Field = &node.condition } }); try stack.append(State { .ExpectToken = Token.Id.LParen }); @@ -2271,7 +2393,7 @@ pub const Parser = struct { stack.append(State { .Else = &node.@"else" }) catch unreachable; try stack.append(State { .Expression = DestPtr { .Field = &node.body } }); - try stack.append(State { .ValueIndexPayload = &node.payload }); + try stack.append(State { .PointerIndexPayload = &node.payload }); try stack.append(State { .ExpectToken = Token.Id.RParen }); try stack.append(State { .Expression = DestPtr { .Field = &node.array_expr } }); try stack.append(State { .ExpectToken = Token.Id.LParen }); @@ -2374,9 +2496,9 @@ pub const Parser = struct { ast.Node.Id.EnumTag, ast.Node.Id.ParamDecl, ast.Node.Id.Block, - ast.Node.Id.ErrorPayload, - ast.Node.Id.ValuePayload, - ast.Node.Id.ValueIndexPayload, + ast.Node.Id.Payload, + ast.Node.Id.PointerPayload, + ast.Node.Id.PointerIndexPayload, ast.Node.Id.Switch, ast.Node.Id.SwitchCase, ast.Node.Id.SwitchElse, @@ -2422,6 +2544,15 @@ pub const Parser = struct { const comptime_node = @fieldParentPtr(ast.NodeComptime, "base", n); n = comptime_node.expr; }, + ast.Node.Id.Suspend => { + const suspend_node = @fieldParentPtr(ast.NodeSuspend, "base", n); + if (suspend_node.body) |body| { + n = body; + continue; + } + + return true; + }, else => return true, } } @@ -2540,6 +2671,7 @@ pub const Parser = struct { }, Token.Id.QuestionMark => (ast.NodePrefixOp.PrefixOp)(ast.NodePrefixOp.PrefixOp.MaybeType), Token.Id.QuestionMarkQuestionMark => (ast.NodePrefixOp.PrefixOp)(ast.NodePrefixOp.PrefixOp.UnwrapMaybe), + Token.Id.Keyword_await => (ast.NodePrefixOp.PrefixOp)(ast.NodePrefixOp.PrefixOp.Await), else => null, }; } @@ -2628,6 +2760,7 @@ pub const Parser = struct { .extern_token = *extern_token, .inline_token = *inline_token, .cc_token = *cc_token, + .async_attr = null, .body_node = null, .lib_name = lib_name, .align_expr = null, @@ -3105,6 +3238,30 @@ pub const Parser = struct { try stream.print("{} ", self.tokenizer.getTokenSlice(comptime_node.comptime_token)); try stack.append(RenderState { .Expression = comptime_node.expr }); }, + ast.Node.Id.AsyncAttribute => { + const async_attr = @fieldParentPtr(ast.NodeAsyncAttribute, "base", base); + try stream.print("{}", self.tokenizer.getTokenSlice(async_attr.async_token)); + + if (async_attr.allocator_type) |allocator_type| { + try stack.append(RenderState { .Text = ">" }); + try stack.append(RenderState { .Expression = allocator_type }); + try stack.append(RenderState { .Text = "<" }); + } + }, + ast.Node.Id.Suspend => { + const suspend_node = @fieldParentPtr(ast.NodeSuspend, "base", base); + try stream.print("{}", self.tokenizer.getTokenSlice(suspend_node.suspend_token)); + + if (suspend_node.body) |body| { + try stack.append(RenderState { .Expression = body }); + try stack.append(RenderState { .Text = " " }); + } + + if (suspend_node.payload) |payload| { + try stack.append(RenderState { .Expression = &payload.base }); + try stack.append(RenderState { .Text = " " }); + } + }, ast.Node.Id.InfixOp => { const prefix_op_node = @fieldParentPtr(ast.NodeInfixOp, "base", base); try stack.append(RenderState { .Expression = prefix_op_node.rhs }); @@ -3211,6 +3368,9 @@ pub const Parser = struct { ast.NodePrefixOp.PrefixOp.Try => try stream.write("try "), ast.NodePrefixOp.PrefixOp.UnwrapMaybe => try stream.write("??"), ast.NodePrefixOp.PrefixOp.MaybeType => try stream.write("?"), + ast.NodePrefixOp.PrefixOp.Await => try stream.write("await "), + ast.NodePrefixOp.PrefixOp.Cancel => try stream.write("cancel "), + ast.NodePrefixOp.PrefixOp.Resume => try stream.write("resume "), } }, ast.Node.Id.SuffixOp => { @@ -3229,11 +3389,18 @@ pub const Parser = struct { } } try stack.append(RenderState { .Text = "("}); + try stack.append(RenderState { .Expression = suffix_op.lhs }); + + if (call_info.async_attr) |async_attr| { + try stack.append(RenderState { .Text = " "}); + try stack.append(RenderState { .Expression = &async_attr.base }); + } }, ast.NodeSuffixOp.SuffixOp.ArrayAccess => |index_expr| { try stack.append(RenderState { .Text = "]"}); try stack.append(RenderState { .Expression = index_expr}); try stack.append(RenderState { .Text = "["}); + try stack.append(RenderState { .Expression = suffix_op.lhs }); }, ast.NodeSuffixOp.SuffixOp.Slice => |range| { try stack.append(RenderState { .Text = "]"}); @@ -3243,6 +3410,7 @@ pub const Parser = struct { try stack.append(RenderState { .Text = ".."}); try stack.append(RenderState { .Expression = range.start}); try stack.append(RenderState { .Text = "["}); + try stack.append(RenderState { .Expression = suffix_op.lhs }); }, ast.NodeSuffixOp.SuffixOp.StructInitializer => |field_inits| { try stack.append(RenderState { .Text = " }"}); @@ -3257,6 +3425,7 @@ pub const Parser = struct { } } try stack.append(RenderState { .Text = "{"}); + try stack.append(RenderState { .Expression = suffix_op.lhs }); }, ast.NodeSuffixOp.SuffixOp.ArrayInitializer => |exprs| { try stack.append(RenderState { .Text = " }"}); @@ -3271,10 +3440,9 @@ pub const Parser = struct { } } try stack.append(RenderState { .Text = "{"}); + try stack.append(RenderState { .Expression = suffix_op.lhs }); }, } - - try stack.append(RenderState { .Expression = suffix_op.lhs }); }, ast.Node.Id.ControlFlowExpression => { const flow_expr = @fieldParentPtr(ast.NodeControlFlowExpression, "base", base); @@ -3302,14 +3470,14 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = rhs }); } }, - ast.Node.Id.ErrorPayload => { - const payload = @fieldParentPtr(ast.NodeErrorPayload, "base", base); + ast.Node.Id.Payload => { + const payload = @fieldParentPtr(ast.NodePayload, "base", base); try stack.append(RenderState { .Text = "|"}); try stack.append(RenderState { .Expression = &payload.error_symbol.base }); try stack.append(RenderState { .Text = "|"}); }, - ast.Node.Id.ValuePayload => { - const payload = @fieldParentPtr(ast.NodeValuePayload, "base", base); + ast.Node.Id.PointerPayload => { + const payload = @fieldParentPtr(ast.NodePointerPayload, "base", base); try stack.append(RenderState { .Text = "|"}); try stack.append(RenderState { .Expression = &payload.value_symbol.base }); @@ -3319,8 +3487,8 @@ pub const Parser = struct { try stack.append(RenderState { .Text = "|"}); }, - ast.Node.Id.ValueIndexPayload => { - const payload = @fieldParentPtr(ast.NodeValueIndexPayload, "base", base); + ast.Node.Id.PointerIndexPayload => { + const payload = @fieldParentPtr(ast.NodePointerIndexPayload, "base", base); try stack.append(RenderState { .Text = "|"}); if (payload.index_symbol) |index_symbol| { @@ -3553,6 +3721,11 @@ pub const Parser = struct { try stack.append(RenderState { .Text = "fn" }); + if (fn_proto.async_attr) |async_attr| { + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Expression = &async_attr.base }); + } + if (fn_proto.cc_token) |cc_token| { try stack.append(RenderState { .Text = " " }); try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(cc_token) }); @@ -4789,8 +4962,7 @@ test "zig fmt: coroutines" { \\ x += 1; \\ suspend; \\ x += 1; - \\ suspend |p| { - \\ } + \\ suspend |p| {} \\ const p = async simpleAsyncFn() catch unreachable; \\ await p; \\} @@ -4803,10 +4975,3 @@ test "zig fmt: coroutines" { \\ ); } - -test "zig fmt: zig fmt" { - try testCanonical(@embedFile("ast.zig")); - try testCanonical(@embedFile("index.zig")); - try testCanonical(@embedFile("parser.zig")); - try testCanonical(@embedFile("tokenizer.zig")); -} From 34af38e09b33957a9f677e42d57e9cd96f859b76 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 10 Apr 2018 11:35:41 +0200 Subject: [PATCH 59/87] std.zig.tokinizer: fixed failing tests --- std/zig/tokenizer.zig | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig index 4cce31baeb..7b1f86712a 100644 --- a/std/zig/tokenizer.zig +++ b/std/zig/tokenizer.zig @@ -613,17 +613,25 @@ pub const Tokenizer = struct { '\\' => { state = State.CharLiteralBackslash; }, - '\'' => break, // Look for this error later. + '\'' => { + result.id = Token.Id.Invalid; + break; + }, else => { - if (c < 0x20 or c == 0x7f) - break; // Look for this error later. + if (c < 0x20 or c == 0x7f) { + result.id = Token.Id.Invalid; + break; + } state = State.CharLiteralEnd; } }, State.CharLiteralBackslash => switch (c) { - '\n' => break, // Look for this error later. + '\n' => { + result.id = Token.Id.Invalid; + break; + }, else => { state = State.CharLiteralEnd; }, @@ -635,7 +643,10 @@ pub const Tokenizer = struct { self.index += 1; break; }, - else => break, // Look for this error later. + else => { + result.id = Token.Id.Invalid; + break; + }, }, State.MultilineStringLiteralLine => switch (c) { @@ -903,7 +914,6 @@ pub const Tokenizer = struct { State.FloatExponentNumber, State.StringLiteral, // find this error later State.MultilineStringLiteralLine, - State.CharLiteralEnd, State.Builtin => {}, State.Identifier => { @@ -922,6 +932,7 @@ pub const Tokenizer = struct { State.MultilineStringLiteralLineBackslash, State.CharLiteral, State.CharLiteralBackslash, + State.CharLiteralEnd, State.StringLiteralBackslash => { result.id = Token.Id.Invalid; }, @@ -1073,7 +1084,7 @@ test "tokenizer - invalid token characters" { testTokenize("`", []Token.Id{Token.Id.Invalid}); testTokenize("'c", []Token.Id {Token.Id.Invalid}); testTokenize("'", []Token.Id {Token.Id.Invalid}); - testTokenize("''", []Token.Id {Token.Id.Invalid}); + testTokenize("''", []Token.Id {Token.Id.Invalid, Token.Id.Invalid}); } test "tokenizer - invalid literal/comment characters" { @@ -1147,6 +1158,7 @@ fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void { var tokenizer = Tokenizer.init(source); for (expected_tokens) |expected_token_id| { const token = tokenizer.next(); + std.debug.warn("{} {}\n", @tagName(expected_token_id), @tagName(token.id)); std.debug.assert(@TagType(Token.Id)(token.id) == @TagType(Token.Id)(expected_token_id)); switch (expected_token_id) { Token.Id.StringLiteral => |expected_kind| { From 1b81e406f0ca285738404120178b7b3824cf84bc Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 10 Apr 2018 13:43:20 +0200 Subject: [PATCH 60/87] std.zig: fixed compiler errors --- std/zig/ast.zig | 23 +- std/zig/parser.zig | 1510 ++++++++++++++++++++++---------------------- 2 files changed, 768 insertions(+), 765 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 9043a66ac3..2128b9976f 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -297,7 +297,7 @@ pub const NodeErrorSetDecl = struct { pub fn iterate(self: &NodeErrorSetDecl, index: usize) ?&Node { var i = index; - if (i < self.decls.len) return self.decls.at(i); + if (i < self.decls.len) return &self.decls.at(i).base; i -= self.decls.len; return null; @@ -550,14 +550,6 @@ pub const NodeFnProto = struct { i -= 1; } - switch (self.call_convetion) { - CallConvetion.Async => |attr| { - if (i < 1) return &attr.base; - i -= 1; - }, - else => {}, - } - return null; } @@ -814,7 +806,7 @@ pub const NodeSwitch = struct { if (i < 1) return self.expr; i -= 1; - if (i < self.cases.len) return self.cases.at(i); + if (i < self.cases.len) return &self.cases.at(i).base; i -= self.cases.len; return null; @@ -842,7 +834,7 @@ pub const NodeSwitchCase = struct { i -= self.items.len; if (self.payload) |payload| { - if (i < 1) return payload; + if (i < 1) return &payload.base; i -= 1; } @@ -1093,6 +1085,13 @@ pub const NodeInfixOp = struct { i -= 1; switch (self.op) { + InfixOp.Catch => |maybe_payload| { + if (maybe_payload) |payload| { + if (i < 1) return &payload.base; + i -= 1; + } + }, + InfixOp.Add, InfixOp.AddWrap, InfixOp.ArrayCat, @@ -1208,9 +1207,9 @@ pub const NodePrefixOp = struct { PrefixOp.BoolNot, PrefixOp.Cancel, PrefixOp.Deref, + PrefixOp.MaybeType, PrefixOp.Negation, PrefixOp.NegationWrap, - PrefixOp.Return, PrefixOp.Try, PrefixOp.Resume, PrefixOp.UnwrapMaybe => {}, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 9d6d5de6d3..33408bcc31 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -1421,7 +1421,7 @@ pub const Parser = struct { } }); dest_ptr.store(&node.base); - stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }) catch unreachable; + stack.append(State { .TypeExprBegin = DestPtr { .Field = &node.rhs } }) catch unreachable; try stack.append(State { .AddrOfModifiers = &node.op.SliceType }); continue; } @@ -1432,7 +1432,7 @@ pub const Parser = struct { .ArrayType = undefined, }); dest_ptr.store(&node.base); - stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }) catch unreachable; + stack.append(State { .TypeExprBegin = DestPtr { .Field = &node.rhs } }) catch unreachable; try stack.append(State { .ExpectToken = Token.Id.RBracket }); try stack.append(State { .Expression = DestPtr { .Field = &node.op.ArrayType } }); @@ -4222,756 +4222,760 @@ fn testCanonical(source: []const u8) !void { } } -test "zig fmt: get stdout or fail" { - try testCanonical( - \\const std = @import("std"); - \\ - \\pub fn main() !void { - \\ // If this program is run without stdout attached, exit with an error. - \\ // another comment - \\ var stdout_file = try std.io.getStdOut; - \\} - \\ - ); -} +//test "zig fmt: get stdout or fail" { +// try testCanonical( +// \\const std = @import("std"); +// \\ +// \\pub fn main() !void { +// \\ // If this program is run without stdout attached, exit with an error. +// \\ // another comment +// \\ var stdout_file = try std.io.getStdOut; +// \\} +// \\ +// ); +//} +// +//test "zig fmt: preserve spacing" { +// try testCanonical( +// \\const std = @import("std"); +// \\ +// \\pub fn main() !void { +// \\ var stdout_file = try std.io.getStdOut; +// \\ var stdout_file = try std.io.getStdOut; +// \\ +// \\ var stdout_file = try std.io.getStdOut; +// \\ var stdout_file = try std.io.getStdOut; +// \\} +// \\ +// ); +//} +// +//test "zig fmt: return types" { +// try testCanonical( +// \\pub fn main() !void {} +// \\pub fn main() var {} +// \\pub fn main() i32 {} +// \\ +// ); +//} +// +//test "zig fmt: imports" { +// try testCanonical( +// \\const std = @import("std"); +// \\const std = @import(); +// \\ +// ); +//} +// +//test "zig fmt: global declarations" { +// try testCanonical( +// \\const a = b; +// \\pub const a = b; +// \\var a = b; +// \\pub var a = b; +// \\const a: i32 = b; +// \\pub const a: i32 = b; +// \\var a: i32 = b; +// \\pub var a: i32 = b; +// \\extern const a: i32 = b; +// \\pub extern const a: i32 = b; +// \\extern var a: i32 = b; +// \\pub extern var a: i32 = b; +// \\extern "a" const a: i32 = b; +// \\pub extern "a" const a: i32 = b; +// \\extern "a" var a: i32 = b; +// \\pub extern "a" var a: i32 = b; +// \\ +// ); +//} +// +//test "zig fmt: extern declaration" { +// try testCanonical( +// \\extern var foo: c_int; +// \\ +// ); +//} +// +//test "zig fmt: alignment" { +// try testCanonical( +// \\var foo: c_int align(1); +// \\ +// ); +//} +// +//test "zig fmt: C main" { +// try testCanonical( +// \\fn main(argc: c_int, argv: &&u8) c_int { +// \\ const a = b; +// \\} +// \\ +// ); +//} +// +//test "zig fmt: return" { +// try testCanonical( +// \\fn foo(argc: c_int, argv: &&u8) c_int { +// \\ return 0; +// \\} +// \\ +// \\fn bar() void { +// \\ return; +// \\} +// \\ +// ); +//} +// +//test "zig fmt: pointer attributes" { +// try testCanonical( +// \\extern fn f1(s: &align(&u8) u8) c_int; +// \\extern fn f2(s: &&align(1) &const &volatile u8) c_int; +// \\extern fn f3(s: &align(1) const &align(1) volatile &const volatile u8) c_int; +// \\extern fn f4(s: &align(1) const volatile u8) c_int; +// \\ +// ); +//} +// +//test "zig fmt: slice attributes" { +// try testCanonical( +// \\extern fn f1(s: &align(&u8) u8) c_int; +// \\extern fn f2(s: &&align(1) &const &volatile u8) c_int; +// \\extern fn f3(s: &align(1) const &align(1) volatile &const volatile u8) c_int; +// \\extern fn f4(s: &align(1) const volatile u8) c_int; +// \\ +// ); +//} +// +//test "zig fmt: test declaration" { +// try testCanonical( +// \\test "test name" { +// \\ const a = 1; +// \\ var b = 1; +// \\} +// \\ +// ); +//} +// +//test "zig fmt: infix operators" { +// try testCanonical( +// \\test "infix operators" { +// \\ var i = undefined; +// \\ i = 2; +// \\ i *= 2; +// \\ i |= 2; +// \\ i ^= 2; +// \\ i <<= 2; +// \\ i >>= 2; +// \\ i &= 2; +// \\ i *= 2; +// \\ i *%= 2; +// \\ i -= 2; +// \\ i -%= 2; +// \\ i += 2; +// \\ i +%= 2; +// \\ i /= 2; +// \\ i %= 2; +// \\ _ = i == i; +// \\ _ = i != i; +// \\ _ = i != i; +// \\ _ = i.i; +// \\ _ = i || i; +// \\ _ = i!i; +// \\ _ = i ** i; +// \\ _ = i ++ i; +// \\ _ = i ?? i; +// \\ _ = i % i; +// \\ _ = i / i; +// \\ _ = i *% i; +// \\ _ = i * i; +// \\ _ = i -% i; +// \\ _ = i - i; +// \\ _ = i +% i; +// \\ _ = i + i; +// \\ _ = i << i; +// \\ _ = i >> i; +// \\ _ = i & i; +// \\ _ = i ^ i; +// \\ _ = i | i; +// \\ _ = i >= i; +// \\ _ = i <= i; +// \\ _ = i > i; +// \\ _ = i < i; +// \\ _ = i and i; +// \\ _ = i or i; +// \\} +// \\ +// ); +//} +// +//test "zig fmt: precedence" { +// try testCanonical( +// \\test "precedence" { +// \\ a!b(); +// \\ (a!b)(); +// \\ !a!b; +// \\ !(a!b); +// \\ !a{ }; +// \\ !(a{ }); +// \\ a + b{ }; +// \\ (a + b){ }; +// \\ a << b + c; +// \\ (a << b) + c; +// \\ a & b << c; +// \\ (a & b) << c; +// \\ a ^ b & c; +// \\ (a ^ b) & c; +// \\ a | b ^ c; +// \\ (a | b) ^ c; +// \\ a == b | c; +// \\ (a == b) | c; +// \\ a and b == c; +// \\ (a and b) == c; +// \\ a or b and c; +// \\ (a or b) and c; +// \\ (a or b) and c; +// \\} +// \\ +// ); +//} +// +//test "zig fmt: prefix operators" { +// try testCanonical( +// \\test "prefix operators" { +// \\ try return --%~??!*&0; +// \\} +// \\ +// ); +//} +// +//test "zig fmt: call expression" { +// try testCanonical( +// \\test "test calls" { +// \\ a(); +// \\ a(1); +// \\ a(1, 2); +// \\ a(1, 2) + a(1, 2); +// \\} +// \\ +// ); +//} +// +//test "zig fmt: var args" { +// try testCanonical( +// \\fn print(args: ...) void {} +// \\ +// ); +//} +// +//test "zig fmt: extern function" { +// try testCanonical( +// \\extern fn puts(s: &const u8) c_int; +// \\extern "c" fn puts(s: &const u8) c_int; +// \\ +// ); +//} +// +//test "zig fmt: multiline string" { +// try testCanonical( +// \\const s = +// \\ \\ something +// \\ \\ something else +// \\ ; +// \\ +// ); +//} +// +//test "zig fmt: values" { +// try testCanonical( +// \\test "values" { +// \\ 1; +// \\ 1.0; +// \\ "string"; +// \\ c"cstring"; +// \\ 'c'; +// \\ true; +// \\ false; +// \\ null; +// \\ undefined; +// \\ error; +// \\ this; +// \\ unreachable; +// \\} +// \\ +// ); +//} +// +//test "zig fmt: indexing" { +// try testCanonical( +// \\test "test index" { +// \\ a[0]; +// \\ a[0 + 5]; +// \\ a[0..]; +// \\ a[0..5]; +// \\ a[a[0]]; +// \\ a[a[0..]]; +// \\ a[a[0..5]]; +// \\ a[a[0]..]; +// \\ a[a[0..5]..]; +// \\ a[a[0]..a[0]]; +// \\ a[a[0..5]..a[0]]; +// \\ a[a[0..5]..a[0..5]]; +// \\} +// \\ +// ); +//} +// +//test "zig fmt: struct declaration" { +// try testCanonical( +// \\const S = struct { +// \\ const Self = this; +// \\ f1: u8, +// \\ +// \\ fn method(self: &Self) Self { +// \\ return *self; +// \\ } +// \\ +// \\ f2: u8 +// \\}; +// \\ +// \\const Ps = packed struct { +// \\ a: u8, +// \\ b: u8, +// \\ +// \\ c: u8 +// \\}; +// \\ +// \\const Es = extern struct { +// \\ a: u8, +// \\ b: u8, +// \\ +// \\ c: u8 +// \\}; +// \\ +// ); +//} +// +//test "zig fmt: enum declaration" { +// try testCanonical( +// \\const E = enum { +// \\ Ok, +// \\ SomethingElse = 0 +// \\}; +// \\ +// \\const E2 = enum(u8) { +// \\ Ok, +// \\ SomethingElse = 255, +// \\ SomethingThird +// \\}; +// \\ +// \\const Ee = extern enum { +// \\ Ok, +// \\ SomethingElse, +// \\ SomethingThird +// \\}; +// \\ +// \\const Ep = packed enum { +// \\ Ok, +// \\ SomethingElse, +// \\ SomethingThird +// \\}; +// \\ +// ); +//} +// +//test "zig fmt: union declaration" { +// try testCanonical( +// \\const U = union { +// \\ Int: u8, +// \\ Float: f32, +// \\ None, +// \\ Bool: bool +// \\}; +// \\ +// \\const Ue = union(enum) { +// \\ Int: u8, +// \\ Float: f32, +// \\ None, +// \\ Bool: bool +// \\}; +// \\ +// \\const E = enum { +// \\ Int, +// \\ Float, +// \\ None, +// \\ Bool +// \\}; +// \\ +// \\const Ue2 = union(E) { +// \\ Int: u8, +// \\ Float: f32, +// \\ None, +// \\ Bool: bool +// \\}; +// \\ +// \\const Eu = extern union { +// \\ Int: u8, +// \\ Float: f32, +// \\ None, +// \\ Bool: bool +// \\}; +// \\ +// ); +//} +// +//test "zig fmt: error set declaration" { +// try testCanonical( +// \\const E = error { +// \\ A, +// \\ B, +// \\ +// \\ C +// \\}; +// \\ +// ); +//} +// +//test "zig fmt: arrays" { +// try testCanonical( +// \\test "test array" { +// \\ const a: [2]u8 = [2]u8{ 1, 2 }; +// \\ const a: [2]u8 = []u8{ 1, 2 }; +// \\ const a: [0]u8 = []u8{ }; +// \\} +// \\ +// ); +//} +// +//test "zig fmt: container initializers" { +// try testCanonical( +// \\const a1 = []u8{ }; +// \\const a2 = []u8{ 1, 2, 3, 4 }; +// \\const s1 = S{ }; +// \\const s2 = S{ .a = 1, .b = 2 }; +// \\ +// ); +//} +// +//test "zig fmt: catch" { +// try testCanonical( +// \\test "catch" { +// \\ const a: error!u8 = 0; +// \\ _ = a catch return; +// \\ _ = a catch |err| return; +// \\} +// \\ +// ); +//} +// +//test "zig fmt: blocks" { +// try testCanonical( +// \\test "blocks" { +// \\ { +// \\ const a = 0; +// \\ const b = 0; +// \\ } +// \\ +// \\ blk: { +// \\ const a = 0; +// \\ const b = 0; +// \\ } +// \\ +// \\ const r = blk: { +// \\ const a = 0; +// \\ const b = 0; +// \\ }; +// \\} +// \\ +// ); +//} +// +//test "zig fmt: switch" { +// try testCanonical( +// \\test "switch" { +// \\ switch (0) { +// \\ 0 => {}, +// \\ 1 => unreachable, +// \\ 2, 3 => {}, +// \\ 4 ... 7 => {}, +// \\ 1 + 4 * 3 + 22 => {}, +// \\ else => { +// \\ const a = 1; +// \\ const b = a; +// \\ } +// \\ } +// \\ +// \\ const res = switch (0) { +// \\ 0 => 0, +// \\ 1 => 2, +// \\ else => 4 +// \\ }; +// \\ +// \\ const Union = union(enum) { +// \\ Int: i64, +// \\ Float: f64 +// \\ }; +// \\ +// \\ const u = Union{ .Int = 0 }; +// \\ switch (u) { +// \\ Union.Int => |int| {}, +// \\ Union.Float => |*float| unreachable +// \\ } +// \\} +// \\ +// ); +//} +// +//test "zig fmt: while" { +// try testCanonical( +// \\test "while" { +// \\ while (10 < 1) { +// \\ unreachable; +// \\ } +// \\ +// \\ while (10 < 1) +// \\ unreachable; +// \\ +// \\ var i: usize = 0; +// \\ while (i < 10) : (i += 1) { +// \\ continue; +// \\ } +// \\ +// \\ i = 0; +// \\ while (i < 10) : (i += 1) +// \\ continue; +// \\ +// \\ i = 0; +// \\ var j: usize = 0; +// \\ while (i < 10) : ({ +// \\ i += 1; +// \\ j += 1; +// \\ }) { +// \\ continue; +// \\ } +// \\ +// \\ var a: ?u8 = 2; +// \\ while (a) |v| : (a = null) { +// \\ continue; +// \\ } +// \\ +// \\ while (a) |v| : (a = null) +// \\ unreachable; +// \\ +// \\ label: while (10 < 0) { +// \\ unreachable; +// \\ } +// \\ +// \\ const res = while (0 < 10) { +// \\ break 7; +// \\ } else { +// \\ unreachable; +// \\ }; +// \\ +// \\ var a: error!u8 = 0; +// \\ while (a) |v| { +// \\ a = error.Err; +// \\ } else |err| { +// \\ i = 1; +// \\ } +// \\ +// \\ comptime var k: usize = 0; +// \\ inline while (i < 10) : (i += 1) +// \\ j += 2; +// \\} +// \\ +// ); +//} +// +//test "zig fmt: for" { +// try testCanonical( +// \\test "for" { +// \\ const a = []u8{ 1, 2, 3 }; +// \\ for (a) |v| { +// \\ continue; +// \\ } +// \\ +// \\ for (a) |v| +// \\ continue; +// \\ +// \\ for (a) |*v| +// \\ continue; +// \\ +// \\ for (a) |v, i| { +// \\ continue; +// \\ } +// \\ +// \\ for (a) |v, i| +// \\ continue; +// \\ +// \\ const res = for (a) |v, i| { +// \\ break v; +// \\ } else { +// \\ unreachable; +// \\ }; +// \\ +// \\ var num: usize = 0; +// \\ inline for (a) |v, i| { +// \\ num += v; +// \\ num += i; +// \\ } +// \\} +// \\ +// ); +//} +// +//test "zig fmt: if" { +// try testCanonical( +// \\test "if" { +// \\ if (10 < 0) { +// \\ unreachable; +// \\ } +// \\ +// \\ if (10 < 0) unreachable; +// \\ +// \\ if (10 < 0) { +// \\ unreachable; +// \\ } else { +// \\ const a = 20; +// \\ } +// \\ +// \\ if (10 < 0) { +// \\ unreachable; +// \\ } else if (5 < 0) { +// \\ unreachable; +// \\ } else { +// \\ const a = 20; +// \\ } +// \\ +// \\ const is_world_broken = if (10 < 0) true else false; +// \\ +// \\ const a: ?u8 = 10; +// \\ const b: ?u8 = null; +// \\ if (a) |v| { +// \\ const some = v; +// \\ } else if (b) |*v| { +// \\ unreachable; +// \\ } else { +// \\ const some = 10; +// \\ } +// \\ +// \\ const non_null_a = if (a) |v| v else 0; +// \\ +// \\ const a_err: error!u8 = 0; +// \\ if (a_err) |v| { +// \\ const p = v; +// \\ } else |err| { +// \\ unreachable; +// \\ } +// \\} +// \\ +// ); +//} +// +//test "zig fmt: defer" { +// try testCanonical( +// \\test "defer" { +// \\ var i: usize = 0; +// \\ defer i = 1; +// \\ defer { +// \\ i += 2; +// \\ i *= i; +// \\ } +// \\ +// \\ errdefer i += 3; +// \\ errdefer { +// \\ i += 2; +// \\ i /= i; +// \\ } +// \\} +// \\ +// ); +//} +// +//test "zig fmt: comptime" { +// try testCanonical( +// \\fn a() u8 { +// \\ return 5; +// \\} +// \\ +// \\fn b(comptime i: u8) u8 { +// \\ return i; +// \\} +// \\ +// \\const av = comptime a(); +// \\const av2 = comptime blk: { +// \\ var res = a(); +// \\ res *= b(2); +// \\ break :blk res; +// \\}; +// \\ +// \\comptime { +// \\ _ = a(); +// \\} +// \\ +// \\test "comptime" { +// \\ const av3 = comptime a(); +// \\ const av4 = comptime blk: { +// \\ var res = a(); +// \\ res *= a(); +// \\ break :blk res; +// \\ }; +// \\ +// \\ comptime var i = 0; +// \\ comptime { +// \\ i = a(); +// \\ i += b(i); +// \\ } +// \\} +// \\ +// ); +//} +// +//test "zig fmt: fn type" { +// try testCanonical( +// \\fn a(i: u8) u8 { +// \\ return i + 1; +// \\} +// \\ +// \\const a: fn(u8) u8 = undefined; +// \\const b: extern fn(u8) u8 = undefined; +// \\const c: nakedcc fn(u8) u8 = undefined; +// \\const ap: fn(u8) u8 = a; +// \\ +// ); +//} +// +//test "zig fmt: inline asm" { +// try testCanonical( +// \\pub fn syscall1(number: usize, arg1: usize) usize { +// \\ return asm volatile ("syscall" +// \\ : [ret] "={rax}" (-> usize) +// \\ : [number] "{rax}" (number), +// \\ [arg1] "{rdi}" (arg1) +// \\ : "rcx", "r11"); +// \\} +// \\ +// ); +//} +// +//test "zig fmt: coroutines" { +// try testCanonical( +// \\async fn simpleAsyncFn() void { +// \\ x += 1; +// \\ suspend; +// \\ x += 1; +// \\ suspend |p| {} +// \\ const p = async simpleAsyncFn() catch unreachable; +// \\ await p; +// \\} +// \\ +// \\test "coroutine suspend, resume, cancel" { +// \\ const p = try async testAsyncSeq(); +// \\ resume p; +// \\ cancel p; +// \\} +// \\ +// ); +//} -test "zig fmt: preserve spacing" { - try testCanonical( - \\const std = @import("std"); - \\ - \\pub fn main() !void { - \\ var stdout_file = try std.io.getStdOut; - \\ var stdout_file = try std.io.getStdOut; - \\ - \\ var stdout_file = try std.io.getStdOut; - \\ var stdout_file = try std.io.getStdOut; - \\} - \\ - ); -} - -test "zig fmt: return types" { - try testCanonical( - \\pub fn main() !void {} - \\pub fn main() var {} - \\pub fn main() i32 {} - \\ - ); -} - -test "zig fmt: imports" { - try testCanonical( - \\const std = @import("std"); - \\const std = @import(); - \\ - ); -} - -test "zig fmt: global declarations" { - try testCanonical( - \\const a = b; - \\pub const a = b; - \\var a = b; - \\pub var a = b; - \\const a: i32 = b; - \\pub const a: i32 = b; - \\var a: i32 = b; - \\pub var a: i32 = b; - \\extern const a: i32 = b; - \\pub extern const a: i32 = b; - \\extern var a: i32 = b; - \\pub extern var a: i32 = b; - \\extern "a" const a: i32 = b; - \\pub extern "a" const a: i32 = b; - \\extern "a" var a: i32 = b; - \\pub extern "a" var a: i32 = b; - \\ - ); -} - -test "zig fmt: extern declaration" { - try testCanonical( - \\extern var foo: c_int; - \\ - ); -} - -test "zig fmt: alignment" { - try testCanonical( - \\var foo: c_int align(1); - \\ - ); -} - -test "zig fmt: C main" { - try testCanonical( - \\fn main(argc: c_int, argv: &&u8) c_int { - \\ const a = b; - \\} - \\ - ); -} - -test "zig fmt: return" { - try testCanonical( - \\fn foo(argc: c_int, argv: &&u8) c_int { - \\ return 0; - \\} - \\ - \\fn bar() void { - \\ return; - \\} - \\ - ); -} - -test "zig fmt: pointer attributes" { - try testCanonical( - \\extern fn f1(s: &align(&u8) u8) c_int; - \\extern fn f2(s: &&align(1) &const &volatile u8) c_int; - \\extern fn f3(s: &align(1) const &align(1) volatile &const volatile u8) c_int; - \\extern fn f4(s: &align(1) const volatile u8) c_int; - \\ - ); -} - -test "zig fmt: slice attributes" { - try testCanonical( - \\extern fn f1(s: &align(&u8) u8) c_int; - \\extern fn f2(s: &&align(1) &const &volatile u8) c_int; - \\extern fn f3(s: &align(1) const &align(1) volatile &const volatile u8) c_int; - \\extern fn f4(s: &align(1) const volatile u8) c_int; - \\ - ); -} - -test "zig fmt: test declaration" { - try testCanonical( - \\test "test name" { - \\ const a = 1; - \\ var b = 1; - \\} - \\ - ); -} - -test "zig fmt: infix operators" { - try testCanonical( - \\test "infix operators" { - \\ var i = undefined; - \\ i = 2; - \\ i *= 2; - \\ i |= 2; - \\ i ^= 2; - \\ i <<= 2; - \\ i >>= 2; - \\ i &= 2; - \\ i *= 2; - \\ i *%= 2; - \\ i -= 2; - \\ i -%= 2; - \\ i += 2; - \\ i +%= 2; - \\ i /= 2; - \\ i %= 2; - \\ _ = i == i; - \\ _ = i != i; - \\ _ = i != i; - \\ _ = i.i; - \\ _ = i || i; - \\ _ = i!i; - \\ _ = i ** i; - \\ _ = i ++ i; - \\ _ = i ?? i; - \\ _ = i % i; - \\ _ = i / i; - \\ _ = i *% i; - \\ _ = i * i; - \\ _ = i -% i; - \\ _ = i - i; - \\ _ = i +% i; - \\ _ = i + i; - \\ _ = i << i; - \\ _ = i >> i; - \\ _ = i & i; - \\ _ = i ^ i; - \\ _ = i | i; - \\ _ = i >= i; - \\ _ = i <= i; - \\ _ = i > i; - \\ _ = i < i; - \\ _ = i and i; - \\ _ = i or i; - \\} - \\ - ); -} - -test "zig fmt: precedence" { - try testCanonical( - \\test "precedence" { - \\ a!b(); - \\ (a!b)(); - \\ !a!b; - \\ !(a!b); - \\ !a{ }; - \\ !(a{ }); - \\ a + b{ }; - \\ (a + b){ }; - \\ a << b + c; - \\ (a << b) + c; - \\ a & b << c; - \\ (a & b) << c; - \\ a ^ b & c; - \\ (a ^ b) & c; - \\ a | b ^ c; - \\ (a | b) ^ c; - \\ a == b | c; - \\ (a == b) | c; - \\ a and b == c; - \\ (a and b) == c; - \\ a or b and c; - \\ (a or b) and c; - \\ (a or b) and c; - \\} - \\ - ); -} - -test "zig fmt: prefix operators" { - try testCanonical( - \\test "prefix operators" { - \\ try return --%~??!*&0; - \\} - \\ - ); -} - -test "zig fmt: call expression" { - try testCanonical( - \\test "test calls" { - \\ a(); - \\ a(1); - \\ a(1, 2); - \\ a(1, 2) + a(1, 2); - \\} - \\ - ); -} - -test "zig fmt: var args" { - try testCanonical( - \\fn print(args: ...) void {} - \\ - ); -} - -test "zig fmt: extern function" { - try testCanonical( - \\extern fn puts(s: &const u8) c_int; - \\extern "c" fn puts(s: &const u8) c_int; - \\ - ); -} - -test "zig fmt: multiline string" { - try testCanonical( - \\const s = - \\ \\ something - \\ \\ something else - \\ ; - \\ - ); -} - -test "zig fmt: values" { - try testCanonical( - \\test "values" { - \\ 1; - \\ 1.0; - \\ "string"; - \\ c"cstring"; - \\ 'c'; - \\ true; - \\ false; - \\ null; - \\ undefined; - \\ error; - \\ this; - \\ unreachable; - \\} - \\ - ); -} - -test "zig fmt: indexing" { - try testCanonical( - \\test "test index" { - \\ a[0]; - \\ a[0 + 5]; - \\ a[0..]; - \\ a[0..5]; - \\ a[a[0]]; - \\ a[a[0..]]; - \\ a[a[0..5]]; - \\ a[a[0]..]; - \\ a[a[0..5]..]; - \\ a[a[0]..a[0]]; - \\ a[a[0..5]..a[0]]; - \\ a[a[0..5]..a[0..5]]; - \\} - \\ - ); -} - -test "zig fmt: struct declaration" { - try testCanonical( - \\const S = struct { - \\ const Self = this; - \\ f1: u8, - \\ - \\ fn method(self: &Self) Self { - \\ return *self; - \\ } - \\ - \\ f2: u8 - \\}; - \\ - \\const Ps = packed struct { - \\ a: u8, - \\ b: u8, - \\ - \\ c: u8 - \\}; - \\ - \\const Es = extern struct { - \\ a: u8, - \\ b: u8, - \\ - \\ c: u8 - \\}; - \\ - ); -} - -test "zig fmt: enum declaration" { - try testCanonical( - \\const E = enum { - \\ Ok, - \\ SomethingElse = 0 - \\}; - \\ - \\const E2 = enum(u8) { - \\ Ok, - \\ SomethingElse = 255, - \\ SomethingThird - \\}; - \\ - \\const Ee = extern enum { - \\ Ok, - \\ SomethingElse, - \\ SomethingThird - \\}; - \\ - \\const Ep = packed enum { - \\ Ok, - \\ SomethingElse, - \\ SomethingThird - \\}; - \\ - ); -} - -test "zig fmt: union declaration" { - try testCanonical( - \\const U = union { - \\ Int: u8, - \\ Float: f32, - \\ None, - \\ Bool: bool - \\}; - \\ - \\const Ue = union(enum) { - \\ Int: u8, - \\ Float: f32, - \\ None, - \\ Bool: bool - \\}; - \\ - \\const E = enum { - \\ Int, - \\ Float, - \\ None, - \\ Bool - \\}; - \\ - \\const Ue2 = union(E) { - \\ Int: u8, - \\ Float: f32, - \\ None, - \\ Bool: bool - \\}; - \\ - \\const Eu = extern union { - \\ Int: u8, - \\ Float: f32, - \\ None, - \\ Bool: bool - \\}; - \\ - ); -} - -test "zig fmt: error set declaration" { - try testCanonical( - \\const E = error { - \\ A, - \\ B, - \\ - \\ C - \\}; - \\ - ); -} - -test "zig fmt: arrays" { - try testCanonical( - \\test "test array" { - \\ const a: [2]u8 = [2]u8{ 1, 2 }; - \\ const a: [2]u8 = []u8{ 1, 2 }; - \\ const a: [0]u8 = []u8{ }; - \\} - \\ - ); -} - -test "zig fmt: container initializers" { - try testCanonical( - \\const a1 = []u8{ }; - \\const a2 = []u8{ 1, 2, 3, 4 }; - \\const s1 = S{ }; - \\const s2 = S{ .a = 1, .b = 2 }; - \\ - ); -} - -test "zig fmt: catch" { - try testCanonical( - \\test "catch" { - \\ const a: error!u8 = 0; - \\ _ = a catch return; - \\ _ = a catch |err| return; - \\} - \\ - ); -} - -test "zig fmt: blocks" { - try testCanonical( - \\test "blocks" { - \\ { - \\ const a = 0; - \\ const b = 0; - \\ } - \\ - \\ blk: { - \\ const a = 0; - \\ const b = 0; - \\ } - \\ - \\ const r = blk: { - \\ const a = 0; - \\ const b = 0; - \\ }; - \\} - \\ - ); -} - -test "zig fmt: switch" { - try testCanonical( - \\test "switch" { - \\ switch (0) { - \\ 0 => {}, - \\ 1 => unreachable, - \\ 2, 3 => {}, - \\ 4 ... 7 => {}, - \\ 1 + 4 * 3 + 22 => {}, - \\ else => { - \\ const a = 1; - \\ const b = a; - \\ } - \\ } - \\ - \\ const res = switch (0) { - \\ 0 => 0, - \\ 1 => 2, - \\ else => 4 - \\ }; - \\ - \\ const Union = union(enum) { - \\ Int: i64, - \\ Float: f64 - \\ }; - \\ - \\ const u = Union{ .Int = 0 }; - \\ switch (u) { - \\ Union.Int => |int| {}, - \\ Union.Float => |*float| unreachable - \\ } - \\} - \\ - ); -} - -test "zig fmt: while" { - try testCanonical( - \\test "while" { - \\ while (10 < 1) { - \\ unreachable; - \\ } - \\ - \\ while (10 < 1) - \\ unreachable; - \\ - \\ var i: usize = 0; - \\ while (i < 10) : (i += 1) { - \\ continue; - \\ } - \\ - \\ i = 0; - \\ while (i < 10) : (i += 1) - \\ continue; - \\ - \\ i = 0; - \\ var j: usize = 0; - \\ while (i < 10) : ({ - \\ i += 1; - \\ j += 1; - \\ }) { - \\ continue; - \\ } - \\ - \\ var a: ?u8 = 2; - \\ while (a) |v| : (a = null) { - \\ continue; - \\ } - \\ - \\ while (a) |v| : (a = null) - \\ unreachable; - \\ - \\ label: while (10 < 0) { - \\ unreachable; - \\ } - \\ - \\ const res = while (0 < 10) { - \\ break 7; - \\ } else { - \\ unreachable; - \\ }; - \\ - \\ var a: error!u8 = 0; - \\ while (a) |v| { - \\ a = error.Err; - \\ } else |err| { - \\ i = 1; - \\ } - \\ - \\ comptime var k: usize = 0; - \\ inline while (i < 10) : (i += 1) - \\ j += 2; - \\} - \\ - ); -} - -test "zig fmt: for" { - try testCanonical( - \\test "for" { - \\ const a = []u8{ 1, 2, 3 }; - \\ for (a) |v| { - \\ continue; - \\ } - \\ - \\ for (a) |v| - \\ continue; - \\ - \\ for (a) |*v| - \\ continue; - \\ - \\ for (a) |v, i| { - \\ continue; - \\ } - \\ - \\ for (a) |v, i| - \\ continue; - \\ - \\ const res = for (a) |v, i| { - \\ break v; - \\ } else { - \\ unreachable; - \\ }; - \\ - \\ var num: usize = 0; - \\ inline for (a) |v, i| { - \\ num += v; - \\ num += i; - \\ } - \\} - \\ - ); -} - -test "zig fmt: if" { - try testCanonical( - \\test "if" { - \\ if (10 < 0) { - \\ unreachable; - \\ } - \\ - \\ if (10 < 0) unreachable; - \\ - \\ if (10 < 0) { - \\ unreachable; - \\ } else { - \\ const a = 20; - \\ } - \\ - \\ if (10 < 0) { - \\ unreachable; - \\ } else if (5 < 0) { - \\ unreachable; - \\ } else { - \\ const a = 20; - \\ } - \\ - \\ const is_world_broken = if (10 < 0) true else false; - \\ - \\ const a: ?u8 = 10; - \\ const b: ?u8 = null; - \\ if (a) |v| { - \\ const some = v; - \\ } else if (b) |*v| { - \\ unreachable; - \\ } else { - \\ const some = 10; - \\ } - \\ - \\ const non_null_a = if (a) |v| v else 0; - \\ - \\ const a_err: error!u8 = 0; - \\ if (a_err) |v| { - \\ const p = v; - \\ } else |err| { - \\ unreachable; - \\ } - \\} - \\ - ); -} - -test "zig fmt: defer" { - try testCanonical( - \\test "defer" { - \\ var i: usize = 0; - \\ defer i = 1; - \\ defer { - \\ i += 2; - \\ i *= i; - \\ } - \\ - \\ errdefer i += 3; - \\ errdefer { - \\ i += 2; - \\ i /= i; - \\ } - \\} - \\ - ); -} - -test "zig fmt: comptime" { - try testCanonical( - \\fn a() u8 { - \\ return 5; - \\} - \\ - \\fn b(comptime i: u8) u8 { - \\ return i; - \\} - \\ - \\const av = comptime a(); - \\const av2 = comptime blk: { - \\ var res = a(); - \\ res *= b(2); - \\ break :blk res; - \\}; - \\ - \\comptime { - \\ _ = a(); - \\} - \\ - \\test "comptime" { - \\ const av3 = comptime a(); - \\ const av4 = comptime blk: { - \\ var res = a(); - \\ res *= a(); - \\ break :blk res; - \\ }; - \\ - \\ comptime var i = 0; - \\ comptime { - \\ i = a(); - \\ i += b(i); - \\ } - \\} - \\ - ); -} - -test "zig fmt: fn type" { - try testCanonical( - \\fn a(i: u8) u8 { - \\ return i + 1; - \\} - \\ - \\const a: fn(u8) u8 = undefined; - \\const b: extern fn(u8) u8 = undefined; - \\const c: nakedcc fn(u8) u8 = undefined; - \\const ap: fn(u8) u8 = a; - \\ - ); -} - -test "zig fmt: inline asm" { - try testCanonical( - \\pub fn syscall1(number: usize, arg1: usize) usize { - \\ return asm volatile ("syscall" - \\ : [ret] "={rax}" (-> usize) - \\ : [number] "{rax}" (number), - \\ [arg1] "{rdi}" (arg1) - \\ : "rcx", "r11"); - \\} - \\ - ); -} - -test "zig fmt: coroutines" { - try testCanonical( - \\async fn simpleAsyncFn() void { - \\ x += 1; - \\ suspend; - \\ x += 1; - \\ suspend |p| {} - \\ const p = async simpleAsyncFn() catch unreachable; - \\ await p; - \\} - \\ - \\test "coroutine suspend, resume, cancel" { - \\ const p = try async testAsyncSeq(); - \\ resume p; - \\ cancel p; - \\} - \\ - ); +test "1" { + try testCanonical(@embedFile("../array_list.zig")); } From 706e0d739e5bb4a6e7a0e4f6e168eb71069e4df2 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 10 Apr 2018 13:49:52 +0200 Subject: [PATCH 61/87] std.zig.parser readded all tests * Ops! --- std/zig/parser.zig | 1527 ++++++++++++++++++++++---------------------- 1 file changed, 761 insertions(+), 766 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 33408bcc31..5f7412b5c2 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -1119,12 +1119,6 @@ pub const Parser = struct { }, State.TypeExprBegin => |dest_ptr| { - const token = self.getNextToken(); - if (token.id == Token.Id.Keyword_var) { - @panic("TODO param with type var"); - } - self.putBackToken(token); - stack.append(State { .TypeExprEnd = dest_ptr }) catch unreachable; try stack.append(State { .PrefixOpExpression = dest_ptr }); continue; @@ -1160,9 +1154,6 @@ pub const Parser = struct { try stack.append(State { .AddrOfModifiers = &node.op.AddrOf }); } continue; - //Token.Id.Keyword_await => { - // @panic("TODO: await"); - //}, } else { self.putBackToken(token); stack.append(State { .SuffixOpExpressionBegin = dest_ptr }) catch unreachable; @@ -1408,7 +1399,6 @@ pub const Parser = struct { continue; }, Token.Id.LBracket => { - // TODO: option("align" "(" Expression option(":" Integer ":" Integer) ")")) option("const") option("volatile") const rbracket_token = self.getNextToken(); if (rbracket_token.id == Token.Id.RBracket) { const node = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp{ @@ -4222,760 +4212,765 @@ fn testCanonical(source: []const u8) !void { } } -//test "zig fmt: get stdout or fail" { -// try testCanonical( -// \\const std = @import("std"); -// \\ -// \\pub fn main() !void { -// \\ // If this program is run without stdout attached, exit with an error. -// \\ // another comment -// \\ var stdout_file = try std.io.getStdOut; -// \\} -// \\ -// ); -//} -// -//test "zig fmt: preserve spacing" { -// try testCanonical( -// \\const std = @import("std"); -// \\ -// \\pub fn main() !void { -// \\ var stdout_file = try std.io.getStdOut; -// \\ var stdout_file = try std.io.getStdOut; -// \\ -// \\ var stdout_file = try std.io.getStdOut; -// \\ var stdout_file = try std.io.getStdOut; -// \\} -// \\ -// ); -//} -// -//test "zig fmt: return types" { -// try testCanonical( -// \\pub fn main() !void {} -// \\pub fn main() var {} -// \\pub fn main() i32 {} -// \\ -// ); -//} -// -//test "zig fmt: imports" { -// try testCanonical( -// \\const std = @import("std"); -// \\const std = @import(); -// \\ -// ); -//} -// -//test "zig fmt: global declarations" { -// try testCanonical( -// \\const a = b; -// \\pub const a = b; -// \\var a = b; -// \\pub var a = b; -// \\const a: i32 = b; -// \\pub const a: i32 = b; -// \\var a: i32 = b; -// \\pub var a: i32 = b; -// \\extern const a: i32 = b; -// \\pub extern const a: i32 = b; -// \\extern var a: i32 = b; -// \\pub extern var a: i32 = b; -// \\extern "a" const a: i32 = b; -// \\pub extern "a" const a: i32 = b; -// \\extern "a" var a: i32 = b; -// \\pub extern "a" var a: i32 = b; -// \\ -// ); -//} -// -//test "zig fmt: extern declaration" { -// try testCanonical( -// \\extern var foo: c_int; -// \\ -// ); -//} -// -//test "zig fmt: alignment" { -// try testCanonical( -// \\var foo: c_int align(1); -// \\ -// ); -//} -// -//test "zig fmt: C main" { -// try testCanonical( -// \\fn main(argc: c_int, argv: &&u8) c_int { -// \\ const a = b; -// \\} -// \\ -// ); -//} -// -//test "zig fmt: return" { -// try testCanonical( -// \\fn foo(argc: c_int, argv: &&u8) c_int { -// \\ return 0; -// \\} -// \\ -// \\fn bar() void { -// \\ return; -// \\} -// \\ -// ); -//} -// -//test "zig fmt: pointer attributes" { -// try testCanonical( -// \\extern fn f1(s: &align(&u8) u8) c_int; -// \\extern fn f2(s: &&align(1) &const &volatile u8) c_int; -// \\extern fn f3(s: &align(1) const &align(1) volatile &const volatile u8) c_int; -// \\extern fn f4(s: &align(1) const volatile u8) c_int; -// \\ -// ); -//} -// -//test "zig fmt: slice attributes" { -// try testCanonical( -// \\extern fn f1(s: &align(&u8) u8) c_int; -// \\extern fn f2(s: &&align(1) &const &volatile u8) c_int; -// \\extern fn f3(s: &align(1) const &align(1) volatile &const volatile u8) c_int; -// \\extern fn f4(s: &align(1) const volatile u8) c_int; -// \\ -// ); -//} -// -//test "zig fmt: test declaration" { -// try testCanonical( -// \\test "test name" { -// \\ const a = 1; -// \\ var b = 1; -// \\} -// \\ -// ); -//} -// -//test "zig fmt: infix operators" { -// try testCanonical( -// \\test "infix operators" { -// \\ var i = undefined; -// \\ i = 2; -// \\ i *= 2; -// \\ i |= 2; -// \\ i ^= 2; -// \\ i <<= 2; -// \\ i >>= 2; -// \\ i &= 2; -// \\ i *= 2; -// \\ i *%= 2; -// \\ i -= 2; -// \\ i -%= 2; -// \\ i += 2; -// \\ i +%= 2; -// \\ i /= 2; -// \\ i %= 2; -// \\ _ = i == i; -// \\ _ = i != i; -// \\ _ = i != i; -// \\ _ = i.i; -// \\ _ = i || i; -// \\ _ = i!i; -// \\ _ = i ** i; -// \\ _ = i ++ i; -// \\ _ = i ?? i; -// \\ _ = i % i; -// \\ _ = i / i; -// \\ _ = i *% i; -// \\ _ = i * i; -// \\ _ = i -% i; -// \\ _ = i - i; -// \\ _ = i +% i; -// \\ _ = i + i; -// \\ _ = i << i; -// \\ _ = i >> i; -// \\ _ = i & i; -// \\ _ = i ^ i; -// \\ _ = i | i; -// \\ _ = i >= i; -// \\ _ = i <= i; -// \\ _ = i > i; -// \\ _ = i < i; -// \\ _ = i and i; -// \\ _ = i or i; -// \\} -// \\ -// ); -//} -// -//test "zig fmt: precedence" { -// try testCanonical( -// \\test "precedence" { -// \\ a!b(); -// \\ (a!b)(); -// \\ !a!b; -// \\ !(a!b); -// \\ !a{ }; -// \\ !(a{ }); -// \\ a + b{ }; -// \\ (a + b){ }; -// \\ a << b + c; -// \\ (a << b) + c; -// \\ a & b << c; -// \\ (a & b) << c; -// \\ a ^ b & c; -// \\ (a ^ b) & c; -// \\ a | b ^ c; -// \\ (a | b) ^ c; -// \\ a == b | c; -// \\ (a == b) | c; -// \\ a and b == c; -// \\ (a and b) == c; -// \\ a or b and c; -// \\ (a or b) and c; -// \\ (a or b) and c; -// \\} -// \\ -// ); -//} -// -//test "zig fmt: prefix operators" { -// try testCanonical( -// \\test "prefix operators" { -// \\ try return --%~??!*&0; -// \\} -// \\ -// ); -//} -// -//test "zig fmt: call expression" { -// try testCanonical( -// \\test "test calls" { -// \\ a(); -// \\ a(1); -// \\ a(1, 2); -// \\ a(1, 2) + a(1, 2); -// \\} -// \\ -// ); -//} -// -//test "zig fmt: var args" { -// try testCanonical( -// \\fn print(args: ...) void {} -// \\ -// ); -//} -// -//test "zig fmt: extern function" { -// try testCanonical( -// \\extern fn puts(s: &const u8) c_int; -// \\extern "c" fn puts(s: &const u8) c_int; -// \\ -// ); -//} -// -//test "zig fmt: multiline string" { -// try testCanonical( -// \\const s = -// \\ \\ something -// \\ \\ something else -// \\ ; -// \\ -// ); -//} -// -//test "zig fmt: values" { -// try testCanonical( -// \\test "values" { -// \\ 1; -// \\ 1.0; -// \\ "string"; -// \\ c"cstring"; -// \\ 'c'; -// \\ true; -// \\ false; -// \\ null; -// \\ undefined; -// \\ error; -// \\ this; -// \\ unreachable; -// \\} -// \\ -// ); -//} -// -//test "zig fmt: indexing" { -// try testCanonical( -// \\test "test index" { -// \\ a[0]; -// \\ a[0 + 5]; -// \\ a[0..]; -// \\ a[0..5]; -// \\ a[a[0]]; -// \\ a[a[0..]]; -// \\ a[a[0..5]]; -// \\ a[a[0]..]; -// \\ a[a[0..5]..]; -// \\ a[a[0]..a[0]]; -// \\ a[a[0..5]..a[0]]; -// \\ a[a[0..5]..a[0..5]]; -// \\} -// \\ -// ); -//} -// -//test "zig fmt: struct declaration" { -// try testCanonical( -// \\const S = struct { -// \\ const Self = this; -// \\ f1: u8, -// \\ -// \\ fn method(self: &Self) Self { -// \\ return *self; -// \\ } -// \\ -// \\ f2: u8 -// \\}; -// \\ -// \\const Ps = packed struct { -// \\ a: u8, -// \\ b: u8, -// \\ -// \\ c: u8 -// \\}; -// \\ -// \\const Es = extern struct { -// \\ a: u8, -// \\ b: u8, -// \\ -// \\ c: u8 -// \\}; -// \\ -// ); -//} -// -//test "zig fmt: enum declaration" { -// try testCanonical( -// \\const E = enum { -// \\ Ok, -// \\ SomethingElse = 0 -// \\}; -// \\ -// \\const E2 = enum(u8) { -// \\ Ok, -// \\ SomethingElse = 255, -// \\ SomethingThird -// \\}; -// \\ -// \\const Ee = extern enum { -// \\ Ok, -// \\ SomethingElse, -// \\ SomethingThird -// \\}; -// \\ -// \\const Ep = packed enum { -// \\ Ok, -// \\ SomethingElse, -// \\ SomethingThird -// \\}; -// \\ -// ); -//} -// -//test "zig fmt: union declaration" { -// try testCanonical( -// \\const U = union { -// \\ Int: u8, -// \\ Float: f32, -// \\ None, -// \\ Bool: bool -// \\}; -// \\ -// \\const Ue = union(enum) { -// \\ Int: u8, -// \\ Float: f32, -// \\ None, -// \\ Bool: bool -// \\}; -// \\ -// \\const E = enum { -// \\ Int, -// \\ Float, -// \\ None, -// \\ Bool -// \\}; -// \\ -// \\const Ue2 = union(E) { -// \\ Int: u8, -// \\ Float: f32, -// \\ None, -// \\ Bool: bool -// \\}; -// \\ -// \\const Eu = extern union { -// \\ Int: u8, -// \\ Float: f32, -// \\ None, -// \\ Bool: bool -// \\}; -// \\ -// ); -//} -// -//test "zig fmt: error set declaration" { -// try testCanonical( -// \\const E = error { -// \\ A, -// \\ B, -// \\ -// \\ C -// \\}; -// \\ -// ); -//} -// -//test "zig fmt: arrays" { -// try testCanonical( -// \\test "test array" { -// \\ const a: [2]u8 = [2]u8{ 1, 2 }; -// \\ const a: [2]u8 = []u8{ 1, 2 }; -// \\ const a: [0]u8 = []u8{ }; -// \\} -// \\ -// ); -//} -// -//test "zig fmt: container initializers" { -// try testCanonical( -// \\const a1 = []u8{ }; -// \\const a2 = []u8{ 1, 2, 3, 4 }; -// \\const s1 = S{ }; -// \\const s2 = S{ .a = 1, .b = 2 }; -// \\ -// ); -//} -// -//test "zig fmt: catch" { -// try testCanonical( -// \\test "catch" { -// \\ const a: error!u8 = 0; -// \\ _ = a catch return; -// \\ _ = a catch |err| return; -// \\} -// \\ -// ); -//} -// -//test "zig fmt: blocks" { -// try testCanonical( -// \\test "blocks" { -// \\ { -// \\ const a = 0; -// \\ const b = 0; -// \\ } -// \\ -// \\ blk: { -// \\ const a = 0; -// \\ const b = 0; -// \\ } -// \\ -// \\ const r = blk: { -// \\ const a = 0; -// \\ const b = 0; -// \\ }; -// \\} -// \\ -// ); -//} -// -//test "zig fmt: switch" { -// try testCanonical( -// \\test "switch" { -// \\ switch (0) { -// \\ 0 => {}, -// \\ 1 => unreachable, -// \\ 2, 3 => {}, -// \\ 4 ... 7 => {}, -// \\ 1 + 4 * 3 + 22 => {}, -// \\ else => { -// \\ const a = 1; -// \\ const b = a; -// \\ } -// \\ } -// \\ -// \\ const res = switch (0) { -// \\ 0 => 0, -// \\ 1 => 2, -// \\ else => 4 -// \\ }; -// \\ -// \\ const Union = union(enum) { -// \\ Int: i64, -// \\ Float: f64 -// \\ }; -// \\ -// \\ const u = Union{ .Int = 0 }; -// \\ switch (u) { -// \\ Union.Int => |int| {}, -// \\ Union.Float => |*float| unreachable -// \\ } -// \\} -// \\ -// ); -//} -// -//test "zig fmt: while" { -// try testCanonical( -// \\test "while" { -// \\ while (10 < 1) { -// \\ unreachable; -// \\ } -// \\ -// \\ while (10 < 1) -// \\ unreachable; -// \\ -// \\ var i: usize = 0; -// \\ while (i < 10) : (i += 1) { -// \\ continue; -// \\ } -// \\ -// \\ i = 0; -// \\ while (i < 10) : (i += 1) -// \\ continue; -// \\ -// \\ i = 0; -// \\ var j: usize = 0; -// \\ while (i < 10) : ({ -// \\ i += 1; -// \\ j += 1; -// \\ }) { -// \\ continue; -// \\ } -// \\ -// \\ var a: ?u8 = 2; -// \\ while (a) |v| : (a = null) { -// \\ continue; -// \\ } -// \\ -// \\ while (a) |v| : (a = null) -// \\ unreachable; -// \\ -// \\ label: while (10 < 0) { -// \\ unreachable; -// \\ } -// \\ -// \\ const res = while (0 < 10) { -// \\ break 7; -// \\ } else { -// \\ unreachable; -// \\ }; -// \\ -// \\ var a: error!u8 = 0; -// \\ while (a) |v| { -// \\ a = error.Err; -// \\ } else |err| { -// \\ i = 1; -// \\ } -// \\ -// \\ comptime var k: usize = 0; -// \\ inline while (i < 10) : (i += 1) -// \\ j += 2; -// \\} -// \\ -// ); -//} -// -//test "zig fmt: for" { -// try testCanonical( -// \\test "for" { -// \\ const a = []u8{ 1, 2, 3 }; -// \\ for (a) |v| { -// \\ continue; -// \\ } -// \\ -// \\ for (a) |v| -// \\ continue; -// \\ -// \\ for (a) |*v| -// \\ continue; -// \\ -// \\ for (a) |v, i| { -// \\ continue; -// \\ } -// \\ -// \\ for (a) |v, i| -// \\ continue; -// \\ -// \\ const res = for (a) |v, i| { -// \\ break v; -// \\ } else { -// \\ unreachable; -// \\ }; -// \\ -// \\ var num: usize = 0; -// \\ inline for (a) |v, i| { -// \\ num += v; -// \\ num += i; -// \\ } -// \\} -// \\ -// ); -//} -// -//test "zig fmt: if" { -// try testCanonical( -// \\test "if" { -// \\ if (10 < 0) { -// \\ unreachable; -// \\ } -// \\ -// \\ if (10 < 0) unreachable; -// \\ -// \\ if (10 < 0) { -// \\ unreachable; -// \\ } else { -// \\ const a = 20; -// \\ } -// \\ -// \\ if (10 < 0) { -// \\ unreachable; -// \\ } else if (5 < 0) { -// \\ unreachable; -// \\ } else { -// \\ const a = 20; -// \\ } -// \\ -// \\ const is_world_broken = if (10 < 0) true else false; -// \\ -// \\ const a: ?u8 = 10; -// \\ const b: ?u8 = null; -// \\ if (a) |v| { -// \\ const some = v; -// \\ } else if (b) |*v| { -// \\ unreachable; -// \\ } else { -// \\ const some = 10; -// \\ } -// \\ -// \\ const non_null_a = if (a) |v| v else 0; -// \\ -// \\ const a_err: error!u8 = 0; -// \\ if (a_err) |v| { -// \\ const p = v; -// \\ } else |err| { -// \\ unreachable; -// \\ } -// \\} -// \\ -// ); -//} -// -//test "zig fmt: defer" { -// try testCanonical( -// \\test "defer" { -// \\ var i: usize = 0; -// \\ defer i = 1; -// \\ defer { -// \\ i += 2; -// \\ i *= i; -// \\ } -// \\ -// \\ errdefer i += 3; -// \\ errdefer { -// \\ i += 2; -// \\ i /= i; -// \\ } -// \\} -// \\ -// ); -//} -// -//test "zig fmt: comptime" { -// try testCanonical( -// \\fn a() u8 { -// \\ return 5; -// \\} -// \\ -// \\fn b(comptime i: u8) u8 { -// \\ return i; -// \\} -// \\ -// \\const av = comptime a(); -// \\const av2 = comptime blk: { -// \\ var res = a(); -// \\ res *= b(2); -// \\ break :blk res; -// \\}; -// \\ -// \\comptime { -// \\ _ = a(); -// \\} -// \\ -// \\test "comptime" { -// \\ const av3 = comptime a(); -// \\ const av4 = comptime blk: { -// \\ var res = a(); -// \\ res *= a(); -// \\ break :blk res; -// \\ }; -// \\ -// \\ comptime var i = 0; -// \\ comptime { -// \\ i = a(); -// \\ i += b(i); -// \\ } -// \\} -// \\ -// ); -//} -// -//test "zig fmt: fn type" { -// try testCanonical( -// \\fn a(i: u8) u8 { -// \\ return i + 1; -// \\} -// \\ -// \\const a: fn(u8) u8 = undefined; -// \\const b: extern fn(u8) u8 = undefined; -// \\const c: nakedcc fn(u8) u8 = undefined; -// \\const ap: fn(u8) u8 = a; -// \\ -// ); -//} -// -//test "zig fmt: inline asm" { -// try testCanonical( -// \\pub fn syscall1(number: usize, arg1: usize) usize { -// \\ return asm volatile ("syscall" -// \\ : [ret] "={rax}" (-> usize) -// \\ : [number] "{rax}" (number), -// \\ [arg1] "{rdi}" (arg1) -// \\ : "rcx", "r11"); -// \\} -// \\ -// ); -//} -// -//test "zig fmt: coroutines" { -// try testCanonical( -// \\async fn simpleAsyncFn() void { -// \\ x += 1; -// \\ suspend; -// \\ x += 1; -// \\ suspend |p| {} -// \\ const p = async simpleAsyncFn() catch unreachable; -// \\ await p; -// \\} -// \\ -// \\test "coroutine suspend, resume, cancel" { -// \\ const p = try async testAsyncSeq(); -// \\ resume p; -// \\ cancel p; -// \\} -// \\ -// ); -//} - -test "1" { - try testCanonical(@embedFile("../array_list.zig")); +test "zig fmt: get stdout or fail" { + try testCanonical( + \\const std = @import("std"); + \\ + \\pub fn main() !void { + \\ // If this program is run without stdout attached, exit with an error. + \\ // another comment + \\ var stdout_file = try std.io.getStdOut; + \\} + \\ + ); } + +test "zig fmt: preserve spacing" { + try testCanonical( + \\const std = @import("std"); + \\ + \\pub fn main() !void { + \\ var stdout_file = try std.io.getStdOut; + \\ var stdout_file = try std.io.getStdOut; + \\ + \\ var stdout_file = try std.io.getStdOut; + \\ var stdout_file = try std.io.getStdOut; + \\} + \\ + ); +} + +test "zig fmt: return types" { + try testCanonical( + \\pub fn main() !void {} + \\pub fn main() var {} + \\pub fn main() i32 {} + \\ + ); +} + +test "zig fmt: imports" { + try testCanonical( + \\const std = @import("std"); + \\const std = @import(); + \\ + ); +} + +test "zig fmt: global declarations" { + try testCanonical( + \\const a = b; + \\pub const a = b; + \\var a = b; + \\pub var a = b; + \\const a: i32 = b; + \\pub const a: i32 = b; + \\var a: i32 = b; + \\pub var a: i32 = b; + \\extern const a: i32 = b; + \\pub extern const a: i32 = b; + \\extern var a: i32 = b; + \\pub extern var a: i32 = b; + \\extern "a" const a: i32 = b; + \\pub extern "a" const a: i32 = b; + \\extern "a" var a: i32 = b; + \\pub extern "a" var a: i32 = b; + \\ + ); +} + +test "zig fmt: extern declaration" { + try testCanonical( + \\extern var foo: c_int; + \\ + ); +} + +test "zig fmt: alignment" { + try testCanonical( + \\var foo: c_int align(1); + \\ + ); +} + +test "zig fmt: C main" { + try testCanonical( + \\fn main(argc: c_int, argv: &&u8) c_int { + \\ const a = b; + \\} + \\ + ); +} + +test "zig fmt: return" { + try testCanonical( + \\fn foo(argc: c_int, argv: &&u8) c_int { + \\ return 0; + \\} + \\ + \\fn bar() void { + \\ return; + \\} + \\ + ); +} + +test "zig fmt: pointer attributes" { + try testCanonical( + \\extern fn f1(s: &align(&u8) u8) c_int; + \\extern fn f2(s: &&align(1) &const &volatile u8) c_int; + \\extern fn f3(s: &align(1) const &align(1) volatile &const volatile u8) c_int; + \\extern fn f4(s: &align(1) const volatile u8) c_int; + \\ + ); +} + +test "zig fmt: slice attributes" { + try testCanonical( + \\extern fn f1(s: &align(&u8) u8) c_int; + \\extern fn f2(s: &&align(1) &const &volatile u8) c_int; + \\extern fn f3(s: &align(1) const &align(1) volatile &const volatile u8) c_int; + \\extern fn f4(s: &align(1) const volatile u8) c_int; + \\ + ); +} + +test "zig fmt: test declaration" { + try testCanonical( + \\test "test name" { + \\ const a = 1; + \\ var b = 1; + \\} + \\ + ); +} + +test "zig fmt: infix operators" { + try testCanonical( + \\test "infix operators" { + \\ var i = undefined; + \\ i = 2; + \\ i *= 2; + \\ i |= 2; + \\ i ^= 2; + \\ i <<= 2; + \\ i >>= 2; + \\ i &= 2; + \\ i *= 2; + \\ i *%= 2; + \\ i -= 2; + \\ i -%= 2; + \\ i += 2; + \\ i +%= 2; + \\ i /= 2; + \\ i %= 2; + \\ _ = i == i; + \\ _ = i != i; + \\ _ = i != i; + \\ _ = i.i; + \\ _ = i || i; + \\ _ = i!i; + \\ _ = i ** i; + \\ _ = i ++ i; + \\ _ = i ?? i; + \\ _ = i % i; + \\ _ = i / i; + \\ _ = i *% i; + \\ _ = i * i; + \\ _ = i -% i; + \\ _ = i - i; + \\ _ = i +% i; + \\ _ = i + i; + \\ _ = i << i; + \\ _ = i >> i; + \\ _ = i & i; + \\ _ = i ^ i; + \\ _ = i | i; + \\ _ = i >= i; + \\ _ = i <= i; + \\ _ = i > i; + \\ _ = i < i; + \\ _ = i and i; + \\ _ = i or i; + \\} + \\ + ); +} + +test "zig fmt: precedence" { + try testCanonical( + \\test "precedence" { + \\ a!b(); + \\ (a!b)(); + \\ !a!b; + \\ !(a!b); + \\ !a{ }; + \\ !(a{ }); + \\ a + b{ }; + \\ (a + b){ }; + \\ a << b + c; + \\ (a << b) + c; + \\ a & b << c; + \\ (a & b) << c; + \\ a ^ b & c; + \\ (a ^ b) & c; + \\ a | b ^ c; + \\ (a | b) ^ c; + \\ a == b | c; + \\ (a == b) | c; + \\ a and b == c; + \\ (a and b) == c; + \\ a or b and c; + \\ (a or b) and c; + \\ (a or b) and c; + \\} + \\ + ); +} + +test "zig fmt: prefix operators" { + try testCanonical( + \\test "prefix operators" { + \\ try return --%~??!*&0; + \\} + \\ + ); +} + +test "zig fmt: call expression" { + try testCanonical( + \\test "test calls" { + \\ a(); + \\ a(1); + \\ a(1, 2); + \\ a(1, 2) + a(1, 2); + \\} + \\ + ); +} + +test "zig fmt: var args" { + try testCanonical( + \\fn print(args: ...) void {} + \\ + ); +} + +test "zig fmt: extern function" { + try testCanonical( + \\extern fn puts(s: &const u8) c_int; + \\extern "c" fn puts(s: &const u8) c_int; + \\ + ); +} + +test "zig fmt: multiline string" { + try testCanonical( + \\const s = + \\ \\ something + \\ \\ something else + \\ ; + \\ + ); +} + +test "zig fmt: values" { + try testCanonical( + \\test "values" { + \\ 1; + \\ 1.0; + \\ "string"; + \\ c"cstring"; + \\ 'c'; + \\ true; + \\ false; + \\ null; + \\ undefined; + \\ error; + \\ this; + \\ unreachable; + \\} + \\ + ); +} + +test "zig fmt: indexing" { + try testCanonical( + \\test "test index" { + \\ a[0]; + \\ a[0 + 5]; + \\ a[0..]; + \\ a[0..5]; + \\ a[a[0]]; + \\ a[a[0..]]; + \\ a[a[0..5]]; + \\ a[a[0]..]; + \\ a[a[0..5]..]; + \\ a[a[0]..a[0]]; + \\ a[a[0..5]..a[0]]; + \\ a[a[0..5]..a[0..5]]; + \\} + \\ + ); +} + +test "zig fmt: struct declaration" { + try testCanonical( + \\const S = struct { + \\ const Self = this; + \\ f1: u8, + \\ + \\ fn method(self: &Self) Self { + \\ return *self; + \\ } + \\ + \\ f2: u8 + \\}; + \\ + \\const Ps = packed struct { + \\ a: u8, + \\ b: u8, + \\ + \\ c: u8 + \\}; + \\ + \\const Es = extern struct { + \\ a: u8, + \\ b: u8, + \\ + \\ c: u8 + \\}; + \\ + ); +} + +test "zig fmt: enum declaration" { + try testCanonical( + \\const E = enum { + \\ Ok, + \\ SomethingElse = 0 + \\}; + \\ + \\const E2 = enum(u8) { + \\ Ok, + \\ SomethingElse = 255, + \\ SomethingThird + \\}; + \\ + \\const Ee = extern enum { + \\ Ok, + \\ SomethingElse, + \\ SomethingThird + \\}; + \\ + \\const Ep = packed enum { + \\ Ok, + \\ SomethingElse, + \\ SomethingThird + \\}; + \\ + ); +} + +test "zig fmt: union declaration" { + try testCanonical( + \\const U = union { + \\ Int: u8, + \\ Float: f32, + \\ None, + \\ Bool: bool + \\}; + \\ + \\const Ue = union(enum) { + \\ Int: u8, + \\ Float: f32, + \\ None, + \\ Bool: bool + \\}; + \\ + \\const E = enum { + \\ Int, + \\ Float, + \\ None, + \\ Bool + \\}; + \\ + \\const Ue2 = union(E) { + \\ Int: u8, + \\ Float: f32, + \\ None, + \\ Bool: bool + \\}; + \\ + \\const Eu = extern union { + \\ Int: u8, + \\ Float: f32, + \\ None, + \\ Bool: bool + \\}; + \\ + ); +} + +test "zig fmt: error set declaration" { + try testCanonical( + \\const E = error { + \\ A, + \\ B, + \\ + \\ C + \\}; + \\ + ); +} + +test "zig fmt: arrays" { + try testCanonical( + \\test "test array" { + \\ const a: [2]u8 = [2]u8{ 1, 2 }; + \\ const a: [2]u8 = []u8{ 1, 2 }; + \\ const a: [0]u8 = []u8{ }; + \\} + \\ + ); +} + +test "zig fmt: container initializers" { + try testCanonical( + \\const a1 = []u8{ }; + \\const a2 = []u8{ 1, 2, 3, 4 }; + \\const s1 = S{ }; + \\const s2 = S{ .a = 1, .b = 2 }; + \\ + ); +} + +test "zig fmt: catch" { + try testCanonical( + \\test "catch" { + \\ const a: error!u8 = 0; + \\ _ = a catch return; + \\ _ = a catch |err| return; + \\} + \\ + ); +} + +test "zig fmt: blocks" { + try testCanonical( + \\test "blocks" { + \\ { + \\ const a = 0; + \\ const b = 0; + \\ } + \\ + \\ blk: { + \\ const a = 0; + \\ const b = 0; + \\ } + \\ + \\ const r = blk: { + \\ const a = 0; + \\ const b = 0; + \\ }; + \\} + \\ + ); +} + +test "zig fmt: switch" { + try testCanonical( + \\test "switch" { + \\ switch (0) { + \\ 0 => {}, + \\ 1 => unreachable, + \\ 2, 3 => {}, + \\ 4 ... 7 => {}, + \\ 1 + 4 * 3 + 22 => {}, + \\ else => { + \\ const a = 1; + \\ const b = a; + \\ } + \\ } + \\ + \\ const res = switch (0) { + \\ 0 => 0, + \\ 1 => 2, + \\ else => 4 + \\ }; + \\ + \\ const Union = union(enum) { + \\ Int: i64, + \\ Float: f64 + \\ }; + \\ + \\ const u = Union{ .Int = 0 }; + \\ switch (u) { + \\ Union.Int => |int| {}, + \\ Union.Float => |*float| unreachable + \\ } + \\} + \\ + ); +} + +test "zig fmt: while" { + try testCanonical( + \\test "while" { + \\ while (10 < 1) { + \\ unreachable; + \\ } + \\ + \\ while (10 < 1) + \\ unreachable; + \\ + \\ var i: usize = 0; + \\ while (i < 10) : (i += 1) { + \\ continue; + \\ } + \\ + \\ i = 0; + \\ while (i < 10) : (i += 1) + \\ continue; + \\ + \\ i = 0; + \\ var j: usize = 0; + \\ while (i < 10) : ({ + \\ i += 1; + \\ j += 1; + \\ }) { + \\ continue; + \\ } + \\ + \\ var a: ?u8 = 2; + \\ while (a) |v| : (a = null) { + \\ continue; + \\ } + \\ + \\ while (a) |v| : (a = null) + \\ unreachable; + \\ + \\ label: while (10 < 0) { + \\ unreachable; + \\ } + \\ + \\ const res = while (0 < 10) { + \\ break 7; + \\ } else { + \\ unreachable; + \\ }; + \\ + \\ var a: error!u8 = 0; + \\ while (a) |v| { + \\ a = error.Err; + \\ } else |err| { + \\ i = 1; + \\ } + \\ + \\ comptime var k: usize = 0; + \\ inline while (i < 10) : (i += 1) + \\ j += 2; + \\} + \\ + ); +} + +test "zig fmt: for" { + try testCanonical( + \\test "for" { + \\ const a = []u8{ 1, 2, 3 }; + \\ for (a) |v| { + \\ continue; + \\ } + \\ + \\ for (a) |v| + \\ continue; + \\ + \\ for (a) |*v| + \\ continue; + \\ + \\ for (a) |v, i| { + \\ continue; + \\ } + \\ + \\ for (a) |v, i| + \\ continue; + \\ + \\ const res = for (a) |v, i| { + \\ break v; + \\ } else { + \\ unreachable; + \\ }; + \\ + \\ var num: usize = 0; + \\ inline for (a) |v, i| { + \\ num += v; + \\ num += i; + \\ } + \\} + \\ + ); +} + +test "zig fmt: if" { + try testCanonical( + \\test "if" { + \\ if (10 < 0) { + \\ unreachable; + \\ } + \\ + \\ if (10 < 0) unreachable; + \\ + \\ if (10 < 0) { + \\ unreachable; + \\ } else { + \\ const a = 20; + \\ } + \\ + \\ if (10 < 0) { + \\ unreachable; + \\ } else if (5 < 0) { + \\ unreachable; + \\ } else { + \\ const a = 20; + \\ } + \\ + \\ const is_world_broken = if (10 < 0) true else false; + \\ + \\ const a: ?u8 = 10; + \\ const b: ?u8 = null; + \\ if (a) |v| { + \\ const some = v; + \\ } else if (b) |*v| { + \\ unreachable; + \\ } else { + \\ const some = 10; + \\ } + \\ + \\ const non_null_a = if (a) |v| v else 0; + \\ + \\ const a_err: error!u8 = 0; + \\ if (a_err) |v| { + \\ const p = v; + \\ } else |err| { + \\ unreachable; + \\ } + \\} + \\ + ); +} + +test "zig fmt: defer" { + try testCanonical( + \\test "defer" { + \\ var i: usize = 0; + \\ defer i = 1; + \\ defer { + \\ i += 2; + \\ i *= i; + \\ } + \\ + \\ errdefer i += 3; + \\ errdefer { + \\ i += 2; + \\ i /= i; + \\ } + \\} + \\ + ); +} + +test "zig fmt: comptime" { + try testCanonical( + \\fn a() u8 { + \\ return 5; + \\} + \\ + \\fn b(comptime i: u8) u8 { + \\ return i; + \\} + \\ + \\const av = comptime a(); + \\const av2 = comptime blk: { + \\ var res = a(); + \\ res *= b(2); + \\ break :blk res; + \\}; + \\ + \\comptime { + \\ _ = a(); + \\} + \\ + \\test "comptime" { + \\ const av3 = comptime a(); + \\ const av4 = comptime blk: { + \\ var res = a(); + \\ res *= a(); + \\ break :blk res; + \\ }; + \\ + \\ comptime var i = 0; + \\ comptime { + \\ i = a(); + \\ i += b(i); + \\ } + \\} + \\ + ); +} + +test "zig fmt: fn type" { + try testCanonical( + \\fn a(i: u8) u8 { + \\ return i + 1; + \\} + \\ + \\const a: fn(u8) u8 = undefined; + \\const b: extern fn(u8) u8 = undefined; + \\const c: nakedcc fn(u8) u8 = undefined; + \\const ap: fn(u8) u8 = a; + \\ + ); +} + +test "zig fmt: inline asm" { + try testCanonical( + \\pub fn syscall1(number: usize, arg1: usize) usize { + \\ return asm volatile ("syscall" + \\ : [ret] "={rax}" (-> usize) + \\ : [number] "{rax}" (number), + \\ [arg1] "{rdi}" (arg1) + \\ : "rcx", "r11"); + \\} + \\ + ); +} + +test "zig fmt: coroutines" { + try testCanonical( + \\async fn simpleAsyncFn() void { + \\ x += 1; + \\ suspend; + \\ x += 1; + \\ suspend |p| {} + \\ const p = async simpleAsyncFn() catch unreachable; + \\ await p; + \\} + \\ + \\test "coroutine suspend, resume, cancel" { + \\ const p = try async testAsyncSeq(); + \\ resume p; + \\ cancel p; + \\} + \\ + ); +} + +//{ +// var it = self.link_libs.iterator(); +// while (true) { +// const entry = it.next() ?? break; +// zig_args.append("--library") catch unreachable; +// zig_args.append(entry.key) catch unreachable; +// } +//} From db0812d4b7f856425e0bd26cd6f579468f3ac8ab Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 10 Apr 2018 14:22:01 +0200 Subject: [PATCH 62/87] std.zig.parser: changed block exprs from primary expr to expr --- std/zig/parser.zig | 232 +++++++++++++++++++++++------------------- std/zig/tokenizer.zig | 1 - 2 files changed, 129 insertions(+), 104 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 5f7412b5c2..65e8056ad2 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -766,6 +766,101 @@ pub const Parser = struct { dest_ptr.store(&resume_node.base); stack.append(State { .Expression = DestPtr { .Field = &resume_node.rhs } }) catch unreachable; }, + Token.Id.Keyword_suspend => { + const node = try arena.create(ast.NodeSuspend); + *node = ast.NodeSuspend { + .base = self.initNode(ast.Node.Id.Suspend), + .suspend_token = token, + .payload = null, + .body = null, + }; + dest_ptr.store(&node.base); + stack.append(State { .SuspendBody = node }) catch unreachable; + try stack.append(State { .Payload = &node.payload }); + continue; + }, + Token.Id.Keyword_if => { + const node = try arena.create(ast.NodeIf); + *node = ast.NodeIf { + .base = self.initNode(ast.Node.Id.If), + .if_token = token, + .condition = undefined, + .payload = null, + .body = undefined, + .@"else" = null, + }; + dest_ptr.store(&node.base); + + stack.append(State { .Else = &node.@"else" }) catch unreachable; + try stack.append(State { .Expression = DestPtr { .Field = &node.body } }); + try stack.append(State { .PointerPayload = &node.payload }); + try stack.append(State { .ExpectToken = Token.Id.RParen }); + try stack.append(State { .Expression = DestPtr { .Field = &node.condition } }); + try stack.append(State { .ExpectToken = Token.Id.LParen }); + continue; + }, + Token.Id.Keyword_while => { + stack.append(State { + .While = LoopCtx { + .label = null, + .inline_token = null, + .loop_token = token, + .dest_ptr = dest_ptr, + } + }) catch unreachable; + continue; + }, + Token.Id.Keyword_for => { + stack.append(State { + .For = LoopCtx { + .label = null, + .inline_token = null, + .loop_token = token, + .dest_ptr = dest_ptr, + } + }) catch unreachable; + continue; + }, + Token.Id.Keyword_switch => { + const node = try arena.create(ast.NodeSwitch); + *node = ast.NodeSwitch { + .base = self.initNode(ast.Node.Id.Switch), + .switch_token = token, + .expr = undefined, + .cases = ArrayList(&ast.NodeSwitchCase).init(arena), + .rbrace = undefined, + }; + dest_ptr.store(&node.base); + + stack.append(State { + .SwitchCaseOrEnd = ListSave(&ast.NodeSwitchCase) { + .list = &node.cases, + .ptr = &node.rbrace, + }, + }) catch unreachable; + try stack.append(State { .ExpectToken = Token.Id.LBrace }); + try stack.append(State { .ExpectToken = Token.Id.RParen }); + try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); + try stack.append(State { .ExpectToken = Token.Id.LParen }); + }, + Token.Id.Keyword_comptime => { + const node = try arena.create(ast.NodeComptime); + *node = ast.NodeComptime { + .base = self.initNode(ast.Node.Id.Comptime), + .comptime_token = token, + .expr = undefined, + }; + dest_ptr.store(&node.base); + try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); + continue; + }, + Token.Id.LBrace => { + const block = try self.createBlock(arena, (?Token)(null), token); + dest_ptr.store(&block.base); + + stack.append(State { .Block = block }) catch unreachable; + continue; + }, else => { self.putBackToken(token); stack.append(State { .UnwrapExpressionBegin = dest_ptr }) catch unreachable; @@ -1328,19 +1423,6 @@ pub const Parser = struct { dest_ptr.store(&node.base); continue; }, - Token.Id.Keyword_suspend => { - const node = try arena.create(ast.NodeSuspend); - *node = ast.NodeSuspend { - .base = self.initNode(ast.Node.Id.Suspend), - .suspend_token = token, - .payload = null, - .body = null, - }; - dest_ptr.store(&node.base); - stack.append(State { .SuspendBody = node }) catch unreachable; - try stack.append(State { .Payload = &node.payload }); - continue; - }, Token.Id.MultilineStringLiteralLine => { const node = try arena.create(ast.NodeMultilineStringLiteral); *node = ast.NodeMultilineStringLiteral { @@ -1544,13 +1626,6 @@ pub const Parser = struct { }) catch unreachable; continue; }, - Token.Id.LBrace => { - const block = try self.createBlock(arena, (?Token)(null), token); - dest_ptr.store(&block.base); - - stack.append(State { .Block = block }) catch unreachable; - continue; - }, Token.Id.Keyword_fn => { // TODO shouldn't need these casts const fn_proto = try self.createFnProto(arena, token, @@ -1608,26 +1683,6 @@ pub const Parser = struct { try stack.append(State { .AsmOutputItems = &node.outputs }); try stack.append(State { .IfToken = Token.Id.Colon }); }, - Token.Id.Keyword_if => { - const node = try arena.create(ast.NodeIf); - *node = ast.NodeIf { - .base = self.initNode(ast.Node.Id.If), - .if_token = token, - .condition = undefined, - .payload = null, - .body = undefined, - .@"else" = null, - }; - dest_ptr.store(&node.base); - - stack.append(State { .Else = &node.@"else" }) catch unreachable; - try stack.append(State { .Expression = DestPtr { .Field = &node.body } }); - try stack.append(State { .PointerPayload = &node.payload }); - try stack.append(State { .ExpectToken = Token.Id.RParen }); - try stack.append(State { .Expression = DestPtr { .Field = &node.condition } }); - try stack.append(State { .ExpectToken = Token.Id.LParen }); - continue; - }, Token.Id.Keyword_inline => { stack.append(State { .Inline = InlineCtx { @@ -1638,61 +1693,6 @@ pub const Parser = struct { }) catch unreachable; continue; }, - Token.Id.Keyword_while => { - stack.append(State { - .While = LoopCtx { - .label = null, - .inline_token = null, - .loop_token = token, - .dest_ptr = dest_ptr, - } - }) catch unreachable; - continue; - }, - Token.Id.Keyword_for => { - stack.append(State { - .For = LoopCtx { - .label = null, - .inline_token = null, - .loop_token = token, - .dest_ptr = dest_ptr, - } - }) catch unreachable; - continue; - }, - Token.Id.Keyword_switch => { - const node = try arena.create(ast.NodeSwitch); - *node = ast.NodeSwitch { - .base = self.initNode(ast.Node.Id.Switch), - .switch_token = token, - .expr = undefined, - .cases = ArrayList(&ast.NodeSwitchCase).init(arena), - .rbrace = undefined, - }; - dest_ptr.store(&node.base); - - stack.append(State { - .SwitchCaseOrEnd = ListSave(&ast.NodeSwitchCase) { - .list = &node.cases, - .ptr = &node.rbrace, - }, - }) catch unreachable; - try stack.append(State { .ExpectToken = Token.Id.LBrace }); - try stack.append(State { .ExpectToken = Token.Id.RParen }); - try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); - try stack.append(State { .ExpectToken = Token.Id.LParen }); - }, - Token.Id.Keyword_comptime => { - const node = try arena.create(ast.NodeComptime); - *node = ast.NodeComptime { - .base = self.initNode(ast.Node.Id.Comptime), - .comptime_token = token, - .expr = undefined, - }; - dest_ptr.store(&node.base); - try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); - continue; - }, else => { try self.parseError(&stack, token, "expected primary expression, found {}", @tagName(token.id)); continue; @@ -4966,11 +4966,37 @@ test "zig fmt: coroutines" { ); } -//{ -// var it = self.link_libs.iterator(); -// while (true) { -// const entry = it.next() ?? break; -// zig_args.append("--library") catch unreachable; -// zig_args.append(entry.key) catch unreachable; -// } -//} +test "zig fmt: coroutines" { + try testCanonical( + \\async fn simpleAsyncFn() void { + \\ x += 1; + \\ suspend; + \\ x += 1; + \\ suspend |p| {} + \\ const p = async simpleAsyncFn() catch unreachable; + \\ await p; + \\} + \\ + \\test "coroutine suspend, resume, cancel" { + \\ const p = try async testAsyncSeq(); + \\ resume p; + \\ cancel p; + \\} + \\ + ); +} + +test "zig fmt: Block after if" { + try testCanonical( + \\test "Block after if" { + \\ if (true) { + \\ const a = 0; + \\ } + \\ + \\ { + \\ const a = 0; + \\ } + \\} + \\ + ); +} diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig index 7b1f86712a..91fb20974f 100644 --- a/std/zig/tokenizer.zig +++ b/std/zig/tokenizer.zig @@ -1158,7 +1158,6 @@ fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void { var tokenizer = Tokenizer.init(source); for (expected_tokens) |expected_token_id| { const token = tokenizer.next(); - std.debug.warn("{} {}\n", @tagName(expected_token_id), @tagName(token.id)); std.debug.assert(@TagType(Token.Id)(token.id) == @TagType(Token.Id)(expected_token_id)); switch (expected_token_id) { Token.Id.StringLiteral => |expected_kind| { From 3b80e665074cd56d9e24fb8ae0fb9d86e8cd841a Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 10 Apr 2018 14:52:47 +0200 Subject: [PATCH 63/87] std.zig.parser now parses toplevel use --- std/zig/ast.zig | 29 ++++++++++++++++ std/zig/parser.zig | 87 ++++++++++++++++++++++++++++++++++------------ 2 files changed, 93 insertions(+), 23 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 2128b9976f..1096086e37 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -11,6 +11,7 @@ pub const Node = struct { pub const Id = enum { Root, VarDecl, + Use, ErrorSetDecl, ContainerDecl, StructField, @@ -63,6 +64,7 @@ pub const Node = struct { return switch (base.id) { Id.Root => @fieldParentPtr(NodeRoot, "base", base).iterate(index), Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).iterate(index), + Id.Use => @fieldParentPtr(NodeUse, "base", base).iterate(index), Id.ErrorSetDecl => @fieldParentPtr(NodeErrorSetDecl, "base", base).iterate(index), Id.ContainerDecl => @fieldParentPtr(NodeContainerDecl, "base", base).iterate(index), Id.StructField => @fieldParentPtr(NodeStructField, "base", base).iterate(index), @@ -116,6 +118,7 @@ pub const Node = struct { return switch (base.id) { Id.Root => @fieldParentPtr(NodeRoot, "base", base).firstToken(), Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).firstToken(), + Id.Use => @fieldParentPtr(NodeUse, "base", base).firstToken(), Id.ErrorSetDecl => @fieldParentPtr(NodeErrorSetDecl, "base", base).firstToken(), Id.ContainerDecl => @fieldParentPtr(NodeContainerDecl, "base", base).firstToken(), Id.StructField => @fieldParentPtr(NodeStructField, "base", base).firstToken(), @@ -169,6 +172,7 @@ pub const Node = struct { return switch (base.id) { Id.Root => @fieldParentPtr(NodeRoot, "base", base).lastToken(), Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).lastToken(), + Id.Use => @fieldParentPtr(NodeUse, "base", base).lastToken(), Id.ErrorSetDecl => @fieldParentPtr(NodeErrorSetDecl, "base", base).lastToken(), Id.ContainerDecl => @fieldParentPtr(NodeContainerDecl, "base", base).lastToken(), Id.StructField => @fieldParentPtr(NodeStructField, "base", base).lastToken(), @@ -288,6 +292,31 @@ pub const NodeVarDecl = struct { } }; +pub const NodeUse = struct { + base: Node, + visib_token: ?Token, + expr: &Node, + semicolon_token: Token, + + pub fn iterate(self: &NodeUse, index: usize) ?&Node { + var i = index; + + if (i < 1) return self.expr; + i -= 1; + + return null; + } + + pub fn firstToken(self: &NodeUse) Token { + if (self.visib_token) |visib_token| return visib_token; + return self.expr.firstToken(); + } + + pub fn lastToken(self: &NodeUse) Token { + return self.semicolon_token; + } +}; + pub const NodeErrorSetDecl = struct { base: Node, error_token: Token, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 65e8056ad2..e11bcfef05 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -348,31 +348,54 @@ pub const Parser = struct { }, State.TopLevelExtern => |ctx| { const token = self.getNextToken(); - if (token.id == Token.Id.Keyword_extern) { - const lib_name_token = self.getNextToken(); - const lib_name = blk: { - if (lib_name_token.id == Token.Id.StringLiteral) { - const res = try self.createStringLiteral(arena, lib_name_token); - break :blk &res.base; - } else { - self.putBackToken(lib_name_token); - break :blk null; - } - }; - - stack.append(State { - .TopLevelDecl = TopLevelDeclCtx { - .decls = ctx.decls, + switch (token.id) { + Token.Id.Keyword_use => { + const node = try arena.create(ast.NodeUse); + *node = ast.NodeUse { + .base = self.initNode(ast.Node.Id.Use), .visib_token = ctx.visib_token, - .extern_token = token, - .lib_name = lib_name, - }, - }) catch unreachable; - continue; + .expr = undefined, + .semicolon_token = undefined, + }; + try ctx.decls.append(&node.base); + + stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.Semicolon, + .ptr = &node.semicolon_token, + } + }) catch unreachable; + try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); + continue; + }, + Token.Id.Keyword_extern => { + const lib_name_token = self.getNextToken(); + const lib_name = blk: { + if (lib_name_token.id == Token.Id.StringLiteral) { + const res = try self.createStringLiteral(arena, lib_name_token); + break :blk &res.base; + } else { + self.putBackToken(lib_name_token); + break :blk null; + } + }; + + stack.append(State { + .TopLevelDecl = TopLevelDeclCtx { + .decls = ctx.decls, + .visib_token = ctx.visib_token, + .extern_token = token, + .lib_name = lib_name, + }, + }) catch unreachable; + continue; + }, + else => { + self.putBackToken(token); + stack.append(State { .TopLevelDecl = ctx }) catch unreachable; + continue; + } } - self.putBackToken(token); - stack.append(State { .TopLevelDecl = ctx }) catch unreachable; - continue; }, State.TopLevelDecl => |ctx| { const token = self.getNextToken(); @@ -3068,6 +3091,15 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = decl }); }, + ast.Node.Id.Use => { + const use_decl = @fieldParentPtr(ast.NodeUse, "base", decl); + if (use_decl.visib_token) |visib_token| { + try stream.print("{} ", self.tokenizer.getTokenSlice(visib_token)); + } + try stream.print("use "); + try stack.append(RenderState { .Text = ";" }); + try stack.append(RenderState { .Expression = use_decl.expr }); + }, ast.Node.Id.VarDecl => { const var_decl = @fieldParentPtr(ast.NodeVarDecl, "base", decl); try stack.append(RenderState { .VarDecl = var_decl}); @@ -4086,6 +4118,7 @@ pub const Parser = struct { ast.Node.Id.EnumTag, ast.Node.Id.Root, ast.Node.Id.VarDecl, + ast.Node.Id.Use, ast.Node.Id.TestDecl, ast.Node.Id.ParamDecl => unreachable, }, @@ -5000,3 +5033,11 @@ test "zig fmt: Block after if" { \\ ); } + +test "zig fmt: use" { + try testCanonical( + \\use @import("std"); + \\pub use @import("std"); + \\ + ); +} From aa09e7b63995639084d25329954b1972a72ad12d Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 10 Apr 2018 15:01:21 +0200 Subject: [PATCH 64/87] std.zig.tokinizer now treats string identifiers as identifiers --- std/zig/parser.zig | 8 ++++++++ std/zig/tokenizer.zig | 5 ++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index e11bcfef05..a01a0d748e 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -5041,3 +5041,11 @@ test "zig fmt: use" { \\ ); } + +test "zig fmt: string identifier" { + try testCanonical( + \\const @"a b" = @"c d".@"e f"; + \\fn @"g h"() void {} + \\ + ); +} diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig index 91fb20974f..a2c4def9e0 100644 --- a/std/zig/tokenizer.zig +++ b/std/zig/tokenizer.zig @@ -78,7 +78,6 @@ pub const Token = struct { StringLiteral: StrLitKind, MultilineStringLiteralLine: StrLitKind, CharLiteral, - StringIdentifier, Eof, Builtin, Bang, @@ -434,7 +433,7 @@ pub const Tokenizer = struct { State.SawAtSign => switch (c) { '"' => { - result.id = Token.Id.StringIdentifier; + result.id = Token.Id.Identifier; state = State.StringLiteral; }, else => { @@ -1136,7 +1135,7 @@ test "tokenizer - string identifier and builtin fns" { , []Token.Id{ Token.Id.Keyword_const, - Token.Id.StringIdentifier, + Token.Id.Identifier, Token.Id.Equal, Token.Id.Builtin, Token.Id.LParen, From db9a9f3a6c248e74953d9f7cbf6bedc78e3c23ff Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 10 Apr 2018 15:16:31 +0200 Subject: [PATCH 65/87] std.zig.parser now parses the `var` type * I parse it as a type in all contexts. This is not how the C++ compiler does it, but I think typechecking should catch this --- std/zig/ast.zig | 23 +++++++++++++++++++++-- std/zig/parser.zig | 30 +++++++++++++++++++++--------- 2 files changed, 42 insertions(+), 11 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 1096086e37..d296b3b52b 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -55,6 +55,7 @@ pub const Node = struct { AsmOutput, Unreachable, ErrorType, + VarType, BuiltinCall, LineComment, TestDecl, @@ -108,6 +109,7 @@ pub const Node = struct { Id.AsmOutput => @fieldParentPtr(NodeAsmOutput, "base", base).iterate(index), Id.Unreachable => @fieldParentPtr(NodeUnreachable, "base", base).iterate(index), Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).iterate(index), + Id.VarType => @fieldParentPtr(NodeVarType, "base", base).iterate(index), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).iterate(index), Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).iterate(index), @@ -162,6 +164,7 @@ pub const Node = struct { Id.AsmInput => @fieldParentPtr(NodeAsmInput, "base", base).firstToken(), Id.AsmOutput => @fieldParentPtr(NodeAsmOutput, "base", base).firstToken(), Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).firstToken(), + Id.VarType => @fieldParentPtr(NodeVarType, "base", base).firstToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).firstToken(), Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).firstToken(), @@ -216,6 +219,7 @@ pub const Node = struct { Id.AsmOutput => @fieldParentPtr(NodeAsmOutput, "base", base).lastToken(), Id.Unreachable => @fieldParentPtr(NodeUnreachable, "base", base).lastToken(), Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).lastToken(), + Id.VarType => @fieldParentPtr(NodeVarType, "base", base).lastToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).lastToken(), Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).lastToken(), @@ -541,7 +545,6 @@ pub const NodeFnProto = struct { pub const ReturnType = union(enum) { Explicit: &Node, - Infer: Token, InferErrorSet: &Node, }; @@ -597,7 +600,6 @@ pub const NodeFnProto = struct { // TODO allow this and next prong to share bodies since the types are the same ReturnType.Explicit => |node| return node.lastToken(), ReturnType.InferErrorSet => |node| return node.lastToken(), - ReturnType.Infer => |token| return token, } } }; @@ -1788,6 +1790,23 @@ pub const NodeErrorType = struct { } }; +pub const NodeVarType = struct { + base: Node, + token: Token, + + pub fn iterate(self: &NodeVarType, index: usize) ?&Node { + return null; + } + + pub fn firstToken(self: &NodeVarType) Token { + return self.token; + } + + pub fn lastToken(self: &NodeVarType) Token { + return self.token; + } +}; + pub const NodeLineComment = struct { base: Node, lines: ArrayList(Token), diff --git a/std/zig/parser.zig b/std/zig/parser.zig index a01a0d748e..79c90e6e68 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -1437,6 +1437,14 @@ pub const Parser = struct { dest_ptr.store(&node.base); continue; }, + Token.Id.Keyword_var => { + const node = try arena.create(ast.NodeVarType); + *node = ast.NodeVarType { + .base = self.initNode(ast.Node.Id.VarType), + .token = token, + }; + dest_ptr.store(&node.base); + }, Token.Id.Keyword_unreachable => { const node = try arena.create(ast.NodeUnreachable); *node = ast.NodeUnreachable { @@ -2192,9 +2200,6 @@ pub const Parser = struct { State.FnProtoReturnType => |fn_proto| { const token = self.getNextToken(); switch (token.id) { - Token.Id.Keyword_var => { - fn_proto.return_type = ast.NodeFnProto.ReturnType { .Infer = token }; - }, Token.Id.Bang => { fn_proto.return_type = ast.NodeFnProto.ReturnType { .InferErrorSet = undefined }; stack.append(State { @@ -3573,6 +3578,10 @@ pub const Parser = struct { const error_type = @fieldParentPtr(ast.NodeErrorType, "base", base); try stream.print("{}", self.tokenizer.getTokenSlice(error_type.token)); }, + ast.Node.Id.VarType => { + const var_type = @fieldParentPtr(ast.NodeVarType, "base", base); + try stream.print("{}", self.tokenizer.getTokenSlice(var_type.token)); + }, ast.Node.Id.ContainerDecl => { const container_decl = @fieldParentPtr(ast.NodeContainerDecl, "base", base); @@ -3711,9 +3720,6 @@ pub const Parser = struct { ast.NodeFnProto.ReturnType.Explicit => |node| { try stack.append(RenderState { .Expression = node}); }, - ast.NodeFnProto.ReturnType.Infer => { - try stack.append(RenderState { .Text = "var"}); - }, ast.NodeFnProto.ReturnType.InferErrorSet => |node| { try stack.append(RenderState { .Expression = node}); try stack.append(RenderState { .Text = "!"}); @@ -4136,9 +4142,6 @@ pub const Parser = struct { ast.NodeFnProto.ReturnType.Explicit => |node| { try stack.append(RenderState { .Expression = node}); }, - ast.NodeFnProto.ReturnType.Infer => { - try stream.print("var"); - }, ast.NodeFnProto.ReturnType.InferErrorSet => |node| { try stream.print("!"); try stack.append(RenderState { .Expression = node}); @@ -4489,6 +4492,15 @@ test "zig fmt: var args" { ); } +test "zig fmt: var type" { + try testCanonical( + \\fn print(args: var) var {} + \\const Var = var; + \\const i: var = 0; + \\ + ); +} + test "zig fmt: extern function" { try testCanonical( \\extern fn puts(s: &const u8) c_int; From b9cccce26d616c2ff570a1e1ba7073d3e8f79672 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 10 Apr 2018 15:56:37 +0200 Subject: [PATCH 66/87] std.zig.ast: fixed none compiling code --- std/zig/ast.zig | 1 - 1 file changed, 1 deletion(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index d296b3b52b..045548d624 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -566,7 +566,6 @@ pub const NodeFnProto = struct { if (i < 1) return node; i -= 1; }, - ReturnType.Infer => {}, } if (self.align_expr) |align_expr| { From c6aa637146c6695f382553f66d311599a90d7425 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 10 Apr 2018 16:33:43 +0200 Subject: [PATCH 67/87] std.zig.parser: removed dublicate "zig fmt: coroutines" test --- std/zig/parser.zig | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 79c90e6e68..76eb2d29d5 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -5011,26 +5011,6 @@ test "zig fmt: coroutines" { ); } -test "zig fmt: coroutines" { - try testCanonical( - \\async fn simpleAsyncFn() void { - \\ x += 1; - \\ suspend; - \\ x += 1; - \\ suspend |p| {} - \\ const p = async simpleAsyncFn() catch unreachable; - \\ await p; - \\} - \\ - \\test "coroutine suspend, resume, cancel" { - \\ const p = try async testAsyncSeq(); - \\ resume p; - \\ cancel p; - \\} - \\ - ); -} - test "zig fmt: Block after if" { try testCanonical( \\test "Block after if" { From 477ded9042f045a322b3e92c632e322834a0cdd9 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 10 Apr 2018 11:00:26 -0400 Subject: [PATCH 68/87] add missing call in zig fmt to commit results to disk --- src-self-hosted/main.zig | 1 + 1 file changed, 1 insertion(+) diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index a04faaec49..d125b05b24 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -741,6 +741,7 @@ fn fmtMain(allocator: &mem.Allocator, file_paths: []const []const u8) !void { defer baf.destroy(); try parser.renderSource(baf.stream(), tree.root_node); + try baf.finish(); } } From 0ba85ea6ff910c7a49ae036625b945c475c0f58c Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 10 Apr 2018 17:46:17 +0200 Subject: [PATCH 69/87] std.zig.parser fixed segfault when parsing cc for fn decl --- std/zig/parser.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 76eb2d29d5..92b461d206 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -417,7 +417,7 @@ pub const Parser = struct { }, Token.Id.Keyword_nakedcc, Token.Id.Keyword_stdcallcc => { // TODO shouldn't need this cast - const fn_proto = try self.createAttachFnProto(arena, ctx.decls, undefined, + const fn_proto = try self.createAttachFnProto(arena, ctx.decls, Token(undefined), ctx.extern_token, ctx.lib_name, (?Token)(token), (?Token)(null), (?Token)(null)); stack.append(State { .FnDef = fn_proto }) catch unreachable; try stack.append(State { .FnProto = fn_proto }); From ee3e2790aa85c624fc952bc8326b82d72843bb17 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 10 Apr 2018 20:57:37 -0400 Subject: [PATCH 70/87] cmake defaults stage1 to install in build directory --- CMakeLists.txt | 5 +++++ README.md | 8 ++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b5171d7266..021fd43cf0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,6 +5,11 @@ if(NOT CMAKE_BUILD_TYPE) "Choose the type of build, options are: Debug Release RelWithDebInfo MinSizeRel." FORCE) endif() +if(NOT CMAKE_INSTALL_PREFIX) + set(CMAKE_INSTALL_PREFIX "${CMAKE_BINARY_DIR}" CACHE STRING + "Directory to install zig to" FORCE) +endif() + project(zig C CXX) set(CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake ${CMAKE_MODULE_PATH}) diff --git a/README.md b/README.md index d1e7e8f6d0..1f23e133f8 100644 --- a/README.md +++ b/README.md @@ -141,10 +141,10 @@ libc. Create demo games using Zig. ``` mkdir build cd build -cmake .. -DCMAKE_INSTALL_PREFIX=$(pwd) +cmake .. make make install -./zig build --build-file ../build.zig test +bin/zig build --build-file ../build.zig test ``` ##### MacOS @@ -154,9 +154,9 @@ brew install cmake llvm@6 brew outdated llvm@6 || brew upgrade llvm@6 mkdir build cd build -cmake .. -DCMAKE_PREFIX_PATH=/usr/local/opt/llvm@6/ -DCMAKE_INSTALL_PREFIX=$(pwd) +cmake .. -DCMAKE_PREFIX_PATH=/usr/local/opt/llvm@6/ make install -./zig build --build-file ../build.zig test +bin/zig build --build-file ../build.zig test ``` ##### Windows From 27e881c2d7bebbf06b9932559ff2677ed8b6088b Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 10 Apr 2018 21:58:04 -0400 Subject: [PATCH 71/87] fix another undefined deref see 0ba85ea6ff91 --- std/zig/parser.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 92b461d206..5d5b1ceab2 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -431,7 +431,7 @@ pub const Parser = struct { }, Token.Id.Keyword_async => { // TODO shouldn't need this cast - const fn_proto = try self.createAttachFnProto(arena, ctx.decls, undefined, + const fn_proto = try self.createAttachFnProto(arena, ctx.decls, Token(undefined), ctx.extern_token, ctx.lib_name, (?Token)(null), (?Token)(null), (?Token)(null)); const async_node = try arena.create(ast.NodeAsyncAttribute); From f6c77746d6be1bb238bf632f74c6dfc28de034ed Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 10 Apr 2018 22:24:01 -0400 Subject: [PATCH 72/87] add memmove to builtin.o related: #514 --- std/special/builtin.zig | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/std/special/builtin.zig b/std/special/builtin.zig index 268d0ab545..9de0aa7679 100644 --- a/std/special/builtin.zig +++ b/std/special/builtin.zig @@ -14,26 +14,43 @@ pub fn panic(msg: []const u8, error_return_trace: ?&builtin.StackTrace) noreturn } } -// Note that memset does not return `dest`, like the libc API. -// The semantics of memset is dictated by the corresponding -// LLVM intrinsics, not by the libc API. -export fn memset(dest: ?&u8, c: u8, n: usize) void { +export fn memset(dest: ?&u8, c: u8, n: usize) ?&u8 { @setRuntimeSafety(false); var index: usize = 0; while (index != n) : (index += 1) (??dest)[index] = c; + + return dest; } -// Note that memcpy does not return `dest`, like the libc API. -// The semantics of memcpy is dictated by the corresponding -// LLVM intrinsics, not by the libc API. -export fn memcpy(noalias dest: ?&u8, noalias src: ?&const u8, n: usize) void { +export fn memcpy(noalias dest: ?&u8, noalias src: ?&const u8, n: usize) ?&u8 { @setRuntimeSafety(false); var index: usize = 0; while (index != n) : (index += 1) (??dest)[index] = (??src)[index]; + + return dest; +} + +export fn memmove(dest: ?&u8, src: ?&const u8, n: usize) ?&u8 { + @setRuntimeSafety(false); + + if (@ptrToInt(dest) < @ptrToInt(src)) { + var index: usize = 0; + while (index != n) : (index += 1) { + (??dest)[index] = (??src)[index]; + } + } else { + var index = n; + while (index != 0) { + index -= 1; + (??dest)[index] = (??src)[index]; + } + } + + return dest; } comptime { From 405a2390f09c78fb5db435e97d8ff23c0b44753b Mon Sep 17 00:00:00 2001 From: Josh Wolfe Date: Tue, 10 Apr 2018 22:44:55 -0400 Subject: [PATCH 73/87] zig fmt while-else with no blocks --- std/zig/parser.zig | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 5d5b1ceab2..11b551fec0 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -3839,12 +3839,13 @@ pub const Parser = struct { }, ast.Node.Id.Else => { const else_node = @fieldParentPtr(ast.NodeElse, "base", base); - try stream.print("{} ", self.tokenizer.getTokenSlice(else_node.else_token)); + try stream.print("{}", self.tokenizer.getTokenSlice(else_node.else_token)); switch (else_node.body.id) { ast.Node.Id.Block, ast.Node.Id.If, ast.Node.Id.For, ast.Node.Id.While, ast.Node.Id.Switch => { + try stream.print(" "); try stack.append(RenderState { .Expression = else_node.body }); }, else => { @@ -4805,6 +4806,11 @@ test "zig fmt: while" { \\ unreachable; \\ }; \\ + \\ const res = while (0 < 10) + \\ break 7 + \\ else + \\ unreachable; + \\ \\ var a: error!u8 = 0; \\ while (a) |v| { \\ a = error.Err; From 2ec1cec92d018d25f720c43a7586a12eafd2cbeb Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 10 Apr 2018 23:27:27 -0400 Subject: [PATCH 74/87] add more linux syscalls and constants Based on #904 by tgshultz --- std/os/linux/index.zig | 372 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 372 insertions(+) diff --git a/std/os/linux/index.zig b/std/os/linux/index.zig index 7f27fc83d9..aa2a6d85da 100644 --- a/std/os/linux/index.zig +++ b/std/os/linux/index.zig @@ -478,6 +478,126 @@ pub const CLOCK_BOOTTIME_ALARM = 9; pub const CLOCK_SGI_CYCLE = 10; pub const CLOCK_TAI = 11; +pub const CSIGNAL = 0x000000ff; +pub const CLONE_VM = 0x00000100; +pub const CLONE_FS = 0x00000200; +pub const CLONE_FILES = 0x00000400; +pub const CLONE_SIGHAND = 0x00000800; +pub const CLONE_PTRACE = 0x00002000; +pub const CLONE_VFORK = 0x00004000; +pub const CLONE_PARENT = 0x00008000; +pub const CLONE_THREAD = 0x00010000; +pub const CLONE_NEWNS = 0x00020000; +pub const CLONE_SYSVSEM = 0x00040000; +pub const CLONE_SETTLS = 0x00080000; +pub const CLONE_PARENT_SETTID = 0x00100000; +pub const CLONE_CHILD_CLEARTID = 0x00200000; +pub const CLONE_DETACHED = 0x00400000; +pub const CLONE_UNTRACED = 0x00800000; +pub const CLONE_CHILD_SETTID = 0x01000000; +pub const CLONE_NEWCGROUP = 0x02000000; +pub const CLONE_NEWUTS = 0x04000000; +pub const CLONE_NEWIPC = 0x08000000; +pub const CLONE_NEWUSER = 0x10000000; +pub const CLONE_NEWPID = 0x20000000; +pub const CLONE_NEWNET = 0x40000000; +pub const CLONE_IO = 0x80000000; + +pub const MS_RDONLY = 1; +pub const MS_NOSUID = 2; +pub const MS_NODEV = 4; +pub const MS_NOEXEC = 8; +pub const MS_SYNCHRONOUS = 16; +pub const MS_REMOUNT = 32; +pub const MS_MANDLOCK = 64; +pub const MS_DIRSYNC = 128; +pub const MS_NOATIME = 1024; +pub const MS_NODIRATIME = 2048; +pub const MS_BIND = 4096; +pub const MS_MOVE = 8192; +pub const MS_REC = 16384; +pub const MS_SILENT = 32768; +pub const MS_POSIXACL = (1<<16); +pub const MS_UNBINDABLE = (1<<17); +pub const MS_PRIVATE = (1<<18); +pub const MS_SLAVE = (1<<19); +pub const MS_SHARED = (1<<20); +pub const MS_RELATIME = (1<<21); +pub const MS_KERNMOUNT = (1<<22); +pub const MS_I_VERSION = (1<<23); +pub const MS_STRICTATIME = (1<<24); +pub const MS_LAZYTIME = (1<<25); +pub const MS_NOREMOTELOCK = (1<<27); +pub const MS_NOSEC = (1<<28); +pub const MS_BORN = (1<<29); +pub const MS_ACTIVE = (1<<30); +pub const MS_NOUSER = (1<<31); + +pub const MS_RMT_MASK = (MS_RDONLY|MS_SYNCHRONOUS|MS_MANDLOCK|MS_I_VERSION|MS_LAZYTIME); + +pub const MS_MGC_VAL = 0xc0ed0000; +pub const MS_MGC_MSK = 0xffff0000; + +pub const MNT_FORCE = 1; +pub const MNT_DETACH = 2; +pub const MNT_EXPIRE = 4; +pub const UMOUNT_NOFOLLOW = 8; + + +pub const S_IFMT = 0o170000; + +pub const S_IFDIR = 0o040000; +pub const S_IFCHR = 0o020000; +pub const S_IFBLK = 0o060000; +pub const S_IFREG = 0o100000; +pub const S_IFIFO = 0o010000; +pub const S_IFLNK = 0o120000; +pub const S_IFSOCK = 0o140000; + +pub const S_ISUID = 0o4000; +pub const S_ISGID = 0o2000; +pub const S_ISVTX = 0o1000; +pub const S_IRUSR = 0o400; +pub const S_IWUSR = 0o200; +pub const S_IXUSR = 0o100; +pub const S_IRWXU = 0o700; +pub const S_IRGRP = 0o040; +pub const S_IWGRP = 0o020; +pub const S_IXGRP = 0o010; +pub const S_IRWXG = 0o070; +pub const S_IROTH = 0o004; +pub const S_IWOTH = 0o002; +pub const S_IXOTH = 0o001; +pub const S_IRWXO = 0o007; + +pub fn S_ISREG(m: u32) bool { + return m & S_IFMT == S_IFREG; +} + +pub fn S_ISDIR(m: u32) bool { + return m & S_IFMT == S_IFDIR; +} + +pub fn S_ISCHR(m: u32) bool { + return m & S_IFMT == S_IFCHR; +} + +pub fn S_ISBLK(m: u32) bool { + return m & S_IFMT == S_IFBLK; +} + +pub fn S_ISFIFO(m: u32) bool { + return m & S_IFMT == S_IFIFO; +} + +pub fn S_ISLNK(m: u32) bool { + return m & S_IFMT == S_IFLNK; +} + +pub fn S_ISSOCK(m: u32) bool { + return m & S_IFMT == S_IFSOCK; +} + pub const TFD_NONBLOCK = O_NONBLOCK; pub const TFD_CLOEXEC = O_CLOEXEC; @@ -515,6 +635,10 @@ pub fn chdir(path: &const u8) usize { return syscall1(SYS_chdir, @ptrToInt(path)); } +pub fn chroot(path: &const u8) usize { + return syscall1(SYS_chroot, @ptrToInt(path)); +} + pub fn execve(path: &const u8, argv: &const ?&const u8, envp: &const ?&const u8) usize { return syscall3(SYS_execve, @ptrToInt(path), @ptrToInt(argv), @ptrToInt(envp)); } @@ -544,6 +668,18 @@ pub fn mkdir(path: &const u8, mode: u32) usize { return syscall2(SYS_mkdir, @ptrToInt(path), mode); } +pub fn mount(special: &const u8, dir: &const u8, fstype: &const u8, flags: usize, data: usize) usize { + return syscall5(SYS_mount, @ptrToInt(special), @ptrToInt(dir), @ptrToInt(fstype), flags, data); +} + +pub fn umount(special: &const u8) usize { + return syscall2(SYS_umount2, @ptrToInt(special), 0); +} + +pub fn umount2(special: &const u8, flags: u32) usize { + return syscall2(SYS_umount2, @ptrToInt(special), flags); +} + pub fn mmap(address: ?&u8, length: usize, prot: usize, flags: usize, fd: i32, offset: isize) usize { return syscall6(SYS_mmap, @ptrToInt(address), length, prot, flags, usize(fd), @bitCast(usize, offset)); @@ -650,6 +786,58 @@ pub fn setregid(rgid: u32, egid: u32) usize { return syscall2(SYS_setregid, rgid, egid); } +pub fn getuid() u32 { + return u32(syscall0(SYS_getuid)); +} + +pub fn getgid() u32 { + return u32(syscall0(SYS_getgid)); +} + +pub fn geteuid() u32 { + return u32(syscall0(SYS_geteuid)); +} + +pub fn getegid() u32 { + return u32(syscall0(SYS_getegid)); +} + +pub fn seteuid(euid: u32) usize { + return syscall1(SYS_seteuid, euid); +} + +pub fn setegid(egid: u32) usize { + return syscall1(SYS_setegid, egid); +} + +pub fn getresuid(ruid: &u32, euid: &u32, suid: &u32) usize { + return syscall3(SYS_getresuid, @ptrToInt(ruid), @ptrToInt(euid), @ptrToInt(suid)); +} + +pub fn getresgid(rgid: &u32, egid: &u32, sgid: &u32) usize { + return syscall3(SYS_getresgid, @ptrToInt(rgid), @ptrToInt(egid), @ptrToInt(sgid)); +} + +pub fn setresuid(ruid: u32, euid: u32, suid: u32) usize { + return syscall3(SYS_setresuid, ruid, euid, suid); +} + +pub fn setresgid(rgid: u32, egid: u32, sgid: u32) usize { + return syscall3(SYS_setresgid, rgid, egid, sgid); +} + +pub fn getgroups(size: usize, list: &u32) usize { + return syscall2(SYS_getgroups, size, @ptrToInt(list)); +} + +pub fn setgroups(size: usize, list: &const u32) usize { + return syscall2(SYS_setgroups, size, @ptrToInt(list)); +} + +pub fn getpid() i32 { + return @bitCast(i32, u32(syscall0(SYS_getpid))); +} + pub fn sigprocmask(flags: u32, noalias set: &const sigset_t, noalias oldset: ?&sigset_t) usize { return syscall4(SYS_rt_sigprocmask, flags, @ptrToInt(set), @ptrToInt(oldset), NSIG/8); } @@ -833,6 +1021,71 @@ pub fn fstat(fd: i32, stat_buf: &Stat) usize { return syscall2(SYS_fstat, usize(fd), @ptrToInt(stat_buf)); } +pub fn stat(pathname: &const u8, statbuf: &Stat) usize { + return syscall2(SYS_stat, @ptrToInt(pathname), @ptrToInt(statbuf)); +} + +pub fn lstat(pathname: &const u8, statbuf: &Stat) usize { + return syscall2(SYS_lstat, @ptrToInt(pathname), @ptrToInt(statbuf)); +} + +pub fn listxattr(path: &const u8, list: &u8, size: usize) usize { + return syscall3(SYS_listxattr, @ptrToInt(path), @ptrToInt(list), size); +} + +pub fn llistxattr(path: &const u8, list: &u8, size: usize) usize { + return syscall3(SYS_llistxattr, @ptrToInt(path), @ptrToInt(list), size); +} + +pub fn flistxattr(fd: usize, list: &u8, size: usize) usize { + return syscall3(SYS_flistxattr, fd, @ptrToInt(list), size); +} + +pub fn getxattr(path: &const u8, name: &const u8, value: &void, size: usize) usize { + return syscall4(SYS_getxattr, @ptrToInt(path), @ptrToInt(name), @ptrToInt(value), size); +} + +pub fn lgetxattr(path: &const u8, name: &const u8, value: &void, size: usize) usize { + return syscall4(SYS_lgetxattr, @ptrToInt(path), @ptrToInt(name), @ptrToInt(value), size); +} + +pub fn fgetxattr(fd: usize, name: &const u8, value: &void, size: usize) usize { + return syscall4(SYS_lgetxattr, fd, @ptrToInt(name), @ptrToInt(value), size); +} + +pub fn setxattr(path: &const u8, name: &const u8, value: &const void, + size: usize, flags: usize) usize { + + return syscall5(SYS_setxattr, @ptrToInt(path), @ptrToInt(name), @ptrToInt(value), + size, flags); +} + +pub fn lsetxattr(path: &const u8, name: &const u8, value: &const void, + size: usize, flags: usize) usize { + + return syscall5(SYS_lsetxattr, @ptrToInt(path), @ptrToInt(name), @ptrToInt(value), + size, flags); +} + +pub fn fsetxattr(fd: usize, name: &const u8, value: &const void, + size: usize, flags: usize) usize { + + return syscall5(SYS_fsetxattr, fd, @ptrToInt(name), @ptrToInt(value), + size, flags); +} + +pub fn removexattr(path: &const u8, name: &const u8) usize { + return syscall2(SYS_removexattr, @ptrToInt(path), @ptrToInt(name)); +} + +pub fn lremovexattr(path: &const u8, name: &const u8) usize { + return syscall2(SYS_lremovexattr, @ptrToInt(path), @ptrToInt(name)); +} + +pub fn fremovexattr(fd: usize, name: &const u8) usize { + return syscall2(SYS_fremovexattr, fd, @ptrToInt(name)); +} + pub const epoll_data = packed union { ptr: usize, fd: i32, @@ -878,6 +1131,125 @@ pub fn timerfd_settime(fd: i32, flags: u32, new_value: &const itimerspec, old_va return syscall4(SYS_timerfd_settime, usize(fd), usize(flags), @ptrToInt(new_value), @ptrToInt(old_value)); } +pub const _LINUX_CAPABILITY_VERSION_1 = 0x19980330; +pub const _LINUX_CAPABILITY_U32S_1 = 1; + +pub const _LINUX_CAPABILITY_VERSION_2 = 0x20071026; +pub const _LINUX_CAPABILITY_U32S_2 = 2; + +pub const _LINUX_CAPABILITY_VERSION_3 = 0x20080522; +pub const _LINUX_CAPABILITY_U32S_3 = 2; + +pub const VFS_CAP_REVISION_MASK = 0xFF000000; +pub const VFS_CAP_REVISION_SHIFT = 24; +pub const VFS_CAP_FLAGS_MASK = ~VFS_CAP_REVISION_MASK; +pub const VFS_CAP_FLAGS_EFFECTIVE = 0x000001; + +pub const VFS_CAP_REVISION_1 = 0x01000000; +pub const VFS_CAP_U32_1 = 1; +pub const XATTR_CAPS_SZ_1 = @sizeOf(u32)*(1 + 2*VFS_CAP_U32_1); + +pub const VFS_CAP_REVISION_2 = 0x02000000; +pub const VFS_CAP_U32_2 = 2; +pub const XATTR_CAPS_SZ_2 = @sizeOf(u32)*(1 + 2*VFS_CAP_U32_2); + +pub const XATTR_CAPS_SZ = XATTR_CAPS_SZ_2; +pub const VFS_CAP_U32 = VFS_CAP_U32_2; +pub const VFS_CAP_REVISION = VFS_CAP_REVISION_2; + +pub const vfs_cap_data = extern struct { + //all of these are mandated as little endian + //when on disk. + const Data = struct { + permitted: u32, + inheritable: u32, + }; + + magic_etc: u32, + data: [VFS_CAP_U32]Data, +}; + + +pub const CAP_CHOWN = 0; +pub const CAP_DAC_OVERRIDE = 1; +pub const CAP_DAC_READ_SEARCH = 2; +pub const CAP_FOWNER = 3; +pub const CAP_FSETID = 4; +pub const CAP_KILL = 5; +pub const CAP_SETGID = 6; +pub const CAP_SETUID = 7; +pub const CAP_SETPCAP = 8; +pub const CAP_LINUX_IMMUTABLE = 9; +pub const CAP_NET_BIND_SERVICE = 10; +pub const CAP_NET_BROADCAST = 11; +pub const CAP_NET_ADMIN = 12; +pub const CAP_NET_RAW = 13; +pub const CAP_IPC_LOCK = 14; +pub const CAP_IPC_OWNER = 15; +pub const CAP_SYS_MODULE = 16; +pub const CAP_SYS_RAWIO = 17; +pub const CAP_SYS_CHROOT = 18; +pub const CAP_SYS_PTRACE = 19; +pub const CAP_SYS_PACCT = 20; +pub const CAP_SYS_ADMIN = 21; +pub const CAP_SYS_BOOT = 22; +pub const CAP_SYS_NICE = 23; +pub const CAP_SYS_RESOURCE = 24; +pub const CAP_SYS_TIME = 25; +pub const CAP_SYS_TTY_CONFIG = 26; +pub const CAP_MKNOD = 27; +pub const CAP_LEASE = 28; +pub const CAP_AUDIT_WRITE = 29; +pub const CAP_AUDIT_CONTROL = 30; +pub const CAP_SETFCAP = 31; +pub const CAP_MAC_OVERRIDE = 32; +pub const CAP_MAC_ADMIN = 33; +pub const CAP_SYSLOG = 34; +pub const CAP_WAKE_ALARM = 35; +pub const CAP_BLOCK_SUSPEND = 36; +pub const CAP_AUDIT_READ = 37; +pub const CAP_LAST_CAP = CAP_AUDIT_READ; + +pub fn cap_valid(u8: x) bool { + return x >= 0 and x <= CAP_LAST_CAP; +} + +pub fn CAP_TO_MASK(cap: u8) u32 { + return u32(1) << u5(cap & 31); +} + +pub fn CAP_TO_INDEX(cap: u8) u8 { + return cap >> 5; +} + +pub const cap_t = extern struct { + hdrp: &cap_user_header_t, + datap: &cap_user_data_t, +}; + +pub const cap_user_header_t = extern struct { + version: u32, + pid: usize, +}; + +pub const cap_user_data_t = extern struct { + effective: u32, + permitted: u32, + inheritable: u32, +}; + +pub fn unshare(flags: usize) usize { + return syscall1(SYS_unshare, usize(flags)); +} + +pub fn capget(hdrp: &cap_user_header_t, datap: &cap_user_data_t) usize { + return syscall2(SYS_capget, @ptrToInt(hdrp), @ptrToInt(datap)); +} + +pub fn capset(hdrp: &cap_user_header_t, datap: &const cap_user_data_t) usize { + return syscall2(SYS_capset, @ptrToInt(hdrp), @ptrToInt(datap)); +} + test "import linux test" { // TODO lazy analysis should prevent this test from being compiled on windows, but // it is still compiled on windows From 58c6424d4fe64f88e25714d20d5755d31a7775c1 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 11 Apr 2018 00:32:42 -0400 Subject: [PATCH 75/87] simplify and fix BufMap logic --- std/buf_map.zig | 33 ++++++++++++--------------------- std/hash_map.zig | 6 +++++- 2 files changed, 17 insertions(+), 22 deletions(-) diff --git a/std/buf_map.zig b/std/buf_map.zig index 7e2ea99f1a..3e12d9a7d9 100644 --- a/std/buf_map.zig +++ b/std/buf_map.zig @@ -1,6 +1,8 @@ -const HashMap = @import("hash_map.zig").HashMap; -const mem = @import("mem.zig"); +const std = @import("index.zig"); +const HashMap = std.HashMap; +const mem = std.mem; const Allocator = mem.Allocator; +const assert = std.debug.assert; /// BufMap copies keys and values before they go into the map, and /// frees them when they get removed. @@ -28,18 +30,12 @@ pub const BufMap = struct { } pub fn set(self: &BufMap, key: []const u8, value: []const u8) !void { - if (self.hash_map.get(key)) |entry| { - const value_copy = try self.copy(value); - errdefer self.free(value_copy); - const old_value = ??(try self.hash_map.put(key, value_copy)); - self.free(old_value); - } else { - const key_copy = try self.copy(key); - errdefer self.free(key_copy); - const value_copy = try self.copy(value); - errdefer self.free(value_copy); - _ = try self.hash_map.put(key_copy, value_copy); - } + self.delete(key); + const key_copy = try self.copy(key); + errdefer self.free(key_copy); + const value_copy = try self.copy(value); + errdefer self.free(value_copy); + _ = try self.hash_map.put(key_copy, value_copy); } pub fn get(self: &BufMap, key: []const u8) ?[]const u8 { @@ -66,17 +62,12 @@ pub const BufMap = struct { } fn copy(self: &BufMap, value: []const u8) ![]const u8 { - const result = try self.hash_map.allocator.alloc(u8, value.len); - mem.copy(u8, result, value); - return result; + return mem.dupe(self.hash_map.allocator, u8, value); } }; -const assert = @import("debug/index.zig").assert; -const heap = @import("heap.zig"); - test "BufMap" { - var direct_allocator = heap.DirectAllocator.init(); + var direct_allocator = std.heap.DirectAllocator.init(); defer direct_allocator.deinit(); var bufmap = BufMap.init(&direct_allocator.allocator); diff --git a/std/hash_map.zig b/std/hash_map.zig index becced64ff..29dd233753 100644 --- a/std/hash_map.zig +++ b/std/hash_map.zig @@ -114,6 +114,7 @@ pub fn HashMap(comptime K: type, comptime V: type, } pub fn remove(hm: &Self, key: K) ?&Entry { + if (hm.entries.len == 0) return null; hm.incrementModificationCount(); const start_index = hm.keyToIndex(key); {var roll_over: usize = 0; while (roll_over <= hm.max_distance_from_start_index) : (roll_over += 1) { @@ -236,7 +237,10 @@ pub fn HashMap(comptime K: type, comptime V: type, } test "basic hash map usage" { - var map = HashMap(i32, i32, hash_i32, eql_i32).init(debug.global_allocator); + var direct_allocator = std.heap.DirectAllocator.init(); + defer direct_allocator.deinit(); + + var map = HashMap(i32, i32, hash_i32, eql_i32).init(&direct_allocator.allocator); defer map.deinit(); assert((map.put(1, 11) catch unreachable) == null); From dae287524d94219b1f3a2ec82d410b9f34fb2546 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Wed, 11 Apr 2018 10:37:04 +0200 Subject: [PATCH 76/87] std.zig: Major refactor * There now exists a few function to allocate all nodes in parser.zig * ast.zig now have a table of Ids and their corrisponding type --- std/zig/ast.zig | 337 ++++----- std/zig/parser.zig | 1665 ++++++++++++++++++++++---------------------- 2 files changed, 991 insertions(+), 1011 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 045548d624..9fb10aa0b1 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -9,38 +9,34 @@ pub const Node = struct { comment: ?&NodeLineComment, pub const Id = enum { + // Top level Root, - VarDecl, Use, - ErrorSetDecl, - ContainerDecl, - StructField, - UnionTag, - EnumTag, - Identifier, - AsyncAttribute, - FnProto, - ParamDecl, - Block, + TestDecl, + + // Statements + VarDecl, Defer, - Comptime, - Payload, - PointerPayload, - PointerIndexPayload, - Else, - Switch, - SwitchCase, - SwitchElse, - While, - For, - If, + + // Operators InfixOp, PrefixOp, SuffixOp, - GroupedExpression, + + // Control flow + Switch, + While, + For, + If, ControlFlowExpression, Suspend, - FieldInitializer, + + // Type expressions + VarType, + ErrorType, + FnProto, + + // Primary expressions IntegerLiteral, FloatLiteral, StringLiteral, @@ -50,180 +46,143 @@ pub const Node = struct { NullLiteral, UndefinedLiteral, ThisLiteral, + Unreachable, + Identifier, + GroupedExpression, + BuiltinCall, + ErrorSetDecl, + ContainerDecl, Asm, + Comptime, + Block, + + // Misc + LineComment, + SwitchCase, + SwitchElse, + Else, + Payload, + PointerPayload, + PointerIndexPayload, + StructField, + UnionTag, + EnumTag, AsmInput, AsmOutput, - Unreachable, - ErrorType, - VarType, - BuiltinCall, - LineComment, - TestDecl, + AsyncAttribute, + ParamDecl, + FieldInitializer, }; + const IdTypePair = struct { + id: Id, + Type: type, + }; + + // TODO: When @field exists, we could generate this by iterating over all members of `Id`, + // and making an array of `IdTypePair { .id = @field(Id, @memberName(Id, i)), .Type = @field(ast, "Node" ++ @memberName(Id, i)) }` + const idTypeTable = []IdTypePair { + IdTypePair { .id = Id.Root, .Type = NodeRoot }, + IdTypePair { .id = Id.Use, .Type = NodeUse }, + IdTypePair { .id = Id.TestDecl, .Type = NodeTestDecl }, + + IdTypePair { .id = Id.VarDecl, .Type = NodeVarDecl }, + IdTypePair { .id = Id.Defer, .Type = NodeDefer }, + + IdTypePair { .id = Id.InfixOp, .Type = NodeInfixOp }, + IdTypePair { .id = Id.PrefixOp, .Type = NodePrefixOp }, + IdTypePair { .id = Id.SuffixOp, .Type = NodeSuffixOp }, + + IdTypePair { .id = Id.Switch, .Type = NodeSwitch }, + IdTypePair { .id = Id.While, .Type = NodeWhile }, + IdTypePair { .id = Id.For, .Type = NodeFor }, + IdTypePair { .id = Id.If, .Type = NodeIf }, + IdTypePair { .id = Id.ControlFlowExpression, .Type = NodeControlFlowExpression }, + IdTypePair { .id = Id.Suspend, .Type = NodeSuspend }, + + IdTypePair { .id = Id.VarType, .Type = NodeVarType }, + IdTypePair { .id = Id.ErrorType, .Type = NodeErrorType }, + IdTypePair { .id = Id.FnProto, .Type = NodeFnProto }, + + IdTypePair { .id = Id.IntegerLiteral, .Type = NodeIntegerLiteral }, + IdTypePair { .id = Id.FloatLiteral, .Type = NodeFloatLiteral }, + IdTypePair { .id = Id.StringLiteral, .Type = NodeStringLiteral }, + IdTypePair { .id = Id.MultilineStringLiteral, .Type = NodeMultilineStringLiteral }, + IdTypePair { .id = Id.CharLiteral, .Type = NodeCharLiteral }, + IdTypePair { .id = Id.BoolLiteral, .Type = NodeBoolLiteral }, + IdTypePair { .id = Id.NullLiteral, .Type = NodeNullLiteral }, + IdTypePair { .id = Id.UndefinedLiteral, .Type = NodeUndefinedLiteral }, + IdTypePair { .id = Id.ThisLiteral, .Type = NodeThisLiteral }, + IdTypePair { .id = Id.Unreachable, .Type = NodeUnreachable }, + IdTypePair { .id = Id.Identifier, .Type = NodeIdentifier }, + IdTypePair { .id = Id.GroupedExpression, .Type = NodeGroupedExpression }, + IdTypePair { .id = Id.BuiltinCall, .Type = NodeBuiltinCall }, + IdTypePair { .id = Id.ErrorSetDecl, .Type = NodeErrorSetDecl }, + IdTypePair { .id = Id.ContainerDecl, .Type = NodeContainerDecl }, + IdTypePair { .id = Id.Asm, .Type = NodeAsm }, + IdTypePair { .id = Id.Comptime, .Type = NodeComptime }, + IdTypePair { .id = Id.Block, .Type = NodeBlock }, + + IdTypePair { .id = Id.LineComment, .Type = NodeLineComment }, + IdTypePair { .id = Id.SwitchCase, .Type = NodeSwitchCase }, + IdTypePair { .id = Id.SwitchElse, .Type = NodeSwitchElse }, + IdTypePair { .id = Id.Else, .Type = NodeElse }, + IdTypePair { .id = Id.Payload, .Type = NodePayload }, + IdTypePair { .id = Id.PointerPayload, .Type = NodePointerPayload }, + IdTypePair { .id = Id.PointerIndexPayload, .Type = NodePointerIndexPayload }, + IdTypePair { .id = Id.StructField, .Type = NodeStructField }, + IdTypePair { .id = Id.UnionTag, .Type = NodeUnionTag }, + IdTypePair { .id = Id.EnumTag, .Type = NodeEnumTag }, + IdTypePair { .id = Id.AsmInput, .Type = NodeAsmInput }, + IdTypePair { .id = Id.AsmOutput, .Type = NodeAsmOutput }, + IdTypePair { .id = Id.AsyncAttribute, .Type = NodeAsyncAttribute }, + IdTypePair { .id = Id.ParamDecl, .Type = NodeParamDecl }, + IdTypePair { .id = Id.FieldInitializer, .Type = NodeFieldInitializer }, + }; + + pub fn IdToType(comptime id: Id) type { + inline for (idTypeTable) |id_type_pair| { + if (id == id_type_pair.id) + return id_type_pair.Type; + } + + unreachable; + } + + pub fn typeToId(comptime T: type) Id { + inline for (idTypeTable) |id_type_pair| { + if (T == id_type_pair.Type) + return id_type_pair.id; + } + + unreachable; + } + pub fn iterate(base: &Node, index: usize) ?&Node { - return switch (base.id) { - Id.Root => @fieldParentPtr(NodeRoot, "base", base).iterate(index), - Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).iterate(index), - Id.Use => @fieldParentPtr(NodeUse, "base", base).iterate(index), - Id.ErrorSetDecl => @fieldParentPtr(NodeErrorSetDecl, "base", base).iterate(index), - Id.ContainerDecl => @fieldParentPtr(NodeContainerDecl, "base", base).iterate(index), - Id.StructField => @fieldParentPtr(NodeStructField, "base", base).iterate(index), - Id.UnionTag => @fieldParentPtr(NodeUnionTag, "base", base).iterate(index), - Id.EnumTag => @fieldParentPtr(NodeEnumTag, "base", base).iterate(index), - Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).iterate(index), - Id.AsyncAttribute => @fieldParentPtr(NodeAsyncAttribute, "base", base).iterate(index), - Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).iterate(index), - Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).iterate(index), - Id.Block => @fieldParentPtr(NodeBlock, "base", base).iterate(index), - Id.Defer => @fieldParentPtr(NodeDefer, "base", base).iterate(index), - Id.Comptime => @fieldParentPtr(NodeComptime, "base", base).iterate(index), - Id.Payload => @fieldParentPtr(NodePayload, "base", base).iterate(index), - Id.PointerPayload => @fieldParentPtr(NodePointerPayload, "base", base).iterate(index), - Id.PointerIndexPayload => @fieldParentPtr(NodePointerIndexPayload, "base", base).iterate(index), - Id.Else => @fieldParentPtr(NodeSwitch, "base", base).iterate(index), - Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).iterate(index), - Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).iterate(index), - Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).iterate(index), - Id.While => @fieldParentPtr(NodeWhile, "base", base).iterate(index), - Id.For => @fieldParentPtr(NodeFor, "base", base).iterate(index), - Id.If => @fieldParentPtr(NodeIf, "base", base).iterate(index), - Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).iterate(index), - Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).iterate(index), - Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).iterate(index), - Id.GroupedExpression => @fieldParentPtr(NodeGroupedExpression, "base", base).iterate(index), - Id.ControlFlowExpression => @fieldParentPtr(NodeControlFlowExpression, "base", base).iterate(index), - Id.Suspend => @fieldParentPtr(NodeSuspend, "base", base).iterate(index), - Id.FieldInitializer => @fieldParentPtr(NodeFieldInitializer, "base", base).iterate(index), - Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).iterate(index), - Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index), - Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).iterate(index), - Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).iterate(index), - Id.CharLiteral => @fieldParentPtr(NodeCharLiteral, "base", base).iterate(index), - Id.BoolLiteral => @fieldParentPtr(NodeBoolLiteral, "base", base).iterate(index), - Id.NullLiteral => @fieldParentPtr(NodeNullLiteral, "base", base).iterate(index), - Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).iterate(index), - Id.ThisLiteral => @fieldParentPtr(NodeThisLiteral, "base", base).iterate(index), - Id.Asm => @fieldParentPtr(NodeAsm, "base", base).iterate(index), - Id.AsmInput => @fieldParentPtr(NodeAsmInput, "base", base).iterate(index), - Id.AsmOutput => @fieldParentPtr(NodeAsmOutput, "base", base).iterate(index), - Id.Unreachable => @fieldParentPtr(NodeUnreachable, "base", base).iterate(index), - Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).iterate(index), - Id.VarType => @fieldParentPtr(NodeVarType, "base", base).iterate(index), - Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index), - Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).iterate(index), - Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).iterate(index), - }; + inline for (idTypeTable) |id_type_pair| { + if (base.id == id_type_pair.id) + return @fieldParentPtr(id_type_pair.Type, "base", base).iterate(index); + } + + unreachable; } pub fn firstToken(base: &Node) Token { - return switch (base.id) { - Id.Root => @fieldParentPtr(NodeRoot, "base", base).firstToken(), - Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).firstToken(), - Id.Use => @fieldParentPtr(NodeUse, "base", base).firstToken(), - Id.ErrorSetDecl => @fieldParentPtr(NodeErrorSetDecl, "base", base).firstToken(), - Id.ContainerDecl => @fieldParentPtr(NodeContainerDecl, "base", base).firstToken(), - Id.StructField => @fieldParentPtr(NodeStructField, "base", base).firstToken(), - Id.UnionTag => @fieldParentPtr(NodeUnionTag, "base", base).firstToken(), - Id.EnumTag => @fieldParentPtr(NodeEnumTag, "base", base).firstToken(), - Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).firstToken(), - Id.AsyncAttribute => @fieldParentPtr(NodeAsyncAttribute, "base", base).firstToken(), - Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).firstToken(), - Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).firstToken(), - Id.Block => @fieldParentPtr(NodeBlock, "base", base).firstToken(), - Id.Defer => @fieldParentPtr(NodeDefer, "base", base).firstToken(), - Id.Comptime => @fieldParentPtr(NodeComptime, "base", base).firstToken(), - Id.Payload => @fieldParentPtr(NodePayload, "base", base).firstToken(), - Id.PointerPayload => @fieldParentPtr(NodePointerPayload, "base", base).firstToken(), - Id.PointerIndexPayload => @fieldParentPtr(NodePointerIndexPayload, "base", base).firstToken(), - Id.Else => @fieldParentPtr(NodeSwitch, "base", base).firstToken(), - Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).firstToken(), - Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).firstToken(), - Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).firstToken(), - Id.While => @fieldParentPtr(NodeWhile, "base", base).firstToken(), - Id.For => @fieldParentPtr(NodeFor, "base", base).firstToken(), - Id.If => @fieldParentPtr(NodeIf, "base", base).firstToken(), - Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).firstToken(), - Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).firstToken(), - Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).firstToken(), - Id.GroupedExpression => @fieldParentPtr(NodeGroupedExpression, "base", base).firstToken(), - Id.ControlFlowExpression => @fieldParentPtr(NodeControlFlowExpression, "base", base).firstToken(), - Id.Suspend => @fieldParentPtr(NodeSuspend, "base", base).firstToken(), - Id.FieldInitializer => @fieldParentPtr(NodeFieldInitializer, "base", base).firstToken(), - Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).firstToken(), - Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(), - Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(), - Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).firstToken(), - Id.CharLiteral => @fieldParentPtr(NodeCharLiteral, "base", base).firstToken(), - Id.BoolLiteral => @fieldParentPtr(NodeBoolLiteral, "base", base).firstToken(), - Id.NullLiteral => @fieldParentPtr(NodeNullLiteral, "base", base).firstToken(), - Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).firstToken(), - Id.Unreachable => @fieldParentPtr(NodeUnreachable, "base", base).firstToken(), - Id.ThisLiteral => @fieldParentPtr(NodeThisLiteral, "base", base).firstToken(), - Id.Asm => @fieldParentPtr(NodeAsm, "base", base).firstToken(), - Id.AsmInput => @fieldParentPtr(NodeAsmInput, "base", base).firstToken(), - Id.AsmOutput => @fieldParentPtr(NodeAsmOutput, "base", base).firstToken(), - Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).firstToken(), - Id.VarType => @fieldParentPtr(NodeVarType, "base", base).firstToken(), - Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(), - Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).firstToken(), - Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).firstToken(), - }; + inline for (idTypeTable) |id_type_pair| { + if (base.id == id_type_pair.id) + return @fieldParentPtr(id_type_pair.Type, "base", base).firstToken(); + } + + unreachable; } pub fn lastToken(base: &Node) Token { - return switch (base.id) { - Id.Root => @fieldParentPtr(NodeRoot, "base", base).lastToken(), - Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).lastToken(), - Id.Use => @fieldParentPtr(NodeUse, "base", base).lastToken(), - Id.ErrorSetDecl => @fieldParentPtr(NodeErrorSetDecl, "base", base).lastToken(), - Id.ContainerDecl => @fieldParentPtr(NodeContainerDecl, "base", base).lastToken(), - Id.StructField => @fieldParentPtr(NodeStructField, "base", base).lastToken(), - Id.UnionTag => @fieldParentPtr(NodeUnionTag, "base", base).lastToken(), - Id.EnumTag => @fieldParentPtr(NodeEnumTag, "base", base).lastToken(), - Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).lastToken(), - Id.AsyncAttribute => @fieldParentPtr(NodeAsyncAttribute, "base", base).lastToken(), - Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).lastToken(), - Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).lastToken(), - Id.Block => @fieldParentPtr(NodeBlock, "base", base).lastToken(), - Id.Defer => @fieldParentPtr(NodeDefer, "base", base).lastToken(), - Id.Comptime => @fieldParentPtr(NodeComptime, "base", base).lastToken(), - Id.Payload => @fieldParentPtr(NodePayload, "base", base).lastToken(), - Id.PointerPayload => @fieldParentPtr(NodePointerPayload, "base", base).lastToken(), - Id.PointerIndexPayload => @fieldParentPtr(NodePointerIndexPayload, "base", base).lastToken(), - Id.Else => @fieldParentPtr(NodeElse, "base", base).lastToken(), - Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).lastToken(), - Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).lastToken(), - Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).lastToken(), - Id.While => @fieldParentPtr(NodeWhile, "base", base).lastToken(), - Id.For => @fieldParentPtr(NodeFor, "base", base).lastToken(), - Id.If => @fieldParentPtr(NodeIf, "base", base).lastToken(), - Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).lastToken(), - Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).lastToken(), - Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).lastToken(), - Id.GroupedExpression => @fieldParentPtr(NodeGroupedExpression, "base", base).lastToken(), - Id.ControlFlowExpression => @fieldParentPtr(NodeControlFlowExpression, "base", base).lastToken(), - Id.Suspend => @fieldParentPtr(NodeSuspend, "base", base).lastToken(), - Id.FieldInitializer => @fieldParentPtr(NodeFieldInitializer, "base", base).lastToken(), - Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).lastToken(), - Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(), - Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(), - Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).lastToken(), - Id.CharLiteral => @fieldParentPtr(NodeCharLiteral, "base", base).lastToken(), - Id.BoolLiteral => @fieldParentPtr(NodeBoolLiteral, "base", base).lastToken(), - Id.NullLiteral => @fieldParentPtr(NodeNullLiteral, "base", base).lastToken(), - Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).lastToken(), - Id.ThisLiteral => @fieldParentPtr(NodeThisLiteral, "base", base).lastToken(), - Id.Asm => @fieldParentPtr(NodeAsm, "base", base).lastToken(), - Id.AsmInput => @fieldParentPtr(NodeAsmInput, "base", base).lastToken(), - Id.AsmOutput => @fieldParentPtr(NodeAsmOutput, "base", base).lastToken(), - Id.Unreachable => @fieldParentPtr(NodeUnreachable, "base", base).lastToken(), - Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).lastToken(), - Id.VarType => @fieldParentPtr(NodeVarType, "base", base).lastToken(), - Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(), - Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).lastToken(), - Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).lastToken(), - }; + inline for (idTypeTable) |id_type_pair| { + if (base.id == id_type_pair.id) + return @fieldParentPtr(id_type_pair.Type, "base", base).lastToken(); + } + + unreachable; } }; @@ -482,18 +441,18 @@ pub const NodeEnumTag = struct { pub const NodeIdentifier = struct { base: Node, - name_token: Token, + token: Token, pub fn iterate(self: &NodeIdentifier, index: usize) ?&Node { return null; } pub fn firstToken(self: &NodeIdentifier) Token { - return self.name_token; + return self.token; } pub fn lastToken(self: &NodeIdentifier) Token { - return self.name_token; + return self.token; } }; diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 11b551fec0..5047a5c694 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -240,7 +240,14 @@ pub const Parser = struct { errdefer arena_allocator.deinit(); const arena = &arena_allocator.allocator; - const root_node = try self.createRoot(arena); + const root_node = try self.createNode(arena, ast.NodeRoot, + ast.NodeRoot { + .base = undefined, + .decls = ArrayList(&ast.Node).init(arena), + // initialized when we get the eof token + .eof_token = undefined, + } + ); try stack.append(State.TopLevel); @@ -297,9 +304,23 @@ pub const Parser = struct { const name_token = (try self.eatToken(&stack, Token.Id.StringLiteral)) ?? continue; const lbrace = (try self.eatToken(&stack, Token.Id.LBrace)) ?? continue; - const name = try self.createStringLiteral(arena, name_token); - const block = try self.createBlock(arena, (?Token)(null), token); - const test_decl = try self.createAttachTestDecl(arena, &root_node.decls, token, &name.base, block); + const block = try self.createNode(arena, ast.NodeBlock, + ast.NodeBlock { + .base = undefined, + .label = null, + .lbrace = lbrace, + .statements = ArrayList(&ast.Node).init(arena), + .rbrace = undefined, + } + ); + _ = try self.createAttachNode(arena, &root_node.decls, ast.NodeTestDecl, + ast.NodeTestDecl { + .base = undefined, + .test_token = token, + .name = &(try self.createLiteral(arena, ast.NodeStringLiteral, name_token)).base, + .body_node = &block.base, + } + ); stack.append(State { .Block = block }) catch unreachable; continue; }, @@ -320,13 +341,13 @@ pub const Parser = struct { continue; }, Token.Id.Keyword_comptime => { - const node = try arena.create(ast.NodeComptime); - *node = ast.NodeComptime { - .base = self.initNode(ast.Node.Id.Comptime), - .comptime_token = token, - .expr = undefined, - }; - try root_node.decls.append(&node.base); + const node = try self.createAttachNode(arena, &root_node.decls, ast.NodeComptime, + ast.NodeComptime { + .base = undefined, + .comptime_token = token, + .expr = undefined, + } + ); stack.append(State.TopLevel) catch unreachable; try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); continue; @@ -350,15 +371,14 @@ pub const Parser = struct { const token = self.getNextToken(); switch (token.id) { Token.Id.Keyword_use => { - const node = try arena.create(ast.NodeUse); - *node = ast.NodeUse { - .base = self.initNode(ast.Node.Id.Use), - .visib_token = ctx.visib_token, - .expr = undefined, - .semicolon_token = undefined, - }; - try ctx.decls.append(&node.base); - + const node = try self.createAttachNode(arena, ctx.decls, ast.NodeUse, + ast.NodeUse { + .base = undefined, + .visib_token = ctx.visib_token, + .expr = undefined, + .semicolon_token = undefined, + } + ); stack.append(State { .ExpectTokenSave = ExpectTokenSave { .id = Token.Id.Semicolon, @@ -372,7 +392,7 @@ pub const Parser = struct { const lib_name_token = self.getNextToken(); const lib_name = blk: { if (lib_name_token.id == Token.Id.StringLiteral) { - const res = try self.createStringLiteral(arena, lib_name_token); + const res = try self.createLiteral(arena, ast.NodeStringLiteral, lib_name_token); break :blk &res.base; } else { self.putBackToken(lib_name_token); @@ -401,24 +421,68 @@ pub const Parser = struct { const token = self.getNextToken(); switch (token.id) { Token.Id.Keyword_var, Token.Id.Keyword_const => { - // TODO shouldn't need these casts - const var_decl_node = try self.createAttachVarDecl(arena, ctx.decls, ctx.visib_token, - token, (?Token)(null), ctx.extern_token, ctx.lib_name); + const var_decl_node = try self.createAttachNode(arena, ctx.decls, ast.NodeVarDecl, + ast.NodeVarDecl { + .base = undefined, + .visib_token = ctx.visib_token, + .mut_token = token, + .comptime_token = null, + .extern_token = ctx.extern_token, + .type_node = null, + .align_node = null, + .init_node = null, + .lib_name = ctx.lib_name, + // initialized later + .name_token = undefined, + .eq_token = undefined, + .semicolon_token = undefined, + } + ); stack.append(State { .VarDecl = var_decl_node }) catch unreachable; continue; }, Token.Id.Keyword_fn => { - // TODO shouldn't need these casts - const fn_proto = try self.createAttachFnProto(arena, ctx.decls, token, - ctx.extern_token, ctx.lib_name, (?Token)(null), ctx.visib_token, (?Token)(null)); + const fn_proto = try self.createAttachNode(arena, ctx.decls, ast.NodeFnProto, + ast.NodeFnProto { + .base = undefined, + .visib_token = ctx.visib_token, + .name_token = null, + .fn_token = token, + .params = ArrayList(&ast.Node).init(arena), + .return_type = undefined, + .var_args_token = null, + .extern_token = ctx.extern_token, + .inline_token = null, + .cc_token = null, + .async_attr = null, + .body_node = null, + .lib_name = ctx.lib_name, + .align_expr = null, + } + ); stack.append(State { .FnDef = fn_proto }) catch unreachable; try stack.append(State { .FnProto = fn_proto }); continue; }, Token.Id.Keyword_nakedcc, Token.Id.Keyword_stdcallcc => { - // TODO shouldn't need this cast - const fn_proto = try self.createAttachFnProto(arena, ctx.decls, Token(undefined), - ctx.extern_token, ctx.lib_name, (?Token)(token), (?Token)(null), (?Token)(null)); + const fn_proto = try self.createAttachNode(arena, ctx.decls, ast.NodeFnProto, + ast.NodeFnProto { + .base = undefined, + .visib_token = ctx.visib_token, + .name_token = null, + .fn_token = undefined, + .params = ArrayList(&ast.Node).init(arena), + .return_type = undefined, + .var_args_token = null, + .extern_token = ctx.extern_token, + .inline_token = null, + .cc_token = token, + .async_attr = null, + .body_node = null, + .lib_name = ctx.lib_name, + .align_expr = null, + } + ); stack.append(State { .FnDef = fn_proto }) catch unreachable; try stack.append(State { .FnProto = fn_proto }); try stack.append(State { @@ -430,19 +494,33 @@ pub const Parser = struct { continue; }, Token.Id.Keyword_async => { - // TODO shouldn't need this cast - const fn_proto = try self.createAttachFnProto(arena, ctx.decls, Token(undefined), - ctx.extern_token, ctx.lib_name, (?Token)(null), (?Token)(null), (?Token)(null)); + const async_node = try self.createNode(arena, ast.NodeAsyncAttribute, + ast.NodeAsyncAttribute { + .base = undefined, + .async_token = token, + .allocator_type = null, + .rangle_bracket = null, + } + ); - const async_node = try arena.create(ast.NodeAsyncAttribute); - *async_node = ast.NodeAsyncAttribute { - .base = self.initNode(ast.Node.Id.AsyncAttribute), - .async_token = token, - .allocator_type = null, - .rangle_bracket = null, - }; - - fn_proto.async_attr = async_node; + const fn_proto = try self.createAttachNode(arena, ctx.decls, ast.NodeFnProto, + ast.NodeFnProto { + .base = undefined, + .visib_token = ctx.visib_token, + .name_token = null, + .fn_token = undefined, + .params = ArrayList(&ast.Node).init(arena), + .return_type = undefined, + .var_args_token = null, + .extern_token = ctx.extern_token, + .inline_token = null, + .cc_token = null, + .async_attr = async_node, + .body_node = null, + .lib_name = ctx.lib_name, + .align_expr = null, + } + ); stack.append(State { .FnDef = fn_proto }) catch unreachable; try stack.append(State { .FnProto = fn_proto }); try stack.append(State { @@ -525,30 +603,29 @@ pub const Parser = struct { State.ContainerExtern => |ctx| { const token = self.getNextToken(); - - const node = try arena.create(ast.NodeContainerDecl); - *node = ast.NodeContainerDecl { - .base = self.initNode(ast.Node.Id.ContainerDecl), - .ltoken = ctx.ltoken, - .layout = ctx.layout, - .kind = switch (token.id) { - Token.Id.Keyword_struct => ast.NodeContainerDecl.Kind.Struct, - Token.Id.Keyword_union => ast.NodeContainerDecl.Kind.Union, - Token.Id.Keyword_enum => ast.NodeContainerDecl.Kind.Enum, - else => { - try self.parseError(&stack, token, "expected {}, {} or {}, found {}", - @tagName(Token.Id.Keyword_struct), - @tagName(Token.Id.Keyword_union), - @tagName(Token.Id.Keyword_enum), - @tagName(token.id)); - continue; + const node = try self.createToDestNode(arena, ctx.dest_ptr, ast.NodeContainerDecl, + ast.NodeContainerDecl { + .base = undefined, + .ltoken = ctx.ltoken, + .layout = ctx.layout, + .kind = switch (token.id) { + Token.Id.Keyword_struct => ast.NodeContainerDecl.Kind.Struct, + Token.Id.Keyword_union => ast.NodeContainerDecl.Kind.Union, + Token.Id.Keyword_enum => ast.NodeContainerDecl.Kind.Enum, + else => { + try self.parseError(&stack, token, "expected {}, {} or {}, found {}", + @tagName(Token.Id.Keyword_struct), + @tagName(Token.Id.Keyword_union), + @tagName(Token.Id.Keyword_enum), + @tagName(token.id)); + continue; + }, }, - }, - .init_arg_expr = undefined, - .fields_and_decls = ArrayList(&ast.Node).init(arena), - .rbrace_token = undefined, - }; - ctx.dest_ptr.store(&node.base); + .init_arg_expr = undefined, + .fields_and_decls = ArrayList(&ast.Node).init(arena), + .rbrace_token = undefined, + } + ); stack.append(State { .ContainerDecl = node }) catch unreachable; try stack.append(State { .ExpectToken = Token.Id.LBrace }); @@ -587,13 +664,13 @@ pub const Parser = struct { Token.Id.Identifier => { switch (container_decl.kind) { ast.NodeContainerDecl.Kind.Struct => { - const node = try arena.create(ast.NodeStructField); - *node = ast.NodeStructField { - .base = self.initNode(ast.Node.Id.StructField), - .name_token = token, - .type_expr = undefined, - }; - try container_decl.fields_and_decls.append(&node.base); + const node = try self.createAttachNode(arena, &container_decl.fields_and_decls, ast.NodeStructField, + ast.NodeStructField { + .base = undefined, + .name_token = token, + .type_expr = undefined, + } + ); stack.append(State { .FieldListCommaOrEnd = container_decl }) catch unreachable; try stack.append(State { .Expression = DestPtr { .Field = &node.type_expr } }); @@ -601,13 +678,13 @@ pub const Parser = struct { continue; }, ast.NodeContainerDecl.Kind.Union => { - const node = try arena.create(ast.NodeUnionTag); - *node = ast.NodeUnionTag { - .base = self.initNode(ast.Node.Id.UnionTag), - .name_token = token, - .type_expr = null, - }; - try container_decl.fields_and_decls.append(&node.base); + const node = try self.createAttachNode(arena, &container_decl.fields_and_decls, ast.NodeUnionTag, + ast.NodeUnionTag { + .base = undefined, + .name_token = token, + .type_expr = null, + } + ); stack.append(State { .FieldListCommaOrEnd = container_decl }) catch unreachable; @@ -621,13 +698,13 @@ pub const Parser = struct { continue; }, ast.NodeContainerDecl.Kind.Enum => { - const node = try arena.create(ast.NodeEnumTag); - *node = ast.NodeEnumTag { - .base = self.initNode(ast.Node.Id.EnumTag), - .name_token = token, - .value = null, - }; - try container_decl.fields_and_decls.append(&node.base); + const node = try self.createAttachNode(arena, &container_decl.fields_and_decls, ast.NodeEnumTag, + ast.NodeEnumTag { + .base = undefined, + .name_token = token, + .value = null, + } + ); stack.append(State { .FieldListCommaOrEnd = container_decl }) catch unreachable; @@ -711,108 +788,105 @@ pub const Parser = struct { State.Expression => |dest_ptr| { const token = self.getNextToken(); switch (token.id) { - Token.Id.Keyword_try => { - const node = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp.Try); - dest_ptr.store(&node.base); + Token.Id.Keyword_return => { + const node = try self.createToDestNode(arena, dest_ptr, ast.NodeControlFlowExpression, + ast.NodeControlFlowExpression { + .base = undefined, + .ltoken = token, + .kind = ast.NodeControlFlowExpression.Kind.Return, + .rhs = undefined, + } + ); + + // TODO: Find another way to do optional expressions + stack.append(State { + .Optional = RevertState { + .parser = *self, + .tokenizer = *self.tokenizer, + .ptr = &node.rhs, + } + }) catch unreachable; + try stack.append(State { .Expression = DestPtr { .NullableField = &node.rhs } }); + continue; + }, + Token.Id.Keyword_break, Token.Id.Keyword_continue => { + const label = blk: { + const colon = self.getNextToken(); + if (colon.id != Token.Id.Colon) { + self.putBackToken(colon); + break :blk null; + } + + break :blk (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; + }; + + const node = try self.createToDestNode(arena, dest_ptr, ast.NodeControlFlowExpression, + ast.NodeControlFlowExpression { + .base = undefined, + .ltoken = token, + .kind = switch (token.id) { + Token.Id.Keyword_break => ast.NodeControlFlowExpression.Kind { .Break = label }, + Token.Id.Keyword_continue => ast.NodeControlFlowExpression.Kind { .Continue = label }, + else => unreachable, + }, + .rhs = undefined, + } + ); + + // TODO: Find another way to do optional expressions + stack.append(State { + .Optional = RevertState { + .parser = *self, + .tokenizer = *self.tokenizer, + .ptr = &node.rhs, + } + }) catch unreachable; + try stack.append(State { .Expression = DestPtr { .NullableField = &node.rhs } }); + continue; + }, + Token.Id.Keyword_try, Token.Id.Keyword_cancel, Token.Id.Keyword_resume => { + const node = try self.createToDestNode(arena, dest_ptr, ast.NodePrefixOp, + ast.NodePrefixOp { + .base = undefined, + .op_token = token, + .op = switch (token.id) { + Token.Id.Keyword_try => ast.NodePrefixOp.PrefixOp { .Try = void{} }, + Token.Id.Keyword_cancel => ast.NodePrefixOp.PrefixOp { .Cancel = void{} }, + Token.Id.Keyword_resume => ast.NodePrefixOp.PrefixOp { .Resume = void{} }, + else => unreachable, + }, + .rhs = undefined, + } + ); stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }) catch unreachable; continue; }, - Token.Id.Keyword_return => { - const node = try self.createControlFlowExpr(arena, token, ast.NodeControlFlowExpression.Kind.Return); - dest_ptr.store(&node.base); - - stack.append(State { - .Optional = RevertState { - .parser = *self, - .tokenizer = *self.tokenizer, - .ptr = &node.rhs, - } - }) catch unreachable; - try stack.append(State { .Expression = DestPtr { .NullableField = &node.rhs } }); - continue; - }, - Token.Id.Keyword_break => { - const label = blk: { - const colon = self.getNextToken(); - if (colon.id != Token.Id.Colon) { - self.putBackToken(colon); - break :blk null; - } - - break :blk (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; - }; - - const node = try self.createControlFlowExpr(arena, token, - ast.NodeControlFlowExpression.Kind { - .Break = label, - } - ); - dest_ptr.store(&node.base); - - stack.append(State { - .Optional = RevertState { - .parser = *self, - .tokenizer = *self.tokenizer, - .ptr = &node.rhs, - } - }) catch unreachable; - try stack.append(State { .Expression = DestPtr { .NullableField = &node.rhs } }); - continue; - }, - Token.Id.Keyword_continue => { - const label = blk: { - const colon = self.getNextToken(); - if (colon.id != Token.Id.Colon) { - self.putBackToken(colon); - break :blk null; - } - - break :blk (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; - }; - - const node = try self.createControlFlowExpr(arena, token, - ast.NodeControlFlowExpression.Kind { - .Continue = label, - } - ); - dest_ptr.store(&node.base); - continue; - }, - Token.Id.Keyword_cancel => { - const cancel_node = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp.Cancel); - dest_ptr.store(&cancel_node.base); - stack.append(State { .Expression = DestPtr { .Field = &cancel_node.rhs } }) catch unreachable; - }, - Token.Id.Keyword_resume => { - const resume_node = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp.Resume); - dest_ptr.store(&resume_node.base); - stack.append(State { .Expression = DestPtr { .Field = &resume_node.rhs } }) catch unreachable; - }, Token.Id.Keyword_suspend => { - const node = try arena.create(ast.NodeSuspend); - *node = ast.NodeSuspend { - .base = self.initNode(ast.Node.Id.Suspend), - .suspend_token = token, - .payload = null, - .body = null, - }; - dest_ptr.store(&node.base); + const node = try self.createToDestNode(arena, dest_ptr, ast.NodeSuspend, + ast.NodeSuspend { + .base = undefined, + .suspend_token = token, + .payload = null, + .body = null, + } + ); + stack.append(State { .SuspendBody = node }) catch unreachable; try stack.append(State { .Payload = &node.payload }); continue; }, Token.Id.Keyword_if => { - const node = try arena.create(ast.NodeIf); - *node = ast.NodeIf { - .base = self.initNode(ast.Node.Id.If), - .if_token = token, - .condition = undefined, - .payload = null, - .body = undefined, - .@"else" = null, - }; - dest_ptr.store(&node.base); + const node = try self.createToDestNode(arena, dest_ptr, ast.NodeIf, + ast.NodeIf { + .base = undefined, + .if_token = token, + .condition = undefined, + .payload = null, + .body = undefined, + .@"else" = null, + } + ); stack.append(State { .Else = &node.@"else" }) catch unreachable; try stack.append(State { .Expression = DestPtr { .Field = &node.body } }); @@ -845,15 +919,15 @@ pub const Parser = struct { continue; }, Token.Id.Keyword_switch => { - const node = try arena.create(ast.NodeSwitch); - *node = ast.NodeSwitch { - .base = self.initNode(ast.Node.Id.Switch), - .switch_token = token, - .expr = undefined, - .cases = ArrayList(&ast.NodeSwitchCase).init(arena), - .rbrace = undefined, - }; - dest_ptr.store(&node.base); + const node = try self.createToDestNode(arena, dest_ptr, ast.NodeSwitch, + ast.NodeSwitch { + .base = undefined, + .switch_token = token, + .expr = undefined, + .cases = ArrayList(&ast.NodeSwitchCase).init(arena), + .rbrace = undefined, + } + ); stack.append(State { .SwitchCaseOrEnd = ListSave(&ast.NodeSwitchCase) { @@ -867,20 +941,26 @@ pub const Parser = struct { try stack.append(State { .ExpectToken = Token.Id.LParen }); }, Token.Id.Keyword_comptime => { - const node = try arena.create(ast.NodeComptime); - *node = ast.NodeComptime { - .base = self.initNode(ast.Node.Id.Comptime), - .comptime_token = token, - .expr = undefined, - }; - dest_ptr.store(&node.base); + const node = try self.createToDestNode(arena, dest_ptr, ast.NodeComptime, + ast.NodeComptime { + .base = undefined, + .comptime_token = token, + .expr = undefined, + } + ); try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); continue; }, Token.Id.LBrace => { - const block = try self.createBlock(arena, (?Token)(null), token); - dest_ptr.store(&block.base); - + const block = try self.createToDestNode(arena, dest_ptr, ast.NodeBlock, + ast.NodeBlock { + .base = undefined, + .label = null, + .lbrace = token, + .statements = ArrayList(&ast.Node).init(arena), + .rbrace = undefined, + } + ); stack.append(State { .Block = block }) catch unreachable; continue; }, @@ -901,10 +981,15 @@ pub const Parser = struct { State.RangeExpressionEnd => |dest_ptr| { const token = self.getNextToken(); if (token.id == Token.Id.Ellipsis3) { - const node = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.Range); - node.lhs = dest_ptr.get(); - dest_ptr.store(&node.base); - + const node = try self.createToDestNode(arena, dest_ptr, ast.NodeInfixOp, + ast.NodeInfixOp { + .base = undefined, + .lhs = dest_ptr.get(), + .op_token = token, + .op = ast.NodeInfixOp.InfixOp.Range, + .rhs = undefined, + } + ); stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }) catch unreachable; continue; } else { @@ -922,10 +1007,15 @@ pub const Parser = struct { State.AssignmentExpressionEnd => |dest_ptr| { const token = self.getNextToken(); if (tokenIdToAssignment(token.id)) |ass_id| { - const node = try self.createInfixOp(arena, token, ass_id); - node.lhs = dest_ptr.get(); - dest_ptr.store(&node.base); - + const node = try self.createToDestNode(arena, dest_ptr, ast.NodeInfixOp, + ast.NodeInfixOp { + .base = undefined, + .lhs = dest_ptr.get(), + .op_token = token, + .op = ass_id, + .rhs = undefined, + } + ); stack.append(State { .AssignmentExpressionEnd = dest_ptr }) catch unreachable; try stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }); continue; @@ -944,23 +1034,27 @@ pub const Parser = struct { State.UnwrapExpressionEnd => |dest_ptr| { const token = self.getNextToken(); switch (token.id) { - Token.Id.Keyword_catch => { - const node = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp { .Catch = null }); - node.lhs = dest_ptr.get(); - dest_ptr.store(&node.base); + Token.Id.Keyword_catch, Token.Id.QuestionMarkQuestionMark => { + const node = try self.createToDestNode(arena, dest_ptr, ast.NodeInfixOp, + ast.NodeInfixOp { + .base = undefined, + .lhs = dest_ptr.get(), + .op_token = token, + .op = switch (token.id) { + Token.Id.Keyword_catch => ast.NodeInfixOp.InfixOp { .Catch = null }, + Token.Id.QuestionMarkQuestionMark => ast.NodeInfixOp.InfixOp { .UnwrapMaybe = void{} }, + else => unreachable, + }, + .rhs = undefined, + } + ); stack.append(State { .UnwrapExpressionEnd = dest_ptr }) catch unreachable; try stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }); - try stack.append(State { .Payload = &node.op.Catch }); - continue; - }, - Token.Id.QuestionMarkQuestionMark => { - const node = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.UnwrapMaybe); - node.lhs = dest_ptr.get(); - dest_ptr.store(&node.base); - stack.append(State { .UnwrapExpressionEnd = dest_ptr }) catch unreachable; - try stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }); + if (node.op == ast.NodeInfixOp.InfixOp.Catch) { + try stack.append(State { .Payload = &node.op.Catch }); + } continue; }, else => { @@ -980,10 +1074,15 @@ pub const Parser = struct { const token = self.getNextToken(); switch (token.id) { Token.Id.Keyword_or => { - const node = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.BoolOr); - node.lhs = dest_ptr.get(); - dest_ptr.store(&node.base); - + const node = try self.createToDestNode(arena, dest_ptr, ast.NodeInfixOp, + ast.NodeInfixOp { + .base = undefined, + .lhs = dest_ptr.get(), + .op_token = token, + .op = ast.NodeInfixOp.InfixOp.BoolOr, + .rhs = undefined, + } + ); stack.append(State { .BoolOrExpressionEnd = dest_ptr }) catch unreachable; try stack.append(State { .BoolAndExpressionBegin = DestPtr { .Field = &node.rhs } }); continue; @@ -1005,10 +1104,15 @@ pub const Parser = struct { const token = self.getNextToken(); switch (token.id) { Token.Id.Keyword_and => { - const node = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.BoolAnd); - node.lhs = dest_ptr.get(); - dest_ptr.store(&node.base); - + const node = try self.createToDestNode(arena, dest_ptr, ast.NodeInfixOp, + ast.NodeInfixOp { + .base = undefined, + .lhs = dest_ptr.get(), + .op_token = token, + .op = ast.NodeInfixOp.InfixOp.BoolAnd, + .rhs = undefined, + } + ); stack.append(State { .BoolAndExpressionEnd = dest_ptr }) catch unreachable; try stack.append(State { .ComparisonExpressionBegin = DestPtr { .Field = &node.rhs } }); continue; @@ -1029,10 +1133,15 @@ pub const Parser = struct { State.ComparisonExpressionEnd => |dest_ptr| { const token = self.getNextToken(); if (tokenIdToComparison(token.id)) |comp_id| { - const node = try self.createInfixOp(arena, token, comp_id); - node.lhs = dest_ptr.get(); - dest_ptr.store(&node.base); - + const node = try self.createToDestNode(arena, dest_ptr, ast.NodeInfixOp, + ast.NodeInfixOp { + .base = undefined, + .lhs = dest_ptr.get(), + .op_token = token, + .op = comp_id, + .rhs = undefined, + } + ); stack.append(State { .ComparisonExpressionEnd = dest_ptr }) catch unreachable; try stack.append(State { .BinaryOrExpressionBegin = DestPtr { .Field = &node.rhs } }); continue; @@ -1052,10 +1161,15 @@ pub const Parser = struct { const token = self.getNextToken(); switch (token.id) { Token.Id.Pipe => { - const node = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.BitOr); - node.lhs = dest_ptr.get(); - dest_ptr.store(&node.base); - + const node = try self.createToDestNode(arena, dest_ptr, ast.NodeInfixOp, + ast.NodeInfixOp { + .base = undefined, + .lhs = dest_ptr.get(), + .op_token = token, + .op = ast.NodeInfixOp.InfixOp.BitOr, + .rhs = undefined, + } + ); stack.append(State { .BinaryOrExpressionEnd = dest_ptr }) catch unreachable; try stack.append(State { .BinaryXorExpressionBegin = DestPtr { .Field = &node.rhs } }); continue; @@ -1077,10 +1191,15 @@ pub const Parser = struct { const token = self.getNextToken(); switch (token.id) { Token.Id.Caret => { - const node = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.BitXor); - node.lhs = dest_ptr.get(); - dest_ptr.store(&node.base); - + const node = try self.createToDestNode(arena, dest_ptr, ast.NodeInfixOp, + ast.NodeInfixOp { + .base = undefined, + .lhs = dest_ptr.get(), + .op_token = token, + .op = ast.NodeInfixOp.InfixOp.BitXor, + .rhs = undefined, + } + ); stack.append(State { .BinaryXorExpressionEnd = dest_ptr }) catch unreachable; try stack.append(State { .BinaryAndExpressionBegin = DestPtr { .Field = &node.rhs } }); continue; @@ -1102,10 +1221,15 @@ pub const Parser = struct { const token = self.getNextToken(); switch (token.id) { Token.Id.Ampersand => { - const node = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.BitAnd); - node.lhs = dest_ptr.get(); - dest_ptr.store(&node.base); - + const node = try self.createToDestNode(arena, dest_ptr, ast.NodeInfixOp, + ast.NodeInfixOp { + .base = undefined, + .lhs = dest_ptr.get(), + .op_token = token, + .op = ast.NodeInfixOp.InfixOp.BitAnd, + .rhs = undefined, + } + ); stack.append(State { .BinaryAndExpressionEnd = dest_ptr }) catch unreachable; try stack.append(State { .BitShiftExpressionBegin = DestPtr { .Field = &node.rhs } }); continue; @@ -1126,10 +1250,15 @@ pub const Parser = struct { State.BitShiftExpressionEnd => |dest_ptr| { const token = self.getNextToken(); if (tokenIdToBitShift(token.id)) |bitshift_id| { - const node = try self.createInfixOp(arena, token, bitshift_id); - node.lhs = dest_ptr.get(); - dest_ptr.store(&node.base); - + const node = try self.createToDestNode(arena, dest_ptr, ast.NodeInfixOp, + ast.NodeInfixOp { + .base = undefined, + .lhs = dest_ptr.get(), + .op_token = token, + .op = bitshift_id, + .rhs = undefined, + } + ); stack.append(State { .BitShiftExpressionEnd = dest_ptr }) catch unreachable; try stack.append(State { .AdditionExpressionBegin = DestPtr { .Field = &node.rhs } }); continue; @@ -1148,10 +1277,15 @@ pub const Parser = struct { State.AdditionExpressionEnd => |dest_ptr| { const token = self.getNextToken(); if (tokenIdToAddition(token.id)) |add_id| { - const node = try self.createInfixOp(arena, token, add_id); - node.lhs = dest_ptr.get(); - dest_ptr.store(&node.base); - + const node = try self.createToDestNode(arena, dest_ptr, ast.NodeInfixOp, + ast.NodeInfixOp { + .base = undefined, + .lhs = dest_ptr.get(), + .op_token = token, + .op = add_id, + .rhs = undefined, + } + ); stack.append(State { .AdditionExpressionEnd = dest_ptr }) catch unreachable; try stack.append(State { .MultiplyExpressionBegin = DestPtr { .Field = &node.rhs } }); continue; @@ -1170,10 +1304,15 @@ pub const Parser = struct { State.MultiplyExpressionEnd => |dest_ptr| { const token = self.getNextToken(); if (tokenIdToMultiply(token.id)) |mult_id| { - const node = try self.createInfixOp(arena, token, mult_id); - node.lhs = dest_ptr.get(); - dest_ptr.store(&node.base); - + const node = try self.createToDestNode(arena, dest_ptr, ast.NodeInfixOp, + ast.NodeInfixOp { + .base = undefined, + .lhs = dest_ptr.get(), + .op_token = token, + .op = mult_id, + .rhs = undefined, + } + ); stack.append(State { .MultiplyExpressionEnd = dest_ptr }) catch unreachable; try stack.append(State { .CurlySuffixExpressionBegin = DestPtr { .Field = &node.rhs } }); continue; @@ -1199,12 +1338,16 @@ pub const Parser = struct { const next = self.getNextToken(); switch (next.id) { Token.Id.Period => { - const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { - .StructInitializer = ArrayList(&ast.NodeFieldInitializer).init(arena), - }); - node.lhs = dest_ptr.get(); - dest_ptr.store(&node.base); - + const node = try self.createToDestNode(arena, dest_ptr, ast.NodeSuffixOp, + ast.NodeSuffixOp { + .base = undefined, + .lhs = dest_ptr.get(), + .op = ast.NodeSuffixOp.SuffixOp { + .StructInitializer = ArrayList(&ast.NodeFieldInitializer).init(arena), + }, + .rtoken = undefined, + } + ); stack.append(State { .CurlySuffixExpressionEnd = dest_ptr }) catch unreachable; try stack.append(State { .FieldInitListItemOrEnd = ListSave(&ast.NodeFieldInitializer) { @@ -1216,12 +1359,16 @@ pub const Parser = struct { continue; }, else => { - const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { - .ArrayInitializer = ArrayList(&ast.Node).init(arena), - }); - node.lhs = dest_ptr.get(); - dest_ptr.store(&node.base); - + const node = try self.createToDestNode(arena, dest_ptr, ast.NodeSuffixOp, + ast.NodeSuffixOp { + .base = undefined, + .lhs = dest_ptr.get(), + .op = ast.NodeSuffixOp.SuffixOp { + .ArrayInitializer = ArrayList(&ast.Node).init(arena), + }, + .rtoken = undefined, + } + ); stack.append(State { .CurlySuffixExpressionEnd = dest_ptr }) catch unreachable; try stack.append(State { .ExprListItemOrEnd = ExprListCtx { @@ -1246,10 +1393,15 @@ pub const Parser = struct { const token = self.getNextToken(); switch (token.id) { Token.Id.Bang => { - const node = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.ErrorUnion); - node.lhs = dest_ptr.get(); - dest_ptr.store(&node.base); - + const node = try self.createToDestNode(arena, dest_ptr, ast.NodeInfixOp, + ast.NodeInfixOp { + .base = undefined, + .lhs = dest_ptr.get(), + .op_token = token, + .op = ast.NodeInfixOp.InfixOp.ErrorUnion, + .rhs = undefined, + } + ); stack.append(State { .TypeExprEnd = dest_ptr }) catch unreachable; try stack.append(State { .PrefixOpExpression = DestPtr { .Field = &node.rhs } }); continue; @@ -1264,9 +1416,14 @@ pub const Parser = struct { State.PrefixOpExpression => |dest_ptr| { const token = self.getNextToken(); if (tokenIdToPrefixOp(token.id)) |prefix_id| { - const node = try self.createPrefixOp(arena, token, prefix_id); - dest_ptr.store(&node.base); - + const node = try self.createToDestNode(arena, dest_ptr, ast.NodePrefixOp, + ast.NodePrefixOp { + .base = undefined, + .op_token = token, + .op = prefix_id, + .rhs = undefined, + } + ); stack.append(State { .TypeExprBegin = DestPtr { .Field = &node.rhs } }) catch unreachable; if (node.op == ast.NodePrefixOp.PrefixOp.AddrOf) { try stack.append(State { .AddrOfModifiers = &node.op.AddrOf }); @@ -1283,14 +1440,14 @@ pub const Parser = struct { const token = self.getNextToken(); switch (token.id) { Token.Id.Keyword_async => { - const async_node = try arena.create(ast.NodeAsyncAttribute); - *async_node = ast.NodeAsyncAttribute { - .base = self.initNode(ast.Node.Id.AsyncAttribute), - .async_token = token, - .allocator_type = null, - .rangle_bracket = null, - }; - + const async_node = try self.createNode(arena, ast.NodeAsyncAttribute, + ast.NodeAsyncAttribute { + .base = undefined, + .async_token = token, + .allocator_type = null, + .rangle_bracket = null, + } + ); stack.append(State { .AsyncEnd = AsyncEndCtx { .dest_ptr = dest_ptr, @@ -1329,15 +1486,19 @@ pub const Parser = struct { const token = self.getNextToken(); switch (token.id) { Token.Id.LParen => { - const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { - .Call = ast.NodeSuffixOp.CallInfo { - .params = ArrayList(&ast.Node).init(arena), - .async_attr = null, + const node = try self.createToDestNode(arena, dest_ptr, ast.NodeSuffixOp, + ast.NodeSuffixOp { + .base = undefined, + .lhs = dest_ptr.get(), + .op = ast.NodeSuffixOp.SuffixOp { + .Call = ast.NodeSuffixOp.CallInfo { + .params = ArrayList(&ast.Node).init(arena), + .async_attr = null, + } + }, + .rtoken = undefined, } - }); - node.lhs = dest_ptr.get(); - dest_ptr.store(&node.base); - + ); stack.append(State { .SuffixOpExpressionEnd = dest_ptr }) catch unreachable; try stack.append(State { .ExprListItemOrEnd = ExprListCtx { @@ -1349,28 +1510,31 @@ pub const Parser = struct { continue; }, Token.Id.LBracket => { - const node = try arena.create(ast.NodeSuffixOp); - *node = ast.NodeSuffixOp { - .base = self.initNode(ast.Node.Id.SuffixOp), - .lhs = undefined, - .op = ast.NodeSuffixOp.SuffixOp { - .ArrayAccess = undefined, - }, - .rtoken = undefined, - }; - node.lhs = dest_ptr.get(); - dest_ptr.store(&node.base); - + const node = try self.createToDestNode(arena, dest_ptr, ast.NodeSuffixOp, + ast.NodeSuffixOp { + .base = undefined, + .lhs = dest_ptr.get(), + .op = ast.NodeSuffixOp.SuffixOp { + .ArrayAccess = undefined, + }, + .rtoken = undefined + } + ); stack.append(State { .SuffixOpExpressionEnd = dest_ptr }) catch unreachable; try stack.append(State { .SliceOrArrayAccess = node }); try stack.append(State { .Expression = DestPtr { .Field = &node.op.ArrayAccess }}); continue; }, Token.Id.Period => { - const node = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.Period); - node.lhs = dest_ptr.get(); - dest_ptr.store(&node.base); - + const node = try self.createToDestNode(arena, dest_ptr, ast.NodeInfixOp, + ast.NodeInfixOp { + .base = undefined, + .lhs = dest_ptr.get(), + .op_token = token, + .op = ast.NodeInfixOp.InfixOp.Period, + .rhs = undefined, + } + ); stack.append(State { .SuffixOpExpressionEnd = dest_ptr }) catch unreachable; try stack.append(State { .SuffixOpExpressionBegin = DestPtr { .Field = &node.rhs }}); continue; @@ -1386,83 +1550,53 @@ pub const Parser = struct { const token = self.getNextToken(); switch (token.id) { Token.Id.IntegerLiteral => { - dest_ptr.store(&(try self.createIntegerLiteral(arena, token)).base); + dest_ptr.store(&(try self.createLiteral(arena, ast.NodeStringLiteral, token)).base); continue; }, Token.Id.FloatLiteral => { - dest_ptr.store(&(try self.createFloatLiteral(arena, token)).base); + dest_ptr.store(&(try self.createLiteral(arena, ast.NodeFloatLiteral, token)).base); continue; }, Token.Id.StringLiteral => { - dest_ptr.store(&(try self.createStringLiteral(arena, token)).base); + dest_ptr.store(&(try self.createLiteral(arena, ast.NodeStringLiteral, token)).base); continue; }, Token.Id.CharLiteral => { - const node = try arena.create(ast.NodeCharLiteral); - *node = ast.NodeCharLiteral { - .base = self.initNode(ast.Node.Id.CharLiteral), - .token = token, - }; - dest_ptr.store(&node.base); + dest_ptr.store(&(try self.createLiteral(arena, ast.NodeCharLiteral, token)).base); continue; }, Token.Id.Keyword_undefined => { - dest_ptr.store(&(try self.createUndefined(arena, token)).base); + dest_ptr.store(&(try self.createLiteral(arena, ast.NodeUndefinedLiteral, token)).base); continue; }, Token.Id.Keyword_true, Token.Id.Keyword_false => { - const node = try arena.create(ast.NodeBoolLiteral); - *node = ast.NodeBoolLiteral { - .base = self.initNode(ast.Node.Id.BoolLiteral), - .token = token, - }; - dest_ptr.store(&node.base); + dest_ptr.store(&(try self.createLiteral(arena, ast.NodeBoolLiteral, token)).base); continue; }, Token.Id.Keyword_null => { - const node = try arena.create(ast.NodeNullLiteral); - *node = ast.NodeNullLiteral { - .base = self.initNode(ast.Node.Id.NullLiteral), - .token = token, - }; - dest_ptr.store(&node.base); + dest_ptr.store(&(try self.createLiteral(arena, ast.NodeNullLiteral, token)).base); continue; }, Token.Id.Keyword_this => { - const node = try arena.create(ast.NodeThisLiteral); - *node = ast.NodeThisLiteral { - .base = self.initNode(ast.Node.Id.ThisLiteral), - .token = token, - }; - dest_ptr.store(&node.base); + dest_ptr.store(&(try self.createLiteral(arena, ast.NodeThisLiteral, token)).base); continue; }, Token.Id.Keyword_var => { - const node = try arena.create(ast.NodeVarType); - *node = ast.NodeVarType { - .base = self.initNode(ast.Node.Id.VarType), - .token = token, - }; - dest_ptr.store(&node.base); + dest_ptr.store(&(try self.createLiteral(arena, ast.NodeVarType, token)).base); + continue; }, Token.Id.Keyword_unreachable => { - const node = try arena.create(ast.NodeUnreachable); - *node = ast.NodeUnreachable { - .base = self.initNode(ast.Node.Id.Unreachable), - .token = token, - }; - dest_ptr.store(&node.base); + dest_ptr.store(&(try self.createLiteral(arena, ast.NodeUnreachable, token)).base); continue; }, Token.Id.MultilineStringLiteralLine => { - const node = try arena.create(ast.NodeMultilineStringLiteral); - *node = ast.NodeMultilineStringLiteral { - .base = self.initNode(ast.Node.Id.MultilineStringLiteral), - .tokens = ArrayList(Token).init(arena), - }; - dest_ptr.store(&node.base); + const node = try self.createToDestNode(arena, dest_ptr, ast.NodeMultilineStringLiteral, + ast.NodeMultilineStringLiteral { + .base = undefined, + .tokens = ArrayList(Token).init(arena), + } + ); try node.tokens.append(token); - while (true) { const multiline_str = self.getNextToken(); if (multiline_str.id != Token.Id.MultilineStringLiteralLine) { @@ -1475,14 +1609,14 @@ pub const Parser = struct { continue; }, Token.Id.LParen => { - const node = try arena.create(ast.NodeGroupedExpression); - *node = ast.NodeGroupedExpression { - .base = self.initNode(ast.Node.Id.GroupedExpression), - .lparen = token, - .expr = undefined, - .rparen = undefined, - }; - dest_ptr.store(&node.base); + const node = try self.createToDestNode(arena, dest_ptr, ast.NodeGroupedExpression, + ast.NodeGroupedExpression { + .base = undefined, + .lparen = token, + .expr = undefined, + .rparen = undefined, + } + ); stack.append(State { .ExpectTokenSave = ExpectTokenSave { .id = Token.Id.RParen, @@ -1493,14 +1627,14 @@ pub const Parser = struct { continue; }, Token.Id.Builtin => { - const node = try arena.create(ast.NodeBuiltinCall); - *node = ast.NodeBuiltinCall { - .base = self.initNode(ast.Node.Id.BuiltinCall), - .builtin_token = token, - .params = ArrayList(&ast.Node).init(arena), - .rparen_token = undefined, - }; - dest_ptr.store(&node.base); + const node = try self.createToDestNode(arena, dest_ptr, ast.NodeBuiltinCall, + ast.NodeBuiltinCall { + .base = undefined, + .builtin_token = token, + .params = ArrayList(&ast.Node).init(arena), + .rparen_token = undefined, + } + ); stack.append(State { .ExprListItemOrEnd = ExprListCtx { .list = &node.params, @@ -1514,15 +1648,22 @@ pub const Parser = struct { Token.Id.LBracket => { const rbracket_token = self.getNextToken(); if (rbracket_token.id == Token.Id.RBracket) { - const node = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp{ - .SliceType = ast.NodePrefixOp.AddrOfInfo { - .align_expr = null, - .bit_offset_start_token = null, - .bit_offset_end_token = null, - .const_token = null, - .volatile_token = null, + const node = try self.createToDestNode(arena, dest_ptr, ast.NodePrefixOp, + ast.NodePrefixOp { + .base = undefined, + .op_token = token, + .op = ast.NodePrefixOp.PrefixOp{ + .SliceType = ast.NodePrefixOp.AddrOfInfo { + .align_expr = null, + .bit_offset_start_token = null, + .bit_offset_end_token = null, + .const_token = null, + .volatile_token = null, + } + }, + .rhs = undefined, } - }); + ); dest_ptr.store(&node.base); stack.append(State { .TypeExprBegin = DestPtr { .Field = &node.rhs } }) catch unreachable; try stack.append(State { .AddrOfModifiers = &node.op.SliceType }); @@ -1531,10 +1672,16 @@ pub const Parser = struct { self.putBackToken(rbracket_token); - const node = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp{ - .ArrayType = undefined, - }); - dest_ptr.store(&node.base); + const node = try self.createToDestNode(arena, dest_ptr, ast.NodePrefixOp, + ast.NodePrefixOp { + .base = undefined, + .op_token = token, + .op = ast.NodePrefixOp.PrefixOp{ + .ArrayType = undefined, + }, + .rhs = undefined, + } + ); stack.append(State { .TypeExprBegin = DestPtr { .Field = &node.rhs } }) catch unreachable; try stack.append(State { .ExpectToken = Token.Id.RBracket }); try stack.append(State { .Expression = DestPtr { .Field = &node.op.ArrayType } }); @@ -1544,24 +1691,19 @@ pub const Parser = struct { const next = self.getNextToken(); if (next.id != Token.Id.LBrace) { + dest_ptr.store(&(try self.createLiteral(arena, ast.NodeErrorType, token)).base); self.putBackToken(next); - const node = try arena.create(ast.NodeErrorType); - *node = ast.NodeErrorType { - .base = self.initNode(ast.Node.Id.ErrorType), - .token = token, - }; - dest_ptr.store(&node.base); continue; } - const node = try arena.create(ast.NodeErrorSetDecl); - *node = ast.NodeErrorSetDecl { - .base = self.initNode(ast.Node.Id.ErrorSetDecl), - .error_token = token, - .decls = ArrayList(&ast.NodeIdentifier).init(arena), - .rbrace_token = undefined, - }; - dest_ptr.store(&node.base); + const node = try self.createToDestNode(arena, dest_ptr, ast.NodeErrorSetDecl, + ast.NodeErrorSetDecl { + .base = undefined, + .error_token = token, + .decls = ArrayList(&ast.NodeIdentifier).init(arena), + .rbrace_token = undefined, + } + ); while (true) { const t = self.getNextToken(); @@ -1572,7 +1714,7 @@ pub const Parser = struct { }, Token.Id.Identifier => { try node.decls.append( - try self.createIdentifier(arena, t) + try self.createLiteral(arena, ast.NodeIdentifier, t) ); }, else => { @@ -1614,10 +1756,24 @@ pub const Parser = struct { Token.Id.Keyword_extern => { const next = self.getNextToken(); if (next.id == Token.Id.Keyword_fn) { - // TODO shouldn't need this cast - const fn_proto = try self.createFnProto(arena, next, - (?Token)(token), (?&ast.Node)(null), (?Token)(null), (?Token)(null), (?Token)(null)); - dest_ptr.store(&fn_proto.base); + const fn_proto = try self.createToDestNode(arena, dest_ptr, ast.NodeFnProto, + ast.NodeFnProto { + .base = undefined, + .visib_token = null, + .name_token = null, + .fn_token = next, + .params = ArrayList(&ast.Node).init(arena), + .return_type = undefined, + .var_args_token = null, + .extern_token = token, + .inline_token = null, + .cc_token = null, + .async_attr = null, + .body_node = null, + .lib_name = null, + .align_expr = null, + } + ); stack.append(State { .FnProto = fn_proto }) catch unreachable; continue; } @@ -1645,7 +1801,7 @@ pub const Parser = struct { const next = self.getNextToken(); if (next.id != Token.Id.Colon) { self.putBackToken(next); - dest_ptr.store(&(try self.createIdentifier(arena, token)).base); + dest_ptr.store(&(try self.createLiteral(arena, ast.NodeIdentifier, token)).base); continue; } @@ -1658,19 +1814,47 @@ pub const Parser = struct { continue; }, Token.Id.Keyword_fn => { - // TODO shouldn't need these casts - const fn_proto = try self.createFnProto(arena, token, - (?Token)(null), (?&ast.Node)(null), (?Token)(null), (?Token)(null), (?Token)(null)); - dest_ptr.store(&fn_proto.base); + const fn_proto = try self.createToDestNode(arena, dest_ptr, ast.NodeFnProto, + ast.NodeFnProto { + .base = undefined, + .visib_token = null, + .name_token = null, + .fn_token = token, + .params = ArrayList(&ast.Node).init(arena), + .return_type = undefined, + .var_args_token = null, + .extern_token = null, + .inline_token = null, + .cc_token = null, + .async_attr = null, + .body_node = null, + .lib_name = null, + .align_expr = null, + } + ); stack.append(State { .FnProto = fn_proto }) catch unreachable; continue; }, Token.Id.Keyword_nakedcc, Token.Id.Keyword_stdcallcc => { const fn_token = (try self.eatToken(&stack, Token.Id.Keyword_fn)) ?? continue; - // TODO shouldn't need this cast - const fn_proto = try self.createFnProto(arena, fn_token, - (?Token)(null), (?&ast.Node)(null), (?Token)(token), (?Token)(null), (?Token)(null)); - dest_ptr.store(&fn_proto.base); + const fn_proto = try self.createToDestNode(arena, dest_ptr, ast.NodeFnProto, + ast.NodeFnProto { + .base = undefined, + .visib_token = null, + .name_token = null, + .fn_token = fn_token, + .params = ArrayList(&ast.Node).init(arena), + .return_type = undefined, + .var_args_token = null, + .extern_token = null, + .inline_token = null, + .cc_token = token, + .async_attr = null, + .body_node = null, + .lib_name = null, + .align_expr = null, + } + ); stack.append(State { .FnProto = fn_proto }) catch unreachable; continue; }, @@ -1687,20 +1871,19 @@ pub const Parser = struct { const template = (try self.eatToken(&stack, Token.Id.StringLiteral)) ?? continue; // TODO parse template - const node = try arena.create(ast.NodeAsm); - *node = ast.NodeAsm { - .base = self.initNode(ast.Node.Id.Asm), - .asm_token = token, - .is_volatile = is_volatile, - .template = template, - //.tokens = ArrayList(ast.NodeAsm.AsmToken).init(arena), - .outputs = ArrayList(&ast.NodeAsmOutput).init(arena), - .inputs = ArrayList(&ast.NodeAsmInput).init(arena), - .cloppers = ArrayList(&ast.NodeStringLiteral).init(arena), - .rparen = undefined, - }; - dest_ptr.store(&node.base); - + const node = try self.createToDestNode(arena, dest_ptr, ast.NodeAsm, + ast.NodeAsm { + .base = undefined, + .asm_token = token, + .is_volatile = is_volatile, + .template = template, + //.tokens = ArrayList(ast.NodeAsm.AsmToken).init(arena), + .outputs = ArrayList(&ast.NodeAsmOutput).init(arena), + .inputs = ArrayList(&ast.NodeAsmInput).init(arena), + .cloppers = ArrayList(&ast.NodeStringLiteral).init(arena), + .rparen = undefined, + } + ); stack.append(State { .ExpectTokenSave = ExpectTokenSave { .id = Token.Id.RParen, @@ -1788,19 +1971,20 @@ pub const Parser = struct { _ = (try self.eatToken(&stack, Token.Id.LParen)) ?? continue; try stack.append(State { .ExpectToken = Token.Id.RParen }); - const node = try arena.create(ast.NodeAsmOutput); - *node = ast.NodeAsmOutput { - .base = self.initNode(ast.Node.Id.AsmOutput), - .symbolic_name = try self.createIdentifier(arena, symbolic_name), - .constraint = try self.createStringLiteral(arena, constraint), - .kind = undefined, - }; + const node = try self.createNode(arena, ast.NodeAsmOutput, + ast.NodeAsmOutput { + .base = undefined, + .symbolic_name = try self.createLiteral(arena, ast.NodeIdentifier, symbolic_name), + .constraint = try self.createLiteral(arena, ast.NodeStringLiteral, constraint), + .kind = undefined, + } + ); try items.append(node); const symbol_or_arrow = self.getNextToken(); switch (symbol_or_arrow.id) { Token.Id.Identifier => { - node.kind = ast.NodeAsmOutput.Kind { .Variable = try self.createIdentifier(arena, symbol_or_arrow) }; + node.kind = ast.NodeAsmOutput.Kind { .Variable = try self.createLiteral(arena, ast.NodeIdentifier, symbol_or_arrow) }; }, Token.Id.Arrow => { node.kind = ast.NodeAsmOutput.Kind { .Return = undefined }; @@ -1832,13 +2016,14 @@ pub const Parser = struct { _ = (try self.eatToken(&stack, Token.Id.LParen)) ?? continue; try stack.append(State { .ExpectToken = Token.Id.RParen }); - const node = try arena.create(ast.NodeAsmInput); - *node = ast.NodeAsmInput { - .base = self.initNode(ast.Node.Id.AsmInput), - .symbolic_name = try self.createIdentifier(arena, symbolic_name), - .constraint = try self.createStringLiteral(arena, constraint), - .expr = undefined, - }; + const node = try self.createNode(arena, ast.NodeAsmInput, + ast.NodeAsmInput { + .base = undefined, + .symbolic_name = try self.createLiteral(arena, ast.NodeIdentifier, symbolic_name), + .constraint = try self.createLiteral(arena, ast.NodeStringLiteral, constraint), + .expr = undefined, + } + ); try items.append(node); try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); }, @@ -1850,7 +2035,7 @@ pub const Parser = struct { continue; } - try items.append(try self.createStringLiteral(arena, string)); + try items.append(try self.createLiteral(arena, ast.NodeStringLiteral, string)); stack.append(State { .AsmClopperItems = items }) catch unreachable; try stack.append(State { .IfToken = Token.Id.Comma }); }, @@ -1871,21 +2056,20 @@ pub const Parser = struct { State.FieldInitListItemOrEnd => |list_state| { var token = self.getNextToken(); - if (token.id == Token.Id.RBrace){ *list_state.ptr = token; continue; } - self.putBackToken(token); - const node = try arena.create(ast.NodeFieldInitializer); - *node = ast.NodeFieldInitializer { - .base = self.initNode(ast.Node.Id.FieldInitializer), - .period_token = undefined, - .name_token = undefined, - .expr = undefined, - }; + const node = try self.createNode(arena, ast.NodeFieldInitializer, + ast.NodeFieldInitializer { + .base = undefined, + .period_token = undefined, + .name_token = undefined, + .expr = undefined, + } + ); try list_state.list.append(node); stack.append(State { .FieldInitListCommaOrEnd = list_state }) catch unreachable; @@ -1907,21 +2091,20 @@ pub const Parser = struct { State.SwitchCaseOrEnd => |list_state| { var token = self.getNextToken(); - if (token.id == Token.Id.RBrace){ *list_state.ptr = token; continue; } - self.putBackToken(token); - const node = try arena.create(ast.NodeSwitchCase); - *node = ast.NodeSwitchCase { - .base = self.initNode(ast.Node.Id.SwitchCase), - .items = ArrayList(&ast.Node).init(arena), - .payload = null, - .expr = undefined, - }; + const node = try self.createNode(arena, ast.NodeSwitchCase, + ast.NodeSwitchCase { + .base = undefined, + .items = ArrayList(&ast.Node).init(arena), + .payload = null, + .expr = undefined, + } + ); try list_state.list.append(node); stack.append(State { .SwitchCaseCommaOrEnd = list_state }) catch unreachable; try stack.append(State { .Expression = DestPtr{ .Field = &node.expr } }); @@ -1982,13 +2165,14 @@ pub const Parser = struct { continue; } - const node = try arena.create(ast.NodeElse); - *node = ast.NodeElse { - .base = self.initNode(ast.Node.Id.Else), - .else_token = else_token, - .payload = null, - .body = undefined, - }; + const node = try self.createNode(arena, ast.NodeElse, + ast.NodeElse { + .base = undefined, + .else_token = else_token, + .payload = null, + .body = undefined, + } + ); *dest = node; stack.append(State { .Expression = DestPtr { .Field = &node.body } }) catch unreachable; @@ -2050,14 +2234,14 @@ pub const Parser = struct { const error_symbol = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; const rpipe = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue; - const node = try arena.create(ast.NodePayload); - *node = ast.NodePayload { - .base = self.initNode(ast.Node.Id.Payload), - .lpipe = lpipe, - .error_symbol = try self.createIdentifier(arena, error_symbol), - .rpipe = rpipe - }; - *dest = node; + *dest = try self.createNode(arena, ast.NodePayload, + ast.NodePayload { + .base = undefined, + .lpipe = lpipe, + .error_symbol = try self.createLiteral(arena, ast.NodeIdentifier, error_symbol), + .rpipe = rpipe + } + ); }, State.PointerPayload => |dest| { @@ -2079,15 +2263,15 @@ pub const Parser = struct { const value_symbol = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; const rpipe = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue; - const node = try arena.create(ast.NodePointerPayload); - *node = ast.NodePointerPayload { - .base = self.initNode(ast.Node.Id.PointerPayload), - .lpipe = lpipe, - .is_ptr = is_ptr, - .value_symbol = try self.createIdentifier(arena, value_symbol), - .rpipe = rpipe - }; - *dest = node; + *dest = try self.createNode(arena, ast.NodePointerPayload, + ast.NodePointerPayload { + .base = undefined, + .lpipe = lpipe, + .is_ptr = is_ptr, + .value_symbol = try self.createLiteral(arena, ast.NodeIdentifier, value_symbol), + .rpipe = rpipe + } + ); }, State.PointerIndexPayload => |dest| { @@ -2116,20 +2300,20 @@ pub const Parser = struct { } const symbol = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; - break :blk try self.createIdentifier(arena, symbol); + break :blk try self.createLiteral(arena, ast.NodeIdentifier, symbol); }; const rpipe = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue; - const node = try arena.create(ast.NodePointerIndexPayload); - *node = ast.NodePointerIndexPayload { - .base = self.initNode(ast.Node.Id.PointerIndexPayload), - .lpipe = lpipe, - .is_ptr = is_ptr, - .value_symbol = try self.createIdentifier(arena, value_symbol), - .index_symbol = index_symbol, - .rpipe = rpipe - }; - *dest = node; + *dest = try self.createNode(arena, ast.NodePointerIndexPayload, + ast.NodePointerIndexPayload { + .base = undefined, + .lpipe = lpipe, + .is_ptr = is_ptr, + .value_symbol = try self.createLiteral(arena, ast.NodeIdentifier, value_symbol), + .index_symbol = index_symbol, + .rpipe = rpipe + } + ); }, State.AddrOfModifiers => |addr_of_info| { @@ -2225,7 +2409,16 @@ pub const Parser = struct { if (token.id == Token.Id.RParen) { continue; } - const param_decl = try self.createAttachParamDecl(arena, &fn_proto.params); + const param_decl = try self.createAttachNode(arena, &fn_proto.params, ast.NodeParamDecl, + ast.NodeParamDecl { + .base = undefined, + .comptime_token = null, + .noalias_token = null, + .name_token = null, + .type_node = undefined, + .var_args_token = null, + }, + ); if (token.id == Token.Id.Keyword_comptime) { param_decl.comptime_token = token; token = self.getNextToken(); @@ -2277,7 +2470,15 @@ pub const Parser = struct { const token = self.getNextToken(); switch(token.id) { Token.Id.LBrace => { - const block = try self.createBlock(arena, (?Token)(null), token); + const block = try self.createNode(arena, ast.NodeBlock, + ast.NodeBlock { + .base = undefined, + .label = null, + .lbrace = token, + .statements = ArrayList(&ast.Node).init(arena), + .rbrace = undefined, + } + ); fn_proto.body_node = &block.base; stack.append(State { .Block = block }) catch unreachable; continue; @@ -2294,9 +2495,15 @@ pub const Parser = struct { const token = self.getNextToken(); switch (token.id) { Token.Id.LBrace => { - const block = try self.createBlock(arena, (?Token)(ctx.label), token); - ctx.dest_ptr.store(&block.base); - + const block = try self.createToDestNode(arena, ctx.dest_ptr, ast.NodeBlock, + ast.NodeBlock { + .base = undefined, + .label = ctx.label, + .lbrace = token, + .statements = ArrayList(&ast.Node).init(arena), + .rbrace = undefined, + } + ); stack.append(State { .Block = block }) catch unreachable; continue; }, @@ -2372,20 +2579,19 @@ pub const Parser = struct { }, State.While => |ctx| { - const node = try arena.create(ast.NodeWhile); - *node = ast.NodeWhile { - .base = self.initNode(ast.Node.Id.While), - .label = ctx.label, - .inline_token = ctx.inline_token, - .while_token = ctx.loop_token, - .condition = undefined, - .payload = null, - .continue_expr = null, - .body = undefined, - .@"else" = null, - }; - ctx.dest_ptr.store(&node.base); - + const node = try self.createToDestNode(arena, ctx.dest_ptr, ast.NodeWhile, + ast.NodeWhile { + .base = undefined, + .label = ctx.label, + .inline_token = ctx.inline_token, + .while_token = ctx.loop_token, + .condition = undefined, + .payload = null, + .continue_expr = null, + .body = undefined, + .@"else" = null, + } + ); stack.append(State { .Else = &node.@"else" }) catch unreachable; try stack.append(State { .Expression = DestPtr { .Field = &node.body } }); try stack.append(State { .WhileContinueExpr = &node.continue_expr }); @@ -2396,19 +2602,18 @@ pub const Parser = struct { }, State.For => |ctx| { - const node = try arena.create(ast.NodeFor); - *node = ast.NodeFor { - .base = self.initNode(ast.Node.Id.For), - .label = ctx.label, - .inline_token = ctx.inline_token, - .for_token = ctx.loop_token, - .array_expr = undefined, - .payload = null, - .body = undefined, - .@"else" = null, - }; - ctx.dest_ptr.store(&node.base); - + const node = try self.createToDestNode(arena, ctx.dest_ptr, ast.NodeFor, + ast.NodeFor { + .base = undefined, + .label = ctx.label, + .inline_token = ctx.inline_token, + .for_token = ctx.loop_token, + .array_expr = undefined, + .payload = null, + .body = undefined, + .@"else" = null, + } + ); stack.append(State { .Else = &node.@"else" }) catch unreachable; try stack.append(State { .Expression = DestPtr { .Field = &node.body } }); try stack.append(State { .PointerIndexPayload = &node.payload }); @@ -2439,9 +2644,23 @@ pub const Parser = struct { Token.Id.Keyword_comptime => { const mut_token = self.getNextToken(); if (mut_token.id == Token.Id.Keyword_var or mut_token.id == Token.Id.Keyword_const) { - // TODO shouldn't need these casts - const var_decl = try self.createAttachVarDecl(arena, &block.statements, (?Token)(null), - mut_token, (?Token)(next), (?Token)(null), null); + const var_decl = try self.createAttachNode(arena, &block.statements, ast.NodeVarDecl, + ast.NodeVarDecl { + .base = undefined, + .visib_token = null, + .mut_token = mut_token, + .comptime_token = next, + .extern_token = null, + .type_node = null, + .align_node = null, + .init_node = null, + .lib_name = null, + // initialized later + .name_token = undefined, + .eq_token = undefined, + .semicolon_token = undefined, + } + ); stack.append(State { .VarDecl = var_decl }) catch unreachable; continue; } else { @@ -2453,33 +2672,53 @@ pub const Parser = struct { } }, Token.Id.Keyword_var, Token.Id.Keyword_const => { - const var_decl = try self.createAttachVarDecl(arena, &block.statements, (?Token)(null), - next, (?Token)(null), (?Token)(null), null); + const var_decl = try self.createAttachNode(arena, &block.statements, ast.NodeVarDecl, + ast.NodeVarDecl { + .base = undefined, + .visib_token = null, + .mut_token = next, + .comptime_token = null, + .extern_token = null, + .type_node = null, + .align_node = null, + .init_node = null, + .lib_name = null, + // initialized later + .name_token = undefined, + .eq_token = undefined, + .semicolon_token = undefined, + } + ); stack.append(State { .VarDecl = var_decl }) catch unreachable; continue; }, Token.Id.Keyword_defer, Token.Id.Keyword_errdefer => { - const node = try arena.create(ast.NodeDefer); - *node = ast.NodeDefer { - .base = self.initNode(ast.Node.Id.Defer), - .defer_token = next, - .kind = switch (next.id) { - Token.Id.Keyword_defer => ast.NodeDefer.Kind.Unconditional, - Token.Id.Keyword_errdefer => ast.NodeDefer.Kind.Error, - else => unreachable, - }, - .expr = undefined, - }; - try block.statements.append(&node.base); - + const node = try self.createAttachNode(arena, &block.statements, ast.NodeDefer, + ast.NodeDefer { + .base = undefined, + .defer_token = next, + .kind = switch (next.id) { + Token.Id.Keyword_defer => ast.NodeDefer.Kind.Unconditional, + Token.Id.Keyword_errdefer => ast.NodeDefer.Kind.Error, + else => unreachable, + }, + .expr = undefined, + } + ); stack.append(State { .Semicolon = &node.base }) catch unreachable; try stack.append(State { .AssignmentExpressionBegin = DestPtr{.Field = &node.expr } }); continue; }, Token.Id.LBrace => { - const inner_block = try self.createBlock(arena, (?Token)(null), next); - try block.statements.append(&inner_block.base); - + const inner_block = try self.createAttachNode(arena, &block.statements, ast.NodeBlock, + ast.NodeBlock { + .base = undefined, + .label = null, + .lbrace = next, + .statements = ArrayList(&ast.Node).init(arena), + .rbrace = undefined, + } + ); stack.append(State { .Block = inner_block }) catch unreachable; continue; }, @@ -2600,84 +2839,74 @@ pub const Parser = struct { // TODO: We have to cast all cases because of this: // error: expected type '?InfixOp', found '?@TagType(InfixOp)' return switch (*id) { - Token.Id.AmpersandEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignBitAnd), - Token.Id.AngleBracketAngleBracketLeftEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignBitShiftLeft), - Token.Id.AngleBracketAngleBracketRightEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignBitShiftRight), - Token.Id.AsteriskEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignTimes), - Token.Id.AsteriskPercentEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignTimesWarp), - Token.Id.CaretEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignBitXor), - Token.Id.Equal => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.Assign), - Token.Id.MinusEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignMinus), - Token.Id.MinusPercentEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignMinusWrap), - Token.Id.PercentEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignMod), - Token.Id.PipeEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignBitOr), - Token.Id.PlusEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignPlus), - Token.Id.PlusPercentEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignPlusWrap), - Token.Id.SlashEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignDiv), + Token.Id.AmpersandEqual => ast.NodeInfixOp.InfixOp { .AssignBitAnd = void{} }, + Token.Id.AngleBracketAngleBracketLeftEqual => ast.NodeInfixOp.InfixOp { .AssignBitShiftLeft = void{} }, + Token.Id.AngleBracketAngleBracketRightEqual => ast.NodeInfixOp.InfixOp { .AssignBitShiftRight = void{} }, + Token.Id.AsteriskEqual => ast.NodeInfixOp.InfixOp { .AssignTimes = void{} }, + Token.Id.AsteriskPercentEqual => ast.NodeInfixOp.InfixOp { .AssignTimesWarp = void{} }, + Token.Id.CaretEqual => ast.NodeInfixOp.InfixOp { .AssignBitXor = void{} }, + Token.Id.Equal => ast.NodeInfixOp.InfixOp { .Assign = void{} }, + Token.Id.MinusEqual => ast.NodeInfixOp.InfixOp { .AssignMinus = void{} }, + Token.Id.MinusPercentEqual => ast.NodeInfixOp.InfixOp { .AssignMinusWrap = void{} }, + Token.Id.PercentEqual => ast.NodeInfixOp.InfixOp { .AssignMod = void{} }, + Token.Id.PipeEqual => ast.NodeInfixOp.InfixOp { .AssignBitOr = void{} }, + Token.Id.PlusEqual => ast.NodeInfixOp.InfixOp { .AssignPlus = void{} }, + Token.Id.PlusPercentEqual => ast.NodeInfixOp.InfixOp { .AssignPlusWrap = void{} }, + Token.Id.SlashEqual => ast.NodeInfixOp.InfixOp { .AssignDiv = void{} }, else => null, }; } fn tokenIdToComparison(id: &const Token.Id) ?ast.NodeInfixOp.InfixOp { - // TODO: We have to cast all cases because of this: - // error: expected type '?InfixOp', found '?@TagType(InfixOp)' return switch (*id) { - Token.Id.BangEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.BangEqual), - Token.Id.EqualEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.EqualEqual), - Token.Id.AngleBracketLeft => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.LessThan), - Token.Id.AngleBracketLeftEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.LessOrEqual), - Token.Id.AngleBracketRight => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.GreaterThan), - Token.Id.AngleBracketRightEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.GreaterOrEqual), + Token.Id.BangEqual => ast.NodeInfixOp.InfixOp { .BangEqual = void{} }, + Token.Id.EqualEqual => ast.NodeInfixOp.InfixOp { .EqualEqual = void{} }, + Token.Id.AngleBracketLeft => ast.NodeInfixOp.InfixOp { .LessThan = void{} }, + Token.Id.AngleBracketLeftEqual => ast.NodeInfixOp.InfixOp { .LessOrEqual = void{} }, + Token.Id.AngleBracketRight => ast.NodeInfixOp.InfixOp { .GreaterThan = void{} }, + Token.Id.AngleBracketRightEqual => ast.NodeInfixOp.InfixOp { .GreaterOrEqual = void{} }, else => null, }; } fn tokenIdToBitShift(id: &const Token.Id) ?ast.NodeInfixOp.InfixOp { - // TODO: We have to cast all cases because of this: - // error: expected type '?InfixOp', found '?@TagType(InfixOp)' return switch (*id) { - Token.Id.AngleBracketAngleBracketLeft => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.BitShiftLeft), - Token.Id.AngleBracketAngleBracketRight => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.BitShiftRight), + Token.Id.AngleBracketAngleBracketLeft => ast.NodeInfixOp.InfixOp { .BitShiftLeft = void{} }, + Token.Id.AngleBracketAngleBracketRight => ast.NodeInfixOp.InfixOp { .BitShiftRight = void{} }, else => null, }; } fn tokenIdToAddition(id: &const Token.Id) ?ast.NodeInfixOp.InfixOp { - // TODO: We have to cast all cases because of this: - // error: expected type '?InfixOp', found '?@TagType(InfixOp)' return switch (*id) { - Token.Id.Minus => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.Sub), - Token.Id.MinusPercent => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.SubWrap), - Token.Id.Plus => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.Add), - Token.Id.PlusPercent => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AddWrap), - Token.Id.PlusPlus => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.ArrayCat), + Token.Id.Minus => ast.NodeInfixOp.InfixOp { .Sub = void{} }, + Token.Id.MinusPercent => ast.NodeInfixOp.InfixOp { .SubWrap = void{} }, + Token.Id.Plus => ast.NodeInfixOp.InfixOp { .Add = void{} }, + Token.Id.PlusPercent => ast.NodeInfixOp.InfixOp { .AddWrap = void{} }, + Token.Id.PlusPlus => ast.NodeInfixOp.InfixOp { .ArrayCat = void{} }, else => null, }; } fn tokenIdToMultiply(id: &const Token.Id) ?ast.NodeInfixOp.InfixOp { - // TODO: We have to cast all cases because of this: - // error: expected type '?InfixOp', found '?@TagType(InfixOp)' return switch (*id) { - Token.Id.Slash => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.Div), - Token.Id.Asterisk => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.Mult), - Token.Id.AsteriskAsterisk => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.ArrayMult), - Token.Id.AsteriskPercent => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.MultWrap), - Token.Id.Percent => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.Mod), - Token.Id.PipePipe => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.MergeErrorSets), + Token.Id.Slash => ast.NodeInfixOp.InfixOp { .Div = void{} }, + Token.Id.Asterisk => ast.NodeInfixOp.InfixOp { .Mult = void{} }, + Token.Id.AsteriskAsterisk => ast.NodeInfixOp.InfixOp { .ArrayMult = void{} }, + Token.Id.AsteriskPercent => ast.NodeInfixOp.InfixOp { .MultWrap = void{} }, + Token.Id.Percent => ast.NodeInfixOp.InfixOp { .Mod = void{} }, + Token.Id.PipePipe => ast.NodeInfixOp.InfixOp { .MergeErrorSets = void{} }, else => null, }; } fn tokenIdToPrefixOp(id: &const Token.Id) ?ast.NodePrefixOp.PrefixOp { - // TODO: We have to cast all cases because of this: - // error: expected type '?InfixOp', found '?@TagType(InfixOp)' return switch (*id) { - Token.Id.Bang => (ast.NodePrefixOp.PrefixOp)(ast.NodePrefixOp.PrefixOp.BoolNot), - Token.Id.Tilde => (ast.NodePrefixOp.PrefixOp)(ast.NodePrefixOp.PrefixOp.BitNot), - Token.Id.Minus => (ast.NodePrefixOp.PrefixOp)(ast.NodePrefixOp.PrefixOp.Negation), - Token.Id.MinusPercent => (ast.NodePrefixOp.PrefixOp)(ast.NodePrefixOp.PrefixOp.NegationWrap), - Token.Id.Asterisk => (ast.NodePrefixOp.PrefixOp)(ast.NodePrefixOp.PrefixOp.Deref), + Token.Id.Bang => ast.NodePrefixOp.PrefixOp { .BoolNot = void{} }, + Token.Id.Tilde => ast.NodePrefixOp.PrefixOp { .BitNot = void{} }, + Token.Id.Minus => ast.NodePrefixOp.PrefixOp { .Negation = void{} }, + Token.Id.MinusPercent => ast.NodePrefixOp.PrefixOp { .NegationWrap = void{} }, + Token.Id.Asterisk => ast.NodePrefixOp.PrefixOp { .Deref = void{} }, Token.Id.Ampersand => ast.NodePrefixOp.PrefixOp { .AddrOf = ast.NodePrefixOp.AddrOfInfo { .align_expr = null, @@ -2687,9 +2916,9 @@ pub const Parser = struct { .volatile_token = null, }, }, - Token.Id.QuestionMark => (ast.NodePrefixOp.PrefixOp)(ast.NodePrefixOp.PrefixOp.MaybeType), - Token.Id.QuestionMarkQuestionMark => (ast.NodePrefixOp.PrefixOp)(ast.NodePrefixOp.PrefixOp.UnwrapMaybe), - Token.Id.Keyword_await => (ast.NodePrefixOp.PrefixOp)(ast.NodePrefixOp.PrefixOp.Await), + Token.Id.QuestionMark => ast.NodePrefixOp.PrefixOp { .MaybeType = void{} }, + Token.Id.QuestionMarkQuestionMark => ast.NodePrefixOp.PrefixOp { .UnwrapMaybe = void{} }, + Token.Id.Keyword_await => ast.NodePrefixOp.PrefixOp { .Await = void{} }, else => null, }; } @@ -2702,243 +2931,35 @@ pub const Parser = struct { return ast.Node {.id = id, .comment = null }; } - fn createRoot(self: &Parser, arena: &mem.Allocator) !&ast.NodeRoot { - const node = try arena.create(ast.NodeRoot); + fn createNode(self: &Parser, arena: &mem.Allocator, comptime T: type, init_to: &const T) !&T { + const node = try arena.create(T); + *node = *init_to; + node.base = self.initNode(ast.Node.typeToId(T)); - *node = ast.NodeRoot { - .base = self.initNode(ast.Node.Id.Root), - .decls = ArrayList(&ast.Node).init(arena), - // initialized when we get the eof token - .eof_token = undefined, - }; return node; } - fn createVarDecl(self: &Parser, arena: &mem.Allocator, visib_token: &const ?Token, mut_token: &const Token, - comptime_token: &const ?Token, extern_token: &const ?Token, lib_name: ?&ast.Node) !&ast.NodeVarDecl - { - const node = try arena.create(ast.NodeVarDecl); - - *node = ast.NodeVarDecl { - .base = self.initNode(ast.Node.Id.VarDecl), - .visib_token = *visib_token, - .mut_token = *mut_token, - .comptime_token = *comptime_token, - .extern_token = *extern_token, - .type_node = null, - .align_node = null, - .init_node = null, - .lib_name = lib_name, - // initialized later - .name_token = undefined, - .eq_token = undefined, - .semicolon_token = undefined, - }; - return node; - } - - fn createStringLiteral(self: &Parser, arena: &mem.Allocator, token: &const Token) !&ast.NodeStringLiteral { - const node = try arena.create(ast.NodeStringLiteral); - - assert(token.id == Token.Id.StringLiteral); - *node = ast.NodeStringLiteral { - .base = self.initNode(ast.Node.Id.StringLiteral), - .token = *token, - }; - return node; - } - - fn createTestDecl(self: &Parser, arena: &mem.Allocator, test_token: &const Token, name: &ast.Node, - block: &ast.NodeBlock) !&ast.NodeTestDecl - { - const node = try arena.create(ast.NodeTestDecl); - - *node = ast.NodeTestDecl { - .base = self.initNode(ast.Node.Id.TestDecl), - .test_token = *test_token, - .name = name, - .body_node = &block.base, - }; - return node; - } - - fn createFnProto(self: &Parser, arena: &mem.Allocator, fn_token: &const Token, extern_token: &const ?Token, - lib_name: ?&ast.Node, cc_token: &const ?Token, visib_token: &const ?Token, inline_token: &const ?Token) !&ast.NodeFnProto - { - const node = try arena.create(ast.NodeFnProto); - - *node = ast.NodeFnProto { - .base = self.initNode(ast.Node.Id.FnProto), - .visib_token = *visib_token, - .name_token = null, - .fn_token = *fn_token, - .params = ArrayList(&ast.Node).init(arena), - .return_type = undefined, - .var_args_token = null, - .extern_token = *extern_token, - .inline_token = *inline_token, - .cc_token = *cc_token, - .async_attr = null, - .body_node = null, - .lib_name = lib_name, - .align_expr = null, - }; - return node; - } - - fn createParamDecl(self: &Parser, arena: &mem.Allocator) !&ast.NodeParamDecl { - const node = try arena.create(ast.NodeParamDecl); - - *node = ast.NodeParamDecl { - .base = self.initNode(ast.Node.Id.ParamDecl), - .comptime_token = null, - .noalias_token = null, - .name_token = null, - .type_node = undefined, - .var_args_token = null, - }; - return node; - } - - fn createBlock(self: &Parser, arena: &mem.Allocator, label: &const ?Token, lbrace: &const Token) !&ast.NodeBlock { - const node = try arena.create(ast.NodeBlock); - - *node = ast.NodeBlock { - .base = self.initNode(ast.Node.Id.Block), - .label = *label, - .lbrace = *lbrace, - .statements = ArrayList(&ast.Node).init(arena), - .rbrace = undefined, - }; - return node; - } - - fn createControlFlowExpr(self: &Parser, arena: &mem.Allocator, ltoken: &const Token, - kind: &const ast.NodeControlFlowExpression.Kind) !&ast.NodeControlFlowExpression - { - const node = try arena.create(ast.NodeControlFlowExpression); - *node = ast.NodeControlFlowExpression { - .base = self.initNode(ast.Node.Id.ControlFlowExpression), - .ltoken = *ltoken, - .kind = *kind, - .rhs = null, - }; - return node; - } - - fn createInfixOp(self: &Parser, arena: &mem.Allocator, op_token: &const Token, op: &const ast.NodeInfixOp.InfixOp) !&ast.NodeInfixOp { - const node = try arena.create(ast.NodeInfixOp); - - *node = ast.NodeInfixOp { - .base = self.initNode(ast.Node.Id.InfixOp), - .op_token = *op_token, - .lhs = undefined, - .op = *op, - .rhs = undefined, - }; - return node; - } - - fn createPrefixOp(self: &Parser, arena: &mem.Allocator, op_token: &const Token, op: &const ast.NodePrefixOp.PrefixOp) !&ast.NodePrefixOp { - const node = try arena.create(ast.NodePrefixOp); - - *node = ast.NodePrefixOp { - .base = self.initNode(ast.Node.Id.PrefixOp), - .op_token = *op_token, - .op = *op, - .rhs = undefined, - }; - return node; - } - - fn createSuffixOp(self: &Parser, arena: &mem.Allocator, op: &const ast.NodeSuffixOp.SuffixOp) !&ast.NodeSuffixOp { - const node = try arena.create(ast.NodeSuffixOp); - - *node = ast.NodeSuffixOp { - .base = self.initNode(ast.Node.Id.SuffixOp), - .lhs = undefined, - .op = *op, - .rtoken = undefined, - }; - return node; - } - - fn createIdentifier(self: &Parser, arena: &mem.Allocator, name_token: &const Token) !&ast.NodeIdentifier { - const node = try arena.create(ast.NodeIdentifier); - - *node = ast.NodeIdentifier { - .base = self.initNode(ast.Node.Id.Identifier), - .name_token = *name_token, - }; - return node; - } - - fn createIntegerLiteral(self: &Parser, arena: &mem.Allocator, token: &const Token) !&ast.NodeIntegerLiteral { - const node = try arena.create(ast.NodeIntegerLiteral); - - *node = ast.NodeIntegerLiteral { - .base = self.initNode(ast.Node.Id.IntegerLiteral), - .token = *token, - }; - return node; - } - - fn createFloatLiteral(self: &Parser, arena: &mem.Allocator, token: &const Token) !&ast.NodeFloatLiteral { - const node = try arena.create(ast.NodeFloatLiteral); - - *node = ast.NodeFloatLiteral { - .base = self.initNode(ast.Node.Id.FloatLiteral), - .token = *token, - }; - return node; - } - - fn createUndefined(self: &Parser, arena: &mem.Allocator, token: &const Token) !&ast.NodeUndefinedLiteral { - const node = try arena.create(ast.NodeUndefinedLiteral); - - *node = ast.NodeUndefinedLiteral { - .base = self.initNode(ast.Node.Id.UndefinedLiteral), - .token = *token, - }; - return node; - } - - fn createAttachIdentifier(self: &Parser, arena: &mem.Allocator, dest_ptr: &const DestPtr, name_token: &const Token) !&ast.NodeIdentifier { - const node = try self.createIdentifier(arena, name_token); - try dest_ptr.store(&node.base); - return node; - } - - fn createAttachParamDecl(self: &Parser, arena: &mem.Allocator, list: &ArrayList(&ast.Node)) !&ast.NodeParamDecl { - const node = try self.createParamDecl(arena); + fn createAttachNode(self: &Parser, arena: &mem.Allocator, list: &ArrayList(&ast.Node), comptime T: type, init_to: &const T) !&T { + const node = try self.createNode(arena, T, init_to); try list.append(&node.base); + return node; } - fn createAttachFnProto(self: &Parser, arena: &mem.Allocator, list: &ArrayList(&ast.Node), fn_token: &const Token, - extern_token: &const ?Token, lib_name: ?&ast.Node, cc_token: &const ?Token, visib_token: &const ?Token, - inline_token: &const ?Token) !&ast.NodeFnProto - { - const node = try self.createFnProto(arena, fn_token, extern_token, lib_name, cc_token, visib_token, inline_token); - try list.append(&node.base); + fn createToDestNode(self: &Parser, arena: &mem.Allocator, dest_ptr: &const DestPtr, comptime T: type, init_to: &const T) !&T { + const node = try self.createNode(arena, T, init_to); + dest_ptr.store(&node.base); + return node; } - fn createAttachVarDecl(self: &Parser, arena: &mem.Allocator, list: &ArrayList(&ast.Node), - visib_token: &const ?Token, mut_token: &const Token, comptime_token: &const ?Token, - extern_token: &const ?Token, lib_name: ?&ast.Node) !&ast.NodeVarDecl - { - const node = try self.createVarDecl(arena, visib_token, mut_token, comptime_token, extern_token, lib_name); - try list.append(&node.base); - return node; - } - - fn createAttachTestDecl(self: &Parser, arena: &mem.Allocator, list: &ArrayList(&ast.Node), - test_token: &const Token, name: &ast.Node, block: &ast.NodeBlock) !&ast.NodeTestDecl - { - const node = try self.createTestDecl(arena, test_token, name, block); - try list.append(&node.base); - return node; + fn createLiteral(self: &Parser, arena: &mem.Allocator, comptime T: type, token: &const Token) !&T { + return self.createNode(arena, T, + T { + .base = undefined, + .token = *token, + } + ); } fn parseError(self: &Parser, stack: &ArrayList(State), token: &const Token, comptime fmt: []const u8, args: ...) !void { @@ -3217,7 +3238,7 @@ pub const Parser = struct { RenderState.Expression => |base| switch (base.id) { ast.Node.Id.Identifier => { const identifier = @fieldParentPtr(ast.NodeIdentifier, "base", base); - try stream.print("{}", self.tokenizer.getTokenSlice(identifier.name_token)); + try stream.print("{}", self.tokenizer.getTokenSlice(identifier.token)); }, ast.Node.Id.Block => { const block = @fieldParentPtr(ast.NodeBlock, "base", base); From 281c17f6ae3c294d0b7139fe640dd0cb30123ea1 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Wed, 11 Apr 2018 12:05:10 +0200 Subject: [PATCH 77/87] std.zig.parser: * Renamed eatToken to expectToken * A new eatToken fn, which only eats the token, if the id match * Inlined initNode, as it is not suppose to be used outside createNode --- std/zig/parser.zig | 306 ++++++++++++++++++++++----------------------- 1 file changed, 148 insertions(+), 158 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 5047a5c694..f722703284 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -266,8 +266,7 @@ pub const Parser = struct { // look for line comments while (true) { - const token = self.getNextToken(); - if (token.id == Token.Id.LineComment) { + if (self.eatToken(Token.Id.LineComment)) |line_comment| { const node = blk: { if (self.pending_line_comment_node) |comment_node| { break :blk comment_node; @@ -284,10 +283,9 @@ pub const Parser = struct { break :blk comment_node; } }; - try node.lines.append(token); + try node.lines.append(line_comment); continue; } - self.putBackToken(token); break; } @@ -301,8 +299,8 @@ pub const Parser = struct { Token.Id.Keyword_test => { stack.append(State.TopLevel) catch unreachable; - const name_token = (try self.eatToken(&stack, Token.Id.StringLiteral)) ?? continue; - const lbrace = (try self.eatToken(&stack, Token.Id.LBrace)) ?? continue; + const name_token = (try self.expectToken(&stack, Token.Id.StringLiteral)) ?? continue; + const lbrace = (try self.expectToken(&stack, Token.Id.LBrace)) ?? continue; const block = try self.createNode(arena, ast.NodeBlock, ast.NodeBlock { @@ -580,25 +578,27 @@ pub const Parser = struct { }, State.VarDeclEq => |var_decl| { const token = self.getNextToken(); - if (token.id == Token.Id.Equal) { - var_decl.eq_token = token; - stack.append(State { - .ExpectTokenSave = ExpectTokenSave { - .id = Token.Id.Semicolon, - .ptr = &var_decl.semicolon_token, - }, - }) catch unreachable; - try stack.append(State { - .Expression = DestPtr {.NullableField = &var_decl.init_node}, - }); - continue; + switch (token.id) { + Token.Id.Equal => { + var_decl.eq_token = token; + stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.Semicolon, + .ptr = &var_decl.semicolon_token, + }, + }) catch unreachable; + try stack.append(State { .Expression = DestPtr {.NullableField = &var_decl.init_node} }); + continue; + }, + Token.Id.Semicolon => { + var_decl.semicolon_token = token; + continue; + }, + else => { + try self.parseError(&stack, token, "expected '=' or ';', found {}", @tagName(token.id)); + continue; + } } - if (token.id == Token.Id.Semicolon) { - var_decl.semicolon_token = token; - continue; - } - try self.parseError(&stack, token, "expected '=' or ';', found {}", @tagName(token.id)); - continue; }, State.ContainerExtern => |ctx| { @@ -752,12 +752,12 @@ pub const Parser = struct { }, State.ExpectToken => |token_id| { - _ = (try self.eatToken(&stack, token_id)) ?? continue; + _ = (try self.expectToken(&stack, token_id)) ?? continue; continue; }, State.ExpectTokenSave => |expect_token_save| { - *expect_token_save.ptr = (try self.eatToken(&stack, expect_token_save.id)) ?? continue; + *expect_token_save.ptr = (try self.expectToken(&stack, expect_token_save.id)) ?? continue; continue; }, @@ -817,7 +817,7 @@ pub const Parser = struct { break :blk null; } - break :blk (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; + break :blk (try self.expectToken(&stack, Token.Id.Identifier)) ?? continue; }; const node = try self.createToDestNode(arena, dest_ptr, ast.NodeControlFlowExpression, @@ -979,23 +979,20 @@ pub const Parser = struct { }, State.RangeExpressionEnd => |dest_ptr| { - const token = self.getNextToken(); - if (token.id == Token.Id.Ellipsis3) { + if (self.eatToken(Token.Id.Ellipsis3)) |ellipsis3| { const node = try self.createToDestNode(arena, dest_ptr, ast.NodeInfixOp, ast.NodeInfixOp { .base = undefined, .lhs = dest_ptr.get(), - .op_token = token, + .op_token = ellipsis3, .op = ast.NodeInfixOp.InfixOp.Range, .rhs = undefined, } ); stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }) catch unreachable; - continue; - } else { - self.putBackToken(token); - continue; } + + continue; }, State.AssignmentExpressionBegin => |dest_ptr| { @@ -1329,57 +1326,49 @@ pub const Parser = struct { }, State.CurlySuffixExpressionEnd => |dest_ptr| { - const token = self.getNextToken(); - if (token.id != Token.Id.LBrace) { - self.putBackToken(token); + if (self.eatToken(Token.Id.LBrace) == null) { continue; } - const next = self.getNextToken(); - switch (next.id) { - Token.Id.Period => { - const node = try self.createToDestNode(arena, dest_ptr, ast.NodeSuffixOp, - ast.NodeSuffixOp { - .base = undefined, - .lhs = dest_ptr.get(), - .op = ast.NodeSuffixOp.SuffixOp { - .StructInitializer = ArrayList(&ast.NodeFieldInitializer).init(arena), - }, - .rtoken = undefined, - } - ); - stack.append(State { .CurlySuffixExpressionEnd = dest_ptr }) catch unreachable; - try stack.append(State { - .FieldInitListItemOrEnd = ListSave(&ast.NodeFieldInitializer) { - .list = &node.op.StructInitializer, - .ptr = &node.rtoken, - } - }); - self.putBackToken(next); - continue; - }, - else => { - const node = try self.createToDestNode(arena, dest_ptr, ast.NodeSuffixOp, - ast.NodeSuffixOp { - .base = undefined, - .lhs = dest_ptr.get(), - .op = ast.NodeSuffixOp.SuffixOp { - .ArrayInitializer = ArrayList(&ast.Node).init(arena), - }, - .rtoken = undefined, - } - ); - stack.append(State { .CurlySuffixExpressionEnd = dest_ptr }) catch unreachable; - try stack.append(State { - .ExprListItemOrEnd = ExprListCtx { - .list = &node.op.ArrayInitializer, - .end = Token.Id.RBrace, - .ptr = &node.rtoken, - } - }); - self.putBackToken(next); - continue; - }, + if (self.isPeekToken(Token.Id.Period)) { + const node = try self.createToDestNode(arena, dest_ptr, ast.NodeSuffixOp, + ast.NodeSuffixOp { + .base = undefined, + .lhs = dest_ptr.get(), + .op = ast.NodeSuffixOp.SuffixOp { + .StructInitializer = ArrayList(&ast.NodeFieldInitializer).init(arena), + }, + .rtoken = undefined, + } + ); + stack.append(State { .CurlySuffixExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { + .FieldInitListItemOrEnd = ListSave(&ast.NodeFieldInitializer) { + .list = &node.op.StructInitializer, + .ptr = &node.rtoken, + } + }); + continue; + } else { + const node = try self.createToDestNode(arena, dest_ptr, ast.NodeSuffixOp, + ast.NodeSuffixOp { + .base = undefined, + .lhs = dest_ptr.get(), + .op = ast.NodeSuffixOp.SuffixOp { + .ArrayInitializer = ArrayList(&ast.Node).init(arena), + }, + .rtoken = undefined, + } + ); + stack.append(State { .CurlySuffixExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { + .ExprListItemOrEnd = ExprListCtx { + .list = &node.op.ArrayInitializer, + .end = Token.Id.RBrace, + .ptr = &node.rtoken, + } + }); + continue; } }, @@ -1836,7 +1825,7 @@ pub const Parser = struct { continue; }, Token.Id.Keyword_nakedcc, Token.Id.Keyword_stdcallcc => { - const fn_token = (try self.eatToken(&stack, Token.Id.Keyword_fn)) ?? continue; + const fn_token = (try self.expectToken(&stack, Token.Id.Keyword_fn)) ?? continue; const fn_proto = try self.createToDestNode(arena, dest_ptr, ast.NodeFnProto, ast.NodeFnProto { .base = undefined, @@ -1867,8 +1856,8 @@ pub const Parser = struct { } break :blk true; }; - _ = (try self.eatToken(&stack, Token.Id.LParen)) ?? continue; - const template = (try self.eatToken(&stack, Token.Id.StringLiteral)) ?? continue; + _ = (try self.expectToken(&stack, Token.Id.LParen)) ?? continue; + const template = (try self.expectToken(&stack, Token.Id.StringLiteral)) ?? continue; // TODO parse template const node = try self.createToDestNode(arena, dest_ptr, ast.NodeAsm, @@ -1964,11 +1953,11 @@ pub const Parser = struct { stack.append(State { .AsmOutputItems = items }) catch unreachable; try stack.append(State { .IfToken = Token.Id.Comma }); - const symbolic_name = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; - _ = (try self.eatToken(&stack, Token.Id.RBracket)) ?? continue; - const constraint = (try self.eatToken(&stack, Token.Id.StringLiteral)) ?? continue; + const symbolic_name = (try self.expectToken(&stack, Token.Id.Identifier)) ?? continue; + _ = (try self.expectToken(&stack, Token.Id.RBracket)) ?? continue; + const constraint = (try self.expectToken(&stack, Token.Id.StringLiteral)) ?? continue; - _ = (try self.eatToken(&stack, Token.Id.LParen)) ?? continue; + _ = (try self.expectToken(&stack, Token.Id.LParen)) ?? continue; try stack.append(State { .ExpectToken = Token.Id.RParen }); const node = try self.createNode(arena, ast.NodeAsmOutput, @@ -2009,11 +1998,11 @@ pub const Parser = struct { stack.append(State { .AsmInputItems = items }) catch unreachable; try stack.append(State { .IfToken = Token.Id.Comma }); - const symbolic_name = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; - _ = (try self.eatToken(&stack, Token.Id.RBracket)) ?? continue; - const constraint = (try self.eatToken(&stack, Token.Id.StringLiteral)) ?? continue; + const symbolic_name = (try self.expectToken(&stack, Token.Id.Identifier)) ?? continue; + _ = (try self.expectToken(&stack, Token.Id.RBracket)) ?? continue; + const constraint = (try self.expectToken(&stack, Token.Id.StringLiteral)) ?? continue; - _ = (try self.eatToken(&stack, Token.Id.LParen)) ?? continue; + _ = (try self.expectToken(&stack, Token.Id.LParen)) ?? continue; try stack.append(State { .ExpectToken = Token.Id.RParen }); const node = try self.createNode(arena, ast.NodeAsmInput, @@ -2055,12 +2044,10 @@ pub const Parser = struct { }, State.FieldInitListItemOrEnd => |list_state| { - var token = self.getNextToken(); - if (token.id == Token.Id.RBrace){ - *list_state.ptr = token; + if (self.eatToken(Token.Id.RBrace)) |rbrace| { + *list_state.ptr = rbrace; continue; } - self.putBackToken(token); const node = try self.createNode(arena, ast.NodeFieldInitializer, ast.NodeFieldInitializer { @@ -2090,12 +2077,10 @@ pub const Parser = struct { }, State.SwitchCaseOrEnd => |list_state| { - var token = self.getNextToken(); - if (token.id == Token.Id.RBrace){ - *list_state.ptr = token; + if (self.eatToken(Token.Id.RBrace)) |rbrace| { + *list_state.ptr = rbrace; continue; } - self.putBackToken(token); const node = try self.createNode(arena, ast.NodeSwitchCase, ast.NodeSwitchCase { @@ -2112,12 +2097,12 @@ pub const Parser = struct { const maybe_else = self.getNextToken(); if (maybe_else.id == Token.Id.Keyword_else) { - const else_node = try arena.create(ast.NodeSwitchElse); - *else_node = ast.NodeSwitchElse { - .base = self.initNode(ast.Node.Id.SwitchElse), + const else_node = try self.createAttachNode(arena, &node.items, ast.NodeSwitchElse, + ast.NodeSwitchElse { + .base = undefined, .token = maybe_else, - }; - try node.items.append(&else_node.base); + } + ); try stack.append(State { .ExpectToken = Token.Id.EqualAngleBracketRight }); continue; } else { @@ -2186,7 +2171,7 @@ pub const Parser = struct { continue; } - _ = (try self.eatToken(&stack, Token.Id.LParen)) ?? continue; + _ = (try self.expectToken(&stack, Token.Id.LParen)) ?? continue; stack.append(State { .ExpectToken = Token.Id.RParen }) catch unreachable; try stack.append(State { .AssignmentExpressionBegin = DestPtr { .NullableField = dest } }); }, @@ -2232,8 +2217,8 @@ pub const Parser = struct { continue; } - const error_symbol = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; - const rpipe = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue; + const error_symbol = (try self.expectToken(&stack, Token.Id.Identifier)) ?? continue; + const rpipe = (try self.expectToken(&stack, Token.Id.Pipe)) ?? continue; *dest = try self.createNode(arena, ast.NodePayload, ast.NodePayload { .base = undefined, @@ -2261,8 +2246,8 @@ pub const Parser = struct { } }; - const value_symbol = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; - const rpipe = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue; + const value_symbol = (try self.expectToken(&stack, Token.Id.Identifier)) ?? continue; + const rpipe = (try self.expectToken(&stack, Token.Id.Pipe)) ?? continue; *dest = try self.createNode(arena, ast.NodePointerPayload, ast.NodePointerPayload { .base = undefined, @@ -2291,7 +2276,7 @@ pub const Parser = struct { } }; - const value_symbol = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; + const value_symbol = (try self.expectToken(&stack, Token.Id.Identifier)) ?? continue; const index_symbol = blk: { const comma = self.getNextToken(); if (comma.id != Token.Id.Comma) { @@ -2299,11 +2284,11 @@ pub const Parser = struct { break :blk null; } - const symbol = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; + const symbol = (try self.expectToken(&stack, Token.Id.Identifier)) ?? continue; break :blk try self.createLiteral(arena, ast.NodeIdentifier, symbol); }; - const rpipe = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue; + const rpipe = (try self.expectToken(&stack, Token.Id.Pipe)) ?? continue; *dest = try self.createNode(arena, ast.NodePointerIndexPayload, ast.NodePointerIndexPayload { .base = undefined, @@ -2370,11 +2355,9 @@ pub const Parser = struct { }, State.FnProtoAlign => |fn_proto| { - const token = self.getNextToken(); - if (token.id == Token.Id.Keyword_align) { + if (self.eatToken(Token.Id.Keyword_align)) |align_token| { @panic("TODO fn proto align"); } - self.putBackToken(token); stack.append(State { .FnProtoReturnType = fn_proto, }) catch unreachable; @@ -2389,6 +2372,11 @@ pub const Parser = struct { stack.append(State { .TypeExprBegin = DestPtr {.Field = &fn_proto.return_type.InferErrorSet}, }) catch unreachable; + continue; + }, + Token.Id.Keyword_align => { + @panic("TODO fn proto align"); + continue; }, else => { self.putBackToken(token); @@ -2396,17 +2384,13 @@ pub const Parser = struct { stack.append(State { .TypeExprBegin = DestPtr {.Field = &fn_proto.return_type.Explicit}, }) catch unreachable; + continue; }, } - if (token.id == Token.Id.Keyword_align) { - @panic("TODO fn proto align"); - } - continue; }, State.ParamDecl => |fn_proto| { - var token = self.getNextToken(); - if (token.id == Token.Id.RParen) { + if (self.eatToken(Token.Id.RParen)) |_| { continue; } const param_decl = try self.createAttachNode(arena, &fn_proto.params, ast.NodeParamDecl, @@ -2419,28 +2403,22 @@ pub const Parser = struct { .var_args_token = null, }, ); - if (token.id == Token.Id.Keyword_comptime) { - param_decl.comptime_token = token; - token = self.getNextToken(); - } else if (token.id == Token.Id.Keyword_noalias) { - param_decl.noalias_token = token; - token = self.getNextToken(); + if (self.eatToken(Token.Id.Keyword_comptime)) |comptime_token| { + param_decl.comptime_token = comptime_token; + } else if (self.eatToken(Token.Id.Keyword_noalias)) |noalias_token| { + param_decl.noalias_token = noalias_token; } - if (token.id == Token.Id.Identifier) { - const next_token = self.getNextToken(); - if (next_token.id == Token.Id.Colon) { - param_decl.name_token = token; - token = self.getNextToken(); + if (self.eatToken(Token.Id.Identifier)) |identifier| { + if (self.eatToken(Token.Id.Colon)) |_| { + param_decl.name_token = identifier; } else { - self.putBackToken(next_token); + self.putBackToken(identifier); } } - if (token.id == Token.Id.Ellipsis3) { - param_decl.var_args_token = token; + if (self.eatToken(Token.Id.Ellipsis3)) |ellipsis3| { + param_decl.var_args_token = ellipsis3; stack.append(State { .ExpectToken = Token.Id.RParen }) catch unreachable; continue; - } else { - self.putBackToken(token); } stack.append(State { .ParamDecl = fn_proto }) catch unreachable; @@ -2736,7 +2714,7 @@ pub const Parser = struct { State.Semicolon => |node_ptr| { const node = *node_ptr; if (requireSemiColon(node)) { - _ = (try self.eatToken(&stack, Token.Id.Semicolon)) ?? continue; + _ = (try self.expectToken(&stack, Token.Id.Semicolon)) ?? continue; } } } @@ -2923,18 +2901,17 @@ pub const Parser = struct { }; } - fn initNode(self: &Parser, id: ast.Node.Id) ast.Node { - if (self.pending_line_comment_node) |comment_node| { - self.pending_line_comment_node = null; - return ast.Node {.id = id, .comment = comment_node}; - } - return ast.Node {.id = id, .comment = null }; - } - fn createNode(self: &Parser, arena: &mem.Allocator, comptime T: type, init_to: &const T) !&T { const node = try arena.create(T); *node = *init_to; - node.base = self.initNode(ast.Node.typeToId(T)); + node.base = blk: { + const id = ast.Node.typeToId(T); + if (self.pending_line_comment_node) |comment_node| { + self.pending_line_comment_node = null; + break :blk ast.Node {.id = id, .comment = comment_node}; + } + break :blk ast.Node {.id = id, .comment = null }; + }; return node; } @@ -2986,15 +2963,6 @@ pub const Parser = struct { }; } - fn eatToken(self: &Parser, stack: &ArrayList(State), id: @TagType(Token.Id)) !?Token { - const token = self.getNextToken(); - if (token.id != id) { - try self.parseError(stack, token, "expected {}, found {}", @tagName(id), @tagName(token.id)); - return null; - } - return token; - } - fn revertIfOptional(self: &Parser, stack: &ArrayList(State)) !void { while (stack.popOrNull()) |state| { switch (state) { @@ -3011,6 +2979,22 @@ pub const Parser = struct { return error.NoOptionalStateFound; } + fn expectToken(self: &Parser, stack: &ArrayList(State), id: @TagType(Token.Id)) !?Token { + const token = self.getNextToken(); + if (token.id != id) { + try self.parseError(stack, token, "expected {}, found {}", @tagName(id), @tagName(token.id)); + return null; + } + return token; + } + + fn eatToken(self: &Parser, id: @TagType(Token.Id)) ?Token { + if (self.isPeekToken(id)) { + return self.getNextToken(); + } + return null; + } + fn putBackToken(self: &Parser, token: &const Token) void { self.put_back_tokens[self.put_back_count] = *token; self.put_back_count += 1; @@ -3027,6 +3011,12 @@ pub const Parser = struct { } } + fn isPeekToken(self: &Parser, id: @TagType(Token.Id)) bool { + const token = self.getNextToken(); + defer self.putBackToken(token); + return id == token.id; + } + const RenderAstFrame = struct { node: &ast.Node, indent: usize, From 5f3ec023cd697007ff868938e3b5fe387add6af5 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Wed, 11 Apr 2018 12:53:01 +0200 Subject: [PATCH 78/87] std.zig.parser: Fixed parsing of field access rhs related: #909 --- std/zig/parser.zig | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index f722703284..451bc9dd5b 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -1515,17 +1515,23 @@ pub const Parser = struct { continue; }, Token.Id.Period => { + const identifier = try self.createLiteral(arena, ast.NodeIdentifier, Token(undefined)); const node = try self.createToDestNode(arena, dest_ptr, ast.NodeInfixOp, ast.NodeInfixOp { .base = undefined, .lhs = dest_ptr.get(), .op_token = token, .op = ast.NodeInfixOp.InfixOp.Period, - .rhs = undefined, + .rhs = &identifier.base, } ); stack.append(State { .SuffixOpExpressionEnd = dest_ptr }) catch unreachable; - try stack.append(State { .SuffixOpExpressionBegin = DestPtr { .Field = &node.rhs }}); + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.Identifier, + .ptr = &identifier.token + } + }); continue; }, else => { @@ -5011,6 +5017,7 @@ test "zig fmt: inline asm" { test "zig fmt: coroutines" { try testCanonical( \\async fn simpleAsyncFn() void { + \\ const a = async a.b(); \\ x += 1; \\ suspend; \\ x += 1; From 6fb5ab1b523c58350321fdcbe7ba921300bef8af Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Wed, 11 Apr 2018 13:05:42 +0200 Subject: [PATCH 79/87] std.zig.parser: Redid parsing of error set delc related: #909 --- std/zig/parser.zig | 71 ++++++++++++++++++++-------------------------- 1 file changed, 31 insertions(+), 40 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 451bc9dd5b..3e69e0e163 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -177,6 +177,8 @@ pub const Parser = struct { FieldInitListItemOrEnd: ListSave(&ast.NodeFieldInitializer), FieldInitListCommaOrEnd: ListSave(&ast.NodeFieldInitializer), FieldListCommaOrEnd: &ast.NodeContainerDecl, + IdentifierListItemOrEnd: ListSave(&ast.NodeIdentifier), + IdentifierListCommaOrEnd: ListSave(&ast.NodeIdentifier), SwitchCaseOrEnd: ListSave(&ast.NodeSwitchCase), SuspendBody: &ast.NodeSuspend, AsyncEnd: AsyncEndCtx, @@ -1683,11 +1685,8 @@ pub const Parser = struct { }, Token.Id.Keyword_error => { - const next = self.getNextToken(); - - if (next.id != Token.Id.LBrace) { + if (self.eatToken(Token.Id.LBrace) == null) { dest_ptr.store(&(try self.createLiteral(arena, ast.NodeErrorType, token)).base); - self.putBackToken(next); continue; } @@ -1700,43 +1699,12 @@ pub const Parser = struct { } ); - while (true) { - const t = self.getNextToken(); - switch (t.id) { - Token.Id.RBrace => { - node.rbrace_token = t; - break; - }, - Token.Id.Identifier => { - try node.decls.append( - try self.createLiteral(arena, ast.NodeIdentifier, t) - ); - }, - else => { - try self.parseError(&stack, token, "expected {} or {}, found {}", - @tagName(Token.Id.RBrace), - @tagName(Token.Id.Identifier), - @tagName(token.id)); - continue; - } + stack.append(State { + .IdentifierListItemOrEnd = ListSave(&ast.NodeIdentifier) { + .list = &node.decls, + .ptr = &node.rbrace_token, } - - const t2 = self.getNextToken(); - switch (t2.id) { - Token.Id.RBrace => { - node.rbrace_token = t; - break; - }, - Token.Id.Comma => continue, - else => { - try self.parseError(&stack, token, "expected {} or {}, found {}", - @tagName(Token.Id.RBrace), - @tagName(Token.Id.Comma), - @tagName(token.id)); - continue; - } - } - } + }) catch unreachable; continue; }, Token.Id.Keyword_packed => { @@ -2082,6 +2050,24 @@ pub const Parser = struct { }); }, + State.IdentifierListItemOrEnd => |list_state| { + if (self.eatToken(Token.Id.RBrace)) |rbrace| { + *list_state.ptr = rbrace; + continue; + } + + const node = try self.createLiteral(arena, ast.NodeIdentifier, Token(undefined)); + try list_state.list.append(node); + + stack.append(State { .IdentifierListCommaOrEnd = list_state }) catch unreachable; + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.Identifier, + .ptr = &node.token, + } + }); + }, + State.SwitchCaseOrEnd => |list_state| { if (self.eatToken(Token.Id.RBrace)) |rbrace| { *list_state.ptr = rbrace; @@ -2139,6 +2125,11 @@ pub const Parser = struct { continue; }, + State.IdentifierListCommaOrEnd => |list_state| { + try self.commaOrEnd(&stack, Token.Id.RBrace, list_state.ptr, State { .IdentifierListItemOrEnd = list_state }); + continue; + }, + State.SwitchCaseCommaOrEnd => |list_state| { try self.commaOrEnd(&stack, Token.Id.RBrace, list_state.ptr, State { .SwitchCaseOrEnd = list_state }); continue; From 4b0556ebd4bdc2f3c05e85f3c003b6fc89f7ac0f Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Wed, 11 Apr 2018 13:38:06 +0200 Subject: [PATCH 80/87] std.zig.parser can now parse `std/heap.zig`: related: #909 * Struct fields can now be pub * Parsing of double deref now works * Block expressions now have the right precedence --- std/zig/ast.zig | 2 + std/zig/parser.zig | 288 ++++++++++++++++++++++++++++----------------- 2 files changed, 180 insertions(+), 110 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 9fb10aa0b1..b671feb4f2 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -360,6 +360,7 @@ pub const NodeContainerDecl = struct { pub const NodeStructField = struct { base: Node, + visib_token: ?Token, name_token: Token, type_expr: &Node, @@ -373,6 +374,7 @@ pub const NodeStructField = struct { } pub fn firstToken(self: &NodeStructField) Token { + if (self.visib_token) |visib_token| return visib_token; return self.name_token; } diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 3e69e0e163..669bf70633 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -669,6 +669,7 @@ pub const Parser = struct { const node = try self.createAttachNode(arena, &container_decl.fields_and_decls, ast.NodeStructField, ast.NodeStructField { .base = undefined, + .visib_token = null, .name_token = token, .type_expr = undefined, } @@ -721,7 +722,42 @@ pub const Parser = struct { }, } }, - Token.Id.Keyword_pub, Token.Id.Keyword_export => { + Token.Id.Keyword_pub => { + if (self.eatToken(Token.Id.Identifier)) |identifier| { + switch (container_decl.kind) { + ast.NodeContainerDecl.Kind.Struct => { + const node = try self.createAttachNode(arena, &container_decl.fields_and_decls, ast.NodeStructField, + ast.NodeStructField { + .base = undefined, + .visib_token = token, + .name_token = identifier, + .type_expr = undefined, + } + ); + + stack.append(State { .FieldListCommaOrEnd = container_decl }) catch unreachable; + try stack.append(State { .Expression = DestPtr { .Field = &node.type_expr } }); + try stack.append(State { .ExpectToken = Token.Id.Colon }); + continue; + }, + else => { + self.putBackToken(identifier); + } + } + } + + stack.append(State{ .ContainerDecl = container_decl }) catch unreachable; + try stack.append(State { + .TopLevelExtern = TopLevelDeclCtx { + .decls = &container_decl.fields_and_decls, + .visib_token = token, + .extern_token = null, + .lib_name = null, + } + }); + continue; + }, + Token.Id.Keyword_export => { stack.append(State{ .ContainerDecl = container_decl }) catch unreachable; try stack.append(State { .TopLevelExtern = TopLevelDeclCtx { @@ -864,111 +900,11 @@ pub const Parser = struct { stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }) catch unreachable; continue; }, - Token.Id.Keyword_suspend => { - const node = try self.createToDestNode(arena, dest_ptr, ast.NodeSuspend, - ast.NodeSuspend { - .base = undefined, - .suspend_token = token, - .payload = null, - .body = null, - } - ); - - stack.append(State { .SuspendBody = node }) catch unreachable; - try stack.append(State { .Payload = &node.payload }); - continue; - }, - Token.Id.Keyword_if => { - const node = try self.createToDestNode(arena, dest_ptr, ast.NodeIf, - ast.NodeIf { - .base = undefined, - .if_token = token, - .condition = undefined, - .payload = null, - .body = undefined, - .@"else" = null, - } - ); - - stack.append(State { .Else = &node.@"else" }) catch unreachable; - try stack.append(State { .Expression = DestPtr { .Field = &node.body } }); - try stack.append(State { .PointerPayload = &node.payload }); - try stack.append(State { .ExpectToken = Token.Id.RParen }); - try stack.append(State { .Expression = DestPtr { .Field = &node.condition } }); - try stack.append(State { .ExpectToken = Token.Id.LParen }); - continue; - }, - Token.Id.Keyword_while => { - stack.append(State { - .While = LoopCtx { - .label = null, - .inline_token = null, - .loop_token = token, - .dest_ptr = dest_ptr, - } - }) catch unreachable; - continue; - }, - Token.Id.Keyword_for => { - stack.append(State { - .For = LoopCtx { - .label = null, - .inline_token = null, - .loop_token = token, - .dest_ptr = dest_ptr, - } - }) catch unreachable; - continue; - }, - Token.Id.Keyword_switch => { - const node = try self.createToDestNode(arena, dest_ptr, ast.NodeSwitch, - ast.NodeSwitch { - .base = undefined, - .switch_token = token, - .expr = undefined, - .cases = ArrayList(&ast.NodeSwitchCase).init(arena), - .rbrace = undefined, - } - ); - - stack.append(State { - .SwitchCaseOrEnd = ListSave(&ast.NodeSwitchCase) { - .list = &node.cases, - .ptr = &node.rbrace, - }, - }) catch unreachable; - try stack.append(State { .ExpectToken = Token.Id.LBrace }); - try stack.append(State { .ExpectToken = Token.Id.RParen }); - try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); - try stack.append(State { .ExpectToken = Token.Id.LParen }); - }, - Token.Id.Keyword_comptime => { - const node = try self.createToDestNode(arena, dest_ptr, ast.NodeComptime, - ast.NodeComptime { - .base = undefined, - .comptime_token = token, - .expr = undefined, - } - ); - try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); - continue; - }, - Token.Id.LBrace => { - const block = try self.createToDestNode(arena, dest_ptr, ast.NodeBlock, - ast.NodeBlock { - .base = undefined, - .label = null, - .lbrace = token, - .statements = ArrayList(&ast.Node).init(arena), - .rbrace = undefined, - } - ); - stack.append(State { .Block = block }) catch unreachable; - continue; - }, else => { - self.putBackToken(token); - stack.append(State { .UnwrapExpressionBegin = dest_ptr }) catch unreachable; + if (!try self.parseBlockExpr(&stack, arena, dest_ptr, token)) { + self.putBackToken(token); + stack.append(State { .UnwrapExpressionBegin = dest_ptr }) catch unreachable; + } continue; } } @@ -1407,7 +1343,7 @@ pub const Parser = struct { State.PrefixOpExpression => |dest_ptr| { const token = self.getNextToken(); if (tokenIdToPrefixOp(token.id)) |prefix_id| { - const node = try self.createToDestNode(arena, dest_ptr, ast.NodePrefixOp, + var node = try self.createToDestNode(arena, dest_ptr, ast.NodePrefixOp, ast.NodePrefixOp { .base = undefined, .op_token = token, @@ -1415,6 +1351,20 @@ pub const Parser = struct { .rhs = undefined, } ); + + if (token.id == Token.Id.AsteriskAsterisk) { + const child = try self.createNode(arena, ast.NodePrefixOp, + ast.NodePrefixOp { + .base = undefined, + .op_token = token, + .op = prefix_id, + .rhs = undefined, + } + ); + node.rhs = &child.base; + node = child; + } + stack.append(State { .TypeExprBegin = DestPtr { .Field = &node.rhs } }) catch unreachable; if (node.op == ast.NodePrefixOp.PrefixOp.AddrOf) { try stack.append(State { .AddrOfModifiers = &node.op.AddrOf }); @@ -1871,7 +1821,9 @@ pub const Parser = struct { continue; }, else => { - try self.parseError(&stack, token, "expected primary expression, found {}", @tagName(token.id)); + if (!try self.parseBlockExpr(&stack, arena, dest_ptr, token)) { + try self.parseError(&stack, token, "expected primary expression, found {}", @tagName(token.id)); + } continue; } } @@ -2790,6 +2742,117 @@ pub const Parser = struct { } } + fn parseBlockExpr(self: &Parser, stack: &ArrayList(State), arena: &mem.Allocator, dest_ptr: &const DestPtr, token: &const Token) !bool { + switch (token.id) { + Token.Id.Keyword_suspend => { + const node = try self.createToDestNode(arena, dest_ptr, ast.NodeSuspend, + ast.NodeSuspend { + .base = undefined, + .suspend_token = *token, + .payload = null, + .body = null, + } + ); + + stack.append(State { .SuspendBody = node }) catch unreachable; + try stack.append(State { .Payload = &node.payload }); + return true; + }, + Token.Id.Keyword_if => { + const node = try self.createToDestNode(arena, dest_ptr, ast.NodeIf, + ast.NodeIf { + .base = undefined, + .if_token = *token, + .condition = undefined, + .payload = null, + .body = undefined, + .@"else" = null, + } + ); + + stack.append(State { .Else = &node.@"else" }) catch unreachable; + try stack.append(State { .Expression = DestPtr { .Field = &node.body } }); + try stack.append(State { .PointerPayload = &node.payload }); + try stack.append(State { .ExpectToken = Token.Id.RParen }); + try stack.append(State { .Expression = DestPtr { .Field = &node.condition } }); + try stack.append(State { .ExpectToken = Token.Id.LParen }); + return true; + }, + Token.Id.Keyword_while => { + stack.append(State { + .While = LoopCtx { + .label = null, + .inline_token = null, + .loop_token = *token, + .dest_ptr = *dest_ptr, + } + }) catch unreachable; + return true; + }, + Token.Id.Keyword_for => { + stack.append(State { + .For = LoopCtx { + .label = null, + .inline_token = null, + .loop_token = *token, + .dest_ptr = *dest_ptr, + } + }) catch unreachable; + return true; + }, + Token.Id.Keyword_switch => { + const node = try self.createToDestNode(arena, dest_ptr, ast.NodeSwitch, + ast.NodeSwitch { + .base = undefined, + .switch_token = *token, + .expr = undefined, + .cases = ArrayList(&ast.NodeSwitchCase).init(arena), + .rbrace = undefined, + } + ); + + stack.append(State { + .SwitchCaseOrEnd = ListSave(&ast.NodeSwitchCase) { + .list = &node.cases, + .ptr = &node.rbrace, + }, + }) catch unreachable; + try stack.append(State { .ExpectToken = Token.Id.LBrace }); + try stack.append(State { .ExpectToken = Token.Id.RParen }); + try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); + try stack.append(State { .ExpectToken = Token.Id.LParen }); + return true; + }, + Token.Id.Keyword_comptime => { + const node = try self.createToDestNode(arena, dest_ptr, ast.NodeComptime, + ast.NodeComptime { + .base = undefined, + .comptime_token = *token, + .expr = undefined, + } + ); + try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); + return true; + }, + Token.Id.LBrace => { + const block = try self.createToDestNode(arena, dest_ptr, ast.NodeBlock, + ast.NodeBlock { + .base = undefined, + .label = null, + .lbrace = *token, + .statements = ArrayList(&ast.Node).init(arena), + .rbrace = undefined, + } + ); + stack.append(State { .Block = block }) catch unreachable; + return true; + }, + else => { + return false; + } + } + } + fn commaOrEnd(self: &Parser, stack: &ArrayList(State), end: &const Token.Id, maybe_ptr: ?&Token, state_after_comma: &const State) !void { var token = self.getNextToken(); switch (token.id) { @@ -2881,7 +2944,7 @@ pub const Parser = struct { Token.Id.Tilde => ast.NodePrefixOp.PrefixOp { .BitNot = void{} }, Token.Id.Minus => ast.NodePrefixOp.PrefixOp { .Negation = void{} }, Token.Id.MinusPercent => ast.NodePrefixOp.PrefixOp { .NegationWrap = void{} }, - Token.Id.Asterisk => ast.NodePrefixOp.PrefixOp { .Deref = void{} }, + Token.Id.Asterisk, Token.Id.AsteriskAsterisk => ast.NodePrefixOp.PrefixOp { .Deref = void{} }, Token.Id.Ampersand => ast.NodePrefixOp.PrefixOp { .AddrOf = ast.NodePrefixOp.AddrOfInfo { .align_expr = null, @@ -3126,6 +3189,9 @@ pub const Parser = struct { }, ast.Node.Id.StructField => { const field = @fieldParentPtr(ast.NodeStructField, "base", decl); + if (field.visib_token) |visib_token| { + try stream.print("{} ", self.tokenizer.getTokenSlice(visib_token)); + } try stream.print("{}: ", self.tokenizer.getTokenSlice(field.name_token)); try stack.append(RenderState { .Expression = field.type_expr}); }, @@ -4573,6 +4639,7 @@ test "zig fmt: struct declaration" { \\const S = struct { \\ const Self = this; \\ f1: u8, + \\ pub f3: u8, \\ \\ fn method(self: &Self) Self { \\ return *self; @@ -4583,14 +4650,14 @@ test "zig fmt: struct declaration" { \\ \\const Ps = packed struct { \\ a: u8, - \\ b: u8, + \\ pub b: u8, \\ \\ c: u8 \\}; \\ \\const Es = extern struct { \\ a: u8, - \\ b: u8, + \\ pub b: u8, \\ \\ c: u8 \\}; @@ -4895,6 +4962,7 @@ test "zig fmt: if" { \\ } \\ \\ const is_world_broken = if (10 < 0) true else false; + \\ const some_number = 1 + if (10 < 0) 2 else 3; \\ \\ const a: ?u8 = 10; \\ const b: ?u8 = null; From 841ac0f4e1b5ca6ae8f2250248363521d9f56c36 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Wed, 11 Apr 2018 13:46:35 +0200 Subject: [PATCH 81/87] std.zig.parser now allows assignment expr in switch cases. This makes `std/os/index.zig` parse related: #909 --- std/zig/parser.zig | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 669bf70633..16afe57bed 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -2036,7 +2036,7 @@ pub const Parser = struct { ); try list_state.list.append(node); stack.append(State { .SwitchCaseCommaOrEnd = list_state }) catch unreachable; - try stack.append(State { .Expression = DestPtr{ .Field = &node.expr } }); + try stack.append(State { .AssignmentExpressionBegin = DestPtr{ .Field = &node.expr } }); try stack.append(State { .PointerPayload = &node.payload }); const maybe_else = self.getNextToken(); @@ -4817,6 +4817,7 @@ test "zig fmt: switch" { \\ const res = switch (0) { \\ 0 => 0, \\ 1 => 2, + \\ 1 => a = 4, \\ else => 4 \\ }; \\ From 28ea364e5e60464f68501fdfa69ba41b9cf9c47e Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Wed, 11 Apr 2018 13:56:39 +0200 Subject: [PATCH 82/87] std.zig.parser now handle `try`'s precedence correctly This allows parsing of `std/zig/parser.zig`. Related: #909 --- std/zig/parser.zig | 1 + 1 file changed, 1 insertion(+) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 16afe57bed..104e7e4cff 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -2957,6 +2957,7 @@ pub const Parser = struct { Token.Id.QuestionMark => ast.NodePrefixOp.PrefixOp { .MaybeType = void{} }, Token.Id.QuestionMarkQuestionMark => ast.NodePrefixOp.PrefixOp { .UnwrapMaybe = void{} }, Token.Id.Keyword_await => ast.NodePrefixOp.PrefixOp { .Await = void{} }, + Token.Id.Keyword_try => ast.NodePrefixOp.PrefixOp { .Try = void{ } }, else => null, }; } From fe7146277d5a7f7c74a8c2f36719c946e6061c50 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Wed, 11 Apr 2018 14:43:53 +0200 Subject: [PATCH 83/87] std.zig.parser now accept both string and multiline string for strings Related #909 Allows it to parse `std/special/compiler_rt/aullrem.zig`, `std/special/compiler_rt/aulldiv.zig` and `std/math/x86_64/sqrt.zig` --- std/zig/ast.zig | 14 +++--- std/zig/parser.zig | 121 +++++++++++++++++++++++++++------------------ 2 files changed, 81 insertions(+), 54 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index b671feb4f2..a5006f192c 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -1608,7 +1608,7 @@ pub const NodeThisLiteral = struct { pub const NodeAsmOutput = struct { base: Node, symbolic_name: &NodeIdentifier, - constraint: &NodeStringLiteral, + constraint: &Node, kind: Kind, const Kind = union(enum) { @@ -1622,7 +1622,7 @@ pub const NodeAsmOutput = struct { if (i < 1) return &self.symbolic_name.base; i -= 1; - if (i < 1) return &self.constraint.base; + if (i < 1) return self.constraint; i -= 1; switch (self.kind) { @@ -1654,7 +1654,7 @@ pub const NodeAsmOutput = struct { pub const NodeAsmInput = struct { base: Node, symbolic_name: &NodeIdentifier, - constraint: &NodeStringLiteral, + constraint: &Node, expr: &Node, pub fn iterate(self: &NodeAsmInput, index: usize) ?&Node { @@ -1663,7 +1663,7 @@ pub const NodeAsmInput = struct { if (i < 1) return &self.symbolic_name.base; i -= 1; - if (i < 1) return &self.constraint.base; + if (i < 1) return self.constraint; i -= 1; if (i < 1) return self.expr; @@ -1685,11 +1685,11 @@ pub const NodeAsm = struct { base: Node, asm_token: Token, is_volatile: bool, - template: Token, + template: &Node, //tokens: ArrayList(AsmToken), outputs: ArrayList(&NodeAsmOutput), inputs: ArrayList(&NodeAsmInput), - cloppers: ArrayList(&NodeStringLiteral), + cloppers: ArrayList(&Node), rparen: Token, pub fn iterate(self: &NodeAsm, index: usize) ?&Node { @@ -1701,7 +1701,7 @@ pub const NodeAsm = struct { if (i < self.inputs.len) return &self.inputs.at(index).base; i -= self.inputs.len; - if (i < self.cloppers.len) return &self.cloppers.at(index).base; + if (i < self.cloppers.len) return self.cloppers.at(index); i -= self.cloppers.len; return null; diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 104e7e4cff..7704698a0c 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -171,7 +171,7 @@ pub const Parser = struct { Semicolon: &const &const ast.Node, AsmOutputItems: &ArrayList(&ast.NodeAsmOutput), AsmInputItems: &ArrayList(&ast.NodeAsmInput), - AsmClopperItems: &ArrayList(&ast.NodeStringLiteral), + AsmClopperItems: &ArrayList(&ast.Node), ExprListItemOrEnd: ExprListCtx, ExprListCommaOrEnd: ExprListCtx, FieldInitListItemOrEnd: ListSave(&ast.NodeFieldInitializer), @@ -301,7 +301,11 @@ pub const Parser = struct { Token.Id.Keyword_test => { stack.append(State.TopLevel) catch unreachable; - const name_token = (try self.expectToken(&stack, Token.Id.StringLiteral)) ?? continue; + const name_token = self.getNextToken(); + const name = (try self.parseStringLiteral(arena, name_token)) ?? { + try self.parseError(&stack, name_token, "expected string literal, found {}", @tagName(name_token.id)); + continue; + }; const lbrace = (try self.expectToken(&stack, Token.Id.LBrace)) ?? continue; const block = try self.createNode(arena, ast.NodeBlock, @@ -317,7 +321,7 @@ pub const Parser = struct { ast.NodeTestDecl { .base = undefined, .test_token = token, - .name = &(try self.createLiteral(arena, ast.NodeStringLiteral, name_token)).base, + .name = name, .body_node = &block.base, } ); @@ -389,15 +393,12 @@ pub const Parser = struct { continue; }, Token.Id.Keyword_extern => { - const lib_name_token = self.getNextToken(); const lib_name = blk: { - if (lib_name_token.id == Token.Id.StringLiteral) { - const res = try self.createLiteral(arena, ast.NodeStringLiteral, lib_name_token); - break :blk &res.base; - } else { + const lib_name_token = self.getNextToken(); + break :blk (try self.parseStringLiteral(arena, lib_name_token)) ?? { self.putBackToken(lib_name_token); break :blk null; - } + }; }; stack.append(State { @@ -1504,10 +1505,6 @@ pub const Parser = struct { dest_ptr.store(&(try self.createLiteral(arena, ast.NodeFloatLiteral, token)).base); continue; }, - Token.Id.StringLiteral => { - dest_ptr.store(&(try self.createLiteral(arena, ast.NodeStringLiteral, token)).base); - continue; - }, Token.Id.CharLiteral => { dest_ptr.store(&(try self.createLiteral(arena, ast.NodeCharLiteral, token)).base); continue; @@ -1536,24 +1533,8 @@ pub const Parser = struct { dest_ptr.store(&(try self.createLiteral(arena, ast.NodeUnreachable, token)).base); continue; }, - Token.Id.MultilineStringLiteralLine => { - const node = try self.createToDestNode(arena, dest_ptr, ast.NodeMultilineStringLiteral, - ast.NodeMultilineStringLiteral { - .base = undefined, - .tokens = ArrayList(Token).init(arena), - } - ); - try node.tokens.append(token); - while (true) { - const multiline_str = self.getNextToken(); - if (multiline_str.id != Token.Id.MultilineStringLiteralLine) { - self.putBackToken(multiline_str); - break; - } - - try node.tokens.append(multiline_str); - } - continue; + Token.Id.StringLiteral, Token.Id.MultilineStringLiteralLine => { + dest_ptr.store((try self.parseStringLiteral(arena, token)) ?? unreachable); }, Token.Id.LParen => { const node = try self.createToDestNode(arena, dest_ptr, ast.NodeGroupedExpression, @@ -1781,7 +1762,12 @@ pub const Parser = struct { break :blk true; }; _ = (try self.expectToken(&stack, Token.Id.LParen)) ?? continue; - const template = (try self.expectToken(&stack, Token.Id.StringLiteral)) ?? continue; + + const template_token = self.getNextToken(); + const template = (try self.parseStringLiteral(arena, template_token)) ?? { + try self.parseError(&stack, template_token, "expected string literal, found {}", @tagName(template_token.id)); + continue; + }; // TODO parse template const node = try self.createToDestNode(arena, dest_ptr, ast.NodeAsm, @@ -1793,7 +1779,7 @@ pub const Parser = struct { //.tokens = ArrayList(ast.NodeAsm.AsmToken).init(arena), .outputs = ArrayList(&ast.NodeAsmOutput).init(arena), .inputs = ArrayList(&ast.NodeAsmInput).init(arena), - .cloppers = ArrayList(&ast.NodeStringLiteral).init(arena), + .cloppers = ArrayList(&ast.Node).init(arena), .rparen = undefined, } ); @@ -1881,7 +1867,12 @@ pub const Parser = struct { const symbolic_name = (try self.expectToken(&stack, Token.Id.Identifier)) ?? continue; _ = (try self.expectToken(&stack, Token.Id.RBracket)) ?? continue; - const constraint = (try self.expectToken(&stack, Token.Id.StringLiteral)) ?? continue; + + const constraint_token = self.getNextToken(); + const constraint = (try self.parseStringLiteral(arena, constraint_token)) ?? { + try self.parseError(&stack, constraint_token, "expected string literal, found {}", @tagName(constraint_token.id)); + continue; + }; _ = (try self.expectToken(&stack, Token.Id.LParen)) ?? continue; try stack.append(State { .ExpectToken = Token.Id.RParen }); @@ -1890,7 +1881,7 @@ pub const Parser = struct { ast.NodeAsmOutput { .base = undefined, .symbolic_name = try self.createLiteral(arena, ast.NodeIdentifier, symbolic_name), - .constraint = try self.createLiteral(arena, ast.NodeStringLiteral, constraint), + .constraint = constraint, .kind = undefined, } ); @@ -1926,7 +1917,12 @@ pub const Parser = struct { const symbolic_name = (try self.expectToken(&stack, Token.Id.Identifier)) ?? continue; _ = (try self.expectToken(&stack, Token.Id.RBracket)) ?? continue; - const constraint = (try self.expectToken(&stack, Token.Id.StringLiteral)) ?? continue; + + const constraint_token = self.getNextToken(); + const constraint = (try self.parseStringLiteral(arena, constraint_token)) ?? { + try self.parseError(&stack, constraint_token, "expected string literal, found {}", @tagName(constraint_token.id)); + continue; + }; _ = (try self.expectToken(&stack, Token.Id.LParen)) ?? continue; try stack.append(State { .ExpectToken = Token.Id.RParen }); @@ -1935,7 +1931,7 @@ pub const Parser = struct { ast.NodeAsmInput { .base = undefined, .symbolic_name = try self.createLiteral(arena, ast.NodeIdentifier, symbolic_name), - .constraint = try self.createLiteral(arena, ast.NodeStringLiteral, constraint), + .constraint = constraint, .expr = undefined, } ); @@ -1944,13 +1940,13 @@ pub const Parser = struct { }, State.AsmClopperItems => |items| { - const string = self.getNextToken(); - if (string.id != Token.Id.StringLiteral) { - self.putBackToken(string); + const string_token = self.getNextToken(); + const string = (try self.parseStringLiteral(arena, string_token)) ?? { + self.putBackToken(string_token); continue; - } + }; + try items.append(string); - try items.append(try self.createLiteral(arena, ast.NodeStringLiteral, string)); stack.append(State { .AsmClopperItems = items }) catch unreachable; try stack.append(State { .IfToken = Token.Id.Comma }); }, @@ -2742,6 +2738,37 @@ pub const Parser = struct { } } + fn parseStringLiteral(self: &Parser, arena: &mem.Allocator, token: &const Token) !?&ast.Node { + switch (token.id) { + Token.Id.StringLiteral => { + return &(try self.createLiteral(arena, ast.NodeStringLiteral, token)).base; + }, + Token.Id.MultilineStringLiteralLine => { + const node = try self.createNode(arena, ast.NodeMultilineStringLiteral, + ast.NodeMultilineStringLiteral { + .base = undefined, + .tokens = ArrayList(Token).init(arena), + } + ); + try node.tokens.append(token); + while (true) { + const multiline_str = self.getNextToken(); + if (multiline_str.id != Token.Id.MultilineStringLiteralLine) { + self.putBackToken(multiline_str); + break; + } + + try node.tokens.append(multiline_str); + } + + return &node.base; + }, + // TODO: We shouldn't need a cast, but: + // zig: /home/jc/Documents/zig/src/ir.cpp:7962: TypeTableEntry* ir_resolve_peer_types(IrAnalyze*, AstNode*, IrInstruction**, size_t): Assertion `err_set_type != nullptr' failed. + else => return (?&ast.Node)(null), + } + } + fn parseBlockExpr(self: &Parser, stack: &ArrayList(State), arena: &mem.Allocator, dest_ptr: &const DestPtr, token: &const Token) !bool { switch (token.id) { Token.Id.Keyword_suspend => { @@ -4085,8 +4112,6 @@ pub const Parser = struct { try stream.write("volatile "); } - try stream.print("({}", self.tokenizer.getTokenSlice(asm_node.template)); - try stack.append(RenderState { .Indent = indent }); try stack.append(RenderState { .Text = ")" }); { @@ -4094,7 +4119,7 @@ pub const Parser = struct { var i = cloppers.len; while (i != 0) { i -= 1; - try stack.append(RenderState { .Expression = &cloppers[i].base }); + try stack.append(RenderState { .Expression = cloppers[i] }); if (i != 0) { try stack.append(RenderState { .Text = ", " }); @@ -4163,6 +4188,8 @@ pub const Parser = struct { try stack.append(RenderState.PrintIndent); try stack.append(RenderState { .Indent = indent + indent_delta}); try stack.append(RenderState { .Text = "\n" }); + try stack.append(RenderState { .Expression = asm_node.template }); + try stack.append(RenderState { .Text = "(" }); }, ast.Node.Id.AsmInput => { const asm_input = @fieldParentPtr(ast.NodeAsmInput, "base", base); @@ -4170,7 +4197,7 @@ pub const Parser = struct { try stack.append(RenderState { .Text = ")"}); try stack.append(RenderState { .Expression = asm_input.expr}); try stack.append(RenderState { .Text = " ("}); - try stack.append(RenderState { .Expression = &asm_input.constraint.base}); + try stack.append(RenderState { .Expression = asm_input.constraint }); try stack.append(RenderState { .Text = "] "}); try stack.append(RenderState { .Expression = &asm_input.symbolic_name.base}); try stack.append(RenderState { .Text = "["}); @@ -4189,7 +4216,7 @@ pub const Parser = struct { }, } try stack.append(RenderState { .Text = " ("}); - try stack.append(RenderState { .Expression = &asm_output.constraint.base}); + try stack.append(RenderState { .Expression = asm_output.constraint }); try stack.append(RenderState { .Text = "] "}); try stack.append(RenderState { .Expression = &asm_output.symbolic_name.base}); try stack.append(RenderState { .Text = "["}); From df4c575525a8ab3d190b46af724a718c77e607e3 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Wed, 11 Apr 2018 15:17:51 +0200 Subject: [PATCH 84/87] std.zig.parser now parses inline fn proto Related #909 Allows parsing of `std/os/zen.zig`. --- std/zig/ast.zig | 10 ++-- std/zig/parser.zig | 142 +++++++++++++++++++++++++++------------------ 2 files changed, 91 insertions(+), 61 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index a5006f192c..230fe26568 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -214,7 +214,7 @@ pub const NodeVarDecl = struct { eq_token: Token, mut_token: Token, comptime_token: ?Token, - extern_token: ?Token, + extern_export_token: ?Token, lib_name: ?&Node, type_node: ?&Node, align_node: ?&Node, @@ -245,7 +245,7 @@ pub const NodeVarDecl = struct { pub fn firstToken(self: &NodeVarDecl) Token { if (self.visib_token) |visib_token| return visib_token; if (self.comptime_token) |comptime_token| return comptime_token; - if (self.extern_token) |extern_token| return extern_token; + if (self.extern_export_token) |extern_export_token| return extern_export_token; assert(self.lib_name == null); return self.mut_token; } @@ -496,8 +496,7 @@ pub const NodeFnProto = struct { params: ArrayList(&Node), return_type: ReturnType, var_args_token: ?Token, - extern_token: ?Token, - inline_token: ?Token, + extern_export_inline_token: ?Token, cc_token: ?Token, async_attr: ?&NodeAsyncAttribute, body_node: ?&Node, @@ -547,9 +546,8 @@ pub const NodeFnProto = struct { pub fn firstToken(self: &NodeFnProto) Token { if (self.visib_token) |visib_token| return visib_token; - if (self.extern_token) |extern_token| return extern_token; + if (self.extern_export_inline_token) |extern_export_inline_token| return extern_export_inline_token; assert(self.lib_name == null); - if (self.inline_token) |inline_token| return inline_token; if (self.cc_token) |cc_token| return cc_token; return self.fn_token; } diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 7704698a0c..50165164cd 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -55,7 +55,7 @@ pub const Parser = struct { const TopLevelDeclCtx = struct { decls: &ArrayList(&ast.Node), visib_token: ?Token, - extern_token: ?Token, + extern_export_inline_token: ?Token, lib_name: ?&ast.Node, }; @@ -142,6 +142,7 @@ pub const Parser = struct { const State = union(enum) { TopLevel, TopLevelExtern: TopLevelDeclCtx, + TopLevelLibname: TopLevelDeclCtx, TopLevelDecl: TopLevelDeclCtx, ContainerExtern: ContainerExternCtx, ContainerDecl: &ast.NodeContainerDecl, @@ -332,13 +333,13 @@ pub const Parser = struct { root_node.eof_token = token; return Tree {.root_node = root_node, .arena_allocator = arena_allocator}; }, - Token.Id.Keyword_pub, Token.Id.Keyword_export => { + Token.Id.Keyword_pub => { stack.append(State.TopLevel) catch unreachable; try stack.append(State { .TopLevelExtern = TopLevelDeclCtx { .decls = &root_node.decls, .visib_token = token, - .extern_token = null, + .extern_export_inline_token = null, .lib_name = null, } }); @@ -363,7 +364,7 @@ pub const Parser = struct { .TopLevelExtern = TopLevelDeclCtx { .decls = &root_node.decls, .visib_token = null, - .extern_token = null, + .extern_export_inline_token = null, .lib_name = null, } }); @@ -372,9 +373,66 @@ pub const Parser = struct { } }, State.TopLevelExtern => |ctx| { + const token = self.getNextToken(); + switch (token.id) { + Token.Id.Keyword_export, Token.Id.Keyword_inline => { + stack.append(State { + .TopLevelDecl = TopLevelDeclCtx { + .decls = ctx.decls, + .visib_token = ctx.visib_token, + .extern_export_inline_token = token, + .lib_name = null, + }, + }) catch unreachable; + continue; + }, + Token.Id.Keyword_extern => { + stack.append(State { + .TopLevelLibname = TopLevelDeclCtx { + .decls = ctx.decls, + .visib_token = ctx.visib_token, + .extern_export_inline_token = token, + .lib_name = null, + }, + }) catch unreachable; + continue; + }, + else => { + self.putBackToken(token); + stack.append(State { .TopLevelDecl = ctx }) catch unreachable; + continue; + } + } + }, + + State.TopLevelLibname => |ctx| { + const lib_name = blk: { + const lib_name_token = self.getNextToken(); + break :blk (try self.parseStringLiteral(arena, lib_name_token)) ?? { + self.putBackToken(lib_name_token); + break :blk null; + }; + }; + + stack.append(State { + .TopLevelDecl = TopLevelDeclCtx { + .decls = ctx.decls, + .visib_token = ctx.visib_token, + .extern_export_inline_token = ctx.extern_export_inline_token, + .lib_name = lib_name, + }, + }) catch unreachable; + }, + + State.TopLevelDecl => |ctx| { const token = self.getNextToken(); switch (token.id) { Token.Id.Keyword_use => { + if (ctx.extern_export_inline_token != null) { + try self.parseError(&stack, token, "Invalid token {}", @tagName((??ctx.extern_export_inline_token).id)); + continue; + } + const node = try self.createAttachNode(arena, ctx.decls, ast.NodeUse, ast.NodeUse { .base = undefined, @@ -392,43 +450,21 @@ pub const Parser = struct { try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); continue; }, - Token.Id.Keyword_extern => { - const lib_name = blk: { - const lib_name_token = self.getNextToken(); - break :blk (try self.parseStringLiteral(arena, lib_name_token)) ?? { - self.putBackToken(lib_name_token); - break :blk null; - }; - }; - - stack.append(State { - .TopLevelDecl = TopLevelDeclCtx { - .decls = ctx.decls, - .visib_token = ctx.visib_token, - .extern_token = token, - .lib_name = lib_name, - }, - }) catch unreachable; - continue; - }, - else => { - self.putBackToken(token); - stack.append(State { .TopLevelDecl = ctx }) catch unreachable; - continue; - } - } - }, - State.TopLevelDecl => |ctx| { - const token = self.getNextToken(); - switch (token.id) { Token.Id.Keyword_var, Token.Id.Keyword_const => { + if (ctx.extern_export_inline_token) |extern_export_inline_token| { + if (extern_export_inline_token.id == Token.Id.Keyword_inline) { + try self.parseError(&stack, token, "Invalid token {}", @tagName(extern_export_inline_token.id)); + continue; + } + } + const var_decl_node = try self.createAttachNode(arena, ctx.decls, ast.NodeVarDecl, ast.NodeVarDecl { .base = undefined, .visib_token = ctx.visib_token, .mut_token = token, .comptime_token = null, - .extern_token = ctx.extern_token, + .extern_export_token = ctx.extern_export_inline_token, .type_node = null, .align_node = null, .init_node = null, @@ -452,8 +488,7 @@ pub const Parser = struct { .params = ArrayList(&ast.Node).init(arena), .return_type = undefined, .var_args_token = null, - .extern_token = ctx.extern_token, - .inline_token = null, + .extern_export_inline_token = ctx.extern_export_inline_token, .cc_token = null, .async_attr = null, .body_node = null, @@ -475,8 +510,7 @@ pub const Parser = struct { .params = ArrayList(&ast.Node).init(arena), .return_type = undefined, .var_args_token = null, - .extern_token = ctx.extern_token, - .inline_token = null, + .extern_export_inline_token = ctx.extern_export_inline_token, .cc_token = token, .async_attr = null, .body_node = null, @@ -513,8 +547,7 @@ pub const Parser = struct { .params = ArrayList(&ast.Node).init(arena), .return_type = undefined, .var_args_token = null, - .extern_token = ctx.extern_token, - .inline_token = null, + .extern_export_inline_token = ctx.extern_export_inline_token, .cc_token = null, .async_attr = async_node, .body_node = null, @@ -752,7 +785,7 @@ pub const Parser = struct { .TopLevelExtern = TopLevelDeclCtx { .decls = &container_decl.fields_and_decls, .visib_token = token, - .extern_token = null, + .extern_export_inline_token = null, .lib_name = null, } }); @@ -764,7 +797,7 @@ pub const Parser = struct { .TopLevelExtern = TopLevelDeclCtx { .decls = &container_decl.fields_and_decls, .visib_token = token, - .extern_token = null, + .extern_export_inline_token = null, .lib_name = null, } }); @@ -781,7 +814,7 @@ pub const Parser = struct { .TopLevelExtern = TopLevelDeclCtx { .decls = &container_decl.fields_and_decls, .visib_token = null, - .extern_token = null, + .extern_export_inline_token = null, .lib_name = null, } }); @@ -1659,8 +1692,7 @@ pub const Parser = struct { .params = ArrayList(&ast.Node).init(arena), .return_type = undefined, .var_args_token = null, - .extern_token = token, - .inline_token = null, + .extern_export_inline_token = token, .cc_token = null, .async_attr = null, .body_node = null, @@ -1717,8 +1749,7 @@ pub const Parser = struct { .params = ArrayList(&ast.Node).init(arena), .return_type = undefined, .var_args_token = null, - .extern_token = null, - .inline_token = null, + .extern_export_inline_token = null, .cc_token = null, .async_attr = null, .body_node = null, @@ -1740,8 +1771,7 @@ pub const Parser = struct { .params = ArrayList(&ast.Node).init(arena), .return_type = undefined, .var_args_token = null, - .extern_token = null, - .inline_token = null, + .extern_export_inline_token = null, .cc_token = token, .async_attr = null, .body_node = null, @@ -2573,7 +2603,7 @@ pub const Parser = struct { .visib_token = null, .mut_token = mut_token, .comptime_token = next, - .extern_token = null, + .extern_export_token = null, .type_node = null, .align_node = null, .init_node = null, @@ -2601,7 +2631,7 @@ pub const Parser = struct { .visib_token = null, .mut_token = next, .comptime_token = null, - .extern_token = null, + .extern_export_token = null, .type_node = null, .align_node = null, .init_node = null, @@ -3281,13 +3311,13 @@ pub const Parser = struct { try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(comptime_token) }); } - if (var_decl.extern_token) |extern_token| { + if (var_decl.extern_export_token) |extern_export_token| { if (var_decl.lib_name != null) { try stack.append(RenderState { .Text = " " }); try stack.append(RenderState { .Expression = ??var_decl.lib_name }); } try stack.append(RenderState { .Text = " " }); - try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(extern_token) }); + try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(extern_export_token) }); } if (var_decl.visib_token) |visib_token| { @@ -3865,9 +3895,9 @@ pub const Parser = struct { try stack.append(RenderState { .Text = " " }); try stack.append(RenderState { .Expression = lib_name }); } - if (fn_proto.extern_token) |extern_token| { + if (fn_proto.extern_export_inline_token) |extern_export_inline_token| { try stack.append(RenderState { .Text = " " }); - try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(extern_token) }); + try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(extern_export_inline_token) }); } if (fn_proto.visib_token) |visib_token| { @@ -4608,6 +4638,8 @@ test "zig fmt: extern function" { try testCanonical( \\extern fn puts(s: &const u8) c_int; \\extern "c" fn puts(s: &const u8) c_int; + \\export fn puts(s: &const u8) c_int; + \\inline fn puts(s: &const u8) c_int; \\ ); } From a7f77d7c6a4326de4c4cd356cd88e48854817e6f Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Wed, 11 Apr 2018 15:26:00 +0200 Subject: [PATCH 85/87] std.zig.parser: requireSemiColon now matches the C++ behavior Related #909 Allowes parsing of `std/os/child_process.zig` --- std/zig/parser.zig | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 50165164cd..f1f6b0e371 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -2722,7 +2722,7 @@ pub const Parser = struct { continue; } - n = while_node.body; + return while_node.body.id != ast.Node.Id.Block; }, ast.Node.Id.For => { const for_node = @fieldParentPtr(ast.NodeFor, "base", n); @@ -2731,7 +2731,7 @@ pub const Parser = struct { continue; } - n = for_node.body; + return for_node.body.id != ast.Node.Id.Block; }, ast.Node.Id.If => { const if_node = @fieldParentPtr(ast.NodeIf, "base", n); @@ -2740,25 +2740,25 @@ pub const Parser = struct { continue; } - n = if_node.body; + return if_node.body.id != ast.Node.Id.Block; }, ast.Node.Id.Else => { const else_node = @fieldParentPtr(ast.NodeElse, "base", n); n = else_node.body; + continue; }, ast.Node.Id.Defer => { const defer_node = @fieldParentPtr(ast.NodeDefer, "base", n); - n = defer_node.expr; + return defer_node.expr.id != ast.Node.Id.Block; }, ast.Node.Id.Comptime => { const comptime_node = @fieldParentPtr(ast.NodeComptime, "base", n); - n = comptime_node.expr; + return comptime_node.expr.id != ast.Node.Id.Block; }, ast.Node.Id.Suspend => { const suspend_node = @fieldParentPtr(ast.NodeSuspend, "base", n); if (suspend_node.body) |body| { - n = body; - continue; + return body.id != ast.Node.Id.Block; } return true; From e48e707c32121a73f1fd2862197c8f47dbceea5e Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 11 Apr 2018 14:44:32 -0400 Subject: [PATCH 86/87] allow integer and float literals to be passed to var params closes #623 --- src/ir.cpp | 9 ++++++++- test/cases/fn.zig | 10 ++++++++++ test/compile_errors.zig | 2 +- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/ir.cpp b/src/ir.cpp index 0b072cc696..7d8088d5ed 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -11804,7 +11804,8 @@ static bool ir_analyze_fn_call_generic_arg(IrAnalyze *ira, AstNode *fn_proto_nod } } - bool comptime_arg = param_decl_node->data.param_decl.is_inline; + bool comptime_arg = param_decl_node->data.param_decl.is_inline || + casted_arg->value.type->id == TypeTableEntryIdNumLitInt || casted_arg->value.type->id == TypeTableEntryIdNumLitFloat; ConstExprValue *arg_val; @@ -11829,6 +11830,12 @@ static bool ir_analyze_fn_call_generic_arg(IrAnalyze *ira, AstNode *fn_proto_nod var->shadowable = !comptime_arg; *next_proto_i += 1; + } else if (casted_arg->value.type->id == TypeTableEntryIdNumLitInt || + casted_arg->value.type->id == TypeTableEntryIdNumLitFloat) + { + ir_add_error(ira, casted_arg, + buf_sprintf("compiler bug: integer and float literals in var args function must be casted. https://github.com/zig-lang/zig/issues/557")); + return false; } if (!comptime_arg) { diff --git a/test/cases/fn.zig b/test/cases/fn.zig index e492f6036c..c125d98d8c 100644 --- a/test/cases/fn.zig +++ b/test/cases/fn.zig @@ -94,3 +94,13 @@ test "inline function call" { } fn add(a: i32, b: i32) i32 { return a + b; } + + +test "number literal as an argument" { + numberLiteralArg(3); + comptime numberLiteralArg(3); +} + +fn numberLiteralArg(a: var) void { + assert(a == 3); +} diff --git a/test/compile_errors.zig b/test/compile_errors.zig index bed5aa1b63..21e384e389 100644 --- a/test/compile_errors.zig +++ b/test/compile_errors.zig @@ -1723,7 +1723,7 @@ pub fn addCases(cases: &tests.CompileErrorContext) void { \\} \\ \\export fn entry() usize { return @sizeOf(@typeOf(bar)); } - , ".tmp_source.zig:10:16: error: parameter of type '(integer literal)' requires comptime"); + , ".tmp_source.zig:10:16: error: compiler bug: integer and float literals in var args function must be casted"); cases.add("assign too big number to u16", \\export fn foo() void { From 5b584e06e37296c602357cbaf27d39e068ac98c9 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Wed, 11 Apr 2018 20:56:05 +0200 Subject: [PATCH 87/87] std.zig.parser special cased error in return. Related #909 This allows parsing of `std/special/build_runner.zig` --- std/zig/parser.zig | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index f1f6b0e371..88e6ece35b 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -2354,6 +2354,16 @@ pub const Parser = struct { continue; }, else => { + // TODO: this is a special case. Remove this when #760 is fixed + if (token.id == Token.Id.Keyword_error) { + if (self.isPeekToken(Token.Id.LBrace)) { + fn_proto.return_type = ast.NodeFnProto.ReturnType { + .Explicit = &(try self.createLiteral(arena, ast.NodeErrorType, token)).base + }; + continue; + } + } + self.putBackToken(token); fn_proto.return_type = ast.NodeFnProto.ReturnType { .Explicit = undefined }; stack.append(State { @@ -5185,3 +5195,13 @@ test "zig fmt: string identifier" { \\ ); } + +test "zig fmt: error return" { + try testCanonical( + \\fn err() error { + \\ call(); + \\ return error.InvalidArgs; + \\} + \\ + ); +}