jimzah/zig
1
0
Fork 0
mirror of https://github.com/ziglang/zig.git synced 2024-11-29 00:22:33 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

self hosted: add tokenizer test fix eof handling

Browse Source
This commit is contained in:
Andrew Kelley 2018-02-13 10:28:55 -05:00
parent dfbb8254ca
commit 2dcff95bd2
1 changed files with 30 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
std/zig/tokenizer.zig
View File

@ -72,6 +72,8 @@ pub const Token = struct {
Eof,
Builtin,
Bang,
Pipe,
PipeEqual,
Equal,
EqualEqual,
BangEqual,
@ -193,6 +195,7 @@ pub const Tokenizer = struct {
StringLiteralBackslash,
Equal,
Bang,
Pipe,
Minus,
Slash,
LineComment,
@ -248,6 +251,9 @@ pub const Tokenizer = struct {
'!' => {
state = State.Bang;
},
'|' => {
state = State.Pipe;
},
'(' => {
result.id = Token.Id.LParen;
self.index += 1;
@ -394,6 +400,18 @@ pub const Tokenizer = struct {
},
},
State.Pipe => switch (c) {
'=' => {
result.id = Token.Id.PipeEqual;
self.index += 1;
break;
},
else => {
result.id = Token.Id.Pipe;
break;
},
},
State.Equal => switch (c) {
'=' => {
result.id = Token.Id.EqualEqual;
@ -525,9 +543,7 @@ pub const Tokenizer = struct {
else => break,
},
}
}
result.end = self.index;
if (self.index == self.buffer.len) {
} else if (self.index == self.buffer.len) {
switch (state) {
State.Start,
State.C,
@ -578,6 +594,9 @@ pub const Tokenizer = struct {
State.Period2 => {
result.id = Token.Id.Ellipsis2;
},
State.Pipe => {
result.id = Token.Id.Pipe;
},
}
}
if (result.id == Token.Id.Eof) {
@ -587,6 +606,7 @@ pub const Tokenizer = struct {
}
}
result.end = self.index;
return result;
}
@ -716,6 +736,13 @@ test "tokenizer - string identifier and builtin fns" {
);
}
test "tokenizer - pipe and then invalid" {
testTokenize("||=", []Token.Id{
Token.Id.Pipe,
Token.Id.PipeEqual,
});
}
fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void {
var tokenizer = Tokenizer.init(source);
for (expected_tokens) |expected_token_id| {