lexer progress

4 files changed, 64 insertions, 20 deletions

diff --git a/src/Lexer.zig b/src/Lexer.zig
index 0b9196c..08ae6a8 100644
--- a/src/Lexer.zig
+++ b/src/Lexer.zig
@@ -6,6 +6,8 @@ const Self = @This();
 allocator: std.mem.Allocator,
 tokens: std.ArrayList(Token),
 s: []const u8,
+line: usize = 0,
+line_start: usize = 0,
 
 const lower = "abcdefghijklmnopqrstuvwxyz";
 const upper = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
@@ -13,7 +15,11 @@ const number = "0123456789";
 const alpha = lower ++ upper;
 const alnum = alpha ++ number;
 const whitespaces = " \t\n\r";
-const punct = "(){}=+-*/";
+const punct = "@(){}=+-*/.;";
+pub const Error = error{
+    UnclosedCharLitteral,
+    UnclosedStringLitteral,
+};
 
 pub fn init(
     allocator: std.mem.Allocator,
@@ -37,25 +43,35 @@ pub fn format(
     _: std.fmt.FormatOptions,
     writer: std.io.AnyWriter,
 ) !void {
-    return writer.print(
-        "Lexer{{ .tokens = {any}, ... }}",
-        .{self.tokens.items},
-    );
+    try writer.print("Lexer{{ .tokens = {{\n", .{});
+    for (self.tokens.items) |e|
+        try writer.print("\t{any},\n", .{e});
+    return writer.print("}}, ... }}", .{});
+}
+
+pub fn minimized(self: Self, writer: std.io.AnyWriter) !void {
+    for (self.tokens.items) |e|
+        try writer.print("{s} ", .{e.s});
+}
+
+fn nextLine(self: *Self, i: usize) void {
+    self.line_start = i;
+    self.line += 1;
 }
 
 pub fn lex(self: *Self) !void {
     var i: usize = 0;
     while (i < self.s.len) {
-        const c = self.s[i];
-        const c2 = if (i < self.s.len - 1) self.s[i + 1] else 0;
-
-        if (match(self.s[i..], whitespaces)) |len| {
-            i += len;
+        if (contains(self.s[i], whitespaces)) {
+            if (self.s[i] == '\n')
+                self.nextLine(i + 1);
+            i += 1;
             continue;
         }
 
-        if (c == '/' and c2 == '/') {
-            i += until(self.s[i + 2 ..], "\n") orelse break;
+        if (self.s[i] == '#') {
+            i += until(self.s[i..], "\n") orelse break;
+            self.nextLine(i);
             continue;
         }
 
@@ -78,13 +94,34 @@ pub fn lex(self: *Self) !void {
             continue;
         }
 
-        std.log.warn("skipped '{c}'", .{c});
+        if (self.s[i] == '\'') {
+            const len = 2 + (until(self.s[i + 1 ..], "'") orelse
+                return Error.UnclosedCharLitteral);
+            try self.newToken(i, len, .character);
+            i += len;
+            continue;
+        }
+
+        if (self.s[i] == '"') {
+            const len = 2 + (until(self.s[i + 1 ..], "\"") orelse
+                return Error.UnclosedStringLitteral);
+            try self.newToken(i, len, .string);
+            i += len;
+            continue;
+        }
+
+        std.log.warn("skipped '{c}'", .{self.s[i]});
         i += 1;
     }
 }
 
 fn newToken(self: *Self, i: usize, len: usize, tag: Token.Tag) !void {
-    return self.tokens.append(Token.init(tag, self.s[i .. i + len]));
+    return self.tokens.append(Token.init(
+        tag,
+        self.s[i .. i + len],
+        i - self.line_start + 1,
+        self.line + 1,
+    ));
 }
 
 fn until(s: []const u8, comptime c: []const u8) ?usize {
diff --git a/src/Token.zig b/src/Token.zig
index cd5257c..bfb1cbd 100644
--- a/src/Token.zig
+++ b/src/Token.zig
@@ -4,19 +4,24 @@ const Self = @This();
 
 tag: Tag,
 s: []const u8,
+x: usize,
+y: usize,
 
 pub const Tag = enum {
     number,
     keyword,
     word,
     punct,
+    character,
     string,
 };
 
-pub fn init(tag: Tag, s: []const u8) Self {
+pub fn init(tag: Tag, s: []const u8, x: usize, y: usize) Self {
     return Self{
         .tag = tag,
         .s = s,
+        .x = x,
+        .y = y,
     };
 }
 
@@ -27,7 +32,7 @@ pub fn format(
     writer: std.io.AnyWriter,
 ) !void {
     return writer.print(
-        "Token{{ .tag = {}, .s = \"{s}\" }}",
-        .{ self.tag, self.s },
+        "Token{{ {}:{}, {}, \"{s}\" }}",
+        .{ self.x, self.y, self.tag, self.s },
     );
 }
diff --git a/src/main.zig b/src/main.zig
index 0b35c9a..9e08fd5 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -22,6 +22,7 @@ pub fn main() !void {
     try lexer.lex();
 
     std.log.debug("{}", .{lexer});
+    try lexer.minimized(std.io.getStdOut().writer().any());
 }
 
 const ArgError = error{
diff --git a/test.o7 b/test.o7
index 0c81f37..531ed70 100644
--- a/test.o7
+++ b/test.o7
@@ -1,3 +1,4 @@
-var x = 25
-// lol
-ptdr = x * 42 - xd23 42d
+mut x = 25; # commentaire
+mut y = x * x + (8 - 2);
+let zied = golem("large");
+let doc = @as(Prof, zied);