lex lit number and word

2 files changed, 56 insertions, 9 deletions

diff --git a/src/Lexer.zig b/src/Lexer.zig
index 9969bf1..2fc5dc9 100644
--- a/src/Lexer.zig
+++ b/src/Lexer.zig
@@ -7,6 +7,13 @@ allocator: std.mem.Allocator,
 tokens: std.ArrayList(Token),
 s: []const u8,
 
+const lower = "abcdefghijklmnopqrstuvwxyz";
+const upper = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
+const number = "0123456789";
+const alpha = lower ++ upper;
+const alnum = alpha ++ number;
+const whitespaces = " \t\n\r";
+
 pub fn init(
     allocator: std.mem.Allocator,
     list_allocator: std.mem.Allocator,
@@ -37,20 +44,60 @@ pub fn format(
 
 pub fn lex(self: *Self) !void {
     var i: usize = 0;
-    while (i < self.s.len) : (i += 1) {
+    while (i < self.s.len) {
         const c = self.s[i];
         const c2 = if (i < self.s.len - 1) self.s[i + 1] else 0;
 
-        if (std.ascii.isWhitespace(c))
+        if (match(self.s[i..], whitespaces)) |len| {
+            i += len;
             continue;
+        }
 
         if (c == '/' and c2 == '/') {
-            i += 2;
-            while (self.s[i] != '\n' and i < self.s.len)
-                i += 1;
+            i += until(self.s[i + 2 ..], "\n") orelse break;
+            continue;
+        }
+
+        if (match(self.s[i..], number)) |len| {
+            const postfix = match(self.s[i + len ..], alnum) orelse 0;
+            try self.tokens.append(Token.init(
+                .number,
+                self.s[i .. i + len + postfix],
+            ));
+            i += len + postfix;
+            continue;
+        }
+
+        if (match(self.s[i..], alnum)) |len| {
+            try self.tokens.append(Token.init(.word, self.s[i .. i + len]));
+            i += len;
             continue;
         }
 
-        try self.tokens.append(Token.init(.punct, self.s[i .. i + 1]));
+        std.log.warn("skipped '{c}'", .{c});
+        i += 1;
+    }
+}
+
+fn until(s: []const u8, comptime c: []const u8) ?usize {
+    for (s, 0..) |e, i|
+        inline for (c) |f|
+            if (e == f)
+                return i;
+    return null;
+}
+
+fn match(s: []const u8, comptime c: []const u8) ?usize {
+    for (s, 0..) |e, i| {
+        var valid = false;
+        inline for (c) |f| {
+            if (e == f) {
+                valid = true;
+                break;
+            }
+        }
+        if (!valid)
+            return if (i == 0) null else i;
     }
+    return if (s.len == 0) null else s.len;
 }
diff --git a/test.o7 b/test.o7
index edd2c05..0c81f37 100644
--- a/test.o7
+++ b/test.o7
@@ -1,3 +1,3 @@
-a  b     c     		d	 e 		f      
-g // simple comment
-h
+var x = 25
+// lol
+ptdr = x * 42 - xd23 42d