1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
|
#include "lexer.h"
#include <ctype.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
static Token *resize_toks(Token *toks, size_t *size)
{
*size += 128;
Token *const new_toks = realloc(toks, sizeof(Token) * (*size + 1));
if (new_toks == NULL) {
perror("resize_toks");
lexer_free(toks);
return NULL;
}
return new_toks;
}
static unsigned int one_wide_tok(const char *s)
{
switch (*s) {
case '(': return TOK_PAREN_OPEN;
case ')': return TOK_PAREN_CLOS;
case '{': return TOK_CURL_OPEN;
case '}': return TOK_CURL_CLOS;
case '[': return TOK_SQUAR_OPEN;
case ']': return TOK_SQUAR_CLOS;
case ':': return TOK_COLON;
case ';': return TOK_SEMICOLON;
case '=': return TOK_ASSIGN;
case ',': return TOK_COMMA;
case '<': return TOK_COMP_LESS;
case '%': return TOK_MODULO;
case '*': return TOK_STAR;
default: return TOK_NONE;
}
}
#define PAIR(a, b) ((int)(a) + (int)(b) * 256)
static unsigned int two_wide_tok(const char *s)
{
switch (PAIR(s[0], s[1])) {
case PAIR('+', '+'): return TOK_INCREMENT;
case PAIR('=', '='): return TOK_COMP_EQ;
case PAIR('!', '='): return TOK_COMP_NEQ;
default: return TOK_NONE;
}
}
Token *lexer(const char *s)
{
size_t size = 128;
Token *toks = calloc(size + 1, sizeof(Token));
size_t tok_i = 0;
size_t column = 1;
size_t line = 1;
if (toks == NULL) {
perror("lexer");
return NULL;
}
while (*s != '\0') {
/* Skip whitespaces. */
while (isspace(*s)) {
if (*s == '\n') {
column = 1;
line += 1;
} else
column += 1;
s += 1;
}
if (*s == '\0')
break;
toks[tok_i].column = column;
toks[tok_i].line = line;
const char *rem_s = s;
if (*s == '"') {
const char *end = strchr(s + 1, '"');
if (end == NULL) {
printf("unclosed string\n");
lexer_free(toks);
return NULL;
}
size_t len = end - s - 1;
toks[tok_i].s = calloc(1, len + 1);
if (toks[tok_i].s == NULL) {
perror("lexer");
lexer_free(toks);
return NULL;
}
toks[tok_i].type = TOK_STRING;
strncpy(toks[tok_i].s, s + 1, len);
tok_i += 1;
s = end + 1;
} else if (isalpha(*s) || *s == '_') {
size_t len = 0;
while (isalnum(s[len]) || s[len] == '_')
len += 1;
toks[tok_i].s = calloc(1, len + 1);
if (toks[tok_i].s == NULL) {
perror("lexer");
lexer_free(toks);
return NULL;
}
strncpy(toks[tok_i].s, s, len);
if (strcmp("fn", toks[tok_i].s) == 0)
toks[tok_i].type = TOK_KW_FN;
else if (strcmp("var", toks[tok_i].s) == 0)
toks[tok_i].type = TOK_KW_VAR;
else if (strcmp("const", toks[tok_i].s) == 0)
toks[tok_i].type = TOK_KW_CONST;
else
toks[tok_i].type = TOK_WORD;
tok_i += 1;
s += len;
} else if (isdigit(*s)) {
size_t len = 0;
while (isdigit(s[len]))
len += 1;
toks[tok_i].v.i = atoi(s);
toks[tok_i].type = TOK_INTEGER;
tok_i += 1;
s += len;
} else if (two_wide_tok(s) != TOK_NONE) {
toks[tok_i].type = two_wide_tok(s);
toks[tok_i].v.c = *s;
tok_i += 1;
s += 2;
} else if (one_wide_tok(s) != TOK_NONE) {
toks[tok_i].type = one_wide_tok(s);
toks[tok_i].v.c = *s;
tok_i += 1;
s += 1;
} else {
printf("wtf is this shit? %c\n", *s);
s += 1;
}
if (tok_i == size) {
toks = resize_toks(toks, &size);
if (toks == NULL)
return NULL;
}
column += s - rem_s;
}
return toks;
}
void lexer_free(Token *toks)
{
if (toks != NULL) {
for (Token *tok = toks; tok->type != TOK_NONE; tok += 1)
token_free(tok);
free(toks);
}
}
void lexer_print(const Token *toks)
{
for (const Token *tok = toks; tok->type != TOK_NONE; tok += 1)
token_print(tok);
}
|
