flatten lexer output

5 files changed, 30 insertions, 7 deletions

diff --git a/lexer.c b/lexer.c
index a3394be..59038e7 100644
--- a/lexer.c
+++ b/lexer.c
@@ -25,7 +25,7 @@ simple_token(char c)
 	case ']': return TOK_CLOS_SQUAR;
 	case '=': return TOK_ASSIGN;
 	case ';': return TOK_END;
-	default: return TOK_NONE;
+	default:  return TOK_NONE;
 	}
 }
 
@@ -38,6 +38,25 @@ slice_match(Slice slice, int (*isfun)(int c))
 	return slice;
 }
 
+static Token *
+flatten(Token *toks)
+{
+	int size = 0;
+	for (Token *t = toks; t != NULL; t = t->next)
+		size += 1;
+	Token *flat = calloc(1, sizeof(Token) * (size + 1));
+	if (flat == NULL)
+		return NULL;
+	int i = 0;
+	for (Token *t = toks; t != NULL; t = t->next) {
+		flat[i] = *t;
+		if (i + 1 < size)
+			flat[i].next = flat + i + 1;
+		i += 1;
+	}
+	return flat;
+}
+
 Token *
 lexer(Slice slice)
 {
@@ -121,5 +140,7 @@ lexer(Slice slice)
 		printf("skipping '%c'\n", slice.str[i]);
 		i += 1;
 	}
-	return toks;
+	Token *flat = flatten(toks);
+	token_destroy(toks);
+	return flat;
 }
diff --git a/lexer.h b/lexer.h
index 5a9efa9..818c2b6 100644
--- a/lexer.h
+++ b/lexer.h
@@ -2,5 +2,5 @@
 #include "token.h"
 #include "slice.h"
 
-// Return NULL on error.
+// Creates an array of tokens from source slice. Return NULL on error.
 Token *lexer(Slice slice);
diff --git a/main.c b/main.c
index c724bd9..48f5189 100644
--- a/main.c
+++ b/main.c
@@ -23,8 +23,8 @@ main(int argc, char **argv)
 	fclose(file);
 
 	Token *tokens = lexer(slice);
-	token_print(tokens);
-	token_destroy(tokens);
+	token_print(tokens, 1);
+	free(tokens);
 
 	if (slice.str != NULL)
 		free(slice.str);
diff --git a/token.c b/token.c
index 51aaab3..11ed955 100644
--- a/token.c
+++ b/token.c
@@ -43,12 +43,14 @@ token_destroy(Token *token)
 }
 
 Token *
-token_print(Token *token)
+token_print(Token *token, int recurse)
 {
 	Token *tok = token;
 	while (token != NULL) {
 		printf("type: %c  slice: ", (char)token->type);
 		slice_print(token->slice);
+		if (!recurse)
+			return tok;
 		token = token->next;
 	}
 	return tok;
diff --git a/token.h b/token.h
index 5934f7b..2c7e5e5 100644
--- a/token.h
+++ b/token.h
@@ -33,4 +33,4 @@ struct Token {
 Token *token_create(Slice slice, unsigned int type);
 Token *token_append(Token **list, Token *elem);
 void token_destroy(Token *token);
-Token *token_print(Token *token);
+Token *token_print(Token *token, int recurse);