From bfdc8d2d843c08bd01517256a63518d03da236f6 Mon Sep 17 00:00:00 2001
From: Marvin Borner
Date: Tue, 27 Jul 2021 21:10:34 +0200
Subject: Started treeify

---
 inc/tokenize.h | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)
 create mode 100644 inc/tokenize.h

(limited to 'inc/tokenize.h')

diff --git a/inc/tokenize.h b/inc/tokenize.h
new file mode 100644
index 0000000..0142bb9
--- /dev/null
+++ b/inc/tokenize.h
@@ -0,0 +1,35 @@
+#ifndef TOKENIZE_H
+#define TOKENIZE_H
+
+#include <context.h>
+
+#define TOKENS_MAX 4096
+
+enum token_type {
+	UNKNOWN,
+
+	TYPE,
+	TYPEDELIM,
+	PARAM,
+
+	IDENT,
+	OPERATOR,
+
+	LPAREN,
+	RPAREN,
+	EQUAL,
+
+	NEWLINE,
+	EOL,
+	END,
+};
+
+struct token {
+	enum token_type type;
+	ctx_string string;
+};
+
+void token_print(struct ctx *ctx, struct token *tok);
+void tokenize(struct ctx *ctx);
+
+#endif
-- 
cgit v1.2.3