Started treeify

1 files changed, 35 insertions, 0 deletions

diff --git a/inc/tokenize.h b/inc/tokenize.h
new file mode 100644
index 0000000..0142bb9
--- /dev/null
+++ b/inc/tokenize.h
@@ -0,0 +1,35 @@
+#ifndef TOKENIZE_H
+#define TOKENIZE_H
+
+#include <context.h>
+
+#define TOKENS_MAX 4096
+
+enum token_type {
+	UNKNOWN,
+
+	TYPE,
+	TYPEDELIM,
+	PARAM,
+
+	IDENT,
+	OPERATOR,
+
+	LPAREN,
+	RPAREN,
+	EQUAL,
+
+	NEWLINE,
+	EOL,
+	END,
+};
+
+struct token {
+	enum token_type type;
+	ctx_string string;
+};
+
+void token_print(struct ctx *ctx, struct token *tok);
+void tokenize(struct ctx *ctx);
+
+#endif