diff options
Diffstat (limited to 'src/tokenize.c')
| -rw-r--r-- | src/tokenize.c | 14 |
1 files changed, 5 insertions, 9 deletions
diff --git a/src/tokenize.c b/src/tokenize.c index 179b251..02059a8 100644 --- a/src/tokenize.c +++ b/src/tokenize.c @@ -46,7 +46,7 @@ static size_t peek_alnum_to(struct ctx *ctx, size_t start, char ch) errln(ctx, "Unexpected end of buffer"); } -static size_t peek_special_to(struct ctx *ctx, size_t start, char ch) +static size_t peek_identifier(struct ctx *ctx, size_t start, char ch) { for (size_t i = start; i < ctx->size; i++) { char cur = ctx->data[i]; @@ -54,8 +54,8 @@ static size_t peek_special_to(struct ctx *ctx, size_t start, char ch) if (cur == ch || cur == ';' || cur == ')') return i; - if (isalnum(cur) || cur < '!' || cur > '~') - errln(ctx, "'%c' is not special", cur); + if (!isalnum(cur) && (cur < '!' || cur > '~')) + errln(ctx, "'%c' is not an identifier", cur); } errln(ctx, "Unexpected end of buffer"); @@ -156,14 +156,10 @@ void tokenize(struct ctx *ctx) continue; } - if (peek_to_is_alnum(ctx, i, ' ')) { // General identifier - size_t end_ident = peek_alnum_to(ctx, i, ' '); + if (peek_identifier(ctx, i, ' ')) { // General identifier + size_t end_ident = peek_to(ctx, i, ' '); token_add(ctx, IDENT, i, end_ident); i = end_ident - 1; - } else { // Special/custom operator - size_t end_operator = peek_special_to(ctx, i, ' '); - token_add(ctx, OPERATOR, i, end_operator); - i = end_operator - 1; } } |