48 lines
1.0 KiB
C
48 lines
1.0 KiB
C
#include "tokenizer.h"
|
|
|
|
Tokenizer tokenizerCreate(const char *source) {
|
|
int line = source ? 1 : 0;
|
|
return (Tokenizer) {source, line};
|
|
}
|
|
|
|
static bool isWhitespace(char c) {
|
|
switch (c) {
|
|
case '\n':
|
|
case ' ':
|
|
case '\r':
|
|
case '\t':
|
|
return true;
|
|
default:
|
|
return false;
|
|
}
|
|
}
|
|
|
|
static void skipWhitespace(Tokenizer *tokenizer) {
|
|
while (1) {
|
|
char c = *tokenizer->current;
|
|
if (c == '\n') {
|
|
tokenizer->line++;
|
|
continue;
|
|
}
|
|
if (!isWhitespace(c))
|
|
break;
|
|
}
|
|
}
|
|
|
|
Token tokenizerScan(Tokenizer *tokenizer) {
|
|
skipWhitespace(tokenizer);
|
|
Token token = {tokenizer->current, 0, tokenizer->line};
|
|
|
|
if (*tokenizer->current == '\0') {
|
|
token.start = NULL;
|
|
return token;
|
|
}
|
|
|
|
while (!isWhitespace(*tokenizer->current) || *tokenizer->current != '\0') {
|
|
tokenizer->current++;
|
|
}
|
|
token.length = (int) (tokenizer->current - token.start);
|
|
|
|
return token;
|
|
}
|