extended lexer, fixed table shrinking

This commit is contained in:
CPunch 2020-11-25 23:34:02 -06:00
parent bb11b3b309
commit 0745fd10a9
6 changed files with 78 additions and 7 deletions

View File

@ -25,12 +25,72 @@ CReservedWord reservedWords[] = {
{TOKEN_WHILE, "while", 5} {TOKEN_WHILE, "while", 5}
}; };
// returns true if current token is a heap allocated buffer
static bool isBuffer(CLexState *state) {
return state->buffer != NULL;
}
// marks the current token as heap allocated & allocates the buffer
static void makeBuffer(CLexState *state) {
state->buffer = cosmoM_xmalloc(state->cstate, sizeof(char) * 32); // start with a 32 character long buffer
state->bufCount = 0;
state->bufCap = 32;
}
static void resetBuffer(CLexState *state) {
state->buffer = NULL;
state->bufCount = 0;
state->bufCap = 0;
}
// cancels the token heap buffer and free's it
static void freeBuffer(CLexState *state) {
cosmoM_freearray(state->cstate, char, state->buffer, state->bufCap);
resetBuffer(state);
}
// adds character to buffer
static void appendBuffer(CLexState *state, char c) {
cosmoM_growarray(state->cstate, char, state->buffer, state->bufCount, state->bufCap);
state->buffer[state->bufCount++] = c;
}
// saves the current character to the buffer, grows the buffer as needed
static void saveBuffer(CLexState *state) {
appendBuffer(state, *state->currentChar);
}
// resets the lex state buffer & returns the allocated buffer as a null terminated string
static char *cutBuffer(CLexState *state) {
// append the null terminator
appendBuffer(state, '\0');
// cache buffer info
char *buf = state->buffer;
size_t count = state->bufCount;
size_t cap = state->bufCap;
// reset lex state buffer!
resetBuffer(state);
// shrink the buffer to only use what we need
return cosmoM_reallocate(state->cstate, buf, cap, count);
}
static CToken makeToken(CLexState *state, CTokenType type) { static CToken makeToken(CLexState *state, CTokenType type) {
CToken token; CToken token;
token.type = type; token.type = type;
token.line = state->line;
if (isBuffer(state)) { // is the buffer heap-allocated?
token.length = state->bufCount;
token.start = cutBuffer(state);
} else {
token.start = state->startChar; token.start = state->startChar;
token.length = state->currentChar - state->startChar; // delta between start & current token.length = state->currentChar - state->startChar; // delta between start & current
token.line = state->line; }
state->lastType = type; state->lastType = type;
@ -44,6 +104,9 @@ static CToken makeError(CLexState *state, const char *msg) {
token.length = strlen(msg); token.length = strlen(msg);
token.line = state->line; token.line = state->line;
if (isBuffer(state))
freeBuffer(state);
return token; return token;
} }
@ -126,9 +189,11 @@ void skipWhitespace(CLexState *state) {
} }
CToken parseString(CLexState *state) { CToken parseString(CLexState *state) {
makeBuffer(state); // buffer mode
while (peek(state) != '"' && !isEnd(state)) { while (peek(state) != '"' && !isEnd(state)) {
if (peek(state) == '\n') // strings can't stretch across lines if (peek(state) == '\n') // strings can't stretch across lines
return makeError(state, "Unterminated string!"); return makeError(state, "Unterminated string!");
saveBuffer(state); // save the character!
next(state); // consume next(state); // consume
} }
@ -170,6 +235,9 @@ CLexState *cosmoL_newLexState(CState *cstate, const char *source) {
state->line = 1; state->line = 1;
state->lastLine = 0; state->lastLine = 0;
state->lastType = TOKEN_ERROR; state->lastType = TOKEN_ERROR;
state->cstate = cstate;
resetBuffer(state);
return state; return state;
} }

View File

@ -78,10 +78,14 @@ typedef struct {
typedef struct { typedef struct {
char *currentChar; char *currentChar;
char *startChar; char *startChar;
char *buffer; // if non-NULL & bufCount > 0, token->start & token->length will be set to buffer & bufCount respectively
size_t bufCount;
size_t bufCap;
int line; // current line int line; // current line
int lastLine; // line of the previous consumed token int lastLine; // line of the previous consumed token
bool isEnd; bool isEnd;
CTokenType lastType; CTokenType lastType;
CState *cstate;
} CLexState; } CLexState;
CLexState *cosmoL_newLexState(CState *state, const char *source); CLexState *cosmoL_newLexState(CState *state, const char *source);

View File

@ -18,7 +18,7 @@
#define cosmoM_growarray(state, type, buf, count, capacity) \ #define cosmoM_growarray(state, type, buf, count, capacity) \
if (count >= capacity || buf == NULL) { \ if (count >= capacity || buf == NULL) { \
int old = capacity; \ int old = capacity; \
capacity = old *GROW_FACTOR; \ capacity = old * GROW_FACTOR; \
buf = (type*)cosmoM_reallocate(state, buf, sizeof(type) *old, sizeof(type) *capacity); \ buf = (type*)cosmoM_reallocate(state, buf, sizeof(type) *old, sizeof(type) *capacity); \
} }

View File

@ -372,7 +372,7 @@ static void number(CParseState *pstate, bool canAssign) {
} }
static void string(CParseState *pstate, bool canAssign) { static void string(CParseState *pstate, bool canAssign) {
CObjString *strObj = cosmoO_copyString(pstate->state, pstate->previous.start + 1, pstate->previous.length - 2); CObjString *strObj = cosmoO_takeString(pstate->state, pstate->previous.start, pstate->previous.length);
writeConstant(pstate, cosmoV_newObj((CObj*)strObj)); writeConstant(pstate, cosmoV_newObj((CObj*)strObj));
} }

View File

@ -152,8 +152,7 @@ static void resizeTbl(CState *state, CTable *tbl, size_t newCapacity) {
bool cosmoT_checkShrink(CState *state, CTable *tbl) { bool cosmoT_checkShrink(CState *state, CTable *tbl) {
// if count > 8 and active entries < tombstones // if count > 8 and active entries < tombstones
if (tbl->count > MIN_TABLE_CAPACITY && (tbl->count - tbl->tombstones < tbl->tombstones || tbl->tombstones > 50)) { if (tbl->count > MIN_TABLE_CAPACITY && (tbl->count - tbl->tombstones < tbl->tombstones || tbl->tombstones > 50)) {
printf("shrinking table!\n"); tbl->tombstones = 0;
getchar();
resizeTbl(state, tbl, nextPow2((tbl->count - tbl->tombstones) * GROW_FACTOR)); // shrink based on active entries to the next pow of 2 resizeTbl(state, tbl, nextPow2((tbl->count - tbl->tombstones) * GROW_FACTOR)); // shrink based on active entries to the next pow of 2
return true; return true;
} }