mirror of
https://github.com/CPunch/Cosmo.git
synced 2024-11-05 08:10:05 +00:00
extended lexer, fixed table shrinking
This commit is contained in:
parent
bb11b3b309
commit
0745fd10a9
70
src/clex.c
70
src/clex.c
@ -25,12 +25,72 @@ CReservedWord reservedWords[] = {
|
|||||||
{TOKEN_WHILE, "while", 5}
|
{TOKEN_WHILE, "while", 5}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// returns true if current token is a heap allocated buffer
|
||||||
|
static bool isBuffer(CLexState *state) {
|
||||||
|
return state->buffer != NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
// marks the current token as heap allocated & allocates the buffer
|
||||||
|
static void makeBuffer(CLexState *state) {
|
||||||
|
state->buffer = cosmoM_xmalloc(state->cstate, sizeof(char) * 32); // start with a 32 character long buffer
|
||||||
|
state->bufCount = 0;
|
||||||
|
state->bufCap = 32;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void resetBuffer(CLexState *state) {
|
||||||
|
state->buffer = NULL;
|
||||||
|
state->bufCount = 0;
|
||||||
|
state->bufCap = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
// cancels the token heap buffer and free's it
|
||||||
|
static void freeBuffer(CLexState *state) {
|
||||||
|
cosmoM_freearray(state->cstate, char, state->buffer, state->bufCap);
|
||||||
|
|
||||||
|
resetBuffer(state);
|
||||||
|
}
|
||||||
|
|
||||||
|
// adds character to buffer
|
||||||
|
static void appendBuffer(CLexState *state, char c) {
|
||||||
|
cosmoM_growarray(state->cstate, char, state->buffer, state->bufCount, state->bufCap);
|
||||||
|
|
||||||
|
state->buffer[state->bufCount++] = c;
|
||||||
|
}
|
||||||
|
|
||||||
|
// saves the current character to the buffer, grows the buffer as needed
|
||||||
|
static void saveBuffer(CLexState *state) {
|
||||||
|
appendBuffer(state, *state->currentChar);
|
||||||
|
}
|
||||||
|
|
||||||
|
// resets the lex state buffer & returns the allocated buffer as a null terminated string
|
||||||
|
static char *cutBuffer(CLexState *state) {
|
||||||
|
// append the null terminator
|
||||||
|
appendBuffer(state, '\0');
|
||||||
|
|
||||||
|
// cache buffer info
|
||||||
|
char *buf = state->buffer;
|
||||||
|
size_t count = state->bufCount;
|
||||||
|
size_t cap = state->bufCap;
|
||||||
|
|
||||||
|
// reset lex state buffer!
|
||||||
|
resetBuffer(state);
|
||||||
|
|
||||||
|
// shrink the buffer to only use what we need
|
||||||
|
return cosmoM_reallocate(state->cstate, buf, cap, count);
|
||||||
|
}
|
||||||
|
|
||||||
static CToken makeToken(CLexState *state, CTokenType type) {
|
static CToken makeToken(CLexState *state, CTokenType type) {
|
||||||
CToken token;
|
CToken token;
|
||||||
token.type = type;
|
token.type = type;
|
||||||
|
token.line = state->line;
|
||||||
|
|
||||||
|
if (isBuffer(state)) { // is the buffer heap-allocated?
|
||||||
|
token.length = state->bufCount;
|
||||||
|
token.start = cutBuffer(state);
|
||||||
|
} else {
|
||||||
token.start = state->startChar;
|
token.start = state->startChar;
|
||||||
token.length = state->currentChar - state->startChar; // delta between start & current
|
token.length = state->currentChar - state->startChar; // delta between start & current
|
||||||
token.line = state->line;
|
}
|
||||||
|
|
||||||
state->lastType = type;
|
state->lastType = type;
|
||||||
|
|
||||||
@ -44,6 +104,9 @@ static CToken makeError(CLexState *state, const char *msg) {
|
|||||||
token.length = strlen(msg);
|
token.length = strlen(msg);
|
||||||
token.line = state->line;
|
token.line = state->line;
|
||||||
|
|
||||||
|
if (isBuffer(state))
|
||||||
|
freeBuffer(state);
|
||||||
|
|
||||||
return token;
|
return token;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -126,9 +189,11 @@ void skipWhitespace(CLexState *state) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
CToken parseString(CLexState *state) {
|
CToken parseString(CLexState *state) {
|
||||||
|
makeBuffer(state); // buffer mode
|
||||||
while (peek(state) != '"' && !isEnd(state)) {
|
while (peek(state) != '"' && !isEnd(state)) {
|
||||||
if (peek(state) == '\n') // strings can't stretch across lines
|
if (peek(state) == '\n') // strings can't stretch across lines
|
||||||
return makeError(state, "Unterminated string!");
|
return makeError(state, "Unterminated string!");
|
||||||
|
saveBuffer(state); // save the character!
|
||||||
next(state); // consume
|
next(state); // consume
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -170,6 +235,9 @@ CLexState *cosmoL_newLexState(CState *cstate, const char *source) {
|
|||||||
state->line = 1;
|
state->line = 1;
|
||||||
state->lastLine = 0;
|
state->lastLine = 0;
|
||||||
state->lastType = TOKEN_ERROR;
|
state->lastType = TOKEN_ERROR;
|
||||||
|
state->cstate = cstate;
|
||||||
|
|
||||||
|
resetBuffer(state);
|
||||||
|
|
||||||
return state;
|
return state;
|
||||||
}
|
}
|
||||||
|
@ -78,10 +78,14 @@ typedef struct {
|
|||||||
typedef struct {
|
typedef struct {
|
||||||
char *currentChar;
|
char *currentChar;
|
||||||
char *startChar;
|
char *startChar;
|
||||||
|
char *buffer; // if non-NULL & bufCount > 0, token->start & token->length will be set to buffer & bufCount respectively
|
||||||
|
size_t bufCount;
|
||||||
|
size_t bufCap;
|
||||||
int line; // current line
|
int line; // current line
|
||||||
int lastLine; // line of the previous consumed token
|
int lastLine; // line of the previous consumed token
|
||||||
bool isEnd;
|
bool isEnd;
|
||||||
CTokenType lastType;
|
CTokenType lastType;
|
||||||
|
CState *cstate;
|
||||||
} CLexState;
|
} CLexState;
|
||||||
|
|
||||||
CLexState *cosmoL_newLexState(CState *state, const char *source);
|
CLexState *cosmoL_newLexState(CState *state, const char *source);
|
||||||
|
@ -18,7 +18,7 @@
|
|||||||
#define cosmoM_growarray(state, type, buf, count, capacity) \
|
#define cosmoM_growarray(state, type, buf, count, capacity) \
|
||||||
if (count >= capacity || buf == NULL) { \
|
if (count >= capacity || buf == NULL) { \
|
||||||
int old = capacity; \
|
int old = capacity; \
|
||||||
capacity = old *GROW_FACTOR; \
|
capacity = old * GROW_FACTOR; \
|
||||||
buf = (type*)cosmoM_reallocate(state, buf, sizeof(type) *old, sizeof(type) *capacity); \
|
buf = (type*)cosmoM_reallocate(state, buf, sizeof(type) *old, sizeof(type) *capacity); \
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -372,7 +372,7 @@ static void number(CParseState *pstate, bool canAssign) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
static void string(CParseState *pstate, bool canAssign) {
|
static void string(CParseState *pstate, bool canAssign) {
|
||||||
CObjString *strObj = cosmoO_copyString(pstate->state, pstate->previous.start + 1, pstate->previous.length - 2);
|
CObjString *strObj = cosmoO_takeString(pstate->state, pstate->previous.start, pstate->previous.length);
|
||||||
writeConstant(pstate, cosmoV_newObj((CObj*)strObj));
|
writeConstant(pstate, cosmoV_newObj((CObj*)strObj));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -152,8 +152,7 @@ static void resizeTbl(CState *state, CTable *tbl, size_t newCapacity) {
|
|||||||
bool cosmoT_checkShrink(CState *state, CTable *tbl) {
|
bool cosmoT_checkShrink(CState *state, CTable *tbl) {
|
||||||
// if count > 8 and active entries < tombstones
|
// if count > 8 and active entries < tombstones
|
||||||
if (tbl->count > MIN_TABLE_CAPACITY && (tbl->count - tbl->tombstones < tbl->tombstones || tbl->tombstones > 50)) {
|
if (tbl->count > MIN_TABLE_CAPACITY && (tbl->count - tbl->tombstones < tbl->tombstones || tbl->tombstones > 50)) {
|
||||||
printf("shrinking table!\n");
|
tbl->tombstones = 0;
|
||||||
getchar();
|
|
||||||
resizeTbl(state, tbl, nextPow2((tbl->count - tbl->tombstones) * GROW_FACTOR)); // shrink based on active entries to the next pow of 2
|
resizeTbl(state, tbl, nextPow2((tbl->count - tbl->tombstones) * GROW_FACTOR)); // shrink based on active entries to the next pow of 2
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user