detect stack overflows

src/cparse.c

@@ -4,6 +4,7 @@
 #include "cchunk.h"
 #include "cdebug.h"
 #include "cmem.h"
+#include "cvm.h"
 
 #include <string.h>
 

src/cstate.h

@@ -33,6 +33,9 @@ typedef struct ArrayCObj {
 typedef struct CState {
     bool panic;
     int freezeGC; // when > 0, GC events will be ignored (for internal use)
+    int frameCount;
+
+    CObjObject *protoObj; // start met obj for all objects (NULL by default)
     CObj *objects; // tracks all of our allocated objects
     CObj *userRoots; // user definable roots, this holds CObjs that should be considered "roots", lets the VM know you are holding a reference to a CObj in your code
     ArrayCObj grayStack; // keeps track of which objects *haven't yet* been traversed in our GC, but *have been* found
@@ -44,12 +47,9 @@ typedef struct CState {
     CTable globals;
 
     CValue *top; // top of the stack
-    CValue stack[STACK_MAX]; // stack
-    CCallFrame callFrame[FRAME_MAX]; // call frames
-    int frameCount;
-
     CObjString *iStrings[ISTRING_MAX]; // strings used internally by the VM, eg. __init, __index & friends
-    CObjObject *protoObj; // start met obj for all objects (NULL by default)
+    CCallFrame callFrame[FRAME_MAX]; // call frames
+    CValue stack[STACK_MAX]; // stack
 } CState;
 
 COSMO_API CState *cosmoV_newState();
@@ -58,25 +58,4 @@ COSMO_API void cosmoV_register(CState *state, int pairs);
 COSMO_API void cosmoV_freeState(CState *state);
 COSMO_API void cosmoV_printStack(CState *state);
 
-// pushes value to the stack
-static inline void cosmoV_pushValue(CState *state, CValue val) {
-    *(state->top++) = val;
-}
-
-// sets stack->top to stack->top - indx
-static inline StkPtr cosmoV_setTop(CState *state, int indx) {
-    state->top -= indx;
-    return state->top;
-}
-
-// returns stack->top - indx - 1
-static inline StkPtr cosmoV_getTop(CState *state, int indx) {
-    return &state->top[-(indx + 1)];
-}
-
-// pops 1 value off the stack
-static inline StkPtr cosmoV_pop(CState *state) {
-    return cosmoV_setTop(state, 1);
-}
-
 #endif

src/cvm.c

@@ -18,6 +18,7 @@ COSMO_API void cosmoV_pushFString(CState *state, const char *format, ...) {
 void cosmoV_error(CState *state, const char *format, ...) {
     if (state->panic)
         return;
+    state->panic = true;
 
     // print stack trace
     for (int i = 0; i < state->frameCount; i++) {
@@ -50,8 +51,7 @@ void cosmoV_error(CState *state, const char *format, ...) {
     va_end(args);
 
     printf("%.*s\n", errString->length, errString->str);
-    cosmoV_printStack(state);
-    state->panic = true;
+    //cosmoV_printStack(state);
 }
 
 CObjUpval *captureUpvalue(CState *state, CValue *local) {
@@ -90,6 +90,11 @@ void closeUpvalues(CState *state, CValue *local) {
 }
 
 void pushCallFrame(CState *state, CObjClosure *closure, int args) {
+    if (state->frameCount >= FRAME_MAX) {
+        cosmoV_error(state, "Callframe overflow!");
+        return;
+    }
+
     CCallFrame *frame = &state->callFrame[state->frameCount++];
     frame->base = state->top - args - 1; // - 1 for the function
     frame->pc = closure->function->chunk.buf;

src/cvm.h

@@ -23,6 +23,42 @@ COSMO_API void cosmoV_error(CState *state, const char *format, ...);
 
 // nice to have wrappers
 
+// pushes value to the stack
+static inline void cosmoV_pushValue(CState *state, CValue val) {
+    ptrdiff_t stackSize = state->top - state->stack;
+
+    // we reserve 8 slots for the error string and whatever c api we might be in
+    if (stackSize >= STACK_MAX - 8) {
+        if (state->panic) { // we're in a panic state, let the 8 reserved slots be filled
+            if (stackSize < STACK_MAX)
+                *(state->top++) = val;
+            
+            return;
+        }
+
+        cosmoV_error(state, "Stack overflow!");
+        return;
+    }
+
+    *(state->top++) = val;
+}
+
+// sets stack->top to stack->top - indx
+static inline StkPtr cosmoV_setTop(CState *state, int indx) {
+    state->top -= indx;
+    return state->top;
+}
+
+// returns stack->top - indx - 1
+static inline StkPtr cosmoV_getTop(CState *state, int indx) {
+    return &state->top[-(indx + 1)];
+}
+
+// pops 1 value off the stack
+static inline StkPtr cosmoV_pop(CState *state) {
+    return cosmoV_setTop(state, 1);
+}
+
 // pushes a cosmo_Number to the stack
 static inline void cosmoV_pushNumber(CState *state, cosmo_Number num) {
     cosmoV_pushValue(state, cosmoV_newNumber(num));