Handle OOM error on stack resize in coroutine.resume and lua_checkstack.

Thanks to Peter Cawley.

(cherry picked from commit a5d2f70)

If the OOM or stack overflow error is obtained during
`lj_state_growstack()` in the `coroutine.resume()` it may raise an error
leading to the incorrect error message and unexpected behaviour for
this corner case. Also, `lua_checkstack()` may raise an OOM error
leading to panic on an unprotected lua_State, for example.

This patch handles these cases by protecting `lj_state_growstack()`.

Sergey Kaplun:
* added the description and the test for the problem

Part of tarantool/tarantool#12134

Reviewed-by: Sergey Bronnikov <sergeyb@tarantool.org>
Signed-off-by: Sergey Kaplun <skaplun@tarantool.org>
(cherry picked from commit 357a260)

Author: Mike Pall

src/lib_base.c

@@ -606,7 +606,10 @@ static int ffh_resume(lua_State *L, lua_State *co, int wrap)
     setstrV(L, L->base-LJ_FR2, lj_err_str(L, em));
     return FFH_RES(2);
   }
-  lj_state_growstack(co, (MSize)(L->top - L->base));
+  if (lj_state_cpgrowstack(co, (MSize)(L->top - L->base)) != LUA_OK) {
+    cTValue *msg = --co->top;
+    lj_err_callermsg(L, strVdata(msg));
+  }
   return FFH_RETRY;
 }
 

src/lj_api.c

@@ -116,7 +116,12 @@ LUA_API int lua_checkstack(lua_State *L, int size)
   if (size > LUAI_MAXCSTACK || (L->top - L->base + size) > LUAI_MAXCSTACK) {
     return 0;  /* Stack overflow. */
   } else if (size > 0) {
-    lj_state_checkstack(L, (MSize)size);
+    int avail = (int)(mref(L->maxstack, TValue) - L->top);
+    if (size > avail &&
+	lj_state_cpgrowstack(L, (MSize)(size - avail)) != LUA_OK) {
+      L->top--;
+      return 0;  /* Out of memory. */
+    }
   }
   return 1;
 }

src/lj_state.c

@@ -170,6 +170,18 @@ void LJ_FASTCALL lj_state_growstack1(lua_State *L)
   lj_state_growstack(L, 1);
 }
 
+static TValue *cpgrowstack(lua_State *co, lua_CFunction dummy, void *ud)
+{
+  UNUSED(dummy);
+  lj_state_growstack(co, *(MSize *)ud);
+  return NULL;
+}
+
+int LJ_FASTCALL lj_state_cpgrowstack(lua_State *L, MSize need)
+{
+  return lj_vm_cpcall(L, NULL, &need, cpgrowstack);
+}
+
 /* Allocate basic stack for new state. */
 static void stack_init(lua_State *L1, lua_State *L)
 {

src/lj_state.h

@@ -18,6 +18,7 @@ LJ_FUNC void lj_state_relimitstack(lua_State *L);
 LJ_FUNC void lj_state_shrinkstack(lua_State *L, MSize used);
 LJ_FUNCA void LJ_FASTCALL lj_state_growstack(lua_State *L, MSize need);
 LJ_FUNC void LJ_FASTCALL lj_state_growstack1(lua_State *L);
+LJ_FUNC int LJ_FASTCALL lj_state_cpgrowstack(lua_State *L, MSize need);
 
 static LJ_AINLINE void lj_state_checkstack(lua_State *L, MSize need)
 {

test/tarantool-c-tests/lj-1066-err-lua-checkstack.test.c

@@ -0,0 +1,84 @@
+#include "lua.h"
+#include "luaconf.h"
+
+#include "test.h"
+#include "utils.h"
+
+#include <stdlib.h>
+
+/* XXX: Still need normal assert for sanity checks. */
+#undef NDEBUG
+#include <assert.h>
+
+static lua_Alloc old_allocf = NULL;
+static void *old_alloc_state = NULL;
+
+/* Always OOM on reallocation (not on allocation). */
+static void *allocf_null_realloc(void *ud, void *ptr, size_t osize,
+				 size_t nsize)
+{
+	assert(old_allocf != NULL);
+	if (ptr != NULL && osize < nsize)
+		return NULL;
+	else
+		return old_allocf(ud, ptr, osize, nsize);
+}
+
+static void enable_allocinject(lua_State *L)
+{
+	assert(old_allocf == NULL);
+	old_allocf = lua_getallocf(L, &old_alloc_state);
+	lua_setallocf(L, allocf_null_realloc, old_alloc_state);
+}
+
+/* Restore the default allocator function. */
+static void disable_allocinject(lua_State *L)
+{
+	assert(old_allocf != NULL);
+	lua_setallocf(L, old_allocf, old_alloc_state);
+	old_allocf = NULL;
+	old_alloc_state = NULL;
+}
+
+static int checkstack_res = -1;
+
+static int test_checkstack(lua_State *L)
+{
+	/*
+	 * There is no stack overflow error, but the OOM error
+	 * due to stack reallocation, before the patch.
+	 */
+	checkstack_res = lua_checkstack(L, LUAI_MAXCSTACK / 2);
+	return 0;
+}
+
+static int oom_on_lua_checkstack(void *test_state)
+{
+	lua_State *L = test_state;
+	/* Use fresh-new coroutine for stack manipulations. */
+	lua_State *L1 = lua_newthread(L);
+
+	enable_allocinject(L);
+	/*
+	 * `L1` should have enough space to `cpcall()` without
+	 * stack reallocation.
+	 */
+	int status = lua_cpcall(L1, test_checkstack, NULL);
+	disable_allocinject(L);
+
+	assert_true(status == LUA_OK);
+	assert_true(checkstack_res == 0);
+
+	return TEST_EXIT_SUCCESS;
+}
+
+int main(void)
+{
+	lua_State *L = utils_lua_init();
+	const struct test_unit tgroup[] = {
+		test_unit_def(oom_on_lua_checkstack),
+	};
+	const int test_result = test_run_group(tgroup, L);
+	utils_lua_close(L);
+	return test_result;
+}

test/tarantool-tests/lj-1066-err-coroutine-resume.test.lua

@@ -0,0 +1,27 @@
+local tap = require('tap')
+
+-- Test file to demonstrate LuaJIT misbehaviour in case of error
+-- on growing stack for the coroutine during `coroutine.resume()`.
+-- See also: https://github.com/LuaJIT/LuaJIT/issues/1066.
+
+local test = tap.test('lj-1066-err-coroutine-resume')
+
+test:plan(2)
+
+local function resume_wrap()
+  local function recurser()
+    recurser(coroutine.yield())
+  end
+  local co = coroutine.create(recurser)
+  -- Cause the stack overflow and throws an error with incorrect
+  -- error message before the patch. Use some arguments to obtain
+  -- the stack overflow faster.
+  while coroutine.resume(co, 1, 2, 3, 4, 5, 6, 7, 8, 9) do end
+end
+
+local status, errmsg = pcall(resume_wrap)
+
+test:is(status, false, 'status is correct')
+test:like(errmsg, 'stack overflow', 'error message is correct')
+
+test:done(true)