Re-enable stack depth checks under ASan (bellard#161)

The default 256 kb stack is too small to run some of the test262 tests
when ASAN is enabled.

Double it to 512 kb and ensure threads created by quickjs have big
enough stacks.

Author: bnoordhuis

quickjs-libc.c

@@ -3409,6 +3409,9 @@ static JSValue js_worker_ctor(JSContext *ctx, JSValueConst new_target,
     pthread_attr_init(&attr);
     /* no join at the end */
     pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
+    // musl libc gives threads 80 kb stacks, much smaller than
+    // JS_DEFAULT_STACK_SIZE (256 kb)
+    pthread_attr_setstacksize(&attr, 2 << 20); // 2 MB, glibc default
     ret = pthread_create(&tid, &attr, worker_func, args);
     pthread_attr_destroy(&attr);
     if (ret != 0) {

quickjs.c

@@ -67,7 +67,7 @@
 #define CONFIG_PRINTF_RNDN
 #endif
 
-#if !defined(EMSCRIPTEN) && !defined(__ASAN__)
+#if !defined(EMSCRIPTEN)
 /* enable stack limitation */
 #define CONFIG_STACK_CHECK
 #endif
@@ -1635,6 +1635,9 @@ JSRuntime *JS_NewRuntime2(const JSMallocFunctions *mf, void *opaque)
     rt->js_class_id_alloc = JS_CLASS_INIT_COUNT;
 
     rt->stack_size = JS_DEFAULT_STACK_SIZE;
+#ifdef __ASAN__
+    rt->stack_size *= 2; // stack frames are bigger under AddressSanitizer
+#endif
     JS_UpdateStackTop(rt);
 
     rt->current_exception = JS_NULL;

run-test262.c

@@ -531,6 +531,7 @@ static JSValue js_agent_start(JSContext *ctx, JSValue this_val,
 {
     const char *script;
     Test262Agent *agent;
+    pthread_attr_t attr;
 
     if (JS_GetContextOpaque(ctx) != NULL)
         return JS_ThrowTypeError(ctx, "cannot be called inside an agent");
@@ -545,7 +546,12 @@ static JSValue js_agent_start(JSContext *ctx, JSValue this_val,
     agent->script = strdup(script);
     JS_FreeCString(ctx, script);
     list_add_tail(&agent->link, &agent_list);
-    pthread_create(&agent->tid, NULL, agent_start, agent);
+    pthread_attr_init(&attr);
+    // musl libc gives threads 80 kb stacks, much smaller than
+    // JS_DEFAULT_STACK_SIZE (256 kb)
+    pthread_attr_setstacksize(&attr, 2 << 20); // 2 MB, glibc default
+    pthread_create(&agent->tid, &attr, agent_start, agent);
+    pthread_attr_destroy(&attr);
     return JS_UNDEFINED;
 }