Support for nogil (Multithreaded Python without the GIL)

Overview

I wanted to test if I could build a stand-alone executable with nuitka inside the nogil proof of concept. It produced an error when trying to build a simple print("hello world") script.

Versions

• nuitka: 0.9.4
• python: nogil-3.9.10

Error Details

See build error inside the CI action logs here: https://github.com/jimkring/nuitka-nogil-testing/runs/7423321112

Nuitka-Scons:INFO: Backend C compiler: gcc (gcc).
In file included from /usr/local/lib/python3.9/site-packages/nuitka/build/include/nuitka/helpers.h:89,
                 from /usr/local/lib/python3.9/site-packages/nuitka/build/include/nuitka/prelude.h:504,
                 from __constants.c:2:
/usr/local/lib/python3.9/site-packages/nuitka/build/include/nuitka/exceptions.h: In function 'ERROR_OCCURRED':
/usr/local/lib/python3.9/site-packages/nuitka/build/include/nuitka/exceptions.h:25:29: error: 'struct _gilstate_runtime_state' has no member named 'tstate_current'

How did you install Nuitka and Python?

Installed nuitka with pip inside a nogil/python docker container. See my Dockerfile

[Maxwell175]

Got it to build with the following patch, but get a segfault when running.

Program received signal SIGSEGV, Segmentation fault.
setEarlyFrozenModulesFileAttribute () at /mnt/ExtraSSD/src/nuitka-nogil/Nuitka/nuitka/build/static_src/MetaPathBasedLoader.c:1904
1904	        if (key != NULL && value != NULL && PyModule_Check(value)) {
(gdb) bt
#0  setEarlyFrozenModulesFileAttribute () at /mnt/ExtraSSD/src/nuitka-nogil/Nuitka/nuitka/build/static_src/MetaPathBasedLoader.c:1904
#1  0x00005555557e5e78 in main (argc=1, argv=0x7fffffffeb48) at static_src/MainProgram.c:1034
#2  0x00007ffff7c33083 in __libc_start_main (main=0x5555557e5d08 <main>, argc=1, argv=0x7fffffffeb48, init=<optimized out>, fini=<optimized out>, rtld_fini=<optimized out>, stack_end=0x7fffffffeb38) at ../csu/libc-start.c:308
#3  0x00005555557d6dee in _start ()

diff --git a/nuitka/build/static_src/CompiledAsyncgenType.c b/nuitka/build/static_src/CompiledAsyncgenType.c
index b36cf3f8d..921202b07 100644
--- a/nuitka/build/static_src/CompiledAsyncgenType.c
+++ b/nuitka/build/static_src/CompiledAsyncgenType.c
@@ -577,6 +577,7 @@ static PyObject *_Nuitka_Asyncgen_throw2(struct Nuitka_AsyncgenObject *asyncgen,
         PRINT_NEW_LINE();
 #endif
 
+#if !PY_NOGIL
         if (PyGen_CheckExact(asyncgen->m_yieldfrom) || PyCoro_CheckExact(asyncgen->m_yieldfrom)) {
             PyGenObject *gen = (PyGenObject *)asyncgen->m_yieldfrom;
 
@@ -584,7 +585,9 @@ static PyObject *_Nuitka_Asyncgen_throw2(struct Nuitka_AsyncgenObject *asyncgen,
             asyncgen->m_running = 1;
             ret = Nuitka_UncompiledGenerator_throw(gen, 1, exception_type, exception_value, exception_tb);
             asyncgen->m_running = 0;
-        } else if (Nuitka_Generator_Check(asyncgen->m_yieldfrom)) {
+        } else
+#endif
+        if (Nuitka_Generator_Check(asyncgen->m_yieldfrom)) {
             struct Nuitka_GeneratorObject *gen = ((struct Nuitka_GeneratorObject *)asyncgen->m_yieldfrom);
             // Transferred exception ownership to "_Nuitka_Generator_throw2".
             asyncgen->m_running = 1;
diff --git a/nuitka/build/static_src/CompiledCoroutineType.c b/nuitka/build/static_src/CompiledCoroutineType.c
index 77940636a..2f2df8efd 100644
--- a/nuitka/build/static_src/CompiledCoroutineType.c
+++ b/nuitka/build/static_src/CompiledCoroutineType.c
@@ -750,6 +750,7 @@ static PyObject *_Nuitka_Coroutine_throw2(struct Nuitka_CoroutineObject *corouti
         PRINT_NEW_LINE();
 #endif
 
+#if !PY_NOGIL
         if (PyGen_CheckExact(coroutine->m_yieldfrom) || PyCoro_CheckExact(coroutine->m_yieldfrom)) {
             PyGenObject *gen = (PyGenObject *)coroutine->m_yieldfrom;
 
@@ -757,7 +758,9 @@ static PyObject *_Nuitka_Coroutine_throw2(struct Nuitka_CoroutineObject *corouti
             coroutine->m_running = 1;
             ret = Nuitka_UncompiledGenerator_throw(gen, 1, exception_type, exception_value, exception_tb);
             coroutine->m_running = 0;
-        } else if (Nuitka_Generator_Check(coroutine->m_yieldfrom)) {
+        } else
+#endif
+        if (Nuitka_Generator_Check(coroutine->m_yieldfrom)) {
             struct Nuitka_GeneratorObject *gen = ((struct Nuitka_GeneratorObject *)coroutine->m_yieldfrom);
             // Transferred exception ownership to "_Nuitka_Generator_throw2".
             coroutine->m_running = 1;
@@ -1457,7 +1460,7 @@ PyObject *Nuitka_Coroutine_New(coroutine_code code, PyObject *module, PyObject *
 
 static int gen_is_coroutine(PyObject *object) {
     if (PyGen_CheckExact(object)) {
-        PyCodeObject *code = (PyCodeObject *)((PyGenObject *)object)->gi_code;
+        PyCodeObject *code = (PyCodeObject *)((PyGenObject *)object)->code;
 
         if (code->co_flags & CO_ITERABLE_COROUTINE) {
             return 1;
diff --git a/nuitka/build/static_src/CompiledGeneratorType.c b/nuitka/build/static_src/CompiledGeneratorType.c
index bf9545f05..22df7a492 100644
--- a/nuitka/build/static_src/CompiledGeneratorType.c
+++ b/nuitka/build/static_src/CompiledGeneratorType.c
@@ -1002,7 +1002,7 @@ static PyObject *_Nuitka_Generator_throw2(struct Nuitka_GeneratorObject *generat
             ret = Nuitka_UncompiledGenerator_throw(gen, 1, exception_type, exception_value, exception_tb);
             generator->m_running = 0;
 #endif
-#if PYTHON_VERSION >= 0x350
+#if PYTHON_VERSION >= 0x350 && !defined(PY_NOGIL)
         } else if (Nuitka_Coroutine_Check(generator->m_yieldfrom)) {
             struct Nuitka_CoroutineObject *coro = ((struct Nuitka_CoroutineObject *)generator->m_yieldfrom);
             // Transferred exception ownership to "_Nuitka_Coroutine_throw2".
diff --git a/nuitka/build/static_src/CompiledGeneratorTypeUncompiledIntegration.c b/nuitka/build/static_src/CompiledGeneratorTypeUncompiledIntegration.c
index 7fe71df13..40b3bf5ae 100644
--- a/nuitka/build/static_src/CompiledGeneratorTypeUncompiledIntegration.c
+++ b/nuitka/build/static_src/CompiledGeneratorTypeUncompiledIntegration.c
@@ -145,7 +145,8 @@ static PyObject *Nuitka_PyGen_Send(PyGenObject *gen, PyObject *arg) {
         Nuitka_SetStopIterationValue(gen->return_value);
         return NULL;
     } else {
-        return gen_wrap_exception(gen);
+        SET_CURRENT_EXCEPTION_TYPE0(PyExc_StopIteration);
+        return NULL;
     }
 #elif PYTHON_VERSION >= 0x3a0
     PyObject *result;
diff --git a/nuitka/build/static_src/MainProgram.c b/nuitka/build/static_src/MainProgram.c
index c90f2049b..dfd9999a6 100644
--- a/nuitka/build/static_src/MainProgram.c
+++ b/nuitka/build/static_src/MainProgram.c
@@ -854,22 +854,6 @@ int main(int argc, char **argv) {
      */
     Py_NoSiteFlag = 1;
 
-    /* Initial command line handling only. */
-
-#if PYTHON_VERSION >= 0x300 && _NUITKA_NATIVE_WCHAR_ARGV == 0
-    NUITKA_PRINT_TRACE("main(): Calling convertCommandLineParameters.");
-    orig_argv = convertCommandLineParameters(argc, argv);
-#elif PYTHON_VERSION < 0x300 && _NUITKA_NATIVE_WCHAR_ARGV == 1
-    orig_argv = getCommandLineToArgvA(GetCommandLineA());
-#else
-orig_argv = argv;
-#endif
-    orig_argc = argc;
-
-    NUITKA_PRINT_TRACE("main(): Calling initial setCommandLineParameters.");
-
-    setCommandLineParameters(argc, argv, true);
-
     /* For Python installations that need the home set, we inject it back here. */
 #if defined(PYTHON_HOME_PATH)
 #if PYTHON_VERSION < 0x300
@@ -924,6 +908,24 @@ orig_argv = argv;
     NUITKA_PRINT_TIMING("main(): Calling Py_Initialize to initialize interpreter.");
     Py_Initialize();
 
+
+
+  /* Initial command line handling only. */
+
+#if PYTHON_VERSION >= 0x300 && _NUITKA_NATIVE_WCHAR_ARGV == 0
+  NUITKA_PRINT_TRACE("main(): Calling convertCommandLineParameters.");
+    orig_argv = convertCommandLineParameters(argc, argv);
+#elif PYTHON_VERSION < 0x300 && _NUITKA_NATIVE_WCHAR_ARGV == 1
+  orig_argv = getCommandLineToArgvA(GetCommandLineA());
+#else
+  orig_argv = argv;
+#endif
+  orig_argc = argc;
+
+  NUITKA_PRINT_TRACE("main(): Calling initial setCommandLineParameters.");
+
+  setCommandLineParameters(argc, argv, true);
+
 #if PYTHON_VERSION >= 0x300 && SYSFLAG_NO_RANDOMIZATION == 1
     if (old_env_hash_seed) {
         undoEnvironmentVariable("PYTHONHASHSEED", old_env_hash_seed);

Notice that the MainControl patch moved the command line parsing under Py_Initialize to avoid another segfault due to the python memory allocator not being initialized in time.

[kayhayen]

Going to have a look, what the memory allocator is being used there, I think we need to know some things early, esp. for multiprocessing.