feat: free-threaded Python (3.13.0+) support (#165)

* Rebase to master

* More thread-safety change/refactoring

* Fix a race between PO cleanup thread and Python

* Add free-threaded build in GH action

* Fix dll names for FT/debug builds

* Cleanup and more comments

* Rollback temp change and add clarifying comments

* Complete release action for FT builds

* Merge ft builds into build.yml

* Use GITHUB_PATH

* Squash windows-t build, add Java ver in matrix

* Fix for invalid syntax for powershell

* Add upload action for JVM core/log

* debug build.yml

* Apply suggestions from code review

Co-authored-by: Chip Kent <5250374+chipkent@users.noreply.github.com>

* Restore a block of code deleted by mistake

* debug Windows FT build

* Add comments, use version directives

* Code cleanup

* Fix a bug in convert() for java array type

* Remove duplicate comment

* Add clarifying comment

* Respond to review comments

* Naming change to be more applicable in FT mode

---------

Co-authored-by: Chip Kent <5250374+chipkent@users.noreply.github.com>

.github/workflows/build.yml

@@ -79,7 +79,6 @@ jobs:
           distribution: 'temurin'
           java-version: '8'
 
-      - run: pip install setuptools
       - run: ${{ matrix.info.cmd }}
 
       - uses: actions/upload-artifact@v4
@@ -135,7 +134,7 @@ jobs:
           path: dist/*.whl
           retention-days: 1
 
-  bdist-wheels-linux-arm64:
+  bdist-wheel-linux-arm64:
     runs-on: 'ubuntu-22.04'
     steps:
       - uses: actions/checkout@v4
@@ -157,13 +156,97 @@ jobs:
 
       - uses: actions/upload-artifact@v4
         with:
-          name: bdist-wheels-linux-arm64
+          name: bdist-wheel-linux-arm64
           path: /tmp/dist/*.whl
           retention-days: 1
 
+  bdist-wheel-t:
+    runs-on: ${{ matrix.info.machine }}
+    strategy:
+      fail-fast: false
+      matrix:
+        info:
+          - { machine: 'ubuntu-20.04', python: '3.13t', java: '8', arch: 'amd64', cmd: '.github/env/Linux/bdist-wheel.sh' }
+          - { machine: 'windows-2022', python: '3.13t', java: '8', arch: 'amd64', cmd: '.\.github\env\Windows\bdist-wheel.ps1' }
+          - { machine: 'macos-13', python: '3.13t', java: '8', arch: 'amd64', cmd: '.github/env/macOS/bdist-wheel.sh' }
+          - { machine: 'macos-14', python: '3.13t', java: '11', arch: 'arm64', cmd: '.github/env/macOS/bdist-wheel.sh' }
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-java@v4
+        id: setup-java
+        with:
+          distribution: 'temurin'
+          java-version: ${{ matrix.info.java }}
+
+      - uses: astral-sh/setup-uv@v3
+      - if : ${{ startsWith(matrix.info.machine, 'windows')}}
+        run: |
+          uv python install ${{ matrix.info.python }}
+          uv venv --python ${{ matrix.info.python }}
+          .venv\Scripts\Activate.ps1
+          uv pip install pip
+          echo PATH=$PATH >> $GITHUB_PATH
+          ${{ matrix.info.cmd }}
+      - if : ${{ ! startsWith(matrix.info.machine, 'windows')}}
+        run: |
+          uv python install ${{ matrix.info.python }}
+          uv venv --python ${{ matrix.info.python }}
+          source .venv/bin/activate
+          uv pip install pip
+          echo PATH=$PATH >> $GITHUB_PATH
+          ${{ matrix.info.cmd }}
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: build-${{ matrix.info.python }}-${{ matrix.info.machine }}-${{ matrix.info.arch }}
+          path: dist/*.whl
+          retention-days: 1
+
+  bdist-wheel-linux-arm64-t:
+    runs-on: ${{ matrix.info.machine }}
+    strategy:
+      fail-fast: false
+      matrix:
+        info:
+          - { machine: 'ubuntu-20.04', python: '3.13t', java: '11', arch: 'aarch64', cmd: '.github/env/Linux/bdist-wheel.sh' }
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Build wheels
+        uses: pypa/cibuildwheel@v2.21.3
+        env:
+          CIBW_FREE_THREADED_SUPPORT: true
+          CIBW_ARCHS_LINUX: "aarch64"
+          CIBW_BUILD: "cp313t-*"
+          CIBW_SKIP: "cp313t-musllinux_aarch64"
+          CIBW_BEFORE_ALL_LINUX: >
+            yum install -y java-${{ matrix.info.java }}-openjdk-devel &&
+            yum install -y wget &&
+            wget https://www.apache.org/dist/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz -P /tmp &&
+            tar xf /tmp/apache-maven-3.8.8-bin.tar.gz -C /opt &&
+            ln -s /opt/apache-maven-3.8.8/bin/mvn /usr/bin/mvn
+          CIBW_ENVIRONMENT: JAVA_HOME=/etc/alternatives/jre_11_openjdk
+          CIBW_REPAIR_WHEEL_COMMAND_LINUX: 'auditwheel repair --exclude libjvm.so -w {dest_dir} {wheel}'
+
+        with:
+          package-dir: .
+          output-dir: dist
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: build-${{ matrix.info.python }}-${{ matrix.info.machine }}-${{ matrix.info.arch }}
+          path: dist/*.whl
+          retention-days: 1
+
   collect-artifacts:
     runs-on: ubuntu-22.04
-    needs: ['sdist', 'bdist-wheel', 'bdist-wheel-universal2-hack', 'bdist-wheels-linux-arm64']
+    needs: ['sdist', 'bdist-wheel', 'bdist-wheel-universal2-hack', 'bdist-wheel-linux-arm64', 'bdist-wheel-t', 'bdist-wheel-linux-arm64-t']
     steps:
       - uses: actions/checkout@v4
       - uses: actions/download-artifact@v4

.github/workflows/check.yml

@@ -29,3 +29,51 @@ jobs:
 
       - name: Run Test
         run: python setup.py test
+
+      - name: Upload JVM Error Logs
+        uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: check-ci-jvm-err
+          path: |
+            **/*_pid*.log
+            **/core.*
+          if-no-files-found: ignore
+
+  test-free-threaded:
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        python: ['3.13t']
+        java: ['8', '11', '17', '21', '23']
+    steps:
+        - uses: actions/checkout@v4
+
+        - uses: actions/setup-java@v4
+          with:
+            distribution: 'temurin'
+            java-version: ${{ matrix.java }}
+
+        - uses: astral-sh/setup-uv@v3
+        - run: |
+            uv python install ${{ matrix.python }}
+            uv venv --python ${{ matrix.python }}
+            source .venv/bin/activate
+            uv pip install pip
+            echo $JAVA_HOME
+            echo PATH=$PATH >> $GITHUB_PATH
+
+        - run: pip install "setuptools < 72"
+
+        - name: Run Free-threaded Test
+          run: python setup.py test
+
+        - name: Upload JVM Error Logs
+          uses: actions/upload-artifact@v4
+          if: failure()
+          with:
+            name: check-ci-jvm-err
+            path: |
+              **/*_pid*.log
+              **/core.*
+            if-no-files-found: ignore

jpyutil.py

@@ -303,9 +303,13 @@ def _find_python_dll_file(fail=False):
     logger.debug("Searching for Python shared library file")
 
     # Prepare list of search directories
-
     search_dirs = [sys.prefix]
 
+    # installed_base/lib needs to be added to the search path for Python 3.13t files
+    installed_base = sysconfig.get_config_var('installed_base')
+    if installed_base:
+        search_dirs.append(os.path.join(installed_base, "lib"))
+
     extra_search_dirs = [sysconfig.get_config_var(name) for name in PYTHON_LIB_DIR_CONFIG_VAR_NAMES]
     for extra_dir in extra_search_dirs:
         if extra_dir and extra_dir not in search_dirs and os.path.exists(extra_dir):
@@ -326,18 +330,29 @@ def _find_python_dll_file(fail=False):
 
     # Prepare list of possible library file names
 
+    # account for Python debug builds
+
+    debug_build = sysconfig.get_config_var('Py_DEBUG')
+
+    # account for Python 3.13+ with GIL disabled
+    dll_suffix = ''
+    if sys.version_info >= (3, 13):
+        if not sys._is_gil_enabled():
+            dll_suffix = 't'
+    dll_suffix += 'd' if debug_build else ''
+
     vmaj = str(sys.version_info.major)
     vmin = str(sys.version_info.minor)
 
     if platform.system() == 'Windows':
-        versions = (vmaj + vmin, vmaj, '')
+        versions = (vmaj + vmin, vmaj, vmaj + vmin + dll_suffix, '')
         file_names = ['python' + v + '.dll' for v in versions]
     elif platform.system() == 'Darwin':
-        versions = (vmaj + "." + vmin, vmaj, '')
+        versions = (vmaj + "." + vmin, vmaj, vmaj + "." + vmin + dll_suffix, '')
         file_names = ['libpython' + v + '.dylib' for v in versions] + \
                      ['libpython' + v + '.so' for v in versions]
     else:
-        versions = (vmaj + "." + vmin, vmaj, '')
+        versions = (vmaj + "." + vmin, vmaj, vmaj + "." + vmin + dll_suffix, '')
         file_names = ['libpython' + v + '.so' for v in versions]
 
     logger.debug("Potential Python shared library file names: %s" % repr(file_names))

setup.py

@@ -97,6 +97,7 @@
     os.path.join(src_test_py_dir, 'jpy_java_embeddable_test.py'),
     os.path.join(src_test_py_dir, 'jpy_obj_test.py'),
     os.path.join(src_test_py_dir, 'jpy_eval_exec_test.py'),
+    os.path.join(src_test_py_dir, 'jpy_mt_eval_exec_test.py'),
 ]
 
 # e.g. jdk_home_dir = '/home/marta/jdk1.7.0_15'

src/main/c/jni/org_jpy_PyLib.c

@@ -499,12 +499,20 @@ void dumpDict(const char* dictName, PyObject* dict)
 
     size = PyDict_Size(dict);
     printf(">> dumpDict: %s.size = %ld\n", dictName, size);
+#if PY_VERSION_HEX >= 0x030D0000 // >=3.13
+    // PyDict_Next is not thread-safe, so we need to protect it with a critical section
+    // https://docs.python.org/3/howto/free-threading-extensions.html#pydict-next
+    Py_BEGIN_CRITICAL_SECTION(dict);
+#endif
     while (PyDict_Next(dict, &pos, &key, &value)) {
         const char* name;
         name = JPy_AS_UTF8(key);
         printf(">> dumpDict: %s[%ld].name = '%s'\n", dictName, i, name);
         i++;
     }
+#if PY_VERSION_HEX >= 0x030D0000 // >=3.13
+    Py_END_CRITICAL_SECTION();
+#endif
 }
 
 /**
@@ -521,7 +529,7 @@ PyObject *getMainGlobals() {
     }
 
     pyGlobals = PyModule_GetDict(pyMainModule); // borrowed ref
-    JPy_INCREF(pyGlobals);
+    JPy_XINCREF(pyGlobals);
 
     return pyGlobals;
 }
@@ -557,7 +565,7 @@ JNIEXPORT jobject JNICALL Java_org_jpy_PyLib_getCurrentGlobals
 
     JPy_BEGIN_GIL_STATE
 
-#if PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION <= 12
+#if PY_VERSION_HEX < 0x030D0000 // < 3.13
     globals = PyEval_GetGlobals(); // borrowed ref
     JPy_XINCREF(globals);
 #else
@@ -588,7 +596,7 @@ JNIEXPORT jobject JNICALL Java_org_jpy_PyLib_getCurrentLocals
 
     JPy_BEGIN_GIL_STATE
 
-#if PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION <= 12
+#if PY_VERSION_HEX < 0x030D0000 // < 3.13
     locals = PyEval_GetLocals(); // borrowed ref
     JPy_XINCREF(locals);
 #else
@@ -1124,7 +1132,7 @@ JNIEXPORT void JNICALL Java_org_jpy_PyLib_incRef
     if (Py_IsInitialized()) {
         JPy_BEGIN_GIL_STATE
 
-        refCount = pyObject->ob_refcnt;
+        refCount = Py_REFCNT(pyObject);
         JPy_DIAG_PRINT(JPy_DIAG_F_MEM, "Java_org_jpy_PyLib_incRef: pyObject=%p, refCount=%d, type='%s'\n", pyObject, refCount, Py_TYPE(pyObject)->tp_name);
         JPy_INCREF(pyObject);
 
@@ -1150,7 +1158,7 @@ JNIEXPORT void JNICALL Java_org_jpy_PyLib_decRef
     if (Py_IsInitialized()) {
         JPy_BEGIN_GIL_STATE
 
-        refCount = pyObject->ob_refcnt;
+        refCount = Py_REFCNT(pyObject);
         if (refCount <= 0) {
             JPy_DIAG_PRINT(JPy_DIAG_F_ALL, "Java_org_jpy_PyLib_decRef: error: refCount <= 0: pyObject=%p, refCount=%d\n", pyObject, refCount);
         } else {
@@ -1183,7 +1191,7 @@ JNIEXPORT void JNICALL Java_org_jpy_PyLib_decRefs
         buf = (*jenv)->GetLongArrayElements(jenv, objIds, &isCopy);
         for (i = 0; i < len; i++) {
             pyObject = (PyObject*) buf[i];
-            refCount = pyObject->ob_refcnt;
+            refCount = Py_REFCNT(pyObject);
             if (refCount <= 0) {
                 JPy_DIAG_PRINT(JPy_DIAG_F_ALL, "Java_org_jpy_PyLib_decRefs: error: refCount <= 0: pyObject=%p, refCount=%d\n", pyObject, refCount);
             } else {

src/main/c/jpy_jmethod.c

@@ -799,6 +799,7 @@ JPy_JMethod* JOverloadedMethod_FindMethod0(JNIEnv* jenv, JPy_JOverloadedMethod*
                               overloadedMethod->declaringClass->javaName, JPy_AS_UTF8(overloadedMethod->name), overloadCount, argCount);
 
     for (i = 0; i < overloadCount; i++) {
+        // borrowed ref, no need to replace with PyList_GetItemRef() because the list won't be changed concurrently
         currMethod = (JPy_JMethod*) PyList_GetItem(overloadedMethod->methodList, i);
 
         if (currMethod->isVarArgs && matchValueMax > 0 && !bestMethod->isVarArgs) {
@@ -950,6 +951,7 @@ int JOverloadedMethod_AddMethod(JPy_JOverloadedMethod* overloadedMethod, JPy_JMe
         // we need to insert this before the first varargs method
         Py_ssize_t size = PyList_Size(overloadedMethod->methodList);
         for (ii = 0; ii < size; ii++) {
+            // borrowed ref, no need to replace with PyList_GetItemRef() because the list won't be changed concurrently
             PyObject *check = PyList_GetItem(overloadedMethod->methodList, ii);
             if (((JPy_JMethod *) check)->isVarArgs) {
                 // this is the first varargs method, so we should go before it

src/main/c/jpy_jobj.c

@@ -72,19 +72,33 @@ PyObject* JObj_FromType(JNIEnv* jenv, JPy_JType* type, jobject objectRef)
     }
 
 
-// we check the type translations dictionary for a callable for this java type name,
+    // we check the type translations dictionary for a callable for this java type name,
     // and apply the returned callable to the wrapped object
+#if PY_VERSION_HEX < 0x030D0000 // < 3.13
+    // borrowed ref
     callable = PyDict_GetItemString(JPy_Type_Translations, type->javaName);
+    JPy_XINCREF(callable);
+#else
+    // https://docs.python.org/3/howto/free-threading-extensions.html#borrowed-references
+    // PyDict_GetItemStringRef() is a thread safe version of PyDict_GetItemString() and returns a new reference
+    if (PyDict_GetItemStringRef(JPy_Type_Translations, type->javaName, &callable) != 1) {
+        callable = NULL;
+    }
+#endif
+
     if (callable != NULL) {
         if (PyCallable_Check(callable)) {
             callableResult = PyObject_CallFunction(callable, "OO", type, obj);
+            JPy_XDECREF(callable);
+            JPy_XDECREF(obj);
             if (callableResult == NULL) {
                 Py_RETURN_NONE;
             } else {
                 return callableResult;
             }
         }
     }
+    JPy_XDECREF(callable);
 
     return (PyObject *)obj;
 }
@@ -103,6 +117,7 @@ int JObj_init_internal(JNIEnv* jenv, JPy_JObj* self, PyObject* args, PyObject* k
 
     type = ((PyObject*) self)->ob_type;
 
+    // borrowed ref, no need to replace with PyDict_GetItemStringRef because tp_dict won't be changed concurrently
     constructor = PyDict_GetItemString(type->tp_dict, JPy_JTYPE_ATTR_NAME_JINIT);
     if (constructor == NULL) {
         PyErr_SetString(PyExc_RuntimeError, "no constructor found (missing JType attribute '" JPy_JTYPE_ATTR_NAME_JINIT "')");