Debugger updates

plugins/com.python.pydev.docs/release_process.md

@@ -21,7 +21,7 @@ python x:\liclipsews\pydev.page\build_homepage.py
 
 ## Commit everything and merge with master (homepage at: x:\liclipsews\pydev.page and X:\liclipsews\liclipsews\Pydev repo)
 
-mu acp "PyDev release 11.0.3"
+mu acp "PyDev release 12.0.0"
 
 ## Do build:
 
@@ -39,8 +39,8 @@ python -m dev add_to_update_site_mirror
 
 ## Tag repository (needed so that GitHub can reference it later)
 
-git tag pydev_11_0_3 -a -m "PyDev 11.0.3"
-git push --tag origin pydev_11_0_3
+git tag pydev_12_0_0 -a -m "PyDev 12.0.0"
+git push --tag origin pydev_12_0_0
 
 ## Create release in Github Releases
 
@@ -49,19 +49,19 @@ python -m dev add_to_github
 Contents in: X:\pydev_build\build_dir\pydev\features\org.python.pydev.p2-repo\target\github
 
 ### Title:
-PyDev 11.0.3
+PyDev 12.0.0
 
 ### Message:
 
-This release contains PyDev 11.0.3
+This release contains PyDev 12.0.0
 
 It's possible to add it as an Eclipse update site using the url:
 
-https://github.com/fabioz/Pydev/releases/download/pydev_11_0_3/
+https://github.com/fabioz/Pydev/releases/download/pydev_12_0_0/
 
 Or get a .zip to install manually by unzipping it in the dropins:
 
-https://github.com/fabioz/Pydev/releases/download/pydev_11_0_3/PyDev.11.0.3.zip
+https://github.com/fabioz/Pydev/releases/download/pydev_12_0_0/PyDev.12.0.0.zip
 
 
 ## Update homepage:

plugins/org.python.pydev.core/pysrc/.github/workflows/pydevd-release-manylinux.yml

@@ -24,7 +24,7 @@ jobs:
       - uses: actions/setup-python@v3
 
       - name: Install cibuildwheel
-        run: python -m pip install cibuildwheel==2.16.2
+        run: python -m pip install cibuildwheel==2.16.5
 
       - name: Remove .so files (will be rebuilt)
         run: rm pydevd_attach_to_process/*.so

plugins/org.python.pydev.core/pysrc/README.md

@@ -1,5 +1,23 @@
 PyDev.Debugger
-==============
+===============
+
+Latest `3.x` version: the PyDev debugger now supports `sys.monitoring` which enables
+really fast tracing on `Python 3.12` (so, if speed is an issue, make sure you upgrade).
+
+The PyDev Debugger is a Python debugger which historically was created to
+work with `PyDev` (in Eclipse).
+
+Over the years (as it's open source -- EPL) it was adopted by other IDEs/companies
+(so, it was integrated into PyCharm and is also bundled in VSCode Python by
+the usage of debugpy).
+
+Note that although it was adopted by other IDEs (and over the years companies of other
+commercial IDEs did provide backing), by far most of the work was done without any
+external backing and the ongoing work on the project relies on community support.
+
+If you like using it, please consider becoming a backer of the project (this is
+done through the `PyDev` umbrella, so please see https://www.pydev.org/about.html
+for how to financially contribute to the project).
 
 The sources for the PyDev.Debugger may be seen at:
 
@@ -14,12 +32,10 @@ debugging a process which runs in another machine, it's possible to `pip install
 `pydevd.settrace(host='10.1.1.1')` to connect the debugger backend to the debugger UI running in the IDE
 (whereas previously the sources had to be manually copied from the IDE installation).
 
-`pydevd` is compatible with Python 3.6 onwards.
+`pydevd` is compatible with Python 3.6 onwards and is tested both with CPython as well as PyPy.
 
 For `Python 2` please keep using `pydevd 2.8.0`.
 
-`pydevd` is tested both with CPython as well as PyPy.
-
 Recent versions contain speedup modules using Cython, which are generated with a few changes in the regular files
 to `cythonize` the files. To update and compile the cython sources (and generate some other auto-generated files),
 `build_tools/build.py` should be run -- note that the resulting .pyx and .c files should be commited.

plugins/org.python.pydev.core/pysrc/_pydev_bundle/_pydev_saved_modules.py

@@ -77,6 +77,9 @@ def check(self, module, expected_attributes):
 
 with VerifyShadowedImport('threading') as verify_shadowed:
     import threading;    verify_shadowed.check(threading, ['Thread', 'settrace', 'setprofile', 'Lock', 'RLock', 'current_thread'])
+    ThreadingEvent = threading.Event
+    ThreadingLock = threading.Lock
+    threading_current_thread = threading.current_thread
 
 with VerifyShadowedImport('time') as verify_shadowed:
     import time;    verify_shadowed.check(time, ['sleep', 'time', 'mktime'])

plugins/org.python.pydev.core/pysrc/_pydev_bundle/fsnotify/__init__.py

@@ -37,10 +37,9 @@ def start_watching():  # Called from thread
 
 Note: changes are only reported for files (added/modified/deleted), not directories.
 '''
-import threading
 import sys
 from os.path import basename
-from _pydev_bundle import pydev_log
+from _pydev_bundle import pydev_log, _pydev_saved_modules
 from os import scandir
 
 try:
@@ -205,7 +204,7 @@ def __init__(self, accept_directory=None, accept_file=None):
             Note: if passed it'll override the `accepted_file_extensions`.
         '''
         self._path_watchers = set()
-        self._disposed = threading.Event()
+        self._disposed = _pydev_saved_modules.ThreadingEvent()
 
         if accept_directory is None:
             accept_directory = lambda dir_path: basename(dir_path) not in self.ignored_dirs

plugins/org.python.pydev.core/pysrc/_pydevd_bundle/pydevd_additional_thread_info.py

@@ -4,16 +4,24 @@
 
 if USE_CYTHON_FLAG in ENV_TRUE_LOWER_VALUES:
     # We must import the cython version if forcing cython
-    from _pydevd_bundle.pydevd_cython_wrapper import PyDBAdditionalThreadInfo, set_additional_thread_info, _set_additional_thread_info_lock  # @UnusedImport
+    from _pydevd_bundle.pydevd_cython_wrapper import (
+        PyDBAdditionalThreadInfo, set_additional_thread_info, _set_additional_thread_info_lock,  # @UnusedImport
+        any_thread_stepping, remove_additional_info)  # @UnusedImport
 
 elif USE_CYTHON_FLAG in ENV_FALSE_LOWER_VALUES:
     # Use the regular version if not forcing cython
-    from _pydevd_bundle.pydevd_additional_thread_info_regular import PyDBAdditionalThreadInfo, set_additional_thread_info, _set_additional_thread_info_lock  # @UnusedImport @Reimport
+    from _pydevd_bundle.pydevd_additional_thread_info_regular import (
+        PyDBAdditionalThreadInfo, set_additional_thread_info, _set_additional_thread_info_lock,  # @UnusedImport @Reimport
+        any_thread_stepping, remove_additional_info)  # @UnusedImport @Reimport
 
 else:
     # Regular: use fallback if not found (message is already given elsewhere).
     try:
-        from _pydevd_bundle.pydevd_cython_wrapper import PyDBAdditionalThreadInfo, set_additional_thread_info, _set_additional_thread_info_lock
+        from _pydevd_bundle.pydevd_cython_wrapper import (
+            PyDBAdditionalThreadInfo, set_additional_thread_info, _set_additional_thread_info_lock,
+            any_thread_stepping, remove_additional_info)
     except ImportError:
-        from _pydevd_bundle.pydevd_additional_thread_info_regular import PyDBAdditionalThreadInfo, set_additional_thread_info, _set_additional_thread_info_lock  # @UnusedImport
+        from _pydevd_bundle.pydevd_additional_thread_info_regular import (
+            PyDBAdditionalThreadInfo, set_additional_thread_info, _set_additional_thread_info_lock,  # @UnusedImport
+            any_thread_stepping, remove_additional_info)  # @UnusedImport
 

plugins/org.python.pydev.core/pysrc/_pydevd_bundle/pydevd_additional_thread_info_regular.py

@@ -1,11 +1,8 @@
 from _pydevd_bundle.pydevd_constants import (STATE_RUN, PYTHON_SUSPEND, SUPPORT_GEVENT, ForkSafeLock,
-    _current_frames)
+    _current_frames, STATE_SUSPEND, get_global_debugger, get_thread_id)
 from _pydev_bundle import pydev_log
-# IFDEF CYTHON
-# pydev_log.debug("Using Cython speedups")
-# ELSE
-from _pydevd_bundle.pydevd_frame import PyDBFrame
-# ENDIF
+from _pydev_bundle._pydev_saved_modules import threading
+import weakref
 
 version = 11
 
@@ -55,6 +52,10 @@ class PyDBAdditionalThreadInfo(object):
         'target_id_to_smart_step_into_variant',
 
         'pydev_use_scoped_step_frame',
+
+        'weak_thread',
+
+        'is_in_wait_loop',
     ]
     # ENDIF
 
@@ -103,16 +104,69 @@ def __init__(self):
         #
         # See: https://github.com/microsoft/debugpy/issues/869#issuecomment-1132141003
         self.pydev_use_scoped_step_frame = False
+        self.weak_thread = None
+
+        # Purpose: detect if this thread is suspended and actually in the wait loop
+        # at this time (otherwise it may be suspended but still didn't reach a point.
+        # to pause).
+        self.is_in_wait_loop = False
+
+# IFDEF CYTHON
+#     cpdef object _get_related_thread(self):
+# ELSE
+    def _get_related_thread(self):
+# ENDIF
+        if self.pydev_notify_kill:  # Already killed
+            return None
+
+        if self.weak_thread is None:
+            return None
+
+        thread = self.weak_thread()
+        if thread is None:
+            return False
+
+        if thread._is_stopped:
+            return None
+
+        if thread._ident is None:  # Can this happen?
+            pydev_log.critical('thread._ident is None in _get_related_thread!')
+            return None
+
+        if threading._active.get(thread._ident) is not thread:
+            return None
 
+        return thread
+
+# IFDEF CYTHON
+#     cpdef bint _is_stepping(self):
+# ELSE
+    def _is_stepping(self):
+# ENDIF
+        if self.pydev_state == STATE_RUN and self.pydev_step_cmd != -1:
+            # This means actually stepping in a step operation.
+            return True
+
+        if self.pydev_state == STATE_SUSPEND and self.is_in_wait_loop:
+            # This means stepping because it was suspended but still didn't
+            # reach a suspension point.
+            return True
+
+        return False
+
+# IFDEF CYTHON
+#     cpdef get_topmost_frame(self, thread):
+# ELSE
     def get_topmost_frame(self, thread):
+# ENDIF
         '''
         Gets the topmost frame for the given thread. Note that it may be None
         and callers should remove the reference to the frame as soon as possible
         to avoid disturbing user code.
         '''
         # sys._current_frames(): dictionary with thread id -> topmost frame
         current_frames = _current_frames()
-        topmost_frame = current_frames.get(thread.ident)
+        topmost_frame = current_frames.get(thread._ident)
         if topmost_frame is None:
             # Note: this is expected for dummy threads (so, getting the topmost frame should be
             # treated as optional).
@@ -128,6 +182,13 @@ def get_topmost_frame(self, thread):
 
         return topmost_frame
 
+# IFDEF CYTHON
+#     cpdef update_stepping_info(self):
+# ELSE
+    def update_stepping_info(self):
+# ENDIF
+        _update_stepping_info(self)
+
     def __str__(self):
         return 'State:%s Stop:%s Cmd: %s Kill:%s' % (
             self.pydev_state, self.pydev_step_stop, self.pydev_step_cmd, self.pydev_notify_kill)
@@ -162,7 +223,81 @@ def set_additional_thread_info(thread):
                 # and add a new entry only after we set thread.additional_info.
                 additional_info = _next_additional_info[0]
                 thread.additional_info = additional_info
+                additional_info.weak_thread = weakref.ref(thread)
+                add_additional_info(additional_info)
                 del _next_additional_info[:]
                 _next_additional_info.append(PyDBAdditionalThreadInfo())
 
     return additional_info
+
+# IFDEF CYTHON
+# cdef set _all_infos
+# cdef set _infos_stepping
+# cdef object _update_infos_lock
+# ELSE
+# ENDIF
+
+
+_all_infos = set()
+_infos_stepping = set()
+_update_infos_lock = ForkSafeLock()
+
+
+# IFDEF CYTHON
+# cdef _update_stepping_info(PyDBAdditionalThreadInfo info):
+# ELSE
+def _update_stepping_info(info):
+# ENDIF
+    global _infos_stepping
+    global _all_infos
+
+    with _update_infos_lock:
+        # Removes entries that are no longer valid.
+        new_all_infos = set()
+        for info in _all_infos:
+            if info._get_related_thread() is not None:
+                new_all_infos.add(info)
+        _all_infos = new_all_infos
+
+        new_stepping = set()
+        for info in _all_infos:
+            if info._is_stepping():
+                new_stepping.add(info)
+        _infos_stepping = new_stepping
+
+    py_db = get_global_debugger()
+    if py_db is not None and not py_db.pydb_disposed:
+        thread = info.weak_thread()
+        if thread is not None:
+            thread_id = get_thread_id(thread)
+            _queue, event = py_db.get_internal_queue_and_event(thread_id)
+            event.set()
+
+
+# IFDEF CYTHON
+# cpdef add_additional_info(PyDBAdditionalThreadInfo info):
+# ELSE
+def add_additional_info(info):
+# ENDIF
+    with _update_infos_lock:
+        _all_infos.add(info)
+        if info._is_stepping():
+            _infos_stepping.add(info)
+
+
+# IFDEF CYTHON
+# cpdef remove_additional_info(PyDBAdditionalThreadInfo info):
+# ELSE
+def remove_additional_info(info):
+# ENDIF
+    with _update_infos_lock:
+        _all_infos.discard(info)
+        _infos_stepping.discard(info)
+
+
+# IFDEF CYTHON
+# cpdef bint any_thread_stepping():
+# ELSE
+def any_thread_stepping():
+# ENDIF
+    return bool(_infos_stepping)

plugins/org.python.pydev.core/pysrc/_pydevd_bundle/pydevd_api.py

@@ -926,6 +926,7 @@ def stop_on_entry(self):
             info = set_additional_thread_info(main_thread)
             info.pydev_original_step_cmd = CMD_STOP_ON_START
             info.pydev_step_cmd = CMD_STEP_INTO_MY_CODE
+            info.update_stepping_info()
             if PYDEVD_USE_SYS_MONITORING:
                 pydevd_sys_monitoring.update_monitor_events(suspend_requested=True)
                 pydevd_sys_monitoring.restart_events()