Rollup of all of the mixed mode changes (#8071)

* Support mixed mode for 3.10 (#8050)

* Fix 3.10 thread state

* Add the other versions elsewhere

* Test with native calling python

* Fix 3.10 thread state

* Add the other versions elsewhere

* Test with native calling python

* Add commit where _cframe was created

* Use an environment variable for python install path

* Get 3.11 to work in mixed mode (#8056)

* Fix 3.10 thread state

* Partially working for 3.11

Locals/Globals not being read correctly. They moved

* Basics for 3.11 working

* Fix stuff broken in 3.10 by 3.11 changes

* Fix tabs

* Fix review comments

* Support for 312 mixed mode debugging (#8063)

* Fix 3.10 thread state

* Partially working for 3.11

Locals/Globals not being read correctly. They moved

* Basics for 3.11 working

* Fix stuff broken in 3.10 by 3.11 changes

* Fix tabs

* Get python breakpoints working again

* Partially working for 3.11

Locals/Globals not being read correctly. They moved

* Get python breakpoints working again

* Get longs to work using new 3.12 long implementation

* Compute dynamic parts of the frame like the f_back and the line number

* Update comment

* Add support for mixed mode debugging in 3.13 (#8070)

* Changes for 3.13 Mixed mode debugging

* Fix step out/over

* Fix example to not hold file handle

Examples/PythonCallingNative/PythonApplication/PythonApplication.py

@@ -1,6 +1,12 @@
 
 import ctypes 
 
+def returnATuple():
+    return (4, 5)
+
+def returnADict():
+    return { "4": 5 }
+
 # Load the DLL
 mylib = ctypes.CDLL('..\\x64\\Debug\\CppDll.dll')
 
@@ -9,4 +15,9 @@
 
 print(f"Result from C++ DLL: {result}")
 
-print("After result is printed")
+print("After result is printed")
+
+x = returnATuple()
+y = returnADict()
+
+print("After tuple")

Examples/PythonCallingNative/PythonApplication/PythonApplication.pyproj

@@ -11,7 +11,7 @@
     <OutputPath>.</OutputPath>
     <Name>PythonApplication</Name>
     <RootNamespace>PythonApplication</RootNamespace>
-    <InterpreterId>Global|PythonCore|3.9</InterpreterId>
+    <InterpreterId>Global|PythonCore|3.13</InterpreterId>
     <LaunchProvider>Standard Python launcher</LaunchProvider>
     <EnableNativeCodeDebugging>True</EnableNativeCodeDebugging>
   </PropertyGroup>
@@ -34,6 +34,10 @@
     </ProjectReference>
   </ItemGroup>
   <ItemGroup>
+    <InterpreterReference Include="Global|PythonCore|3.10" />
+    <InterpreterReference Include="Global|PythonCore|3.11" />
+    <InterpreterReference Include="Global|PythonCore|3.12" />
+    <InterpreterReference Include="Global|PythonCore|3.13" />
     <InterpreterReference Include="Global|PythonCore|3.9" />
   </ItemGroup>
   <Import Project="$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)\Python Tools\Microsoft.PythonTools.targets" />

Examples/PythonNative/PythonNative.cpp

@@ -13,12 +13,13 @@ DWORD WINAPI runner(LPVOID lpParam)
     std::filesystem::path cwd = std::filesystem::current_path();
     std::filesystem::path startFile = cwd / "runner.py";
 
-    const char * startFileStr = startFile.string().c_str();
+    std::string str = startFile.string();
+    const char * startFileStr = str.c_str();
 
     PyObject* startObj = Py_BuildValue("s", startFileStr);
     FILE* file = _Py_fopen_obj(startObj, "rb+");
     if (file != NULL) {
-        PyRun_SimpleFile(file, startFileStr);
+        PyRun_SimpleFileEx(file, startFileStr, 1);
     }
 
     Py_Finalize();

Examples/PythonNative/PythonNative.vcxproj

@@ -104,14 +104,14 @@
       <SDLCheck>true</SDLCheck>
       <PreprocessorDefinitions>_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
-      <AdditionalIncludeDirectories>C:\Users\rchiodo\AppData\Local\Programs\Python\Python311\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>$(LOCALAPPDATA)\Programs\Python\Python313\include;$(LOCALAPPDATA)\Programs\Python\Python312\include;$(LOCALAPPDATA)\Programs\Python\Python311\include;$(LOCALAPPDATA)\Programs\Python\Python310\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <LanguageStandard>stdcpp17</LanguageStandard>
       <LanguageStandard_C>stdc17</LanguageStandard_C>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>
       <GenerateDebugInformation>true</GenerateDebugInformation>
-      <AdditionalLibraryDirectories>C:\Users\rchiodo\AppData\Local\Programs\Python\Python311\libs;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
+      <AdditionalLibraryDirectories>$(LOCALAPPDATA)\Programs\Python\Python313\libs;$(LOCALAPPDATA)\Programs\Python\Python312\libs;$(LOCALAPPDATA)\Programs\Python\Python311\libs;$(LOCALAPPDATA)\Programs\Python\Python310\libs;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
     </Link>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
@@ -122,7 +122,7 @@
       <SDLCheck>true</SDLCheck>
       <PreprocessorDefinitions>NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
-      <AdditionalIncludeDirectories>C:\Users\rchiodo\AppData\Local\Programs\Python\Python311\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>$(LOCALAPPDATA)\Programs\Python\Python310\include;$(LOCALAPPDATA)\Programs\Python\Python311\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <LanguageStandard>stdcpp17</LanguageStandard>
       <LanguageStandard_C>stdc17</LanguageStandard_C>
     </ClCompile>
@@ -131,7 +131,7 @@
       <EnableCOMDATFolding>true</EnableCOMDATFolding>
       <OptimizeReferences>true</OptimizeReferences>
       <GenerateDebugInformation>true</GenerateDebugInformation>
-      <AdditionalLibraryDirectories>C:\Users\rchiodo\AppData\Local\Programs\Python\Python311\libs;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
+      <AdditionalLibraryDirectories>$(LOCALAPPDATA)\Programs\Python\Python310\libs;$(LOCALAPPDATA)\Programs\Python\Python311\libs;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
     </Link>
   </ItemDefinitionGroup>
   <ItemGroup>

Python/Product/Cookiecutter/Shared/Interpreters/PythonLanguageVersion.cs

@@ -39,7 +39,10 @@ public enum PythonLanguageVersion {
         V37 = 0x0307,
         V38 = 0x0308,
         V39 = 0x0309,
-        V310 = 0x0310
+        V310 = 0x030a,
+        V311 = 0x030b,
+        V312 = 0x030c,
+        V313 = 0x030d,
     }
 
     public static class PythonLanguageVersionExtensions {
@@ -87,6 +90,9 @@ public static PythonLanguageVersion ToLanguageVersion(this Version version) {
                         case 8: return PythonLanguageVersion.V38;
                         case 9: return PythonLanguageVersion.V39;
                         case 10: return PythonLanguageVersion.V310;
+                        case 11: return PythonLanguageVersion.V311;
+                        case 12: return PythonLanguageVersion.V312;
+                        case 13: return PythonLanguageVersion.V313;
                     }
                     break;
             }

Python/Product/Debugger.Concord/CallStackFilter.cs

@@ -93,7 +93,7 @@ public DkmStackWalkFrame[] FilterNextFrame(DkmStackContext stackContext, DkmStac
             PyCodeObject code = pythonFrame.f_code.Read();
             var loc = new SourceLocation(
                 code.co_filename.Read().ToStringOrNull(),
-                pythonFrame.f_lineno.Read(),
+                pythonFrame.ComputeLineNumber(stackContext.InspectionSession, nativeFrame, stackContext.FormatOptions.EvaluationFlags),
                 code.co_name.Read().ToStringOrNull(),
                 nativeFrame.InstructionAddress as DkmNativeInstructionAddress);
 

Python/Product/Debugger.Concord/CppExpressionEvaluator.cs

@@ -32,7 +32,10 @@ internal class CppExpressionEvaluator {
         private readonly DkmStackWalkFrame _nativeFrame;
         private readonly DkmInspectionContext _cppInspectionContext;
 
-        public CppExpressionEvaluator(DkmInspectionContext inspectionContext, DkmStackWalkFrame stackFrame) {
+        public CppExpressionEvaluator(DkmInspectionContext inspectionContext, DkmStackWalkFrame stackFrame) 
+            : this(inspectionContext.InspectionSession, inspectionContext.Radix, stackFrame) { 
+        }
+        public CppExpressionEvaluator(DkmInspectionSession inspectionSession, uint radix, DkmStackWalkFrame stackFrame, DkmEvaluationFlags flags = DkmEvaluationFlags.TreatAsExpression | DkmEvaluationFlags.NoSideEffects) {
             _process = stackFrame.Process;
             var thread = stackFrame.Thread;
 
@@ -53,16 +56,15 @@ public CppExpressionEvaluator(DkmInspectionContext inspectionContext, DkmStackWa
                     DkmStackWalkFrameFlags.None, null, stackFrame.Registers, null);
             }
 
-            _cppInspectionContext = DkmInspectionContext.Create(inspectionContext.InspectionSession, _process.GetNativeRuntimeInstance(), thread, Timeout,
-                DkmEvaluationFlags.TreatAsExpression | DkmEvaluationFlags.NoSideEffects, DkmFuncEvalFlags.None, inspectionContext.Radix, CppLanguage, null);
+            _cppInspectionContext = DkmInspectionContext.Create(inspectionSession, _process.GetNativeRuntimeInstance(), thread, Timeout,
+                flags, DkmFuncEvalFlags.None, radix, CppLanguage, null);
         }
 
-        public CppExpressionEvaluator(DkmThread thread, ulong frameBase, ulong vframe) {
+        public CppExpressionEvaluator(DkmThread thread, ulong frameBase, ulong vframe, DkmEvaluationFlags flags = DkmEvaluationFlags.TreatAsExpression | DkmEvaluationFlags.NoSideEffects) {
             _process = thread.Process;
-
             var inspectionSession = DkmInspectionSession.Create(_process, null);
             _cppInspectionContext = DkmInspectionContext.Create(inspectionSession, _process.GetNativeRuntimeInstance(), thread, Timeout,
-                DkmEvaluationFlags.TreatAsExpression | DkmEvaluationFlags.NoSideEffects, DkmFuncEvalFlags.None, 10, CppLanguage, null);
+                flags, DkmFuncEvalFlags.None, 10, CppLanguage, null);
 
             const int CV_ALLREG_VFRAME = 0x00007536;
             var vframeReg = DkmUnwoundRegister.Create(CV_ALLREG_VFRAME, new ReadOnlyCollection<byte>(BitConverter.GetBytes(vframe)));
@@ -79,8 +81,8 @@ public static string GetExpressionForObject(string moduleName, string typeName,
             return expr;
         }
 
-        public DkmEvaluationResult TryEvaluate(string expr) {
-            using (var cppExpr = DkmLanguageExpression.Create(CppLanguage, DkmEvaluationFlags.NoSideEffects, expr, null)) {
+        public DkmEvaluationResult TryEvaluate(string expr, DkmEvaluationFlags flags = DkmEvaluationFlags.NoSideEffects) {
+            using (var cppExpr = DkmLanguageExpression.Create(CppLanguage, flags, expr, null)) {
                 DkmEvaluationResult cppEvalResult = null;
                 var cppWorkList = DkmWorkList.Create(null);
                 _cppInspectionContext.EvaluateExpression(cppWorkList, cppExpr, _nativeFrame, (result) => {
@@ -95,18 +97,18 @@ public DkmEvaluationResult TryEvaluateObject(string moduleName, string typeName,
             return TryEvaluate(GetExpressionForObject(moduleName, typeName, address, tail));
         }
 
-        public string Evaluate(string expr) {
-            var er = TryEvaluate(expr);
+        public string Evaluate(string expr, DkmEvaluationFlags flags = DkmEvaluationFlags.NoSideEffects) {
+            var er = TryEvaluate(expr, flags);
             var ser = er as DkmSuccessEvaluationResult;
             if (ser == null) {
                 throw new CppEvaluationException(er);
             }
             return ser.Value;
         }
 
-        public int EvaluateInt32(string expr) {
+        public int EvaluateInt32(string expr, DkmEvaluationFlags flags = DkmEvaluationFlags.NoSideEffects) {
             try {
-                return int.Parse(Evaluate("(__int32)(" + expr + ")"));
+                return int.Parse(Evaluate("(__int32)(" + expr + ")", flags));
             } catch (FormatException) {
                 throw new CppEvaluationException();
             }

Python/Product/Debugger.Concord/Debugger.Concord.csproj

@@ -107,6 +107,16 @@
     <Compile Include="Proxies\DataProxy.cs" />
     <Compile Include="ExpressionEvaluator.cs" />
     <Compile Include="LocalStackWalkingComponent.cs" />
+    <Compile Include="Proxies\Structs\CFrameProxy.cs" />
+    <Compile Include="Proxies\Structs\ImportState.cs" />
+    <Compile Include="Proxies\Structs\PyCodeObject310.cs" />
+    <Compile Include="Proxies\Structs\PyCodeObject311.cs" />
+    <Compile Include="Proxies\Structs\PyDictObject.cs" />
+    <Compile Include="Proxies\Structs\PyDictObject311.cs" />
+    <Compile Include="Proxies\Structs\PyFrameObject310.cs" />
+    <Compile Include="Proxies\Structs\PyFrameObject311.cs" />
+    <Compile Include="Proxies\Structs\PyFunctionObject.cs" />
+    <Compile Include="Proxies\Structs\PyInterpreterFrame.cs" />
     <Compile Include="Proxies\Structs\PyRuntimeState.cs" />
     <Compile Include="Proxies\Structs\PyEllipsisObject.cs" />
     <Compile Include="Proxies\Structs\PyComplexObject.cs" />
@@ -118,6 +128,9 @@
     <Compile Include="Proxies\Structs\PyBoolObject.cs" />
     <Compile Include="Proxies\Structs\PySetObject.cs" />
     <Compile Include="Proxies\Structs\PyCellObject.cs" />
+    <Compile Include="Proxies\Structs\PyThreads.cs" />
+    <Compile Include="Proxies\Structs\PyUnicodeObject311.cs" />
+    <Compile Include="Proxies\Structs\PyUnicodeObject312.cs" />
     <Compile Include="PythonRuntimeInfo.cs" />
     <Compile Include="StackFrameDataItem.cs" />
     <Compile Include="ValueStore.cs" />
@@ -138,7 +151,7 @@
     <Compile Include="Proxies\Structs\PyInterpreterState.cs" />
     <Compile Include="Proxies\StructProxy.cs" />
     <Compile Include="Proxies\Structs\PyCodeObject.cs" />
-    <Compile Include="Proxies\Structs\PyDictObject.cs" />
+    <Compile Include="Proxies\Structs\PyDictObject310.cs" />
     <Compile Include="Proxies\Structs\PyFrameObject.cs" />
     <Compile Include="Proxies\Structs\PyListObject.cs" />
     <Compile Include="Proxies\Structs\PyLongObject.cs" />

Python/Product/Debugger.Concord/DebuggerOptions.cs

@@ -20,6 +20,9 @@
 namespace Microsoft.PythonTools.Debugger.Concord {
     public static class DebuggerOptions {
         // These are intentionally not implemented as auto-properties to enable easily changing them at runtime, including when stopped in native code.
+        //
+        // These show up in commands in the debugger watch window if you set the registry value:
+        // Key: HKCU/Software/Microsoft/PythonTools/Debugger - Value: PythonDeveloper: DWORD = 1
         private static bool _showNativePythonFrames;
         private static bool _usePythonStepping;
         private static bool _showCppViewNodes;
@@ -29,7 +32,7 @@ public static class DebuggerOptions {
 
         public static bool ShowNativePythonFrames {
             get {
-                return _showNativePythonFrames; // Enable this to show native (C++) frames including our trace helper
+                return _showNativePythonFrames; // Enable this to show native (C++) frames including our trace helper and CPython code
             }
             set {
                 _showNativePythonFrames = value;
@@ -39,7 +42,7 @@ public static bool ShowNativePythonFrames {
 
         public static bool UsePythonStepping {
             get {
-                return _usePythonStepping; // Disable this to step through the TraceHelper dll in the launched VS
+                return _usePythonStepping; // Disable this to step through the TraceHelper dll (or CPython) in the launched VS
             }
             set {
                 _usePythonStepping = value;

Python/Product/Debugger.Concord/ModuleManager.cs

@@ -76,9 +76,14 @@ public ModuleManager(DkmProcess process) {
             _pyrtInfo = process.GetPythonRuntimeInfo();
 
             LoadInitialPythonModules();
-            string pyCodeFunctionName = (_pyrtInfo.LanguageVersion < PythonLanguageVersion.V38) ? "PyCode_New" : "PyCode_NewWithPosOnlyArgs";
-            LocalComponent.CreateRuntimeDllFunctionBreakpoint(_pyrtInfo.DLLs.Python, pyCodeFunctionName, PythonDllBreakpointHandlers.PyCode_New, enable: true, debugStart: true);
-            LocalComponent.CreateRuntimeDllFunctionBreakpoint(_pyrtInfo.DLLs.Python, "PyCode_NewEmpty", PythonDllBreakpointHandlers.PyCode_NewEmpty, enable: true, debugStart: true);
+
+            if (_pyrtInfo.LanguageVersion <= PythonLanguageVersion.V310) {
+                LocalComponent.CreateRuntimeDllFunctionBreakpoint(_pyrtInfo.DLLs.Python, "PyCode_NewWithPosOnlyArgs", PythonDllBreakpointHandlers.PyCode_New, enable: true, debugStart: true);
+                LocalComponent.CreateRuntimeDllFunctionBreakpoint(_pyrtInfo.DLLs.Python, "PyCode_NewEmpty", PythonDllBreakpointHandlers.PyCode_NewEmpty, enable: true, debugStart: true);
+            } else {
+                // In 3.11, the PyCode_New functions were no longer used. Instead, an internal _PyCode_New function is used to create a code object.
+                LocalComponent.CreateRuntimeDllFunctionBreakpoint(_pyrtInfo.DLLs.Python, "_PyCode_New", PythonDllBreakpointHandlers._PyCode_New, enable: true, debugStart: true);
+            }
         }
 
         private void LoadInitialPythonModules() {
@@ -160,6 +165,27 @@ public static void PyCode_New(DkmThread thread, ulong frameBase, ulong vframe, u
                 }.SendLower(process);
             }
 
+            public static void _PyCode_New(DkmThread thread, ulong frameBase, ulong vframe, ulong returnAddress) {
+                var process = thread.Process;
+                var cppEval = new CppExpressionEvaluator(thread, frameBase, vframe);
+
+                var filenamePtr = cppEval.EvaluateUInt64("con->filename");
+                var filenameObj = PyObject.FromAddress(process, filenamePtr) as IPyBaseStringObject;
+                if (filenameObj == null) {
+                    return;
+                }
+
+                string filename = filenameObj.ToString();
+                if (process.GetPythonRuntimeInstance().GetModuleInstances().Any(mi => mi.FullName == filename)) {
+                    return;
+                }
+
+                new RemoteComponent.CreateModuleRequest {
+                    ModuleId = Guid.NewGuid(),
+                    FileName = filename
+                }.SendLower(process);
+            }
+
             public static void PyCode_NewEmpty(DkmThread thread, ulong frameBase, ulong vframe, ulong returnAddress) {
                 var process = thread.Process;
                 var cppEval = new CppExpressionEvaluator(thread, frameBase, vframe);

Python/Product/Debugger.Concord/Proxies/DataProxy.cs

@@ -58,21 +58,24 @@ private static class FactoryBuilder<TProxy> where TProxy : IDataProxy {
             public static readonly FactoryFunc Factory;
 
             static FactoryBuilder() {
-                FactoryFunc nonPolymorphicFactory;
-                var ctor = typeof(TProxy).GetConstructor(new[] { typeof(DkmProcess), typeof(ulong) });
-                if (ctor != null) {
-                    var processParam = Expression.Parameter(typeof(DkmProcess));
-                    var addressParam = Expression.Parameter(typeof(ulong));
-                    var polymorphicParam = Expression.Parameter(typeof(bool));
-                    nonPolymorphicFactory = Expression.Lambda<FactoryFunc>(
-                        Expression.New(ctor, processParam, addressParam),
-                        new[] { processParam, addressParam, polymorphicParam })
-                        .Compile();
-                } else {
-                    nonPolymorphicFactory = (process, address, polymorphic) => {
-                        Debug.Fail("IDebuggeeReference-derived type " + typeof(TProxy).Name + " does not have a (DkmProcess, ulong) constructor.");
-                        throw new NotSupportedException();
-                    };
+                FactoryFunc nonPolymorphicFactory = (process, address, polymorphic) => {
+                    Debug.Fail("IDebuggeeReference-derived type " + typeof(TProxy).Name + " does not have a (DkmProcess, ulong) constructor or cannot be instantiated.");
+                    throw new NotSupportedException();
+                };
+                var type = typeof(TProxy);
+
+                // Make sure we have a constructor that takes a DkmProcess and a ulong.  If we don't, we can't instantiate the type.
+                if (!type.IsAbstract) { 
+                    var ctor = type.GetConstructor(new[] { typeof(DkmProcess), typeof(ulong) });
+                    if (ctor != null) {
+                        var processParam = Expression.Parameter(typeof(DkmProcess));
+                        var addressParam = Expression.Parameter(typeof(ulong));
+                        var polymorphicParam = Expression.Parameter(typeof(bool));
+                        nonPolymorphicFactory = Expression.Lambda<FactoryFunc>(
+                            Expression.New(ctor, processParam, addressParam),
+                            new[] { processParam, addressParam, polymorphicParam })
+                            .Compile();
+                    }
                 }
 
                 if (typeof(IPyObject).IsAssignableFrom(typeof(TProxy))) {