From 8b1d41a17ad2d2d586ab773d6e55ec43f9c21c13 Mon Sep 17 00:00:00 2001 From: Rich Chiodo Date: Mon, 18 Nov 2024 09:50:12 -0800 Subject: [PATCH] Rollup of all of the mixed mode changes (#8071) * Support mixed mode for 3.10 (#8050) * Fix 3.10 thread state * Add the other versions elsewhere * Test with native calling python * Fix 3.10 thread state * Add the other versions elsewhere * Test with native calling python * Add commit where _cframe was created * Use an environment variable for python install path * Get 3.11 to work in mixed mode (#8056) * Fix 3.10 thread state * Partially working for 3.11 Locals/Globals not being read correctly. They moved * Basics for 3.11 working * Fix stuff broken in 3.10 by 3.11 changes * Fix tabs * Fix review comments * Support for 312 mixed mode debugging (#8063) * Fix 3.10 thread state * Partially working for 3.11 Locals/Globals not being read correctly. They moved * Basics for 3.11 working * Fix stuff broken in 3.10 by 3.11 changes * Fix tabs * Get python breakpoints working again * Partially working for 3.11 Locals/Globals not being read correctly. They moved * Get python breakpoints working again * Get longs to work using new 3.12 long implementation * Compute dynamic parts of the frame like the f_back and the line number * Update comment * Add support for mixed mode debugging in 3.13 (#8070) * Changes for 3.13 Mixed mode debugging * Fix step out/over * Fix example to not hold file handle --- .../PythonApplication/PythonApplication.py | 13 +- .../PythonApplication.pyproj | 6 +- Examples/PythonNative/PythonNative.cpp | 5 +- Examples/PythonNative/PythonNative.vcxproj | 8 +- .../Interpreters/PythonLanguageVersion.cs | 8 +- .../Debugger.Concord/CallStackFilter.cs | 2 +- .../CppExpressionEvaluator.cs | 26 +- .../Debugger.Concord/Debugger.Concord.csproj | 15 +- .../Debugger.Concord/DebuggerOptions.cs | 7 +- .../Product/Debugger.Concord/ModuleManager.cs | 32 ++- .../Debugger.Concord/Proxies/DataProxy.cs | 33 +-- .../Debugger.Concord/Proxies/StructProxy.cs | 6 +- .../Proxies/Structs/CFrameProxy.cs | 45 ++++ .../Proxies/Structs/ImportState.cs | 27 ++ .../Proxies/Structs/PyCodeObject.cs | 50 +--- .../Proxies/Structs/PyCodeObject310.cs | 60 +++++ .../Proxies/Structs/PyCodeObject311.cs | 134 ++++++++++ .../Proxies/Structs/PyDictObject.cs | 137 +---------- .../Proxies/Structs/PyDictObject310.cs | 157 ++++++++++++ .../Proxies/Structs/PyDictObject311.cs | 208 ++++++++++++++++ .../Proxies/Structs/PyFrameObject.cs | 57 ++--- .../Proxies/Structs/PyFrameObject310.cs | 57 +++++ .../Proxies/Structs/PyFrameObject311.cs | 92 +++++++ .../Proxies/Structs/PyFunctionObject.cs | 50 ++++ .../Proxies/Structs/PyInterpreterFrame.cs | 109 ++++++++ .../Proxies/Structs/PyInterpreterState.cs | 62 +++-- .../Proxies/Structs/PyLongObject.cs | 151 +++++++++++- .../Proxies/Structs/PyObject.cs | 11 +- .../Proxies/Structs/PyRuntimeState.cs | 18 ++ .../Proxies/Structs/PyThreadState.cs | 91 +++++-- .../Proxies/Structs/PyThreads.cs | 41 ++++ .../Proxies/Structs/PyTupleObject.cs | 2 +- .../Proxies/Structs/PyUnicodeObject.cs | 220 +---------------- .../Proxies/Structs/PyUnicodeObject311.cs | 232 ++++++++++++++++++ .../Proxies/Structs/PyUnicodeObject312.cs | 219 +++++++++++++++++ .../Debugger.Concord/PythonRuntimeInfo.cs | 3 + .../TraceManagerLocalHelper.cs | 160 +++++++++--- Python/Product/DebuggerHelper/trace.cpp | 116 +++++++-- .../Utilities.Python.Analysis/PythonPaths.cs | 6 + package.json | 2 +- 40 files changed, 2106 insertions(+), 572 deletions(-) create mode 100644 Python/Product/Debugger.Concord/Proxies/Structs/CFrameProxy.cs create mode 100644 Python/Product/Debugger.Concord/Proxies/Structs/ImportState.cs create mode 100644 Python/Product/Debugger.Concord/Proxies/Structs/PyCodeObject310.cs create mode 100644 Python/Product/Debugger.Concord/Proxies/Structs/PyCodeObject311.cs create mode 100644 Python/Product/Debugger.Concord/Proxies/Structs/PyDictObject310.cs create mode 100644 Python/Product/Debugger.Concord/Proxies/Structs/PyDictObject311.cs create mode 100644 Python/Product/Debugger.Concord/Proxies/Structs/PyFrameObject310.cs create mode 100644 Python/Product/Debugger.Concord/Proxies/Structs/PyFrameObject311.cs create mode 100644 Python/Product/Debugger.Concord/Proxies/Structs/PyFunctionObject.cs create mode 100644 Python/Product/Debugger.Concord/Proxies/Structs/PyInterpreterFrame.cs create mode 100644 Python/Product/Debugger.Concord/Proxies/Structs/PyThreads.cs create mode 100644 Python/Product/Debugger.Concord/Proxies/Structs/PyUnicodeObject311.cs create mode 100644 Python/Product/Debugger.Concord/Proxies/Structs/PyUnicodeObject312.cs diff --git a/Examples/PythonCallingNative/PythonApplication/PythonApplication.py b/Examples/PythonCallingNative/PythonApplication/PythonApplication.py index d900c68aab..f0e52db402 100644 --- a/Examples/PythonCallingNative/PythonApplication/PythonApplication.py +++ b/Examples/PythonCallingNative/PythonApplication/PythonApplication.py @@ -1,6 +1,12 @@ import ctypes +def returnATuple(): + return (4, 5) + +def returnADict(): + return { "4": 5 } + # Load the DLL mylib = ctypes.CDLL('..\\x64\\Debug\\CppDll.dll') @@ -9,4 +15,9 @@ print(f"Result from C++ DLL: {result}") -print("After result is printed") \ No newline at end of file +print("After result is printed") + +x = returnATuple() +y = returnADict() + +print("After tuple") \ No newline at end of file diff --git a/Examples/PythonCallingNative/PythonApplication/PythonApplication.pyproj b/Examples/PythonCallingNative/PythonApplication/PythonApplication.pyproj index fc1d81083b..dae27d9f55 100644 --- a/Examples/PythonCallingNative/PythonApplication/PythonApplication.pyproj +++ b/Examples/PythonCallingNative/PythonApplication/PythonApplication.pyproj @@ -11,7 +11,7 @@ . PythonApplication PythonApplication - Global|PythonCore|3.9 + Global|PythonCore|3.13 Standard Python launcher True @@ -34,6 +34,10 @@ + + + + diff --git a/Examples/PythonNative/PythonNative.cpp b/Examples/PythonNative/PythonNative.cpp index 5899825e35..5019f859b6 100644 --- a/Examples/PythonNative/PythonNative.cpp +++ b/Examples/PythonNative/PythonNative.cpp @@ -13,12 +13,13 @@ DWORD WINAPI runner(LPVOID lpParam) std::filesystem::path cwd = std::filesystem::current_path(); std::filesystem::path startFile = cwd / "runner.py"; - const char * startFileStr = startFile.string().c_str(); + std::string str = startFile.string(); + const char * startFileStr = str.c_str(); PyObject* startObj = Py_BuildValue("s", startFileStr); FILE* file = _Py_fopen_obj(startObj, "rb+"); if (file != NULL) { - PyRun_SimpleFile(file, startFileStr); + PyRun_SimpleFileEx(file, startFileStr, 1); } Py_Finalize(); diff --git a/Examples/PythonNative/PythonNative.vcxproj b/Examples/PythonNative/PythonNative.vcxproj index 8c410f4ab8..565b470286 100644 --- a/Examples/PythonNative/PythonNative.vcxproj +++ b/Examples/PythonNative/PythonNative.vcxproj @@ -104,14 +104,14 @@ true _DEBUG;_CONSOLE;%(PreprocessorDefinitions) true - C:\Users\rchiodo\AppData\Local\Programs\Python\Python311\include;%(AdditionalIncludeDirectories) + $(LOCALAPPDATA)\Programs\Python\Python313\include;$(LOCALAPPDATA)\Programs\Python\Python312\include;$(LOCALAPPDATA)\Programs\Python\Python311\include;$(LOCALAPPDATA)\Programs\Python\Python310\include;%(AdditionalIncludeDirectories) stdcpp17 stdc17 Console true - C:\Users\rchiodo\AppData\Local\Programs\Python\Python311\libs;%(AdditionalLibraryDirectories) + $(LOCALAPPDATA)\Programs\Python\Python313\libs;$(LOCALAPPDATA)\Programs\Python\Python312\libs;$(LOCALAPPDATA)\Programs\Python\Python311\libs;$(LOCALAPPDATA)\Programs\Python\Python310\libs;%(AdditionalLibraryDirectories) @@ -122,7 +122,7 @@ true NDEBUG;_CONSOLE;%(PreprocessorDefinitions) true - C:\Users\rchiodo\AppData\Local\Programs\Python\Python311\include;%(AdditionalIncludeDirectories) + $(LOCALAPPDATA)\Programs\Python\Python310\include;$(LOCALAPPDATA)\Programs\Python\Python311\include;%(AdditionalIncludeDirectories) stdcpp17 stdc17 @@ -131,7 +131,7 @@ true true true - C:\Users\rchiodo\AppData\Local\Programs\Python\Python311\libs;%(AdditionalLibraryDirectories) + $(LOCALAPPDATA)\Programs\Python\Python310\libs;$(LOCALAPPDATA)\Programs\Python\Python311\libs;%(AdditionalLibraryDirectories) diff --git a/Python/Product/Cookiecutter/Shared/Interpreters/PythonLanguageVersion.cs b/Python/Product/Cookiecutter/Shared/Interpreters/PythonLanguageVersion.cs index 73a200bf56..1ff391a37b 100644 --- a/Python/Product/Cookiecutter/Shared/Interpreters/PythonLanguageVersion.cs +++ b/Python/Product/Cookiecutter/Shared/Interpreters/PythonLanguageVersion.cs @@ -39,7 +39,10 @@ public enum PythonLanguageVersion { V37 = 0x0307, V38 = 0x0308, V39 = 0x0309, - V310 = 0x0310 + V310 = 0x030a, + V311 = 0x030b, + V312 = 0x030c, + V313 = 0x030d, } public static class PythonLanguageVersionExtensions { @@ -87,6 +90,9 @@ public static PythonLanguageVersion ToLanguageVersion(this Version version) { case 8: return PythonLanguageVersion.V38; case 9: return PythonLanguageVersion.V39; case 10: return PythonLanguageVersion.V310; + case 11: return PythonLanguageVersion.V311; + case 12: return PythonLanguageVersion.V312; + case 13: return PythonLanguageVersion.V313; } break; } diff --git a/Python/Product/Debugger.Concord/CallStackFilter.cs b/Python/Product/Debugger.Concord/CallStackFilter.cs index ec1d4d84a0..f9cabb893b 100644 --- a/Python/Product/Debugger.Concord/CallStackFilter.cs +++ b/Python/Product/Debugger.Concord/CallStackFilter.cs @@ -93,7 +93,7 @@ public DkmStackWalkFrame[] FilterNextFrame(DkmStackContext stackContext, DkmStac PyCodeObject code = pythonFrame.f_code.Read(); var loc = new SourceLocation( code.co_filename.Read().ToStringOrNull(), - pythonFrame.f_lineno.Read(), + pythonFrame.ComputeLineNumber(stackContext.InspectionSession, nativeFrame, stackContext.FormatOptions.EvaluationFlags), code.co_name.Read().ToStringOrNull(), nativeFrame.InstructionAddress as DkmNativeInstructionAddress); diff --git a/Python/Product/Debugger.Concord/CppExpressionEvaluator.cs b/Python/Product/Debugger.Concord/CppExpressionEvaluator.cs index c9dd71769c..2d91f4576f 100644 --- a/Python/Product/Debugger.Concord/CppExpressionEvaluator.cs +++ b/Python/Product/Debugger.Concord/CppExpressionEvaluator.cs @@ -32,7 +32,10 @@ internal class CppExpressionEvaluator { private readonly DkmStackWalkFrame _nativeFrame; private readonly DkmInspectionContext _cppInspectionContext; - public CppExpressionEvaluator(DkmInspectionContext inspectionContext, DkmStackWalkFrame stackFrame) { + public CppExpressionEvaluator(DkmInspectionContext inspectionContext, DkmStackWalkFrame stackFrame) + : this(inspectionContext.InspectionSession, inspectionContext.Radix, stackFrame) { + } + public CppExpressionEvaluator(DkmInspectionSession inspectionSession, uint radix, DkmStackWalkFrame stackFrame, DkmEvaluationFlags flags = DkmEvaluationFlags.TreatAsExpression | DkmEvaluationFlags.NoSideEffects) { _process = stackFrame.Process; var thread = stackFrame.Thread; @@ -53,16 +56,15 @@ public CppExpressionEvaluator(DkmInspectionContext inspectionContext, DkmStackWa DkmStackWalkFrameFlags.None, null, stackFrame.Registers, null); } - _cppInspectionContext = DkmInspectionContext.Create(inspectionContext.InspectionSession, _process.GetNativeRuntimeInstance(), thread, Timeout, - DkmEvaluationFlags.TreatAsExpression | DkmEvaluationFlags.NoSideEffects, DkmFuncEvalFlags.None, inspectionContext.Radix, CppLanguage, null); + _cppInspectionContext = DkmInspectionContext.Create(inspectionSession, _process.GetNativeRuntimeInstance(), thread, Timeout, + flags, DkmFuncEvalFlags.None, radix, CppLanguage, null); } - public CppExpressionEvaluator(DkmThread thread, ulong frameBase, ulong vframe) { + public CppExpressionEvaluator(DkmThread thread, ulong frameBase, ulong vframe, DkmEvaluationFlags flags = DkmEvaluationFlags.TreatAsExpression | DkmEvaluationFlags.NoSideEffects) { _process = thread.Process; - var inspectionSession = DkmInspectionSession.Create(_process, null); _cppInspectionContext = DkmInspectionContext.Create(inspectionSession, _process.GetNativeRuntimeInstance(), thread, Timeout, - DkmEvaluationFlags.TreatAsExpression | DkmEvaluationFlags.NoSideEffects, DkmFuncEvalFlags.None, 10, CppLanguage, null); + flags, DkmFuncEvalFlags.None, 10, CppLanguage, null); const int CV_ALLREG_VFRAME = 0x00007536; var vframeReg = DkmUnwoundRegister.Create(CV_ALLREG_VFRAME, new ReadOnlyCollection(BitConverter.GetBytes(vframe))); @@ -79,8 +81,8 @@ public static string GetExpressionForObject(string moduleName, string typeName, return expr; } - public DkmEvaluationResult TryEvaluate(string expr) { - using (var cppExpr = DkmLanguageExpression.Create(CppLanguage, DkmEvaluationFlags.NoSideEffects, expr, null)) { + public DkmEvaluationResult TryEvaluate(string expr, DkmEvaluationFlags flags = DkmEvaluationFlags.NoSideEffects) { + using (var cppExpr = DkmLanguageExpression.Create(CppLanguage, flags, expr, null)) { DkmEvaluationResult cppEvalResult = null; var cppWorkList = DkmWorkList.Create(null); _cppInspectionContext.EvaluateExpression(cppWorkList, cppExpr, _nativeFrame, (result) => { @@ -95,8 +97,8 @@ public DkmEvaluationResult TryEvaluateObject(string moduleName, string typeName, return TryEvaluate(GetExpressionForObject(moduleName, typeName, address, tail)); } - public string Evaluate(string expr) { - var er = TryEvaluate(expr); + public string Evaluate(string expr, DkmEvaluationFlags flags = DkmEvaluationFlags.NoSideEffects) { + var er = TryEvaluate(expr, flags); var ser = er as DkmSuccessEvaluationResult; if (ser == null) { throw new CppEvaluationException(er); @@ -104,9 +106,9 @@ public string Evaluate(string expr) { return ser.Value; } - public int EvaluateInt32(string expr) { + public int EvaluateInt32(string expr, DkmEvaluationFlags flags = DkmEvaluationFlags.NoSideEffects) { try { - return int.Parse(Evaluate("(__int32)(" + expr + ")")); + return int.Parse(Evaluate("(__int32)(" + expr + ")", flags)); } catch (FormatException) { throw new CppEvaluationException(); } diff --git a/Python/Product/Debugger.Concord/Debugger.Concord.csproj b/Python/Product/Debugger.Concord/Debugger.Concord.csproj index 34fa4736db..0e2e94cc9e 100644 --- a/Python/Product/Debugger.Concord/Debugger.Concord.csproj +++ b/Python/Product/Debugger.Concord/Debugger.Concord.csproj @@ -107,6 +107,16 @@ + + + + + + + + + + @@ -118,6 +128,9 @@ + + + @@ -138,7 +151,7 @@ - + diff --git a/Python/Product/Debugger.Concord/DebuggerOptions.cs b/Python/Product/Debugger.Concord/DebuggerOptions.cs index c78712764d..b41ab1c0aa 100644 --- a/Python/Product/Debugger.Concord/DebuggerOptions.cs +++ b/Python/Product/Debugger.Concord/DebuggerOptions.cs @@ -20,6 +20,9 @@ namespace Microsoft.PythonTools.Debugger.Concord { public static class DebuggerOptions { // These are intentionally not implemented as auto-properties to enable easily changing them at runtime, including when stopped in native code. + // + // These show up in commands in the debugger watch window if you set the registry value: + // Key: HKCU/Software/Microsoft/PythonTools/Debugger - Value: PythonDeveloper: DWORD = 1 private static bool _showNativePythonFrames; private static bool _usePythonStepping; private static bool _showCppViewNodes; @@ -29,7 +32,7 @@ public static class DebuggerOptions { public static bool ShowNativePythonFrames { get { - return _showNativePythonFrames; // Enable this to show native (C++) frames including our trace helper + return _showNativePythonFrames; // Enable this to show native (C++) frames including our trace helper and CPython code } set { _showNativePythonFrames = value; @@ -39,7 +42,7 @@ public static bool ShowNativePythonFrames { public static bool UsePythonStepping { get { - return _usePythonStepping; // Disable this to step through the TraceHelper dll in the launched VS + return _usePythonStepping; // Disable this to step through the TraceHelper dll (or CPython) in the launched VS } set { _usePythonStepping = value; diff --git a/Python/Product/Debugger.Concord/ModuleManager.cs b/Python/Product/Debugger.Concord/ModuleManager.cs index 1dbe7f9ee4..6d8c526bde 100644 --- a/Python/Product/Debugger.Concord/ModuleManager.cs +++ b/Python/Product/Debugger.Concord/ModuleManager.cs @@ -76,9 +76,14 @@ public ModuleManager(DkmProcess process) { _pyrtInfo = process.GetPythonRuntimeInfo(); LoadInitialPythonModules(); - string pyCodeFunctionName = (_pyrtInfo.LanguageVersion < PythonLanguageVersion.V38) ? "PyCode_New" : "PyCode_NewWithPosOnlyArgs"; - LocalComponent.CreateRuntimeDllFunctionBreakpoint(_pyrtInfo.DLLs.Python, pyCodeFunctionName, PythonDllBreakpointHandlers.PyCode_New, enable: true, debugStart: true); - LocalComponent.CreateRuntimeDllFunctionBreakpoint(_pyrtInfo.DLLs.Python, "PyCode_NewEmpty", PythonDllBreakpointHandlers.PyCode_NewEmpty, enable: true, debugStart: true); + + if (_pyrtInfo.LanguageVersion <= PythonLanguageVersion.V310) { + LocalComponent.CreateRuntimeDllFunctionBreakpoint(_pyrtInfo.DLLs.Python, "PyCode_NewWithPosOnlyArgs", PythonDllBreakpointHandlers.PyCode_New, enable: true, debugStart: true); + LocalComponent.CreateRuntimeDllFunctionBreakpoint(_pyrtInfo.DLLs.Python, "PyCode_NewEmpty", PythonDllBreakpointHandlers.PyCode_NewEmpty, enable: true, debugStart: true); + } else { + // In 3.11, the PyCode_New functions were no longer used. Instead, an internal _PyCode_New function is used to create a code object. + LocalComponent.CreateRuntimeDllFunctionBreakpoint(_pyrtInfo.DLLs.Python, "_PyCode_New", PythonDllBreakpointHandlers._PyCode_New, enable: true, debugStart: true); + } } private void LoadInitialPythonModules() { @@ -160,6 +165,27 @@ public static void PyCode_New(DkmThread thread, ulong frameBase, ulong vframe, u }.SendLower(process); } + public static void _PyCode_New(DkmThread thread, ulong frameBase, ulong vframe, ulong returnAddress) { + var process = thread.Process; + var cppEval = new CppExpressionEvaluator(thread, frameBase, vframe); + + var filenamePtr = cppEval.EvaluateUInt64("con->filename"); + var filenameObj = PyObject.FromAddress(process, filenamePtr) as IPyBaseStringObject; + if (filenameObj == null) { + return; + } + + string filename = filenameObj.ToString(); + if (process.GetPythonRuntimeInstance().GetModuleInstances().Any(mi => mi.FullName == filename)) { + return; + } + + new RemoteComponent.CreateModuleRequest { + ModuleId = Guid.NewGuid(), + FileName = filename + }.SendLower(process); + } + public static void PyCode_NewEmpty(DkmThread thread, ulong frameBase, ulong vframe, ulong returnAddress) { var process = thread.Process; var cppEval = new CppExpressionEvaluator(thread, frameBase, vframe); diff --git a/Python/Product/Debugger.Concord/Proxies/DataProxy.cs b/Python/Product/Debugger.Concord/Proxies/DataProxy.cs index 5fa9cf9c8e..150a9058bc 100644 --- a/Python/Product/Debugger.Concord/Proxies/DataProxy.cs +++ b/Python/Product/Debugger.Concord/Proxies/DataProxy.cs @@ -58,21 +58,24 @@ private static class FactoryBuilder where TProxy : IDataProxy { public static readonly FactoryFunc Factory; static FactoryBuilder() { - FactoryFunc nonPolymorphicFactory; - var ctor = typeof(TProxy).GetConstructor(new[] { typeof(DkmProcess), typeof(ulong) }); - if (ctor != null) { - var processParam = Expression.Parameter(typeof(DkmProcess)); - var addressParam = Expression.Parameter(typeof(ulong)); - var polymorphicParam = Expression.Parameter(typeof(bool)); - nonPolymorphicFactory = Expression.Lambda( - Expression.New(ctor, processParam, addressParam), - new[] { processParam, addressParam, polymorphicParam }) - .Compile(); - } else { - nonPolymorphicFactory = (process, address, polymorphic) => { - Debug.Fail("IDebuggeeReference-derived type " + typeof(TProxy).Name + " does not have a (DkmProcess, ulong) constructor."); - throw new NotSupportedException(); - }; + FactoryFunc nonPolymorphicFactory = (process, address, polymorphic) => { + Debug.Fail("IDebuggeeReference-derived type " + typeof(TProxy).Name + " does not have a (DkmProcess, ulong) constructor or cannot be instantiated."); + throw new NotSupportedException(); + }; + var type = typeof(TProxy); + + // Make sure we have a constructor that takes a DkmProcess and a ulong. If we don't, we can't instantiate the type. + if (!type.IsAbstract) { + var ctor = type.GetConstructor(new[] { typeof(DkmProcess), typeof(ulong) }); + if (ctor != null) { + var processParam = Expression.Parameter(typeof(DkmProcess)); + var addressParam = Expression.Parameter(typeof(ulong)); + var polymorphicParam = Expression.Parameter(typeof(bool)); + nonPolymorphicFactory = Expression.Lambda( + Expression.New(ctor, processParam, addressParam), + new[] { processParam, addressParam, polymorphicParam }) + .Compile(); + } } if (typeof(IPyObject).IsAssignableFrom(typeof(TProxy))) { diff --git a/Python/Product/Debugger.Concord/Proxies/StructProxy.cs b/Python/Product/Debugger.Concord/Proxies/StructProxy.cs index 78b196caff..673c5e9ca2 100644 --- a/Python/Product/Debugger.Concord/Proxies/StructProxy.cs +++ b/Python/Product/Debugger.Concord/Proxies/StructProxy.cs @@ -20,6 +20,7 @@ using Microsoft.Dia; using Microsoft.PythonTools.Common.Parsing; using Microsoft.VisualStudio.Debugger; +using Microsoft.VisualStudio.Debugger.Interop; namespace Microsoft.PythonTools.Debugger.Concord.Proxies { [AttributeUsage(AttributeTargets.Class, AllowMultiple = true)] @@ -139,11 +140,6 @@ public override int GetHashCode() { return new { _process, _address }.GetHashCode(); } - protected TProxy GetFieldProxy(StructField? field, bool polymorphic = true) - where TProxy : IDataProxy { - return field.HasValue ? GetFieldProxy(field.Value) : default(TProxy); - } - protected TProxy GetFieldProxy(StructField field, bool polymorphic = true) where TProxy : IDataProxy { return DataProxy.Create(Process, Address.OffsetBy(field.Offset), polymorphic); diff --git a/Python/Product/Debugger.Concord/Proxies/Structs/CFrameProxy.cs b/Python/Product/Debugger.Concord/Proxies/Structs/CFrameProxy.cs new file mode 100644 index 0000000000..b5f979afdf --- /dev/null +++ b/Python/Product/Debugger.Concord/Proxies/Structs/CFrameProxy.cs @@ -0,0 +1,45 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using Microsoft.PythonTools.Common.Parsing; +using Microsoft.VisualStudio.Debugger; + +namespace Microsoft.PythonTools.Debugger.Concord.Proxies.Structs { + [StructProxy(StructName = "_cframe", MinVersion = PythonLanguageVersion.V310, MaxVersion = PythonLanguageVersion.V310)] + [StructProxy(StructName = "_PyCFrame", MinVersion = PythonLanguageVersion.V311)] + internal class CFrameProxy : StructProxy { + internal class Fields { + [FieldProxy(MaxVersion = PythonLanguageVersion.V311)] + public StructField use_tracing; + public StructField> previous; + [FieldProxy(MinVersion = PythonLanguageVersion.V311)] + public StructField> current_frame; + } + + private readonly Fields _fields; + + public CFrameProxy(DkmProcess process, ulong address) + : base(process, address) { + InitializeStruct(this, out _fields); + } + + public Int32Proxy use_tracing { + get { return GetFieldProxy(_fields.use_tracing); } + } + + public PointerProxy current_frame => GetFieldProxy(_fields.current_frame); + } +} diff --git a/Python/Product/Debugger.Concord/Proxies/Structs/ImportState.cs b/Python/Product/Debugger.Concord/Proxies/Structs/ImportState.cs new file mode 100644 index 0000000000..872c291044 --- /dev/null +++ b/Python/Product/Debugger.Concord/Proxies/Structs/ImportState.cs @@ -0,0 +1,27 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.PythonTools.Common.Parsing; +using Microsoft.VisualStudio.Debugger; + +namespace Microsoft.PythonTools.Debugger.Concord.Proxies.Structs { + [StructProxy(StructName = "_import_state", MinVersion = PythonLanguageVersion.V312)] + internal class ImportState : StructProxy { + internal class Fields { + public StructField> modules; + } + + private readonly Fields _fields; + + public ImportState(DkmProcess process, ulong address) + : base(process, address) { + InitializeStruct(this, out _fields); + } + + public PointerProxy modules { + get { return GetFieldProxy(_fields.modules); } + } + } +} diff --git a/Python/Product/Debugger.Concord/Proxies/Structs/PyCodeObject.cs b/Python/Product/Debugger.Concord/Proxies/Structs/PyCodeObject.cs index 4a2cdfe53a..c0883a99e3 100644 --- a/Python/Product/Debugger.Concord/Proxies/Structs/PyCodeObject.cs +++ b/Python/Product/Debugger.Concord/Proxies/Structs/PyCodeObject.cs @@ -14,60 +14,30 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using Microsoft.PythonTools.Common.Parsing; using Microsoft.VisualStudio.Debugger; namespace Microsoft.PythonTools.Debugger.Concord.Proxies.Structs { - internal class PyCodeObject : PyObject { - public class Fields { - public StructField co_nlocals; - public StructField> co_names; - public StructField> co_varnames; - public StructField> co_freevars; - public StructField> co_cellvars; - public StructField> co_filename; - public StructField> co_name; - public StructField co_firstlineno; - } - - private readonly Fields _fields; - + internal abstract class PyCodeObject : PyObject { public PyCodeObject(DkmProcess process, ulong address) : base(process, address) { - InitializeStruct(this, out _fields); - CheckPyType(); } - public Int32Proxy co_nlocals { - get { return GetFieldProxy(_fields.co_nlocals); } - } + public abstract Int32Proxy co_nlocals { get; } - public PointerProxy co_names { - get { return GetFieldProxy(_fields.co_names); } - } + public abstract PointerProxy co_names { get; } - public PointerProxy co_varnames { - get { return GetFieldProxy(_fields.co_varnames); } - } + public abstract IWritableDataProxy co_varnames { get; } - public PointerProxy co_freevars { - get { return GetFieldProxy(_fields.co_freevars); } - } + public abstract IWritableDataProxy co_freevars { get; } - public PointerProxy co_cellvars { - get { return GetFieldProxy(_fields.co_cellvars); } - } + public abstract IWritableDataProxy co_cellvars { get; } - public PointerProxy co_filename { - get { return GetFieldProxy(_fields.co_filename); } - } + public abstract PointerProxy co_filename { get; } - public PointerProxy co_name { - get { return GetFieldProxy(_fields.co_name); } - } + public abstract PointerProxy co_name { get; } - public Int32Proxy co_firstlineno { - get { return GetFieldProxy(_fields.co_firstlineno); } - } + public abstract Int32Proxy co_firstlineno { get; } public override void Repr(ReprBuilder builder) { string name = co_name.TryRead().ToStringOrNull() ?? "???"; diff --git a/Python/Product/Debugger.Concord/Proxies/Structs/PyCodeObject310.cs b/Python/Product/Debugger.Concord/Proxies/Structs/PyCodeObject310.cs new file mode 100644 index 0000000000..c4ca468338 --- /dev/null +++ b/Python/Product/Debugger.Concord/Proxies/Structs/PyCodeObject310.cs @@ -0,0 +1,60 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using Microsoft.PythonTools.Common.Parsing; +using Microsoft.VisualStudio.Debugger; + +namespace Microsoft.PythonTools.Debugger.Concord.Proxies.Structs { + [StructProxy(StructName = "PyCodeObject", MaxVersion = PythonLanguageVersion.V310)] + [PyType(MaxVersion = PythonLanguageVersion.V310, VariableName = "PyCode_Type")] + internal class PyCodeObject310 : PyCodeObject { + public class Fields { + public StructField co_nlocals; + public StructField> co_names; + public StructField> co_varnames; + public StructField> co_freevars; + public StructField> co_cellvars; + public StructField> co_filename; + public StructField> co_name; + public StructField co_firstlineno; + } + + private readonly Fields _fields; + + public override Int32Proxy co_nlocals => GetFieldProxy(_fields.co_nlocals); + + public override PointerProxy co_names => GetFieldProxy(_fields.co_names); + + public override IWritableDataProxy co_varnames => GetFieldProxy(_fields.co_varnames); + + public override IWritableDataProxy co_freevars => GetFieldProxy(_fields.co_freevars); + + public override IWritableDataProxy co_cellvars => GetFieldProxy(_fields.co_cellvars); + + public override PointerProxy co_filename => GetFieldProxy(_fields.co_filename); + + public override PointerProxy co_name => GetFieldProxy(_fields.co_name); + + public override Int32Proxy co_firstlineno => GetFieldProxy(_fields.co_firstlineno); + + public PyCodeObject310(DkmProcess process, ulong address) + : base(process, address) { + InitializeStruct(this, out _fields); + CheckPyType(); + } + + } +} diff --git a/Python/Product/Debugger.Concord/Proxies/Structs/PyCodeObject311.cs b/Python/Product/Debugger.Concord/Proxies/Structs/PyCodeObject311.cs new file mode 100644 index 0000000000..c3d20ae719 --- /dev/null +++ b/Python/Product/Debugger.Concord/Proxies/Structs/PyCodeObject311.cs @@ -0,0 +1,134 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using Microsoft.PythonTools.Common.Parsing; +using Microsoft.VisualStudio.Debugger; + +namespace Microsoft.PythonTools.Debugger.Concord.Proxies.Structs { + [StructProxy(StructName = "PyCodeObject", MinVersion = PythonLanguageVersion.V311)] + [PyType(MinVersion = PythonLanguageVersion.V311, VariableName = "PyCode_Type")] + internal class PyCodeObject311 : PyCodeObject { + public class Fields { + public StructField co_nlocals; + public StructField> co_names; + public StructField> co_filename; + public StructField> co_name; + public StructField co_firstlineno; + public StructField> co_localsplusnames; + public StructField> co_localspluskinds; + public StructField> co_code_adaptive; + public StructField _co_firsttraceable; + } + + private readonly Fields _fields; + + public override Int32Proxy co_nlocals => GetFieldProxy(_fields.co_nlocals); + + public override PointerProxy co_names => GetFieldProxy(_fields.co_names); + + public override IWritableDataProxy co_varnames => GetVariableTupleProxy(VariableKind.CO_FAST_LOCAL); + + public override IWritableDataProxy co_freevars => GetVariableTupleProxy(VariableKind.CO_FAST_FREE); + + public override IWritableDataProxy co_cellvars => GetVariableTupleProxy(VariableKind.CO_FAST_CELL); + + public override PointerProxy co_filename => GetFieldProxy(_fields.co_filename); + + public override PointerProxy co_name => GetFieldProxy(_fields.co_name); + + public override Int32Proxy co_firstlineno => GetFieldProxy(_fields.co_firstlineno); + + public ArrayProxy co_code_adaptive => GetFieldProxy(_fields.co_code_adaptive); + + public Int32Proxy _co_firsttraceable => GetFieldProxy(_fields._co_firsttraceable); + + public PyCodeObject311(DkmProcess process, ulong address) + : base(process, address) { + InitializeStruct(this, out _fields); + CheckPyType(); + } + + [Flags] + internal enum VariableKind { + CO_FAST_LOCAL = 0x20, + CO_FAST_CELL = 0x40, + CO_FAST_FREE = 0x80 + } + + private IWritableDataProxy GetVariableTupleProxy(VariableKind kind) { + return new PyVariableTuplePointerProxy(Process, kind, GetFieldProxy(_fields.co_localsplusnames), GetFieldProxy(_fields.co_localspluskinds)); + } + + // In 3.11, the co_varnames, co_freevars, and co_cellvars were all removed. In order to not have to change our usage + // of them, this class and the PyVariableTuple were created in order to pretend to behave like the original co_varnames etc. + internal class PyVariableTuplePointerProxy : IWritableDataProxy { + private readonly PyVariableTuple _pseudoTuple; + private readonly PointerProxy _realTuple; + private readonly DkmProcess _process; + + public PyVariableTuplePointerProxy(DkmProcess process, VariableKind kind, PointerProxy localsplusnames, PointerProxy localspluskinds) { + _pseudoTuple = new PyVariableTuple(process, kind, localsplusnames, localspluskinds); + _realTuple = localsplusnames; + _process = process; + } + + public DkmProcess Process => _process; + + public ulong Address => _realTuple.Address; + + public long ObjectSize => _realTuple.ObjectSize; + + public PyTupleObject Read() => _pseudoTuple; + + public void Write(PyTupleObject value) => throw new NotImplementedException(); + public void Write(object value) => throw new NotImplementedException(); + object IValueStore.Read() => _pseudoTuple; + } + + [PyType(Hidden = true)] + internal class PyVariableTuple : PyTupleObject { + private readonly VariableKind _kind; + private readonly PointerProxy _localsplusnames; + private readonly PointerProxy _localspluskinds; + private readonly DkmProcess _process; + + public PyVariableTuple(DkmProcess process, VariableKind kind, PointerProxy localsplusnames, PointerProxy localspluskinds) + : base(process, localsplusnames.Read().Address) { + _kind = kind; + _localsplusnames = localsplusnames; + _localspluskinds = localspluskinds; + _process = process; + } + + public override IEnumerable> ReadElements() { + var localsplusnames = new List>(_localsplusnames.Read().ReadElements()); + var localspluskinds = _localspluskinds.Read().ToBytes(); + Debug.Assert(localsplusnames.Count == localspluskinds.Length); + for (var i = 0; i < localsplusnames.Count; i++) { + var name = localsplusnames[i]; + var kind = (VariableKind)localspluskinds[i]; + if ((kind & _kind) == _kind) { + yield return name; + } + } + } + + } + } +} diff --git a/Python/Product/Debugger.Concord/Proxies/Structs/PyDictObject.cs b/Python/Product/Debugger.Concord/Proxies/Structs/PyDictObject.cs index a5b9b7045f..32e6459782 100644 --- a/Python/Product/Debugger.Concord/Proxies/Structs/PyDictObject.cs +++ b/Python/Product/Debugger.Concord/Proxies/Structs/PyDictObject.cs @@ -1,4 +1,4 @@ -// Python Tools for Visual Studio +// Python Tools for Visual Studio // Copyright(c) Microsoft Corporation // All rights reserved. // @@ -21,68 +21,12 @@ using Microsoft.VisualStudio.Debugger.Evaluation; namespace Microsoft.PythonTools.Debugger.Concord.Proxies.Structs { - [StructProxy(StructName = "PyDictObject", MinVersion = PythonLanguageVersion.V39)] - [PyType(VariableName = "PyDict_Type", MinVersion = PythonLanguageVersion.V39)] - internal class PyDictObject : PyObject { - private class Fields { - public StructField> ma_keys; - public StructField>>> ma_values; - } - - private readonly Fields _fields; - - public PyDictObject(DkmProcess process, ulong address) + internal abstract class PyDictObject : PyObject { + protected PyDictObject(DkmProcess process, ulong address) : base(process, address) { - InitializeStruct(this, out _fields); - CheckPyType(); - } - - public PointerProxy ma_keys { - get { return GetFieldProxy(_fields.ma_keys); } } - public PointerProxy>> ma_values { - get { return GetFieldProxy(_fields.ma_values); } - } - - public IEnumerable>> ReadElements() { - if (ma_keys.IsNull) { - yield break; - } - - var keys = ma_keys.Read(); - var size = keys.dk_size.Read(); - if (size <= 0) { - yield break; - } - - var n = keys.dk_nentries.Read(); - var dk_entries = keys.dk_entries; - var entry = dk_entries[0]; - - if (!ma_values.IsNull) { - var values = ma_values.Read(); - var value = values[0]; - - for (int i = 0; i < n; ++i) { - var key = entry.me_key; - if (!value.IsNull && !key.IsNull) { - yield return new KeyValuePair>(key.Read(), value); - } - entry = entry.GetAdjacentProxy(1); - value = value.GetAdjacentProxy(1); - } - } else { - for (int i = 0; i < n; ++i) { - var key = entry.me_key; - var value = entry.me_value; - if (!key.IsNull && !value.IsNull) { - yield return new KeyValuePair>(key.Read(), value); - } - entry = entry.GetAdjacentProxy(1); - } - } - } + public abstract IEnumerable>> ReadElements(); public override void Repr(ReprBuilder builder) { var count = ReadElements().Count(); @@ -114,75 +58,4 @@ public override IEnumerable GetDebugChildren(ReprOptions } } - [StructProxy(MinVersion = PythonLanguageVersion.V39, StructName = "_dictkeysobject")] - internal class PyDictKeysObject : StructProxy { - public class Fields { - public StructField dk_size; - public StructField dk_nentries; - public StructField> dk_indices; - } - - public readonly Fields _fields; - - public SSizeTProxy dk_size { - get { return GetFieldProxy(_fields.dk_size); } - } - - public SSizeTProxy dk_nentries { - get { return GetFieldProxy(_fields.dk_nentries); } - } - - public ArrayProxy dk_entries { - get { - // dk_entries is located after dk_indices, which is - // variable length depending on the size of the table. - long size = dk_size.Read(); - long offset = _fields.dk_indices.Offset; - if (size <= 0) { - return default(ArrayProxy); - } else if (size <= 0xFF) { - offset += size; - } else if (size <= 0xFFFF) { - offset += size * 2; - } else if (size <= 0xFFFFFFFF) { - offset += size * 4; - } else { - offset += size * 8; - } - - return DataProxy.Create>( - Process, - Address.OffsetBy(offset) - ); - } - } - - public PyDictKeysObject(DkmProcess process, ulong address) - : base(process, address) { - InitializeStruct(this, out _fields); - } - } - - [StructProxy(MinVersion = PythonLanguageVersion.V39)] - internal class PyDictKeyEntry : StructProxy { - private class Fields { - public StructField> me_key; - public StructField> me_value; - } - - private readonly Fields _fields; - - public PyDictKeyEntry(DkmProcess process, ulong address) - : base(process, address) { - InitializeStruct(this, out _fields); - } - - public PointerProxy me_key { - get { return GetFieldProxy(_fields.me_key); } - } - - public PointerProxy me_value { - get { return GetFieldProxy(_fields.me_value); } - } - } -} +} \ No newline at end of file diff --git a/Python/Product/Debugger.Concord/Proxies/Structs/PyDictObject310.cs b/Python/Product/Debugger.Concord/Proxies/Structs/PyDictObject310.cs new file mode 100644 index 0000000000..f62051d891 --- /dev/null +++ b/Python/Product/Debugger.Concord/Proxies/Structs/PyDictObject310.cs @@ -0,0 +1,157 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.PythonTools.Common.Parsing; +using Microsoft.VisualStudio.Debugger; +using Microsoft.VisualStudio.Debugger.Evaluation; + +namespace Microsoft.PythonTools.Debugger.Concord.Proxies.Structs { + [StructProxy(StructName = "PyDictObject", MinVersion = PythonLanguageVersion.V39, MaxVersion = PythonLanguageVersion.V310 )] + [PyType(VariableName = "PyDict_Type", MinVersion = PythonLanguageVersion.V39, MaxVersion = PythonLanguageVersion.V310)] + internal class PyDictObject310 : PyDictObject { + private class Fields { + public StructField> ma_keys; + public StructField>>> ma_values; + } + + private readonly Fields _fields; + + public PyDictObject310(DkmProcess process, ulong address) + : base(process, address) { + InitializeStruct(this, out _fields); + CheckPyType(); + } + + public PointerProxy ma_keys { + get { return GetFieldProxy(_fields.ma_keys); } + } + + public PointerProxy>> ma_values { + get { return GetFieldProxy(_fields.ma_values); } + } + + public override IEnumerable>> ReadElements() { + if (ma_keys.IsNull) { + yield break; + } + + var keys = ma_keys.Read(); + var size = keys.dk_size.Read(); + if (size <= 0) { + yield break; + } + + var n = keys.dk_nentries.Read(); + var dk_entries = keys.dk_entries; + var entry = dk_entries.First(); + + if (!ma_values.IsNull) { + var values = ma_values.Read(); + var value = values[0]; + + for (int i = 0; i < n; ++i) { + var key = entry.me_key; + if (!value.IsNull && !key.IsNull) { + yield return new KeyValuePair>(key.Read(), value); + } + entry = entry.GetAdjacentProxy(1); + value = value.GetAdjacentProxy(1); + } + } else { + for (int i = 0; i < n; ++i) { + var key = entry.me_key; + var value = entry.me_value; + if (!key.IsNull && !value.IsNull) { + yield return new KeyValuePair>(key.Read(), value); + } + entry = entry.GetAdjacentProxy(1); + } + } + } + } + + [StructProxy(MinVersion = PythonLanguageVersion.V39, MaxVersion = PythonLanguageVersion.V310, StructName = "_dictkeysobject")] + internal class PyDictKeysObject310 : StructProxy { + public class Fields { + public StructField dk_size; + public StructField dk_nentries; + public StructField> dk_indices; + } + + public readonly Fields _fields; + + public SSizeTProxy dk_size => GetFieldProxy(_fields.dk_size); + + public SSizeTProxy dk_nentries { + get { return GetFieldProxy(_fields.dk_nentries); } + } + + public IEnumerable dk_entries { + get { + // dk_entries is located after dk_indices, which is + // variable length depending on the size of the table. + long size = dk_size.Read(); + long offset = _fields.dk_indices.Offset; + if (size <= 0) { + return default(ArrayProxy); + } else if (size <= 0xFF) { + offset += size; + } else if (size <= 0xFFFF) { + offset += size * 2; + } else if (size <= 0xFFFFFFFF) { + offset += size * 4; + } else { + offset += size * 8; + } + + return DataProxy.Create>( + Process, + Address.OffsetBy(offset) + ); + } + } + + public PyDictKeysObject310(DkmProcess process, ulong address) + : base(process, address) { + InitializeStruct(this, out _fields); + } + } + + [StructProxy(MinVersion = PythonLanguageVersion.V39, StructName = "PyDictKeyEntry")] + internal class PyDictKeyEntry310 : StructProxy { + private class Fields { + public StructField> me_key; + public StructField> me_value; + } + + private readonly Fields _fields; + + public PyDictKeyEntry310(DkmProcess process, ulong address) + : base(process, address) { + InitializeStruct(this, out _fields); + } + + public PointerProxy me_key { + get { return GetFieldProxy(_fields.me_key); } + } + + public PointerProxy me_value { + get { return GetFieldProxy(_fields.me_value); } + } + } +} diff --git a/Python/Product/Debugger.Concord/Proxies/Structs/PyDictObject311.cs b/Python/Product/Debugger.Concord/Proxies/Structs/PyDictObject311.cs new file mode 100644 index 0000000000..0b4658c150 --- /dev/null +++ b/Python/Product/Debugger.Concord/Proxies/Structs/PyDictObject311.cs @@ -0,0 +1,208 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.PythonTools.Common.Parsing; +using Microsoft.VisualStudio.Debugger; +using Microsoft.VisualStudio.Debugger.Evaluation; + +namespace Microsoft.PythonTools.Debugger.Concord.Proxies.Structs { + [StructProxy(StructName = "PyDictObject", MinVersion = PythonLanguageVersion.V311)] + [PyType(VariableName = "PyDict_Type", MinVersion = PythonLanguageVersion.V311)] + internal class PyDictObject311 : PyDictObject { + private class Fields { + public StructField> ma_keys; + public StructField>>> ma_values; + } + + private readonly Fields _fields; + + public PyDictObject311(DkmProcess process, ulong address) + : base(process, address) { + InitializeStruct(this, out _fields); + CheckPyType(); + } + + public PointerProxy ma_keys { + get { return GetFieldProxy(_fields.ma_keys); } + } + + public PointerProxy>> ma_values { + get { return GetFieldProxy(_fields.ma_values); } + } + + public override IEnumerable>> ReadElements() { + if (ma_keys.IsNull) { + yield break; + } + + var keys = ma_keys.Read(); + var size = 1 << (int)keys.dk_log2_size.Read(); + if (size <= 0) { + yield break; + } + + var n = keys.dk_nentries.Read(); + if (!ma_values.IsNull) { + var values = ma_values.Read(); + var value = values[0]; + + for (int i = 0; i < n; ++i) { + var entry = GetDkEntry(i); + var key = entry.me_key; + if (!value.IsNull && !key.IsNull) { + yield return new KeyValuePair>(key.Read(), value); + } + value = value.GetAdjacentProxy(1); + } + } else { + for (int i = 0; i < n; ++i) { + var entry = GetDkEntry(i); + var key = entry.me_key; + var value = entry.me_value; + if (!key.IsNull && !value.IsNull) { + yield return new KeyValuePair>(key.Read(), value); + } + } + } + } + + private IDictKeyEntry GetDkEntry(int position) { + var keys = ma_keys.Read(); + if (keys.dk_kind.Read() == 0) { + var entries = keys.dk_general_entries; + return entries[position]; + } else { + var entries = keys.dk_unicode_entries; + return entries[position]; + } + } + + } + + [StructProxy(MinVersion = PythonLanguageVersion.V311, StructName = "_dictkeysobject")] + internal class PyDictKeysObject311 : StructProxy { + public class Fields { + public StructField dk_log2_size; + public StructField dk_log2_index_bytes; + public StructField dk_nentries; + public StructField> dk_indices; + public StructField dk_kind; + } + + public readonly Fields _fields; + + public SSizeTProxy dk_log2_size => GetFieldProxy(_fields.dk_log2_size); + public SSizeTProxy dk_log2_index_bytes => GetFieldProxy(_fields.dk_log2_index_bytes); + + public ByteProxy dk_kind => GetFieldProxy(_fields.dk_kind); + + public SSizeTProxy dk_nentries { + get { return GetFieldProxy(_fields.dk_nentries); } + } + + public ArrayProxy dk_general_entries { + get { + // dk_entries is located after dk_indices, which is + // variable length depending on the size of the table. + long log_size = dk_log2_index_bytes.Read(); + long offset = _fields.dk_indices.Offset; + offset += 1 << (int)log_size; + + return DataProxy.Create>( + Process, + Address.OffsetBy(offset) + ); + } + } + + public ArrayProxy dk_unicode_entries { + get { + // dk_entries is located after dk_indices, which is + // variable length depending on the size of the table. + long log_size = dk_log2_index_bytes.Read(); + long offset = _fields.dk_indices.Offset; + offset += 1 << (int)log_size; + + return DataProxy.Create>( + Process, + Address.OffsetBy(offset) + ); + } + } + + public PyDictKeysObject311(DkmProcess process, ulong address) + : base(process, address) { + InitializeStruct(this, out _fields); + } + } + + internal interface IDictKeyEntry : IDataProxy { + public PointerProxy me_key { get; } + + public PointerProxy me_value { get; } + } + + [StructProxy(MinVersion = PythonLanguageVersion.V311, StructName ="PyDictKeyEntry")] + internal class PyDictKeyEntry311 : StructProxy, IDictKeyEntry { + private class Fields { + public StructField> me_key; + public StructField> me_value; + } + + private readonly Fields _fields; + + public PyDictKeyEntry311(DkmProcess process, ulong address) + : base(process, address) { + InitializeStruct(this, out _fields); + } + + public PointerProxy me_key { + get { return GetFieldProxy(_fields.me_key); } + } + + public PointerProxy me_value { + get { return GetFieldProxy(_fields.me_value); } + } + + public IDictKeyEntry Read() => this; + } + [StructProxy(MinVersion = PythonLanguageVersion.V311)] + internal class PyDictUnicodeEntry : StructProxy, IDictKeyEntry { + private class Fields { + public StructField> me_key; + public StructField> me_value; + } + + private readonly Fields _fields; + + public PyDictUnicodeEntry(DkmProcess process, ulong address) + : base(process, address) { + InitializeStruct(this, out _fields); + } + + public PointerProxy me_key { + get { return GetFieldProxy(_fields.me_key); } + } + + public PointerProxy me_value { + get { return GetFieldProxy(_fields.me_value); } + } + + public IDictKeyEntry Read() => this; + } +} diff --git a/Python/Product/Debugger.Concord/Proxies/Structs/PyFrameObject.cs b/Python/Product/Debugger.Concord/Proxies/Structs/PyFrameObject.cs index a7036a74d6..01989f32f0 100644 --- a/Python/Product/Debugger.Concord/Proxies/Structs/PyFrameObject.cs +++ b/Python/Product/Debugger.Concord/Proxies/Structs/PyFrameObject.cs @@ -14,37 +14,16 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; -using System.Collections.Generic; -using System.Diagnostics; using System.Linq; -using System.Runtime.CompilerServices; -using Microsoft.PythonTools.Common.Core.OS; using Microsoft.PythonTools.Common.Parsing; using Microsoft.VisualStudio.Debugger; using Microsoft.VisualStudio.Debugger.CallStack; using Microsoft.VisualStudio.Debugger.Evaluation; -using static Microsoft.VisualStudio.Threading.SingleThreadedSynchronizationContext; namespace Microsoft.PythonTools.Debugger.Concord.Proxies.Structs { - [StructProxy(MinVersion = PythonLanguageVersion.V39, StructName = "_frame")] - internal class PyFrameObject : PyVarObject { - internal class Fields { - public StructField> f_back; - public StructField> f_code; - public StructField> f_globals; - public StructField> f_locals; - public StructField f_lineno; - public StructField>> f_localsplus; - } - - private readonly Fields _fields; - + internal abstract class PyFrameObject : PyVarObject { public PyFrameObject(DkmProcess process, ulong address) : base(process, address) { - var pythonInfo = process.GetPythonRuntimeInfo(); - InitializeStruct(this, out _fields); - CheckPyType(); } @@ -77,34 +56,26 @@ public static unsafe PyFrameObject TryCreate(DkmStackWalkFrame frame, int? previ var framePtrAddress = PyFrameObject.GetFramePtrAddress(frame, previousFrameCount); if (framePtrAddress != 0) { - return new PyFrameObject(frame.Process, framePtrAddress); + var pythonInfo = process.GetPythonRuntimeInfo(); + if (pythonInfo.LanguageVersion < PythonLanguageVersion.V311) { + return new PyFrameObject310(frame.Process, framePtrAddress); + } + return new PyFrameObject311(frame.Process, framePtrAddress); } return null; } - public PointerProxy f_back { - get { return GetFieldProxy(_fields.f_back); } - } + public abstract PointerProxy f_back { get; } - public PointerProxy f_code { - get { return GetFieldProxy(_fields.f_code); } - } + public abstract PointerProxy f_code { get; } - public PointerProxy f_globals { - get { return GetFieldProxy(_fields.f_globals); } - } + public abstract PointerProxy f_globals { get; } - public PointerProxy f_locals { - get { return GetFieldProxy(_fields.f_locals); } - } + public abstract PointerProxy f_locals { get; } - public Int32Proxy f_lineno { - get { return GetFieldProxy(_fields.f_lineno); } - } + public abstract int ComputeLineNumber(DkmInspectionSession inspectionSession, DkmStackWalkFrame frame, DkmEvaluationFlags flags); - public ArrayProxy> f_localsplus { - get { return GetFieldProxy(_fields.f_localsplus); } - } + public abstract ArrayProxy> f_localsplus { get; } private static ulong GetFramePtrAddress(DkmStackWalkFrame frame, int? previousFrameCount) { // Frame address may already be stored in the frame, check the data. @@ -115,12 +86,12 @@ private static ulong GetFramePtrAddress(DkmStackWalkFrame frame, int? previousFr var process = frame.Process; var tid = frame.Thread.SystemPart.Id; PyThreadState tstate = PyThreadState.GetThreadStates(process).FirstOrDefault(ts => ts.thread_id.Read() == tid); - PyFrameObject pyFrame = tstate.frame.Read(); + PyFrameObject pyFrame = tstate.frame.TryRead(); if (pyFrame != null) { // This pyFrame should be the topmost frame. We need to go down the callstack // based on the number of previous frames that were already found. var numberBack = previousFrameCount != null ? previousFrameCount.Value : 0; - while (numberBack > 0) { + while (numberBack > 0 && pyFrame.f_back.Process != null) { pyFrame = pyFrame.f_back.Read(); numberBack--; } diff --git a/Python/Product/Debugger.Concord/Proxies/Structs/PyFrameObject310.cs b/Python/Product/Debugger.Concord/Proxies/Structs/PyFrameObject310.cs new file mode 100644 index 0000000000..f15bfec02d --- /dev/null +++ b/Python/Product/Debugger.Concord/Proxies/Structs/PyFrameObject310.cs @@ -0,0 +1,57 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using Microsoft.VisualStudio.Debugger; +using Microsoft.PythonTools.Common.Parsing; +using Microsoft.VisualStudio.Debugger.CallStack; +using Microsoft.VisualStudio.Debugger.Evaluation; + +namespace Microsoft.PythonTools.Debugger.Concord.Proxies.Structs { + [StructProxy(StructName = "_frame", MaxVersion = PythonLanguageVersion.V310)] + [PyType(MaxVersion = PythonLanguageVersion.V310, VariableName = "PyFrame_Type")] + internal class PyFrameObject310 : PyFrameObject { + internal class Fields { + public StructField> f_back; + public StructField> f_code; + public StructField> f_globals; + public StructField> f_locals; + public StructField> f_trace; + public StructField f_lineno; + public StructField>> f_localsplus; + } + + private readonly Fields _fields; + + public override PointerProxy f_back => GetFieldProxy(_fields.f_back); + + public override PointerProxy f_code => GetFieldProxy(_fields.f_code); + + public override PointerProxy f_globals => GetFieldProxy(_fields.f_globals); + + public override PointerProxy f_locals => GetFieldProxy(_fields.f_locals); + + public override ArrayProxy> f_localsplus => GetFieldProxy(_fields.f_localsplus); + + public override int ComputeLineNumber(DkmInspectionSession inspectionSession, DkmStackWalkFrame frame, DkmEvaluationFlags flags) => GetFieldProxy(_fields.f_lineno).Read(); + + public PyFrameObject310(DkmProcess process, ulong address) + : base(process, address) { + var pythonInfo = process.GetPythonRuntimeInfo(); + InitializeStruct(this, out _fields); + CheckPyType(); + } + } +} diff --git a/Python/Product/Debugger.Concord/Proxies/Structs/PyFrameObject311.cs b/Python/Product/Debugger.Concord/Proxies/Structs/PyFrameObject311.cs new file mode 100644 index 0000000000..15672e8f8d --- /dev/null +++ b/Python/Product/Debugger.Concord/Proxies/Structs/PyFrameObject311.cs @@ -0,0 +1,92 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using Microsoft.PythonTools.Common.Parsing; +using Microsoft.VisualStudio.Debugger; +using Microsoft.VisualStudio.Debugger.CallStack; +using Microsoft.VisualStudio.Debugger.Evaluation; + +namespace Microsoft.PythonTools.Debugger.Concord.Proxies.Structs { + [StructProxy(StructName = "_frame", MinVersion = PythonLanguageVersion.V311)] + [PyType(MinVersion = PythonLanguageVersion.V311, VariableName = "PyFrame_Type")] + internal class PyFrameObject311 : PyFrameObject { + internal class Fields { + public StructField> f_back; + public StructField> f_frame; + public StructField> f_trace; + public StructField f_lineno; + } + + private readonly Fields _fields; + + public PyFrameObject311(DkmProcess process, ulong address) + : base(process, address) { + var pythonInfo = process.GetPythonRuntimeInfo(); + InitializeStruct(this, out _fields); + CheckPyType(); + } + + public PointerProxy f_frame { + get { return GetFieldProxy(_fields.f_frame); } + } + + private PyInterpreterFrame GetFrame() { + return f_frame.TryRead(); + } + + public override PointerProxy f_back { + get { + // See the code here https://github.com/python/cpython/blob/a0866f4c81ecc057d4521e8e7a02f4e1fff175a1/Objects/frameobject.c#L1491, + // f_back can be null when there is actually a frame because it's created lazily. + var back = GetFieldProxy(_fields.f_back); + if (back.IsNull) { + back = GetFrame().FindBackFrame(); + } + return back; + } + } + + public override PointerProxy f_code => GetFrame().f_code; + + public override PointerProxy f_globals => GetFrame().f_globals; + + public override PointerProxy f_locals => GetFrame().f_locals; + + public override ArrayProxy> f_localsplus => GetFrame().f_localsplus; + + public override int ComputeLineNumber(DkmInspectionSession inspectionSession, DkmStackWalkFrame frame, DkmEvaluationFlags flags) { + var setLineNumber = GetFieldProxy(_fields.f_lineno).Read(); + if (setLineNumber == 0 && flags != DkmEvaluationFlags.None) { + // We need to use the CppExpressionEvaluator to compute the line number from + // our frame object. This function here: https://github.com/python/cpython/blob/46710ca5f263936a2e36fa5d0f140cf9f50b2618/Objects/frameobject.c#L40-L41 + // + // However only do this when stopped at a breakpoint and evaluating the frame. Otherwise we it will fail and cause stepping + // to think the frame we eval is where we should stop. + var evaluator = new CppExpressionEvaluator(inspectionSession, 10, frame, DkmEvaluationFlags.TreatAsExpression); + var funcAddr = Process.GetPythonRuntimeInfo().DLLs.Python.GetFunctionAddress("PyFrame_GetLineNumber"); + var frameAddr = Address; + try { + setLineNumber = evaluator.EvaluateInt32(string.Format("((int (*)(void *)){0})({1})", funcAddr, frameAddr), DkmEvaluationFlags.EnableExtendedSideEffects); + } catch (CppEvaluationException) { + // This means we can't evaluate right now, just leave as zero + setLineNumber = 0; + } + } + + return setLineNumber; + } + } +} diff --git a/Python/Product/Debugger.Concord/Proxies/Structs/PyFunctionObject.cs b/Python/Product/Debugger.Concord/Proxies/Structs/PyFunctionObject.cs new file mode 100644 index 0000000000..3a27d9c451 --- /dev/null +++ b/Python/Product/Debugger.Concord/Proxies/Structs/PyFunctionObject.cs @@ -0,0 +1,50 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using Microsoft.PythonTools.Common.Parsing; +using Microsoft.VisualStudio.Debugger; + +namespace Microsoft.PythonTools.Debugger.Concord.Proxies.Structs { + [StructProxy(StructName = "PyFunctionObject", MinVersion = PythonLanguageVersion.V310)] + internal class PyFunctionObject : PyObject { + internal class Fields { + public StructField> func_globals; + public StructField> func_builtins; + public StructField> func_name; + public StructField> func_qualname; + public StructField> func_code; + public StructField> func_defaults; + public StructField> func_kwdefaults; + public StructField> func_closure; + public StructField> func_doc; + public StructField> func_dict; + public StructField> func_weakreflist; + public StructField> func_module; + public StructField> func_annotations; + public StructField> vectorcall; + [FieldProxy(MinVersion = PythonLanguageVersion.V311)] + public StructField func_version; + } + + private readonly Fields _fields; + + public PyFunctionObject(DkmProcess process, ulong address) + : base(process, address) { + InitializeStruct(this, out _fields); + CheckPyType(); + } + } +} diff --git a/Python/Product/Debugger.Concord/Proxies/Structs/PyInterpreterFrame.cs b/Python/Product/Debugger.Concord/Proxies/Structs/PyInterpreterFrame.cs new file mode 100644 index 0000000000..b7b4f3ca96 --- /dev/null +++ b/Python/Product/Debugger.Concord/Proxies/Structs/PyInterpreterFrame.cs @@ -0,0 +1,109 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Diagnostics; +using Microsoft.PythonTools.Common.Parsing; +using Microsoft.VisualStudio.Debugger; + +namespace Microsoft.PythonTools.Debugger.Concord.Proxies.Structs { + // This was added in commit https://github.com/python/cpython/commit/ae0a2b756255629140efcbe57fc2e714f0267aa3 + [StructProxy(StructName = "_PyInterpreterFrame", MinVersion = PythonLanguageVersion.V311)] + internal class PyInterpreterFrame : StructProxy { + internal class Fields { + public StructField> f_globals; + public StructField> f_builtins; + public StructField> f_locals; + [FieldProxy(MaxVersion = PythonLanguageVersion.V312)] + public StructField> f_code; + [FieldProxy(MinVersion = PythonLanguageVersion.V313)] + public StructField> f_executable; + public StructField> frame_obj; + public StructField> previous; + public StructField>> localsplus; + public StructField owner; + } + + private const int FRAME_OWNED_BY_THREAD = 0; + private const int FRAME_OWNED_BY_GENERATOR = 1; + private const int FRAME_OWNED_BY_FRAME_OBJECT = 2; + private const int FRAME_OWNED_BY_CSTACK = 3; + + private readonly Fields _fields; + + public PyInterpreterFrame(DkmProcess process, ulong address) + : base(process, address) { + InitializeStruct(this, out _fields); + } + + public PointerProxy f_code { + get { + // In 3.13, the f_code was renamed to f_executable + if (_fields.f_code.Process != null) { + return GetFieldProxy(_fields.f_code); + } + return GetFieldProxy(_fields.f_executable); + } + } + + public PointerProxy f_globals { + get { return GetFieldProxy(_fields.f_globals); } + } + + public PointerProxy f_locals { + get { return GetFieldProxy(_fields.f_locals); } + } + + public ArrayProxy> f_localsplus { + get { return GetFieldProxy(_fields.localsplus); } + } + + public PointerProxy frame_obj { + get { return GetFieldProxy(_fields.frame_obj); } + } + + public PointerProxy previous => GetFieldProxy(_fields.previous); + + private bool OwnedByThread() { + if (Process.GetPythonRuntimeInfo().LanguageVersion <= PythonLanguageVersion.V310) { + return true; + } + var owner = (GetFieldProxy(_fields.owner) as IValueStore).Read(); + var charOwner = owner.ToString()[0]; + return (int)charOwner == FRAME_OWNED_BY_THREAD; + } + + public PointerProxy FindBackFrame() { + // Trace.WriteLine("Searching for back frame ..."); + var frame = previous.TryRead(); + while (frame != null && !frame.OwnedByThread()) { + frame = frame.previous.TryRead(); + } + + // Make sure this frame_obj is pointing to this frame. Since the f_back + // of a PyFrameObject can be null even if it exists, the PyFrameObject we find + // through the list of PyInterpreterFrames may not have been created. We need + // to make sure it points to the PyInterpreterFrame we found so that subsequent calls + // to get things like its f_locals will work. + if (frame != null && !frame.frame_obj.IsNull) { + var obj = frame.frame_obj.Read() as PyFrameObject311; + obj.f_frame.Write(frame); + return frame.frame_obj; + } + return default(PointerProxy); + } + } +} diff --git a/Python/Product/Debugger.Concord/Proxies/Structs/PyInterpreterState.cs b/Python/Product/Debugger.Concord/Proxies/Structs/PyInterpreterState.cs index 89a6108b9c..002a4e11d4 100644 --- a/Python/Product/Debugger.Concord/Proxies/Structs/PyInterpreterState.cs +++ b/Python/Product/Debugger.Concord/Proxies/Structs/PyInterpreterState.cs @@ -15,6 +15,7 @@ // permissions and limitations under the License. using System.Collections.Generic; +using System.Diagnostics; using Microsoft.PythonTools.Common.Parsing; using Microsoft.VisualStudio.Debugger; @@ -23,8 +24,14 @@ namespace Microsoft.PythonTools.Debugger.Concord.Proxies.Structs { internal class PyInterpreterState : StructProxy { private class Fields { public StructField> next; + [FieldProxy(MaxVersion = PythonLanguageVersion.V310)] public StructField> tstate_head; + [FieldProxy(MinVersion = PythonLanguageVersion.V311)] + public StructField threads; + [FieldProxy(MaxVersion = PythonLanguageVersion.V311)] public StructField> modules; + [FieldProxy(MinVersion = PythonLanguageVersion.V312)] + public StructField imports; public StructField eval_frame; public StructField ceval; } @@ -49,11 +56,23 @@ public PointerProxy next { } public PointerProxy tstate_head { - get { return GetFieldProxy(_fields.tstate_head); } + get { + if (_fields.tstate_head.Process != null) { + return GetFieldProxy(_fields.tstate_head); + } + var threads = GetFieldProxy(_fields.threads); + return threads.head; + } } public PointerProxy modules { - get { return GetFieldProxy(_fields.modules); } + get { + if (_fields.modules.Process != null) { + return GetFieldProxy(_fields.modules); + } + var imports = GetFieldProxy(_fields.imports); + return imports.modules; + } } public PointerProxy eval_frame { @@ -62,29 +81,31 @@ public PointerProxy eval_frame { public ceval_state ceval => GetFieldProxy(_fields.ceval); - private class InterpHeadHolder : DkmDataItem { - public readonly PointerProxy Proxy; - - public InterpHeadHolder(DkmProcess process) { - var pyrtInfo = process.GetPythonRuntimeInfo(); - Proxy = pyrtInfo.GetRuntimeState()?.interpreters.head - ?? pyrtInfo.DLLs.Python.GetStaticVariable>("interp_head"); - } - } - - public static PointerProxy interp_head(DkmProcess process) { - return process.GetOrCreateDataItem(() => new InterpHeadHolder(process)).Proxy; - } public static IEnumerable GetInterpreterStates(DkmProcess process) { - for (var interp = interp_head(process).TryRead(); interp != null; interp = interp.next.TryRead()) { - yield return interp; + var pyrtInfo = process.GetPythonRuntimeInfo(); + var runtimeState = pyrtInfo.GetRuntimeState(); + var interpreters = runtimeState.interpreters; + var head = interpreters.head.TryRead(); + while (head != null) { + yield return head; + head = head.next.TryRead(); } } - public IEnumerable GetThreadStates() { - for (var tstate = tstate_head.TryRead(); tstate != null; tstate = tstate.next.TryRead()) { - yield return tstate; + public IEnumerable GetThreadStates(DkmProcess process) { + var pyrtInfo = process.GetPythonRuntimeInfo(); + if (pyrtInfo.LanguageVersion <= PythonLanguageVersion.V310) { + for (var tstate = tstate_head.TryRead(); tstate != null; tstate = tstate.next.TryRead()) { + yield return tstate; + } + } else { + var threads = GetFieldProxy(_fields.threads); + var head = threads.head.TryRead(); + while (head != null && head.Address != 0) { + yield return head; + head = head.next.TryRead(); + } } } @@ -92,6 +113,7 @@ public IEnumerable GetThreadStates() { public class ceval_state : StructProxy { private class Fields { public StructField recursion_limit; + [FieldProxy(MaxVersion = PythonLanguageVersion.V39)] public StructField tracing_possible; } diff --git a/Python/Product/Debugger.Concord/Proxies/Structs/PyLongObject.cs b/Python/Product/Debugger.Concord/Proxies/Structs/PyLongObject.cs index 8ae5a9fe23..ef127637ef 100644 --- a/Python/Product/Debugger.Concord/Proxies/Structs/PyLongObject.cs +++ b/Python/Product/Debugger.Concord/Proxies/Structs/PyLongObject.cs @@ -17,12 +17,16 @@ using System; using System.Diagnostics; using System.Numerics; +using Microsoft.PythonTools.Common.Parsing; using Microsoft.VisualStudio.Debugger; namespace Microsoft.PythonTools.Debugger.Concord.Proxies.Structs { internal class PyLongObject : PyVarObject { private class Fields { + [FieldProxy(MaxVersion = Common.Parsing.PythonLanguageVersion.V311)] public StructField ob_digit; // this is actually either uint16 or uint32, depending on Python bitness + [FieldProxy(MinVersion = PythonLanguageVersion.V312)] + public StructField<_PyLongValue> long_value; } private readonly Fields _fields; @@ -40,6 +44,16 @@ protected PyLongObject(DkmProcess process, ulong address, bool checkType) } public static PyLongObject Create(DkmProcess process, BigInteger value) { + // Use two different methods. PyLongObjects changed in 3.12. Instead of inheritance, we'll use an if statement so + // that we don't have to change classes derived from PyLongObject. + if (process.GetPythonRuntimeInfo().LanguageVersion < PythonLanguageVersion.V312) { + return Create11(process, value); + } else { + return Create12(process, value); + } + } + + private static PyLongObject Create11(DkmProcess process, BigInteger value) { var allocator = process.GetDataItem(); Debug.Assert(allocator != null); @@ -77,12 +91,69 @@ public static PyLongObject Create(DkmProcess process, BigInteger value) { return result; } + + private static PyLongObject Create12(DkmProcess process, BigInteger value) { + var allocator = process.GetDataItem(); + Debug.Assert(allocator != null); + + var bitsInDigit = process.Is64Bit() ? 30 : 15; + var bytesInDigit = process.Is64Bit() ? 4 : 2; + + var absValue = BigInteger.Abs(value); + long numDigits = 0; + for (var t = absValue; t != 0;) { + ++numDigits; + t >>= bitsInDigit; + } + + var result = allocator.Allocate(numDigits * bytesInDigit); + + // Size comes from here: + // https://github.com/python/cpython/blob/a0866f4c81ecc057d4521e8e7a02f4e1fff175a1/Objects/longobject.c#L158 + var fields = StructProxy.GetStructFields<_PyLongValue, _PyLongValue.Fields>(process); + long ob_size = numDigits * bytesInDigit + fields.ob_digit.Offset; + result.ob_size.Write(ob_size); + + // Digits are stored in the long_value.lv_data field in 3.12 + result.long_value.WriteTag((ulong)numDigits, value == 0, value < 0); + + // Then we write the data out one digit at a time + if (bitsInDigit == 15) { + for (var digitPtr = new UInt16Proxy(process, result.ob_digit.Address); absValue != 0; digitPtr = digitPtr.GetAdjacentProxy(1)) { + digitPtr.Write((ushort)(absValue % (1 << bitsInDigit))); + absValue >>= bitsInDigit; + } + } else { + for (var digitPtr = new UInt32Proxy(process, result.ob_digit.Address); absValue != 0; digitPtr = digitPtr.GetAdjacentProxy(1)) { + digitPtr.Write((uint)(absValue % (1 << bitsInDigit))); + absValue >>= bitsInDigit; + } + } + + return result; + } private ByteProxy ob_digit { - get { return GetFieldProxy(_fields.ob_digit); } + get { + if (_fields.ob_digit.Process != null) { + return GetFieldProxy(_fields.ob_digit); + } else { + return GetFieldProxy(_fields.long_value).ob_digit; + } + } } public BigInteger ToBigInteger() { + if (Process.GetPythonRuntimeInfo().LanguageVersion < PythonLanguageVersion.V312) { + return ToBigInteger11(); + } else { + return ToBigInteger12(); + } + } + + public _PyLongValue long_value => GetFieldProxy(_fields.long_value); + + private BigInteger ToBigInteger11() { var bitsInDigit = Process.Is64Bit() ? 30 : 15; long ob_size = this.ob_size.Read(); @@ -112,8 +183,86 @@ public BigInteger ToBigInteger() { return ob_size > 0 ? result : -result; } + public BigInteger ToBigInteger12() { + var bitsInDigit = Process.Is64Bit() ? 30 : 15; + var long_value = GetFieldProxy(_fields.long_value); + long count = long_value.digit_count; + + // Read and parse digits in reverse, starting from the most significant ones. + var result = new BigInteger(0); + if (bitsInDigit == 15) { + var digitPtr = new UInt16Proxy(Process, ob_digit.Address).GetAdjacentProxy(count); + for (long i = 0; i != count; ++i) { + digitPtr = digitPtr.GetAdjacentProxy(-1); + result <<= bitsInDigit; + result += digitPtr.Read(); + } + } else { + var digitPtr = new UInt32Proxy(Process, ob_digit.Address).GetAdjacentProxy(count); + for (long i = 0; i != count; ++i) { + digitPtr = digitPtr.GetAdjacentProxy(-1); + result <<= bitsInDigit; + result += digitPtr.Read(); + } + } + + return long_value.is_negative ? -result : result; + } + public override void Repr(ReprBuilder builder) { builder.AppendLiteral(ToBigInteger()); } + + [StructProxy(MinVersion = PythonLanguageVersion.V312, StructName = "_PyLongValue")] + public class _PyLongValue : StructProxy { + public class Fields { + public StructField lv_tag; + public StructField ob_digit; + } + + private const int SIGN_ZERO = 1; + private const int SIGN_NEGATIVE = 2; + private const int NON_SIZE_BITS = 3; + private const int SIGN_MASK = 3; + + private readonly Fields _fields; + + public _PyLongValue(DkmProcess process, ulong address) + : base(process, address) { + InitializeStruct(this, out _fields); + } + + public ByteProxy ob_digit => GetFieldProxy(_fields.ob_digit); + + public UInt64Proxy lv_tag => GetFieldProxy(_fields.lv_tag); + + public void WriteTag(ulong numberDigits, bool isZero, bool isNegative) { + ulong lv_tag = numberDigits << NON_SIZE_BITS; + if (isZero) { + lv_tag |= SIGN_ZERO; + } else if (isNegative) { + lv_tag |= SIGN_NEGATIVE; + } + this.lv_tag.Write(lv_tag); + } + + public uint digit_count { + get { + return (uint)(lv_tag.Read() >> NON_SIZE_BITS); + } + + set { + lv_tag.Write((value << NON_SIZE_BITS) | (lv_tag.Read() & SIGN_MASK)); + } + } + + public bool is_negative { + get { + return (lv_tag.Read() & SIGN_MASK) == SIGN_NEGATIVE; + } + } + } } + + } diff --git a/Python/Product/Debugger.Concord/Proxies/Structs/PyObject.cs b/Python/Product/Debugger.Concord/Proxies/Structs/PyObject.cs index 52bf695c1c..693cab2c92 100644 --- a/Python/Product/Debugger.Concord/Proxies/Structs/PyObject.cs +++ b/Python/Product/Debugger.Concord/Proxies/Structs/PyObject.cs @@ -30,6 +30,7 @@ internal class PyTypeAttribute : Attribute { public string VariableName { get; set; } public PythonLanguageVersion MinVersion { get; set; } public PythonLanguageVersion MaxVersion { get; set; } + public bool Hidden { get; set; } } internal interface IPyObject : IValueStore, IDataProxy { @@ -81,6 +82,9 @@ public ProxyTypes(DkmProcess process) { if (pyTypeAttr.MaxVersion != PythonLanguageVersion.None && langVer > pyTypeAttr.MaxVersion) { continue; } + if (pyTypeAttr.Hidden) { + continue; + } typeVarName = pyTypeAttr.VariableName ?? ComputeVariableName(proxyType); break; @@ -169,7 +173,12 @@ public PointerProxy? __dict__ { } else if (dictoffset < 0) { var varObj = this as PyVarObject; if (varObj == null) { - throw new InvalidDataException(); + // Prior to 3.11 this should have been an error. In 3.11 many objects have negative tp_dictoffset + if (Process.GetPythonRuntimeInfo().LanguageVersion <= PythonLanguageVersion.V310) { + Debug.Fail("Non-var object with negative tp_dictoffset."); + throw new InvalidOperationException(); + } + return null; } long size = ob_type.tp_basicsize.Read(); diff --git a/Python/Product/Debugger.Concord/Proxies/Structs/PyRuntimeState.cs b/Python/Product/Debugger.Concord/Proxies/Structs/PyRuntimeState.cs index e78f83b345..aad94c256b 100644 --- a/Python/Product/Debugger.Concord/Proxies/Structs/PyRuntimeState.cs +++ b/Python/Product/Debugger.Concord/Proxies/Structs/PyRuntimeState.cs @@ -32,6 +32,7 @@ private class Fields { public StructField core_initialized; public StructField initialized; public StructField interpreters; + public StructField gilstate; } private readonly Fields _fields; @@ -44,6 +45,7 @@ public PyRuntimeState(DkmProcess process, ulong address) public BoolProxy core_initialized => GetFieldProxy(_fields.core_initialized); public BoolProxy initialized => GetFieldProxy(_fields.initialized); public pyinterpreters interpreters => GetFieldProxy(_fields.interpreters); + public gilstate_runtime_state gilstate => GetFieldProxy(_fields.gilstate); [StructProxy(MinVersion = PythonLanguageVersion.V39, StructName = "pyinterpreters")] @@ -64,6 +66,22 @@ public pyinterpreters(DkmProcess process, ulong address) public PointerProxy main => GetFieldProxy(_fields.main); } + [StructProxy(MinVersion = PythonLanguageVersion.V312, StructName = "_gilstate_runtime_state")] + public class gilstate_runtime_state : StructProxy { + public class Fields { + public StructField check_enabled; + } + + private readonly Fields _fields; + + public gilstate_runtime_state(DkmProcess process, ulong address) + : base(process, address) { + InitializeStruct(this, out _fields); + } + + public Int32Proxy check_enabled => GetFieldProxy(_fields.check_enabled); + } + } diff --git a/Python/Product/Debugger.Concord/Proxies/Structs/PyThreadState.cs b/Python/Product/Debugger.Concord/Proxies/Structs/PyThreadState.cs index e3b6f1059e..d4f3516e7c 100644 --- a/Python/Product/Debugger.Concord/Proxies/Structs/PyThreadState.cs +++ b/Python/Product/Debugger.Concord/Proxies/Structs/PyThreadState.cs @@ -24,15 +24,27 @@ namespace Microsoft.PythonTools.Debugger.Concord.Proxies.Structs { internal class PyThreadState : StructProxy { private class Fields { public StructField> next; + [FieldProxy(MaxVersion = PythonLanguageVersion.V310)] public StructField> frame; + [FieldProxy(MinVersion = PythonLanguageVersion.V311)] + public StructField> interp; + [FieldProxy(MaxVersion = PythonLanguageVersion.V39)] public StructField use_tracing; + [FieldProxy(MinVersion = PythonLanguageVersion.V310, MaxVersion = PythonLanguageVersion.V312)] + public StructField> cframe; public StructField c_tracefunc; + [FieldProxy(MaxVersion = PythonLanguageVersion.V311)] public StructField> curexc_type; + [FieldProxy(MaxVersion = PythonLanguageVersion.V311)] public StructField> curexc_value; - public StructField> curexc_traceback; - public StructField exc_state; + [FieldProxy(MinVersion = PythonLanguageVersion.V312)] + public StructField> current_exception; public StructField> exc_info; public StructField thread_id; + [FieldProxy(MinVersion = PythonLanguageVersion.V312)] + public StructField tracing; // Indicates if sys.monitoring is set, not something we set here. + [FieldProxy(MinVersion = PythonLanguageVersion.V313)] + public StructField> current_frame; } private readonly Fields _fields; @@ -54,11 +66,27 @@ public PointerProxy next { } public PointerProxy frame { - get { return GetFieldProxy(_fields.frame); } + get { + if (_fields.frame.Process != null) { + return GetFieldProxy(_fields.frame); + } + + // In 3.11, the current frame was moved into the cframe + if (_fields.cframe.Process != null) { + var cframe = GetFieldProxy(_fields.cframe).Read(); + var interpFrame311 = cframe.current_frame.TryRead(); + return interpFrame311.frame_obj; + } + + // In 3.13, cframe was removed and the current_frame was just placed + // in the thread. + var interpFrame = GetFieldProxy(_fields.current_frame).TryRead(); + return interpFrame.frame_obj; + } } - public Int32Proxy use_tracing { - get { return GetFieldProxy(_fields.use_tracing); } + public PointerProxy cframe { + get { return GetFieldProxy(_fields.cframe); } } public PointerProxy c_tracefunc { @@ -66,33 +94,52 @@ public PointerProxy c_tracefunc { } public PointerProxy curexc_type { - get { return GetFieldProxy(_fields.curexc_type); } + get { + if (_fields.curexc_type.Process != null) { + return GetFieldProxy(_fields.curexc_type); + } + + // In 3.12, the current exception was stored by itself instead of separately with the + // type and value. + var exc = GetFieldProxy(_fields.current_exception).Read(); + return exc.ob_type.ReinterpretCast(); + } } public PointerProxy curexc_value { - get { return GetFieldProxy(_fields.curexc_value); } - } - - public PointerProxy curexc_traceback { - get { return GetFieldProxy(_fields.curexc_traceback); } + get { + if (_fields.curexc_value.Process != null) { + return GetFieldProxy(_fields.curexc_value); + } + + // In 3.12, the current exception was stored by itself instead of separately with the + // type and value. + return GetFieldProxy(_fields.current_exception).ReinterpretCast(); + } } - public PointerProxy exc_type(PythonLanguageVersion version) => - GetFieldProxy(_fields.exc_state).exc_type; - - public PointerProxy exc_value(PythonLanguageVersion version) => - GetFieldProxy(_fields.exc_state).exc_value; - - public PointerProxy exc_traceback(PythonLanguageVersion version) => - GetFieldProxy(_fields.exc_state).exc_traceback; - - public Int32Proxy thread_id { get { return GetFieldProxy(_fields.thread_id); } } public static IEnumerable GetThreadStates(DkmProcess process) { - return PyInterpreterState.GetInterpreterStates(process).SelectMany(interp => interp.GetThreadStates()); + return PyInterpreterState.GetInterpreterStates(process).SelectMany(interp => interp.GetThreadStates(process)); + } + + public void RegisterTracing(ulong traceFunc) { + // In 3.10, the use_tracing flag sets tracing for the thread. + if (_fields.use_tracing.Process != null) { + GetFieldProxy(_fields.use_tracing).Write(1); + } + // In 3.11 the cframe has the use_tracing flag, but must be set to 255. + if (_fields.cframe.Process != null && Process.GetPythonRuntimeInfo().LanguageVersion == PythonLanguageVersion.V311) { + var frame = cframe.Read(); + frame.use_tracing.Write(255); + } + // In 3.12, there's no longer a use_tracing flag. We have to register tracing through an API instead. + if (Process.GetPythonRuntimeInfo().LanguageVersion <= PythonLanguageVersion.V311) { + c_tracefunc.Write(traceFunc); + } } diff --git a/Python/Product/Debugger.Concord/Proxies/Structs/PyThreads.cs b/Python/Product/Debugger.Concord/Proxies/Structs/PyThreads.cs new file mode 100644 index 0000000000..80df29eb2f --- /dev/null +++ b/Python/Product/Debugger.Concord/Proxies/Structs/PyThreads.cs @@ -0,0 +1,41 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using Microsoft.PythonTools.Common.Parsing; +using Microsoft.VisualStudio.Debugger; + +namespace Microsoft.PythonTools.Debugger.Concord.Proxies.Structs { + [StructProxy(StructName = "pythreads", MinVersion = PythonLanguageVersion.V311)] + internal class PyThreads: StructProxy { + internal class Fields { + public StructField next_unique_id; + public StructField> head; + public StructField count; + public StructField stacksize; + } + + private readonly Fields _fields; + + public PyThreads(DkmProcess process, ulong address) + : base(process, address) { + InitializeStruct(this, out _fields); + } + + public PointerProxy head { + get { return GetFieldProxy(_fields.head); } + } + } +} diff --git a/Python/Product/Debugger.Concord/Proxies/Structs/PyTupleObject.cs b/Python/Product/Debugger.Concord/Proxies/Structs/PyTupleObject.cs index 87d137d8d4..de25166460 100644 --- a/Python/Product/Debugger.Concord/Proxies/Structs/PyTupleObject.cs +++ b/Python/Product/Debugger.Concord/Proxies/Structs/PyTupleObject.cs @@ -38,7 +38,7 @@ public ArrayProxy> ob_item { get { return GetFieldProxy(_fields.ob_item); } } - public IEnumerable> ReadElements() { + public virtual IEnumerable> ReadElements() { return ob_item.Take(ob_size.Read()); } diff --git a/Python/Product/Debugger.Concord/Proxies/Structs/PyUnicodeObject.cs b/Python/Product/Debugger.Concord/Proxies/Structs/PyUnicodeObject.cs index 37fee86207..a3e73cec0f 100644 --- a/Python/Product/Debugger.Concord/Proxies/Structs/PyUnicodeObject.cs +++ b/Python/Product/Debugger.Concord/Proxies/Structs/PyUnicodeObject.cs @@ -23,176 +23,11 @@ using Microsoft.VisualStudio.Debugger.Evaluation; namespace Microsoft.PythonTools.Debugger.Concord.Proxies.Structs { - [StructProxy(MinVersion = PythonLanguageVersion.V39, StructName = "PyUnicodeObject")] - [PyType(MinVersion = PythonLanguageVersion.V39, VariableName = "PyUnicode_Type")] - internal class PyUnicodeObject : PyVarObject, IPyBaseStringObject { - private static readonly Encoding _latin1 = Encoding.GetEncoding("Latin1"); - - private enum PyUnicode_Kind { - PyUnicode_WCHAR_KIND = 0, - PyUnicode_1BYTE_KIND = 1, - PyUnicode_2BYTE_KIND = 2, - PyUnicode_4BYTE_KIND = 4 - } - - private enum Interned { - SSTATE_NOT_INTERNED = 0, - SSTATE_INTERNED_MORTAL = 1, - SSTATE_INTERNED_IMMORTAL = 2 - } - - private struct State { - private static readonly BitVector32.Section - internedSection = BitVector32.CreateSection(2), - kindSection = BitVector32.CreateSection(4, internedSection), - compactSection = BitVector32.CreateSection(1, kindSection), - asciiSection = BitVector32.CreateSection(1, compactSection), - readySection = BitVector32.CreateSection(1, asciiSection); - - private BitVector32 _state; - - private State(byte state) { - _state = new BitVector32(state); - } - - public static explicit operator State(byte state) { - return new State(state); - } - - public static explicit operator byte(State state) { - return (byte)state._state.Data; - } - - public Interned interned { - get { return (Interned)_state[internedSection]; } - set { _state[internedSection] = (int)value; } - } - - public PyUnicode_Kind kind { - get { return (PyUnicode_Kind)_state[kindSection]; } - set { _state[kindSection] = (int)value; } - } - - public bool compact { - get { return _state[compactSection] != 0; } - set { _state[compactSection] = value ? 1 : 0; } - } - - public bool ascii { - get { return _state[asciiSection] != 0; } - set { _state[asciiSection] = value ? 1 : 0; } - } - - public bool ready { - get { return _state[readySection] != 0; } - set { _state[readySection] = value ? 1 : 0; } - } - } - - public class Fields { - public StructField data; - } - - private readonly Fields _fields; - private readonly PyASCIIObject _asciiObject; - private readonly PyCompactUnicodeObject _compactObject; + internal abstract class PyUnicodeObject : PyVarObject, IPyBaseStringObject { + protected static readonly Encoding _latin1 = Encoding.GetEncoding("Latin1"); public PyUnicodeObject(DkmProcess process, ulong address) : base(process, address) { - InitializeStruct(this, out _fields); - CheckPyType(); - - _asciiObject = new PyASCIIObject(process, address); - _compactObject = new PyCompactUnicodeObject(process, address); - } - - public static PyUnicodeObject Create(DkmProcess process, string value) { - var allocator = process.GetDataItem(); - Debug.Assert(allocator != null); - - var result = allocator.Allocate(value.Length * sizeof(char)); - - result._asciiObject.hash.Write(-1); - result._asciiObject.length.Write(value.Length); - result._compactObject.wstr_length.Write(value.Length); - - var state = new State { - interned = Interned.SSTATE_NOT_INTERNED, - kind = PyUnicode_Kind.PyUnicode_2BYTE_KIND, - compact = true, - ascii = false, - ready = true - }; - result._asciiObject.state.Write((byte)state); - - ulong dataPtr = result.Address.OffsetBy(StructProxy.SizeOf(process)); - result._asciiObject.wstr.Write(dataPtr); - process.WriteMemory(dataPtr, Encoding.Unicode.GetBytes(value)); - - return result; - } - - - public override string ToString() { - byte[] buf; - - State state = (State)_asciiObject.state.Read(); - if (state.ascii) { - state.kind = PyUnicode_Kind.PyUnicode_1BYTE_KIND; - } - - if (!state.ready) { - ulong wstr = _asciiObject.wstr.Read(); - if (wstr == 0) { - return null; - } - - uint wstr_length = checked((uint)_compactObject.wstr_length.Read()); - if (wstr_length == 0) { - return ""; - } - - buf = new byte[wstr_length * 2]; - Process.ReadMemory(wstr, DkmReadMemoryFlags.None, buf); - return Encoding.Unicode.GetString(buf, 0, buf.Length); - } - - int length = checked((int)_asciiObject.length.Read()); - if (length == 0) { - return ""; - } - - ulong data; - if (!state.compact) { - data = GetFieldProxy(_fields.data).Read(); - } else if (state.ascii) { - data = Address.OffsetBy(StructProxy.SizeOf(Process)); - } else { - data = Address.OffsetBy(StructProxy.SizeOf(Process)); - } - if (data == 0) { - return null; - } - - buf = new byte[length * (int)state.kind]; - Process.ReadMemory(data, DkmReadMemoryFlags.None, buf); - Encoding enc; - switch (state.kind) { - case PyUnicode_Kind.PyUnicode_1BYTE_KIND: - enc = _latin1; - break; - case PyUnicode_Kind.PyUnicode_2BYTE_KIND: - enc = Encoding.Unicode; - break; - case PyUnicode_Kind.PyUnicode_4BYTE_KIND: - enc = Encoding.UTF32; - break; - default: - Debug.Fail("Unsupported PyUnicode_Kind " + state.kind); - return null; - } - - return enc.GetString(buf, 0, buf.Length); } public override void Repr(ReprBuilder builder) { @@ -214,54 +49,15 @@ public override IEnumerable GetDebugChildren(ReprOptions public static explicit operator string(PyUnicodeObject obj) { return (object)obj == null ? null : obj.ToString(); } - } - - internal class PyASCIIObject : StructProxy { - public class Fields { - public StructField length; - public StructField hash; - public StructField state; - public StructField wstr; - } - - private readonly Fields _fields; - - public PyASCIIObject(DkmProcess process, ulong address) - : base(process, address) { - InitializeStruct(this, out _fields); - } - - public SSizeTProxy length { - get { return GetFieldProxy(_fields.length); } - } - public SSizeTProxy hash { - get { return GetFieldProxy(_fields.hash); } - } - - public ByteProxy state { - get { return GetFieldProxy(_fields.state); } - } - - public PointerProxy wstr { - get { return GetFieldProxy(_fields.wstr); } + public static PyUnicodeObject Create(DkmProcess process, string value) { + if (process.GetPythonRuntimeInfo().LanguageVersion <= PythonLanguageVersion.V311) { + return PyUnicodeObject311.Create311(process, value); + } else { + return PyUnicodeObject312.Create312(process, value); + } } } - internal class PyCompactUnicodeObject : StructProxy { - public class Fields { - public StructField wstr_length; - } - private readonly Fields _fields; - - public PyCompactUnicodeObject(DkmProcess process, ulong address) - : base(process, address) { - InitializeStruct(this, out _fields); - } - - public SSizeTProxy wstr_length { - get { return GetFieldProxy(_fields.wstr_length); } - } - } } diff --git a/Python/Product/Debugger.Concord/Proxies/Structs/PyUnicodeObject311.cs b/Python/Product/Debugger.Concord/Proxies/Structs/PyUnicodeObject311.cs new file mode 100644 index 0000000000..e13ef18b76 --- /dev/null +++ b/Python/Product/Debugger.Concord/Proxies/Structs/PyUnicodeObject311.cs @@ -0,0 +1,232 @@ +using System; +using System.Collections.Generic; +using System.Collections.Specialized; +using System.Diagnostics; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.PythonTools.Common.Parsing; +using Microsoft.VisualStudio.Debugger; + +namespace Microsoft.PythonTools.Debugger.Concord.Proxies.Structs { + [StructProxy(MinVersion = PythonLanguageVersion.V39, MaxVersion = PythonLanguageVersion.V311, StructName = "PyUnicodeObject")] + [PyType(MinVersion = PythonLanguageVersion.V39, MaxVersion = PythonLanguageVersion.V311, VariableName = "PyUnicode_Type")] + internal class PyUnicodeObject311 : PyUnicodeObject { + private enum PyUnicode_Kind { + PyUnicode_WCHAR_KIND = 0, + PyUnicode_1BYTE_KIND = 1, + PyUnicode_2BYTE_KIND = 2, + PyUnicode_4BYTE_KIND = 4 + } + + private enum Interned { + SSTATE_NOT_INTERNED = 0, + SSTATE_INTERNED_MORTAL = 1, + SSTATE_INTERNED_IMMORTAL = 2 + } + + private struct State { + private static readonly BitVector32.Section + internedSection = BitVector32.CreateSection(2), + kindSection = BitVector32.CreateSection(4, internedSection), + compactSection = BitVector32.CreateSection(1, kindSection), + asciiSection = BitVector32.CreateSection(1, compactSection), + readySection = BitVector32.CreateSection(1, asciiSection); + + private BitVector32 _state; + + private State(byte state) { + _state = new BitVector32(state); + } + + public static explicit operator State(byte state) { + return new State(state); + } + + public static explicit operator byte(State state) { + return (byte)state._state.Data; + } + + public Interned interned { + get { return (Interned)_state[internedSection]; } + set { _state[internedSection] = (int)value; } + } + + public PyUnicode_Kind kind { + get { return (PyUnicode_Kind)_state[kindSection]; } + set { _state[kindSection] = (int)value; } + } + + public bool compact { + get { return _state[compactSection] != 0; } + set { _state[compactSection] = value ? 1 : 0; } + } + + public bool ascii { + get { return _state[asciiSection] != 0; } + set { _state[asciiSection] = value ? 1 : 0; } + } + + public bool ready { + get { return _state[readySection] != 0; } + set { _state[readySection] = value ? 1 : 0; } + } + } + + public class Fields { + public StructField data; + } + + private readonly Fields _fields; + private readonly PyASCIIObject311 _asciiObject; + private readonly PyCompactUnicodeObject311 _compactObject; + + public PyUnicodeObject311(DkmProcess process, ulong address) + : base(process, address) { + InitializeStruct(this, out _fields); + CheckPyType(); + + _asciiObject = new PyASCIIObject311(process, address); + _compactObject = new PyCompactUnicodeObject311(process, address); + } + + public static PyUnicodeObject311 Create311(DkmProcess process, string value) { + var allocator = process.GetDataItem(); + Debug.Assert(allocator != null); + + var result = allocator.Allocate(value.Length * sizeof(char)); + + result._asciiObject.hash.Write(-1); + result._asciiObject.length.Write(value.Length); + result._compactObject.wstr_length.Write(value.Length); + + var state = new State { + interned = Interned.SSTATE_NOT_INTERNED, + kind = PyUnicode_Kind.PyUnicode_2BYTE_KIND, + compact = true, + ascii = false, + ready = true + }; + result._asciiObject.state.Write((byte)state); + + ulong dataPtr = result.Address.OffsetBy(StructProxy.SizeOf(process)); + result._asciiObject.wstr.Write(dataPtr); + process.WriteMemory(dataPtr, Encoding.Unicode.GetBytes(value)); + + return result; + } + + public override string ToString() { + byte[] buf; + + State state = (State)_asciiObject.state.Read(); + if (state.ascii) { + state.kind = PyUnicode_Kind.PyUnicode_1BYTE_KIND; + } + + if (!state.ready) { + ulong wstr = _asciiObject.wstr.Read(); + if (wstr == 0) { + return null; + } + + uint wstr_length = checked((uint)_compactObject.wstr_length.Read()); + if (wstr_length == 0) { + return ""; + } + + buf = new byte[wstr_length * 2]; + Process.ReadMemory(wstr, DkmReadMemoryFlags.None, buf); + return Encoding.Unicode.GetString(buf, 0, buf.Length); + } + + int length = checked((int)_asciiObject.length.Read()); + if (length == 0) { + return ""; + } + + ulong data; + if (!state.compact) { + data = GetFieldProxy(_fields.data).Read(); + } else if (state.ascii) { + data = Address.OffsetBy(StructProxy.SizeOf(Process)); + } else { + data = Address.OffsetBy(StructProxy.SizeOf(Process)); + } + if (data == 0) { + return null; + } + + buf = new byte[length * (int)state.kind]; + Process.ReadMemory(data, DkmReadMemoryFlags.None, buf); + Encoding enc; + switch (state.kind) { + case PyUnicode_Kind.PyUnicode_1BYTE_KIND: + enc = _latin1; + break; + case PyUnicode_Kind.PyUnicode_2BYTE_KIND: + enc = Encoding.Unicode; + break; + case PyUnicode_Kind.PyUnicode_4BYTE_KIND: + enc = Encoding.UTF32; + break; + default: + Debug.Fail("Unsupported PyUnicode_Kind " + state.kind); + return null; + } + + return enc.GetString(buf, 0, buf.Length); + } + } + + [StructProxy(MaxVersion = PythonLanguageVersion.V311, StructName = "PyASCIIObject")] + [PyType(MinVersion = PythonLanguageVersion.V39, MaxVersion = PythonLanguageVersion.V311, VariableName = "PyASCII_Type")] + internal class PyASCIIObject311 : StructProxy { + public class Fields { + public StructField length; + public StructField hash; + public StructField state; + public StructField wstr; + } + + private readonly Fields _fields; + + public PyASCIIObject311(DkmProcess process, ulong address) + : base(process, address) { + InitializeStruct(this, out _fields); + } + + public SSizeTProxy length { + get { return GetFieldProxy(_fields.length); } + } + + public SSizeTProxy hash { + get { return GetFieldProxy(_fields.hash); } + } + + public ByteProxy state { + get { return GetFieldProxy(_fields.state); } + } + + public PointerProxy wstr => GetFieldProxy(_fields.wstr); + } + + [StructProxy(MaxVersion = PythonLanguageVersion.V311, StructName = "PyCompactUnicodeObject")] + [PyType(MinVersion = PythonLanguageVersion.V39, MaxVersion = PythonLanguageVersion.V311, VariableName = "PyCompactUnicode_Type")] + internal class PyCompactUnicodeObject311 : StructProxy { + public class Fields { + public StructField wstr_length; + } + + private readonly Fields _fields; + + public PyCompactUnicodeObject311(DkmProcess process, ulong address) + : base(process, address) { + InitializeStruct(this, out _fields); + } + + public SSizeTProxy wstr_length { + get { return GetFieldProxy(_fields.wstr_length); } + } + } +} diff --git a/Python/Product/Debugger.Concord/Proxies/Structs/PyUnicodeObject312.cs b/Python/Product/Debugger.Concord/Proxies/Structs/PyUnicodeObject312.cs new file mode 100644 index 0000000000..8883e981cc --- /dev/null +++ b/Python/Product/Debugger.Concord/Proxies/Structs/PyUnicodeObject312.cs @@ -0,0 +1,219 @@ +using System; +using System.Collections.Generic; +using System.Collections.Specialized; +using System.Diagnostics; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.PythonTools.Common.Core.OS; +using Microsoft.PythonTools.Common.Parsing; +using Microsoft.VisualStudio.Debugger; + +namespace Microsoft.PythonTools.Debugger.Concord.Proxies.Structs { + [StructProxy(MinVersion = PythonLanguageVersion.V312, StructName = "PyUnicodeObject")] + [PyType(MinVersion = PythonLanguageVersion.V312, VariableName = "PyUnicode_Type")] + internal class PyUnicodeObject312 : PyUnicodeObject { + private enum PyUnicode_Kind { + PyUnicode_1BYTE_KIND = 1, + PyUnicode_2BYTE_KIND = 2, + PyUnicode_4BYTE_KIND = 4 + } + + private enum Interned { + SSTATE_NOT_INTERNED = 0, + SSTATE_INTERNED_MORTAL = 1, + SSTATE_INTERNED_IMMORTAL = 2 + } + + private struct State { + private static readonly BitVector32.Section + internedSection = BitVector32.CreateSection(2), + kindSection = BitVector32.CreateSection(4, internedSection), + compactSection = BitVector32.CreateSection(1, kindSection), + asciiSection = BitVector32.CreateSection(1, compactSection), + readySection = BitVector32.CreateSection(1, asciiSection); + + private BitVector32 _state; + + private State(byte state) { + _state = new BitVector32(state); + } + + public static explicit operator State(byte state) { + return new State(state); + } + + public static explicit operator byte(State state) { + return (byte)state._state.Data; + } + + public Interned interned { + get { return (Interned)_state[internedSection]; } + set { _state[internedSection] = (int)value; } + } + + public PyUnicode_Kind kind { + get { return (PyUnicode_Kind)_state[kindSection]; } + set { _state[kindSection] = (int)value; } + } + + public bool compact { + get { return _state[compactSection] != 0; } + set { _state[compactSection] = value ? 1 : 0; } + } + + public bool ascii { + get { return _state[asciiSection] != 0; } + set { _state[asciiSection] = value ? 1 : 0; } + } + + public bool ready { + get { return _state[readySection] != 0; } + set { _state[readySection] = value ? 1 : 0; } + } + } + + public class Fields { + public StructField data; + } + + private readonly Fields _fields; + private readonly PyASCIIObject312 _asciiObject; + private readonly PyCompactUnicodeObject312 _compactObject; + + public PyUnicodeObject312(DkmProcess process, ulong address) + : base(process, address) { + InitializeStruct(this, out _fields); + CheckPyType(); + + _asciiObject = new PyASCIIObject312(process, address); + _compactObject = new PyCompactUnicodeObject312(process, address); + } + + public static PyUnicodeObject312 Create312(DkmProcess process, string value) { + var allocator = process.GetDataItem(); + Debug.Assert(allocator != null); + + var result = allocator.Allocate(value.Length * sizeof(char)); + + result._asciiObject.hash.Write(-1); + result._asciiObject.length.Write(value.Length); + result._compactObject.utf8_length.Write(value.Length); + var kind = value.Length != Encoding.UTF8.GetByteCount(value) ? PyUnicode_Kind.PyUnicode_2BYTE_KIND : PyUnicode_Kind.PyUnicode_1BYTE_KIND; + + var state = new State { + interned = Interned.SSTATE_NOT_INTERNED, + kind = kind, + compact = true, + ascii = false, + ready = true + }; + result._asciiObject.state.Write((byte)state); + + ulong dataPtr = result.Address.OffsetBy(StructProxy.SizeOf(process)); + result._compactObject.utf8.Write(dataPtr); + process.WriteMemory(dataPtr, Encoding.UTF8.GetBytes(value)); + + return result; + } + + public override string ToString() { + byte[] buf; + + State state = (State)_asciiObject.state.Read(); + if (state.ascii) { + state.kind = PyUnicode_Kind.PyUnicode_1BYTE_KIND; + } + + int length = checked((int)_asciiObject.length.Read()); + if (length == 0) { + return ""; + } + + ulong data; + if (!state.compact) { + data = GetFieldProxy(_fields.data).Read(); + } else if (state.ascii) { + data = Address.OffsetBy(StructProxy.SizeOf(Process)); + } else { + data = Address.OffsetBy(StructProxy.SizeOf(Process)); + } + if (data == 0) { + return null; + } + + buf = new byte[length * (int)state.kind]; + Process.ReadMemory(data, DkmReadMemoryFlags.None, buf); + Encoding enc; + switch (state.kind) { + case PyUnicode_Kind.PyUnicode_1BYTE_KIND: + enc = _latin1; + break; + case PyUnicode_Kind.PyUnicode_2BYTE_KIND: + enc = Encoding.Unicode; + break; + case PyUnicode_Kind.PyUnicode_4BYTE_KIND: + enc = Encoding.UTF32; + break; + default: + Debug.Fail("Unsupported PyUnicode_Kind " + state.kind); + return null; + } + + return enc.GetString(buf, 0, buf.Length); + } + } + + [StructProxy(MinVersion = PythonLanguageVersion.V312, StructName = "PyASCIIObject")] + [PyType(MinVersion = PythonLanguageVersion.V312, VariableName = "PyASCII_Type")] + internal class PyASCIIObject312 : StructProxy { + public class Fields { + public StructField length; + public StructField hash; + public StructField state; + } + + private readonly Fields _fields; + + public PyASCIIObject312(DkmProcess process, ulong address) + : base(process, address) { + InitializeStruct(this, out _fields); + } + + public SSizeTProxy length { + get { return GetFieldProxy(_fields.length); } + } + + public SSizeTProxy hash { + get { return GetFieldProxy(_fields.hash); } + } + + public ByteProxy state { + get { return GetFieldProxy(_fields.state); } + } + } + + [StructProxy(MinVersion = PythonLanguageVersion.V312, StructName = "PyCompactUnicodeObject")] + [PyType(MinVersion = PythonLanguageVersion.V312, VariableName = "PyCompactUnicode_Type")] + internal class PyCompactUnicodeObject312 : StructProxy { + public class Fields { + public StructField utf8_length; + public StructField utf8; + } + + private readonly Fields _fields; + + public PyCompactUnicodeObject312(DkmProcess process, ulong address) + : base(process, address) { + InitializeStruct(this, out _fields); + } + + public SSizeTProxy utf8_length { + get { return GetFieldProxy(_fields.utf8_length); } + } + + public PointerProxy utf8 { + get { return GetFieldProxy(_fields.utf8); } + } + } +} diff --git a/Python/Product/Debugger.Concord/PythonRuntimeInfo.cs b/Python/Product/Debugger.Concord/PythonRuntimeInfo.cs index f35a3a7a8a..06f3355f18 100644 --- a/Python/Product/Debugger.Concord/PythonRuntimeInfo.cs +++ b/Python/Product/Debugger.Concord/PythonRuntimeInfo.cs @@ -76,6 +76,9 @@ public static PythonLanguageVersion GetPythonLanguageVersion(DkmNativeModuleInst case "38": return PythonLanguageVersion.V38; case "39": return PythonLanguageVersion.V39; case "310": return PythonLanguageVersion.V310; + case "311": return PythonLanguageVersion.V311; + case "312": return PythonLanguageVersion.V312; + case "313": return PythonLanguageVersion.V313; default: return PythonLanguageVersion.None; } } diff --git a/Python/Product/Debugger.Concord/TraceManagerLocalHelper.cs b/Python/Product/Debugger.Concord/TraceManagerLocalHelper.cs index 01dc9da3b0..14a39cbc37 100644 --- a/Python/Product/Debugger.Concord/TraceManagerLocalHelper.cs +++ b/Python/Product/Debugger.Concord/TraceManagerLocalHelper.cs @@ -62,28 +62,48 @@ public PyVarObject_FieldOffsets(DkmProcess process) { // Layout of this struct must always remain in sync with DebuggerHelper/trace.cpp. [StructLayout(LayoutKind.Sequential, Pack = 8)] private struct PyCodeObject_FieldOffsets { - public readonly long co_varnames, co_filename, co_name; + public readonly long co_filename, co_name; public PyCodeObject_FieldOffsets(DkmProcess process) { - var fields = StructProxy.GetStructFields(process); - co_varnames = fields.co_varnames.Offset; - co_filename = fields.co_filename.Offset; - co_name = fields.co_name.Offset; + if (process.GetPythonRuntimeInfo().LanguageVersion <= PythonLanguageVersion.V310) { + var fields = StructProxy.GetStructFields(process); + co_filename = fields.co_filename.Offset; + co_name = fields.co_name.Offset; + } else { + var fields = StructProxy.GetStructFields(process); + co_filename = fields.co_filename.Offset; + co_name = fields.co_name.Offset; + } } } // Layout of this struct must always remain in sync with DebuggerHelper/trace.cpp. [StructLayout(LayoutKind.Sequential, Pack = 8)] private struct PyFrameObject_FieldOffsets { - public readonly long f_back, f_code, f_globals, f_locals, f_lineno; + public readonly long f_back, f_code, f_globals, f_locals, f_lineno, f_frame; public PyFrameObject_FieldOffsets(DkmProcess process) { - var fields = StructProxy.GetStructFields(process); - f_back = fields.f_back.Offset; - f_code = fields.f_code.Offset; - f_globals = fields.f_globals.Offset; - f_locals = fields.f_locals.Offset; - f_lineno = fields.f_lineno.Offset; + // For 310, these are on the _frame struct itself. + if (process.GetPythonRuntimeInfo().LanguageVersion <= PythonLanguageVersion.V310) { + var fields = StructProxy.GetStructFields(process); + f_back = fields.f_back.Offset; + f_code = fields.f_code.Offset; + f_globals = fields.f_globals.Offset; + f_locals = fields.f_locals.Offset; + f_lineno = fields.f_lineno.Offset; + f_frame = 0; + return; + } + + // For 311 and higher, they are on the PyInterpreterFrame struct which is pointed to by the _frame struct. + var _frameFields = StructProxy.GetStructFields(process); + var _interpreterFields = StructProxy.GetStructFields(process); + f_frame = _frameFields.f_frame.Offset; + f_back = _frameFields.f_back.Offset; + f_code = _interpreterFields.f_code.Process != null ? _interpreterFields.f_code.Offset : _interpreterFields.f_executable.Offset; + f_globals = _interpreterFields.f_globals.Offset; + f_locals = _interpreterFields.f_locals.Offset; + f_lineno = _frameFields.f_lineno.Offset; } } @@ -102,22 +122,43 @@ public PyBytesObject_FieldOffsets(DkmProcess process) { [StructLayout(LayoutKind.Sequential, Pack = 8)] private struct PyUnicodeObject_FieldOffsets { public readonly long sizeof_PyASCIIObject, sizeof_PyCompactUnicodeObject; - public readonly long length, state, wstr, wstr_length, data; + public readonly long length, state, wstr, wstr_length, utf8, utf8_length, data; public PyUnicodeObject_FieldOffsets(DkmProcess process) { - sizeof_PyASCIIObject = StructProxy.SizeOf(process); - sizeof_PyCompactUnicodeObject = StructProxy.SizeOf(process); - - var asciiFields = StructProxy.GetStructFields(process); - length = asciiFields.length.Offset; - state = asciiFields.state.Offset; - wstr = asciiFields.wstr.Offset; - - var compactFields = StructProxy.GetStructFields(process); - wstr_length = compactFields.wstr_length.Offset; + if (process.GetPythonRuntimeInfo().LanguageVersion <= PythonLanguageVersion.V311) { + sizeof_PyASCIIObject = StructProxy.SizeOf(process); + sizeof_PyCompactUnicodeObject = StructProxy.SizeOf(process); + + var asciiFields = StructProxy.GetStructFields(process); + length = asciiFields.length.Offset; + state = asciiFields.state.Offset; + wstr = asciiFields.wstr.Offset; + utf8 = 0; + utf8_length = 0; + + var compactFields = StructProxy.GetStructFields(process); + wstr_length = compactFields.wstr_length.Offset; + + var unicodeFields = StructProxy.GetStructFields(process); + data = unicodeFields.data.Offset; + } else { + sizeof_PyASCIIObject = StructProxy.SizeOf(process); + sizeof_PyCompactUnicodeObject = StructProxy.SizeOf(process); + + var asciiFields = StructProxy.GetStructFields(process); + length = asciiFields.length.Offset; + state = asciiFields.state.Offset; + wstr = 0; + + var compactFields = StructProxy.GetStructFields(process); + wstr_length = 0; + utf8_length = compactFields.utf8_length.Offset; + utf8 = compactFields.utf8.Offset; + + var unicodeFields = StructProxy.GetStructFields(process); + data = unicodeFields.data.Offset; + } - var unicodeFields = StructProxy.GetStructFields(process); - data = unicodeFields.data.Offset; } } @@ -149,7 +190,9 @@ private struct Types { public Types(DkmProcess process, PythonRuntimeInfo pyrtInfo) { PyBytes_Type = PyObject.GetPyType(process).Address; - PyUnicode_Type = PyObject.GetPyType(process).Address; + PyUnicode_Type = process.GetPythonRuntimeInfo().LanguageVersion <= PythonLanguageVersion.V311 ? + PyObject.GetPyType(process).Address : + PyObject.GetPyType(process).Address; } } @@ -163,6 +206,11 @@ private struct FunctionPointers { public ulong PyErr_Restore; public ulong PyErr_Occurred; public ulong PyObject_Str; + public ulong PyEval_SetTraceAllThreads; + public ulong PyGILState_Ensure; + public ulong PyGILState_Release; + public ulong Py_Initialize; + public ulong Py_Finalize; public FunctionPointers(DkmProcess process, PythonRuntimeInfo pyrtInfo) { Py_DecRef = pyrtInfo.DLLs.Python.GetFunctionAddress("Py_DecRef"); @@ -172,6 +220,13 @@ public FunctionPointers(DkmProcess process, PythonRuntimeInfo pyrtInfo) { PyErr_Restore = pyrtInfo.DLLs.Python.GetFunctionAddress("PyErr_Restore"); PyErr_Occurred = pyrtInfo.DLLs.Python.GetFunctionAddress("PyErr_Occurred"); PyObject_Str = pyrtInfo.DLLs.Python.GetFunctionAddress("PyObject_Str"); + PyEval_SetTraceAllThreads = pyrtInfo.LanguageVersion >= PythonLanguageVersion.V312 ? + pyrtInfo.DLLs.Python.GetFunctionAddress("PyEval_SetTraceAllThreads") : + 0; + PyGILState_Ensure = pyrtInfo.DLLs.Python.GetFunctionAddress("PyGILState_Ensure"); + PyGILState_Release = pyrtInfo.DLLs.Python.GetFunctionAddress("PyGILState_Release"); + Py_Initialize = pyrtInfo.DLLs.Python.GetFunctionAddress("Py_Initialize"); + Py_Finalize = pyrtInfo.DLLs.Python.GetFunctionAddress("Py_Finalize"); } } @@ -180,6 +235,7 @@ public FunctionPointers(DkmProcess process, PythonRuntimeInfo pyrtInfo) { private readonly PythonDllBreakpointHandlers _handlers; private readonly DkmNativeInstructionAddress _traceFunc; private readonly DkmNativeInstructionAddress _evalFrameFunc; + private readonly DkmNativeInstructionAddress _pyEval_FrameDefault; private readonly PointerProxy _defaultEvalFrameFunc; private readonly ByteProxy _isTracing; @@ -222,6 +278,7 @@ public unsafe TraceManagerLocalHelper(DkmProcess process, Kind kind) { _process = process; _pyrtInfo = process.GetPythonRuntimeInfo(); + _pyEval_FrameDefault = _pyrtInfo.DLLs.Python.GetExportedFunctionAddress("_PyEval_EvalFrameDefault"); _traceFunc = _pyrtInfo.DLLs.DebuggerHelper.GetExportedFunctionAddress("TraceFunc"); _evalFrameFunc = _pyrtInfo.DLLs.DebuggerHelper.GetExportedFunctionAddress("EvalFrameFunc"); @@ -243,7 +300,7 @@ public unsafe TraceManagerLocalHelper(DkmProcess process, Kind kind) { if (_pyrtInfo.LanguageVersion >= PythonLanguageVersion.V36) { RegisterJITTracing(interp); } - foreach (var tstate in interp.GetThreadStates()) { + foreach (var tstate in interp.GetThreadStates(process)) { RegisterTracing(tstate); } } @@ -251,6 +308,7 @@ public unsafe TraceManagerLocalHelper(DkmProcess process, Kind kind) { _handlers = new PythonDllBreakpointHandlers(this); LocalComponent.CreateRuntimeDllFunctionExitBreakpoints(_pyrtInfo.DLLs.Python, "new_threadstate", _handlers.new_threadstate, enable: true); LocalComponent.CreateRuntimeDllFunctionExitBreakpoints(_pyrtInfo.DLLs.Python, "PyInterpreterState_New", _handlers.PyInterpreterState_New, enable: true); + LocalComponent.CreateRuntimeDllFunctionExitBreakpoints(_pyrtInfo.DLLs.Python, "_PyInterpreterState_New", _handlers._PyInterpreterState_New, enable: true); foreach (var methodInfo in _handlers.GetType().GetMethods()) { var stepInAttr = (StepInGateAttribute)Attribute.GetCustomAttribute(methodInfo, typeof(StepInGateAttribute)); @@ -269,9 +327,15 @@ public unsafe TraceManagerLocalHelper(DkmProcess process, Kind kind) { } } - private IEnumerable GetTracingPossible(PythonRuntimeInfo pyrtInfo, DkmProcess process) => - from interp in PyInterpreterState.GetInterpreterStates(process) + private IEnumerable GetTracingPossible(PythonRuntimeInfo pyrtInfo, DkmProcess process) { + // On 3.10 and above the tracing_possible is determined on each thread by the cframe object, so we don't need to + // check the interpreter state (it's no longer set there). + if (pyrtInfo.LanguageVersion > PythonLanguageVersion.V39) { + return Enumerable.Empty(); + } + return from interp in PyInterpreterState.GetInterpreterStates(process) select interp.ceval.tracing_possible; + } private void AddStepInGate(StepInGateHandler handler, DkmNativeModuleInstance module, string funcName, bool hasMultipleExitPoints) { var gate = new StepInGate { @@ -288,8 +352,7 @@ public void OnCTypesLoaded(DkmNativeModuleInstance moduleInstance) { } public unsafe void RegisterTracing(PyThreadState tstate) { - tstate.use_tracing.Write(1); - tstate.c_tracefunc.Write(_traceFunc.GetPointer()); + tstate.RegisterTracing(_traceFunc.GetPointer()); foreach (var pyTracingPossible in GetTracingPossible(_pyrtInfo, _process)) { pyTracingPossible.Write(pyTracingPossible.Read() + 1); } @@ -300,9 +363,16 @@ public unsafe void RegisterJITTracing(PyInterpreterState istate) { Debug.Assert(_pyrtInfo.LanguageVersion >= PythonLanguageVersion.V36); var current = istate.eval_frame.Read(); - if (current != _evalFrameFunc.GetPointer()) { + var evalFrameAddr = _evalFrameFunc.GetPointer(); + + if (current == 0) { + // This means the eval_frame is set to the default. Write + // this as our _defaultEvalFrameFunc + _defaultEvalFrameFunc.Write(_pyEval_FrameDefault.GetPointer()); + istate.eval_frame.Write(evalFrameAddr); + } else if (current != evalFrameAddr) { _defaultEvalFrameFunc.Write(current); - istate.eval_frame.Write(_evalFrameFunc.GetPointer()); + istate.eval_frame.Write(evalFrameAddr); } } @@ -481,6 +551,15 @@ public void PyInterpreterState_New(DkmThread thread, ulong frameBase, ulong vfra } } + public void _PyInterpreterState_New(DkmThread thread, ulong frameBase, ulong vframe, ulong returnAddress) { + // The new interpreter should be the 'head' of the list of interpreters + // (this function actually returns a status code, not the interpreter state). + var head = PyInterpreterState.GetInterpreterStates(thread.Process).First(); + if (head != null && head.Process != null) { + _owner.RegisterJITTracing(head); + } + } + // This step-in gate is not marked [StepInGate] because it doesn't live in pythonXX.dll, and so we register it manually. public void _call_function_pointer(DkmThread thread, ulong frameBase, ulong vframe, bool useRegisters) { var cppEval = new CppExpressionEvaluator(thread, frameBase, vframe); @@ -488,7 +567,7 @@ public void _call_function_pointer(DkmThread thread, ulong frameBase, ulong vfra _owner.OnPotentialRuntimeExit(thread, pProc); } - [StepInGate] + [StepInGate(MaxVersion = PythonLanguageVersion.V310)] public void call_function(DkmThread thread, ulong frameBase, ulong vframe, bool useRegisters) { var process = thread.Process; var cppEval = new CppExpressionEvaluator(thread, frameBase, vframe); @@ -511,6 +590,19 @@ public void call_function(DkmThread thread, ulong frameBase, ulong vframe, bool _owner.OnPotentialRuntimeExit(thread, ml_meth); } + [StepInGate(MinVersion = PythonLanguageVersion.V311)] + public void PyObject_Vectorcall(DkmThread thread, ulong frameBase, ulong vframe, bool useRegisters) { + var process = thread.Process; + var cppEval = new CppExpressionEvaluator(thread, frameBase, vframe); + ulong func = cppEval.EvaluateUInt64(useRegisters ? "@rdx" : "callable"); + var obj = PyObject.FromAddress(process, func); + ulong ml_meth = cppEval.EvaluateUInt64( + "((PyObject*){0})->ob_type == &PyCFunction_Type ? ((PyCFunctionObject*){0})->m_ml->ml_meth : 0", + func); + + _owner.OnPotentialRuntimeExit(thread, ml_meth); + } + [StepInGate] public void PyCFunction_Call(DkmThread thread, ulong frameBase, ulong vframe, bool useRegisters) { var process = thread.Process; diff --git a/Python/Product/DebuggerHelper/trace.cpp b/Python/Product/DebuggerHelper/trace.cpp index c5819ac052..5f2814d92e 100644 --- a/Python/Product/DebuggerHelper/trace.cpp +++ b/Python/Product/DebuggerHelper/trace.cpp @@ -51,17 +51,17 @@ struct { int64_t ob_size; } PyVarObject; struct { - int64_t f_back, f_code, f_globals, f_locals, f_lineno; + int64_t f_back, f_code, f_globals, f_locals, f_lineno, f_frame; } PyFrameObject; struct { - int64_t co_varnames, co_filename, co_name; + int64_t co_filename, co_name; } PyCodeObject; struct { int64_t ob_sval; } PyBytesObject; struct { - int64_t PyAsciiObjectData, PyCompactUnicodeObjectData; - int64_t length, state, wstr, wstr_length, data; + int64_t sizeof_PyAsciiObjectData, sizeof_PyCompactUnicodeObjectData; + int64_t length, state, wstr, wstr_length, utf8, utf8_length, data; } PyUnicodeObject; } fieldOffsets; @@ -82,6 +82,11 @@ struct { uint64_t PyErr_Restore; uint64_t PyErr_Occurred; uint64_t PyObject_Str; + uint64_t PyEval_SetTraceAllThreads; + uint64_t PyGILState_Ensure; + uint64_t PyGILState_Release; + uint64_t Py_Initialize; + uint64_t Py_Finalize; } functionPointers; void Py_DecRef(PyObject* a1) { @@ -112,6 +117,33 @@ PyObject* PyObject_Str(PyObject* o) { return reinterpret_cast(functionPointers.PyObject_Str)(o); } +/* Py_tracefunc return -1 when raising an exception, or 0 for success. */ +typedef int (*Py_tracefunc)(void* obj, void* frame, int what, void* arg); + +void +PyEval_SetTraceAllThreads(Py_tracefunc func, PyObject* arg) { + return reinterpret_cast(functionPointers.PyEval_SetTraceAllThreads)(func, arg); +} + +typedef +enum { PyGILState_LOCKED, PyGILState_UNLOCKED } PyGILState_STATE; + +PyGILState_STATE PyGILState_Ensure() { + return reinterpret_cast(functionPointers.PyGILState_Ensure)(); +} + +void PyGILState_Release(PyGILState_STATE state) { + return reinterpret_cast(functionPointers.PyGILState_Release)(state); +} + +void Py_Initialize() { + return reinterpret_cast(functionPointers.Py_Initialize)(); +} + +void Py_Finalize() { + return reinterpret_cast(functionPointers.Py_Finalize)(); +} + // A string provided by the debugger (e.g. for file names). This is actually a variable-length struct, // with _countof(data) == length + 1 - the extra wchar_t is the null terminator. struct DebuggerString { @@ -256,7 +288,7 @@ bool StringEquals(const DebuggerString* debuggerString, const void* pyString) { state = ReadField(pyString, fieldOffsets.PyUnicodeObject.state); - if (!ready) { + if (!ready && fieldOffsets.PyUnicodeObject.wstr != 0) { auto wstr = ReadField(pyString, fieldOffsets.PyUnicodeObject.wstr); if (!wstr) { return false; @@ -279,9 +311,9 @@ bool StringEquals(const DebuggerString* debuggerString, const void* pyString) { if (!compact) { data = ReadField(pyString, fieldOffsets.PyUnicodeObject.data); } else if (ascii) { - data = reinterpret_cast(pyString) + fieldOffsets.PyUnicodeObject.PyAsciiObjectData; + data = reinterpret_cast(pyString) + fieldOffsets.PyUnicodeObject.sizeof_PyAsciiObjectData; } else { - data = reinterpret_cast(pyString) + fieldOffsets.PyUnicodeObject.PyCompactUnicodeObjectData; + data = reinterpret_cast(pyString) + fieldOffsets.PyUnicodeObject.sizeof_PyCompactUnicodeObjectData; } if (kind == 2) { @@ -353,8 +385,20 @@ void EvalLoop(void (*bp)()) { auto frame = reinterpret_cast(evalLoopFrame); PyFrame_FastToLocals(frame); - auto f_globals = ReadField(frame, fieldOffsets.PyFrameObject.f_globals); - auto f_locals = ReadField(frame, fieldOffsets.PyFrameObject.f_locals); + PyObject* f_globals = NULL; + PyObject* f_locals = NULL; + if (fieldOffsets.PyFrameObject.f_frame == 0) { + // We're on 3.10 or earlier. f_globals and f_locals is directly off of the frame. + f_globals = ReadField(frame, fieldOffsets.PyFrameObject.f_globals); + f_locals = ReadField(frame, fieldOffsets.PyFrameObject.f_locals); + } + else { + // We're on 3.11 or later. f_frame (PyInterpreterFrame) is off of the frame. It has + // the f_globals and f_locals object. + void* f_frame = ReadField(frame, fieldOffsets.PyFrameObject.f_frame); + f_globals = ReadField(f_frame, fieldOffsets.PyFrameObject.f_globals); + f_locals = ReadField(f_frame, fieldOffsets.PyFrameObject.f_locals); + } PyObject *orig_exc_type, *orig_exc_value, *orig_exc_tb; PyErr_Fetch(&orig_exc_type, &orig_exc_value, &orig_exc_tb); @@ -425,7 +469,7 @@ static void TraceLine(void* frame) { const auto& bpData = breakpointData[iData]; int f_lineno = ReadField(frame, fieldOffsets.PyFrameObject.f_lineno); - if (f_lineno > bpData.maxLineNumber) { + if (f_lineno > bpData.maxLineNumber || f_lineno <= 0) { return; } @@ -435,8 +479,19 @@ static void TraceLine(void* frame) { return; } - void* f_code = ReadField(frame, fieldOffsets.PyFrameObject.f_code); - void* co_filename = ReadField(f_code, fieldOffsets.PyCodeObject.co_filename); + void* co_filename = nullptr; + if (fieldOffsets.PyFrameObject.f_frame == 0) { + // We're on 3.10 or earlier. f_code is directly off of the frame. + void* f_code = ReadField(frame, fieldOffsets.PyFrameObject.f_code); + co_filename = ReadField(f_code, fieldOffsets.PyCodeObject.co_filename); + } + else { + // We're on 3.11 or later. f_frame (PyInterpreterFrame) is off of the frame. It has + // the f_code object. + void* f_frame = ReadField(frame, fieldOffsets.PyFrameObject.f_frame); + void* f_code = ReadField(f_frame, fieldOffsets.PyFrameObject.f_code); + co_filename = ReadField(f_code, fieldOffsets.PyCodeObject.co_filename); + } if (co_filename == nullptr) { return; } @@ -540,19 +595,46 @@ int TraceFunc(void* obj, void* frame, int what, void* arg) { return 0; } -// In Python 3.9 the eval function added the thread state. So -// we need a new function def to accomodate that change. -typedef void* (*_PyFrameEvalFunction)(void*,void*, int); +void* InitialEvalFrameFunc(void* ts, void* f, int throwFlag); + +typedef void* (*_PyFrameEvalFunction)(void*, void*, int); +_PyFrameEvalFunction CurrentEvalFrameFunc = InitialEvalFrameFunc; + __declspec(dllexport) _PyFrameEvalFunction DefaultEvalFrameFunc = nullptr; -__declspec(dllexport) -void* EvalFrameFunc(void* ts, void* f, int throwFlag) +// Initial EvalFrameFunc that is used to set the trace function. +volatile unsigned long _isTracing = 0; +void *InitialEvalFrameFunc(void* ts, void* f, int throwFlag) { + // If were in 3.12, we need to set the trace function ourselves. + // This is because just writing to the use_tracing flag is no longer enough. Internally CPython + // doesn't just trace everything if the flag is set. Instead tracing now uses the sys.monitoring + // api under the covers. That's a lot more than just flipping a flag. + if (functionPointers.PyEval_SetTraceAllThreads != 0 && ::InterlockedCompareExchange(&_isTracing, 1, 0) == 0) { + auto gilState = PyGILState_Ensure(); + PyEval_SetTraceAllThreads(TraceFunc, nullptr); + PyGILState_Release(gilState); + } + + // Rewrite the current EvalFrameFunc so we don't bother attempting to register the trace function again. + CurrentEvalFrameFunc = DefaultEvalFrameFunc; + if (DefaultEvalFrameFunc) return (*DefaultEvalFrameFunc)(ts, f, throwFlag); return nullptr; } + + +// Function that is inserted into the current thread state by the debugger as the function to call +// in order to evaluate a frame. This is done so that the debugger can intercept the call +__declspec(dllexport) +void* EvalFrameFunc(void* ts, void* f, int throwFlag) +{ + return (*CurrentEvalFrameFunc)(ts, f, throwFlag); +} + + } diff --git a/Python/Tests/Utilities.Python.Analysis/PythonPaths.cs b/Python/Tests/Utilities.Python.Analysis/PythonPaths.cs index 41432e43d2..7ccbae4723 100644 --- a/Python/Tests/Utilities.Python.Analysis/PythonPaths.cs +++ b/Python/Tests/Utilities.Python.Analysis/PythonPaths.cs @@ -46,6 +46,9 @@ public class PythonPaths { public static readonly PythonVersion Python38 = GetCPythonVersion(PythonLanguageVersion.V38, InterpreterArchitecture.x86); public static readonly PythonVersion Python39 = GetCPythonVersion(PythonLanguageVersion.V39, InterpreterArchitecture.x86); public static readonly PythonVersion Python310 = GetCPythonVersion(PythonLanguageVersion.V310, InterpreterArchitecture.x86); + public static readonly PythonVersion Python311 = GetCPythonVersion(PythonLanguageVersion.V311, InterpreterArchitecture.x86); + public static readonly PythonVersion Python312 = GetCPythonVersion(PythonLanguageVersion.V312, InterpreterArchitecture.x86); + public static readonly PythonVersion Python313 = GetCPythonVersion(PythonLanguageVersion.V313, InterpreterArchitecture.x86); public static readonly PythonVersion Python27_x64 = GetCPythonVersion(PythonLanguageVersion.V27, InterpreterArchitecture.x64); public static readonly PythonVersion Python35_x64 = GetCPythonVersion(PythonLanguageVersion.V35, InterpreterArchitecture.x64); public static readonly PythonVersion Python36_x64 = GetCPythonVersion(PythonLanguageVersion.V36, InterpreterArchitecture.x64); @@ -53,6 +56,9 @@ public class PythonPaths { public static readonly PythonVersion Python38_x64 = GetCPythonVersion(PythonLanguageVersion.V38, InterpreterArchitecture.x64); public static readonly PythonVersion Python39_x64 = GetCPythonVersion(PythonLanguageVersion.V39, InterpreterArchitecture.x64); public static readonly PythonVersion Python310_x64 = GetCPythonVersion(PythonLanguageVersion.V310, InterpreterArchitecture.x64); + public static readonly PythonVersion Python311_x64 = GetCPythonVersion(PythonLanguageVersion.V311, InterpreterArchitecture.x64); + public static readonly PythonVersion Python312_x64 = GetCPythonVersion(PythonLanguageVersion.V312, InterpreterArchitecture.x64); + public static readonly PythonVersion Python313_x64 = GetCPythonVersion(PythonLanguageVersion.V313, InterpreterArchitecture.x64); public static readonly PythonVersion Anaconda27 = GetAnacondaVersion(PythonLanguageVersion.V27, InterpreterArchitecture.x86); public static readonly PythonVersion Anaconda27_x64 = GetAnacondaVersion(PythonLanguageVersion.V27, InterpreterArchitecture.x64); public static readonly PythonVersion Anaconda36 = GetAnacondaVersion(PythonLanguageVersion.V36, InterpreterArchitecture.x86); diff --git a/package.json b/package.json index fe8b152145..c5c573c929 100644 --- a/package.json +++ b/package.json @@ -2,6 +2,6 @@ "name": "ptvs", "private": true, "devDependencies": { - "@pylance/pylance": "2024.11.1" + "@pylance/pylance": "2024.11.2" } }