Merge pull request #2241 from js6i/gpu-capture-pipe

Added mechanism to trigger GPU capture without attaching a debugger.

MoltenVK/MoltenVK/API/mvk_private_api.h

@@ -124,9 +124,10 @@ typedef enum MVKConfigTraceVulkanCalls {
 
 /** Identifies the scope for Metal to run an automatic GPU capture for diagnostic debugging purposes. */
 typedef enum MVKConfigAutoGPUCaptureScope {
- MVK_CONFIG_AUTO_GPU_CAPTURE_SCOPE_NONE = 0, /**< No automatic GPU capture. */
- MVK_CONFIG_AUTO_GPU_CAPTURE_SCOPE_DEVICE = 1, /**< Automatically capture all GPU activity during the lifetime of a VkDevice. */
- MVK_CONFIG_AUTO_GPU_CAPTURE_SCOPE_FRAME = 2, /**< Automatically capture all GPU activity during the rendering and presentation of the first frame. */
+ MVK_CONFIG_AUTO_GPU_CAPTURE_SCOPE_NONE = 0, /**< No automatic GPU capture. */
+ MVK_CONFIG_AUTO_GPU_CAPTURE_SCOPE_DEVICE = 1, /**< Automatically capture all GPU activity during the lifetime of a VkDevice. */
+ MVK_CONFIG_AUTO_GPU_CAPTURE_SCOPE_FRAME = 2, /**< Automatically capture all GPU activity during the rendering and presentation of the first frame. */
+ MVK_CONFIG_AUTO_GPU_CAPTURE_SCOPE_ON_DEMAND = 3, /**< Capture all GPU activity when signaled on a temporary named pipe. */
  MVK_CONFIG_AUTO_GPU_CAPTURE_SCOPE_MAX_ENUM = 0x7FFFFFFF
 } MVKConfigAutoGPUCaptureScope;
 

MoltenVK/MoltenVK/GPUObjects/MVKDevice.h

@@ -773,7 +773,8 @@ class MVKDevice : public MVKDispatchableVulkanAPIObject {
  *
  * The mtlCaptureObject must be one of:
  * - MTLDevice for scope MVK_CONFIG_AUTO_GPU_CAPTURE_SCOPE_DEVICE
- * - MTLCommandQueue for scope MVK_CONFIG_AUTO_GPU_CAPTURE_SCOPE_FRAME.
+ * - MTLCommandQueue for scopes MVK_CONFIG_AUTO_GPU_CAPTURE_SCOPE_FRAME
+ * and MVK_CONFIG_AUTO_GPU_CAPTURE_SCOPE_ON_DEMAND.
  */
  void startAutoGPUCapture(MVKConfigAutoGPUCaptureScope autoGPUCaptureScope, id mtlCaptureObject);
 
@@ -836,6 +837,7 @@ class MVKDevice : public MVKDispatchableVulkanAPIObject {
  void getDescriptorVariableDescriptorCountLayoutSupport(const VkDescriptorSetLayoutCreateInfo* pCreateInfo,
  VkDescriptorSetLayoutSupport* pSupport,
  VkDescriptorSetVariableDescriptorCountLayoutSupport* pVarDescSetCountSupport);
+ bool readGPUCapturePipe() { char dummy; return _capturePipeFileDesc >= 0 && read(_capturePipeFileDesc, &dummy, 1) > 0; };
 
  MVKPhysicalDevice* _physicalDevice = nullptr;
  MVKExtensionList _enabledExtensions;
@@ -862,10 +864,12 @@ class MVKDevice : public MVKDispatchableVulkanAPIObject {
  std::mutex _sem4Lock;
  std::mutex _perfLock;
  std::mutex _vizLock;
+ std::string _capturePipeFileName;
  id<MTLBuffer> _globalVisibilityResultMTLBuffer = nil;
  id<MTLSamplerState> _defaultMTLSamplerState = nil;
  id<MTLBuffer> _dummyBlitMTLBuffer = nil;
  uint32_t _globalVisibilityQueryCount = 0;
+ int _capturePipeFileDesc = -1;
  bool _isPerformanceTracking = false;
  bool _isCurrentlyAutoGPUCapturing = false;
  bool _isUsingMetalArgumentBuffers = false;

MoltenVK/MoltenVK/GPUObjects/MVKDevice.mm

@@ -37,6 +37,7 @@
 
 #import "CAMetalLayer+MoltenVK.h"
 
+#include <sys/stat.h>
 #include <cmath>
 
 using namespace std;
@@ -4726,6 +4727,8 @@ static uint32_t mvkGetEntryProperty(io_registry_entry_t entry, CFStringRef prope
 
  if (_isCurrentlyAutoGPUCapturing || (getMVKConfig().autoGPUCaptureScope != autoGPUCaptureScope)) { return; }
 
+ if (autoGPUCaptureScope == MVK_CONFIG_AUTO_GPU_CAPTURE_SCOPE_ON_DEMAND && !readGPUCapturePipe()) return;
+
  _isCurrentlyAutoGPUCapturing = true;
 
  @autoreleasepool {
@@ -4773,6 +4776,7 @@ static uint32_t mvkGetEntryProperty(io_registry_entry_t entry, CFStringRef prope
 
 void MVKDevice::stopAutoGPUCapture(MVKConfigAutoGPUCaptureScope autoGPUCaptureScope) {
  if (_isCurrentlyAutoGPUCapturing && getMVKConfig().autoGPUCaptureScope == autoGPUCaptureScope) {
+ if (autoGPUCaptureScope == MVK_CONFIG_AUTO_GPU_CAPTURE_SCOPE_ON_DEMAND && readGPUCapturePipe()) return;
  [[MTLCaptureManager sharedCaptureManager] stopCapture];
  _isCurrentlyAutoGPUCapturing = false;
  }
@@ -4880,6 +4884,30 @@ static uint32_t mvkGetEntryProperty(io_registry_entry_t entry, CFStringRef prope
 
  startAutoGPUCapture(MVK_CONFIG_AUTO_GPU_CAPTURE_SCOPE_DEVICE, _physicalDevice->_mtlDevice);
 
+ if (getMVKConfig().autoGPUCaptureScope == MVK_CONFIG_AUTO_GPU_CAPTURE_SCOPE_ON_DEMAND) {
+ for (int tries = 0; tries < 3; ++tries) {
+ char pipeName[] = "/tmp/MoltenVKCapturePipe-XXXXXXXXXX";
+
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wdeprecated-declarations"
+ mktemp(pipeName);
+#pragma clang diagnostic pop
+
+ if (mkfifo(pipeName, 0600) < 0) {
+ if (errno == EEXIST) continue; // Name collision, retry.
+ reportMessage(MVK_CONFIG_LOG_LEVEL_ERROR, "Could not create named pipe for signaling GPU capture: %s\n", strerror(errno));
+ } else {
+ _capturePipeFileName = std::string(pipeName);
+ _capturePipeFileDesc = open(pipeName, O_RDONLY | O_NONBLOCK);
+ if (_capturePipeFileDesc < 0) {
+ reportMessage(MVK_CONFIG_LOG_LEVEL_ERROR, "Could not open named pipe for signaling GPU capture at path %s: %s\n", pipeName, strerror(errno));
+ }
+ }
+
+ break;
+ }
+ }
+
  MVKLogInfo("Created VkDevice to run on GPU %s with the following %d Vulkan extensions enabled:%s",
  getName(), _enabledExtensions.getEnabledCount(), _enabledExtensions.enabledNamesString("\n\t\t", true).c_str());
 }
@@ -5206,6 +5234,8 @@ static uint32_t mvkGetEntryProperty(io_registry_entry_t entry, CFStringRef prope
 
  stopAutoGPUCapture(MVK_CONFIG_AUTO_GPU_CAPTURE_SCOPE_DEVICE);
 
+ if (!_capturePipeFileName.empty()) { unlink(_capturePipeFileName.c_str()); }
+
  mvkDestroyContainerContents(_privateDataSlots);
 
  MVKLogInfo("Destroyed VkDevice on GPU %s with %d Vulkan extensions enabled.",

MoltenVK/MoltenVK/GPUObjects/MVKQueue.mm

@@ -705,6 +705,12 @@
  setConfigurationResult(_presentInfo[i].presentableImage->presentCAMetalDrawable(mtlCmdBuff, _presentInfo[i]));
  }
 
+ if (_queue->_queueFamily->getIndex() == getMVKConfig().defaultGPUCaptureScopeQueueFamilyIndex &&
+ _queue->_index == getMVKConfig().defaultGPUCaptureScopeQueueIndex) {
+ getDevice()->stopAutoGPUCapture(MVK_CONFIG_AUTO_GPU_CAPTURE_SCOPE_ON_DEMAND);
+ getDevice()->startAutoGPUCapture(MVK_CONFIG_AUTO_GPU_CAPTURE_SCOPE_ON_DEMAND, _queue->getMTLCommandQueue());
+ }
+
  if ( !mtlCmdBuff ) { setConfigurationResult(VK_ERROR_OUT_OF_POOL_MEMORY); } // Check after images may set error.
 
  // Add completion callback to the MTLCommandBuffer to call finish(),