Capture exe with renderdoc and pass --vulkan_renderdoc_capture_all.

src/xenia/gpu/vulkan/vulkan_command_processor.cc

@@ -163,6 +163,11 @@ bool VulkanCommandProcessor::IssueDraw(PrimitiveType primitive_type,
                                        IndexBufferInfo* index_buffer_info) {
   auto& regs = *register_file_;
 
+  // TODO(benvanik): move to CP or to host (trace dump, etc).
+  if (FLAGS_vulkan_renderdoc_capture_all && device_->is_renderdoc_attached()) {
+    device_->BeginRenderDocFrameCapture();
+  }
+
 #if FINE_GRAINED_DRAW_SCOPES
   SCOPE_profile_cpu_f("gpu");
 #endif  // FINE_GRAINED_DRAW_SCOPES
@@ -313,6 +318,11 @@ bool VulkanCommandProcessor::IssueDraw(PrimitiveType primitive_type,
   }
   vkDestroyFence(*device_, fence, nullptr);
 
+  // TODO(benvanik): move to CP or to host (trace dump, etc).
+  if (FLAGS_vulkan_renderdoc_capture_all && device_->is_renderdoc_attached()) {
+    device_->EndRenderDocFrameCapture();
+  }
+
   return true;
 }
 

src/xenia/gpu/vulkan/vulkan_gpu_flags.cc

@@ -8,3 +8,6 @@
  */
 
 #include "xenia/gpu/vulkan/vulkan_gpu_flags.h"
+
+DEFINE_bool(vulkan_renderdoc_capture_all, false,
+            "Capture everything with RenderDoc.");

src/xenia/gpu/vulkan/vulkan_gpu_flags.h

@@ -14,4 +14,6 @@
 
 #define FINE_GRAINED_DRAW_SCOPES 1
 
+DECLARE_bool(vulkan_renderdoc_capture_all);
+
 #endif  // XENIA_GPU_VULKAN_VULKAN_GPU_FLAGS_H_

src/xenia/ui/vulkan/vulkan_device.cc

@@ -15,12 +15,15 @@
 #include <mutex>
 #include <string>
 
+#include "third_party/renderdoc/renderdoc_app.h"
+
 #include "xenia/base/assert.h"
 #include "xenia/base/logging.h"
 #include "xenia/base/math.h"
 #include "xenia/base/profiling.h"
 #include "xenia/ui/vulkan/vulkan.h"
 #include "xenia/ui/vulkan/vulkan_immediate_drawer.h"
+#include "xenia/ui/vulkan/vulkan_instance.h"
 #include "xenia/ui/vulkan/vulkan_util.h"
 #include "xenia/ui/window.h"
 
@@ -212,6 +215,30 @@ void VulkanDevice::ReleaseQueue(VkQueue queue) {
   free_queues_.push_back(queue);
 }
 
+bool VulkanDevice::is_renderdoc_attached() const {
+  return instance_->is_renderdoc_attached();
+}
+
+void VulkanDevice::BeginRenderDocFrameCapture() {
+  auto api = reinterpret_cast<RENDERDOC_API_1_0_1*>(instance_->renderdoc_api());
+  if (!api) {
+    return;
+  }
+  assert_true(api->IsFrameCapturing() == 0);
+
+  api->StartFrameCapture(nullptr, nullptr);
+}
+
+void VulkanDevice::EndRenderDocFrameCapture() {
+  auto api = reinterpret_cast<RENDERDOC_API_1_0_1*>(instance_->renderdoc_api());
+  if (!api) {
+    return;
+  }
+  assert_true(api->IsFrameCapturing() == 1);
+
+  api->EndFrameCapture(nullptr, nullptr);
+}
+
 VkDeviceMemory VulkanDevice::AllocateMemory(
     const VkMemoryRequirements& requirements, VkFlags required_properties) {
   // Search memory types to find one matching our requirements and our

src/xenia/ui/vulkan/vulkan_device.h

@@ -75,6 +75,15 @@ class VulkanDevice {
   // This method is thread safe.
   void ReleaseQueue(VkQueue queue);
 
+  // True if RenderDoc is attached and available for use.
+  bool is_renderdoc_attached() const;
+  // Begins capturing the current frame in RenderDoc, if it is attached.
+  // Must be paired with EndRenderDocCapture. Multiple frames cannot be
+  // captured at the same time.
+  void BeginRenderDocFrameCapture();
+  // Ends a capture.
+  void EndRenderDocFrameCapture();
+
   // Allocates memory of the given size matching the required properties.
   VkDeviceMemory AllocateMemory(
       const VkMemoryRequirements& requirements,

src/xenia/ui/vulkan/vulkan_instance.cc

@@ -15,6 +15,8 @@
 #include <mutex>
 #include <string>
 
+#include "third_party/renderdoc/renderdoc_app.h"
+
 #include "xenia/base/assert.h"
 #include "xenia/base/logging.h"
 #include "xenia/base/math.h"
@@ -58,6 +60,9 @@ bool VulkanInstance::Initialize(Window* any_target_window) {
   auto version = Version::Parse(VK_API_VERSION);
   XELOGVK("Initializing Vulkan %s...", version.pretty_string.c_str());
 
+  // Hook into renderdoc, if it's available.
+  EnableRenderDoc();
+
   // Get all of the global layers and extensions provided by the system.
   if (!QueryGlobals()) {
     XELOGE("Failed to query instance globals");
@@ -81,6 +86,45 @@ bool VulkanInstance::Initialize(Window* any_target_window) {
   return true;
 }
 
+bool VulkanInstance::EnableRenderDoc() {
+  // RenderDoc injects itself into our process, so we should be able to get it.
+  pRENDERDOC_GetAPI get_api = nullptr;
+#if XE_PLATFORM_WIN32
+  auto module_handle = GetModuleHandle(L"renderdoc.dll");
+  if (!module_handle) {
+    XELOGI("RenderDoc support requested but it is not attached");
+    return false;
+  }
+  get_api = reinterpret_cast<pRENDERDOC_GetAPI>(
+      GetProcAddress(module_handle, "RENDERDOC_GetAPI"));
+#else
+// TODO(benvanik): dlsym/etc - abstracted in base/.
+#endif  // XE_PLATFORM_32
+  if (!get_api) {
+    XELOGI("RenderDoc support requested but it is not attached");
+    return false;
+  }
+
+  // Request all API function pointers.
+  if (!get_api(eRENDERDOC_API_Version_1_0_1,
+               reinterpret_cast<void**>(&renderdoc_api_))) {
+    XELOGE("RenderDoc found but was unable to get API - version mismatch?");
+    return false;
+  }
+  auto api = reinterpret_cast<RENDERDOC_API_1_0_1*>(renderdoc_api_);
+
+  // Query version.
+  int major;
+  int minor;
+  int patch;
+  api->GetAPIVersion(&major, &minor, &patch);
+  XELOGI("RenderDoc attached; %d.%d.%d", major, minor, patch);
+
+  is_renderdoc_attached_ = true;
+
+  return true;
+}
+
 bool VulkanInstance::QueryGlobals() {
   // Scan global layers and accumulate properties.
   // We do this in a loop so that we can allocate the required amount of

src/xenia/ui/vulkan/vulkan_instance.h

@@ -58,7 +58,15 @@ class VulkanInstance {
     return available_devices_;
   }
 
+  // True if RenderDoc is attached and available for use.
+  bool is_renderdoc_attached() const { return is_renderdoc_attached_; }
+  // RenderDoc API handle, if attached.
+  void* renderdoc_api() const { return renderdoc_api_; }
+
  private:
+  // Attempts to enable RenderDoc via the API, if it is attached.
+  bool EnableRenderDoc();
+
   // Queries the system to find global extensions and layers.
   bool QueryGlobals();
 
@@ -86,6 +94,9 @@ class VulkanInstance {
   std::vector<DeviceInfo> available_devices_;
 
   VkDebugReportCallbackEXT dbg_report_callback_ = nullptr;
+
+  void* renderdoc_api_ = nullptr;
+  bool is_renderdoc_attached_ = false;
 };
 
 }  // namespace vulkan