A new debugger.

Lots of bugs/rough edges/etc - issues will be filed.
Old-style debugging still works (just use --emit_source_annotations to get
the helpful movs back and --break_on_instruction will still fire).

src/xenia/base/exception_handler_win.cc

@@ -9,53 +9,118 @@
 
 #include "xenia/base/exception_handler.h"
 
+#include "xenia/base/assert.h"
+#include "xenia/base/math.h"
 #include "xenia/base/platform_win.h"
 
 namespace xe {
-std::vector<ExceptionHandler::Handler> ExceptionHandler::handlers_;
+
+// Handle of the added VectoredExceptionHandler.
+void* veh_handle_ = nullptr;
+// Handle of the added VectoredContinueHandler.
+void* vch_handle_ = nullptr;
+
+// This can be as large as needed, but isn't often needed.
+// As we will be sometimes firing many exceptions we want to avoid having to
+// scan the table too much or invoke many custom handlers.
+constexpr size_t kMaxHandlerCount = 8;
+
+// All custom handlers, left-aligned and null terminated.
+// Executed in order.
+std::pair<ExceptionHandler::Handler, void*> handlers_[kMaxHandlerCount];
 
 LONG CALLBACK ExceptionHandlerCallback(PEXCEPTION_POINTERS ex_info) {
-  // Visual Studio SetThreadName
+  // Visual Studio SetThreadName.
   if (ex_info->ExceptionRecord->ExceptionCode == 0x406D1388) {
     return EXCEPTION_CONTINUE_SEARCH;
   }
 
-  auto code = ex_info->ExceptionRecord->ExceptionCode;
-  ExceptionHandler::Info info;
-  info.pc = ex_info->ContextRecord->Rip;
-  info.thread_context = ex_info->ContextRecord;
+  // TODO(benvanik): avoid this by mapping X64Context virtual?
+  X64Context thread_context;
+  thread_context.rip = ex_info->ContextRecord->Rip;
+  thread_context.eflags = ex_info->ContextRecord->EFlags;
+  std::memcpy(thread_context.int_registers, &ex_info->ContextRecord->Rax,
+              sizeof(thread_context.int_registers));
+  std::memcpy(thread_context.xmm_registers, &ex_info->ContextRecord->Xmm0,
+              sizeof(thread_context.xmm_registers));
 
-  switch (code) {
-    case STATUS_ACCESS_VIOLATION:
-      info.code = ExceptionHandler::Info::kAccessViolation;
-      info.fault_address = ex_info->ExceptionRecord->ExceptionInformation[1];
+  // http://msdn.microsoft.com/en-us/library/ms679331(v=vs.85).aspx
+  // http://msdn.microsoft.com/en-us/library/aa363082(v=vs.85).aspx
+  Exception ex;
+  switch (ex_info->ExceptionRecord->ExceptionCode) {
+    case STATUS_ILLEGAL_INSTRUCTION:
+      ex.InitializeIllegalInstruction(&thread_context);
       break;
+    case STATUS_ACCESS_VIOLATION:
+      ex.InitializeAccessViolation(
+          &thread_context, ex_info->ExceptionRecord->ExceptionInformation[1]);
+      break;
+    default:
+      // Unknown/unhandled type.
+      return EXCEPTION_CONTINUE_SEARCH;
   }
 
-  // Only call a handler if we support this type of exception.
-  if (info.code != ExceptionHandler::Info::kInvalidException) {
-    for (auto handler : ExceptionHandler::handlers()) {
-      if (handler(&info)) {
-        // Exception handled.
-        return EXCEPTION_CONTINUE_EXECUTION;
-      }
+  for (size_t i = 0; i < xe::countof(handlers_) && handlers_[i].first; ++i) {
+    if (handlers_[i].first(&ex, handlers_[i].second)) {
+      // Exception handled.
+      // TODO(benvanik): update all thread state? Dirty flags?
+      ex_info->ContextRecord->Rip = thread_context.rip;
+      return EXCEPTION_CONTINUE_EXECUTION;
     }
   }
-
   return EXCEPTION_CONTINUE_SEARCH;
 }
 
-bool ExceptionHandler::Initialize() {
-  AddVectoredExceptionHandler(0, ExceptionHandlerCallback);
+void ExceptionHandler::Install(Handler fn, void* data) {
+  if (!veh_handle_) {
+    veh_handle_ = AddVectoredExceptionHandler(1, ExceptionHandlerCallback);
 
-  // TODO: Do we need a continue handler if a debugger is attached?
-  return true;
+    if (IsDebuggerPresent()) {
+      // TODO(benvanik): do we need a continue handler if a debugger is
+      // attached?
+      // vch_handle_ = AddVectoredContinueHandler(1, ExceptionHandlerCallback);
+    }
+  }
+
+  for (size_t i = 0; i < xe::countof(handlers_); ++i) {
+    if (!handlers_[i].first) {
+      handlers_[i].first = fn;
+      handlers_[i].second = data;
+      return;
+    }
+  }
+  assert_always("Too many exception handlers installed");
 }
 
-uint32_t ExceptionHandler::Install(std::function<bool(Info* ex_info)> fn) {
-  handlers_.push_back(fn);
+void ExceptionHandler::Uninstall(Handler fn, void* data) {
+  for (size_t i = 0; i < xe::countof(handlers_); ++i) {
+    if (handlers_[i].first == fn && handlers_[i].second == data) {
+      for (; i < xe::countof(handlers_) - 1; ++i) {
+        handlers_[i] = handlers_[i + 1];
+      }
+      handlers_[i].first = nullptr;
+      handlers_[i].second = nullptr;
+      break;
+    }
+  }
 
-  // TODO: ID support!
-  return 0;
+  bool has_any = false;
+  for (size_t i = 0; i < xe::countof(handlers_); ++i) {
+    if (handlers_[i].first) {
+      has_any = true;
+      break;
+    }
+  }
+  if (!has_any) {
+    if (veh_handle_) {
+      RemoveVectoredExceptionHandler(veh_handle_);
+      veh_handle_ = nullptr;
+    }
+    if (vch_handle_) {
+      RemoveVectoredContinueHandler(vch_handle_);
+      vch_handle_ = nullptr;
+    }
+  }
 }
-}
+
+}  // namespace xe