[Vulkan] Use VK_EXT_custom_border_color for YCbCr texture border colors

Co-authored-by: Herman S. <429230+has207@users.noreply.github.com>

src/xenia/gpu/vulkan/vulkan_texture_cache.cc

@@ -862,11 +862,43 @@ VkSampler VulkanTextureCache::UseSampler(SamplerParameters parameters,
   } else {
     sampler_create_info.maxLod = VK_LOD_CLAMP_NONE;
   }
-  // TODO(Triang3l): Custom border colors for CrYCb / YCrCb.
+  // The two YCbCr border colors are not expressible as fixed Vulkan border
+  // color enums. Use a custom border color when supported, otherwise fall back
+  // to transparent black (matching the alpha at least).
+  VkSamplerCustomBorderColorCreateInfoEXT custom_border_color = {
+      VK_STRUCTURE_TYPE_SAMPLER_CUSTOM_BORDER_COLOR_CREATE_INFO_EXT};
+  const bool custom_border_color_supported =
+      vulkan_device->properties().customBorderColors &&
+      vulkan_device->properties().customBorderColorWithoutFormat;
   switch (parameters.border_color) {
     case xenos::BorderColor::k_ABGR_White:
       sampler_create_info.borderColor = VK_BORDER_COLOR_FLOAT_OPAQUE_WHITE;
       break;
+    case xenos::BorderColor::k_ACBYCR_Black:
+    case xenos::BorderColor::k_ACBCRY_Black:
+      if (custom_border_color_supported) {
+        float* const color = custom_border_color.customBorderColor.float32;
+        if (parameters.border_color == xenos::BorderColor::k_ACBYCR_Black) {
+          // (Cr, Y, Cb) unsigned.
+          color[0] = 0.5f;
+          color[1] = 0.0f;
+          color[2] = 0.5f;
+        } else {
+          // (Y, Cr, Cb) unsigned.
+          color[0] = 0.0f;
+          color[1] = 0.5f;
+          color[2] = 0.5f;
+        }
+        color[3] = 0.0f;
+        custom_border_color.format = VK_FORMAT_UNDEFINED;
+        custom_border_color.pNext = sampler_create_info.pNext;
+        sampler_create_info.pNext = &custom_border_color;
+        sampler_create_info.borderColor = VK_BORDER_COLOR_FLOAT_CUSTOM_EXT;
+      } else {
+        sampler_create_info.borderColor =
+            VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK;
+      }
+      break;
     default:
       sampler_create_info.borderColor = VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK;
       break;

src/xenia/ui/vulkan/vulkan_device.cc

@@ -181,6 +181,7 @@ std::unique_ptr<VulkanDevice> VulkanDevice::CreateIfSupported(
   bool ext_EXT_fragment_shader_interlock = false;
   bool ext_1_3_EXT_shader_demote_to_helper_invocation = false;
   bool ext_EXT_non_seamless_cube_map = false;
+  bool ext_EXT_custom_border_color = false;
   if (with_gpu_emulation) {
     // #15.
     XE_UI_VULKAN_LOCAL_PROMOTED_EXTENSION(KHR_sampler_mirror_clamp_to_edge, 1,
@@ -204,6 +205,8 @@ std::unique_ptr<VulkanDevice> VulkanDevice::CreateIfSupported(
           EXT_shader_demote_to_helper_invocation, 1, 3)
       // #423.
       XE_UI_VULKAN_LOCAL_EXTENSION(EXT_non_seamless_cube_map)
+      // #288. Custom sampler border colors (for YCbCr border colors).
+      XE_UI_VULKAN_LOCAL_EXTENSION(EXT_custom_border_color)
     }
     if (properties.apiVersion >= VK_MAKE_API_VERSION(0, 1, 1, 0)) {
       // #237.
@@ -311,6 +314,10 @@ std::unique_ptr<VulkanDevice> VulkanDevice::CreateIfSupported(
       VkPhysicalDeviceNonSeamlessCubeMapFeaturesEXT,
       VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_NON_SEAMLESS_CUBE_MAP_FEATURES_EXT>
       features_EXT_non_seamless_cube_map;
+  VulkanFeatures<
+      VkPhysicalDeviceCustomBorderColorFeaturesEXT,
+      VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_CUSTOM_BORDER_COLOR_FEATURES_EXT>
+      features_EXT_custom_border_color;
 
   if (get_physical_device_properties2_supported) {
     if (properties.apiVersion >= VK_MAKE_API_VERSION(0, 1, 2, 0)) {
@@ -347,6 +354,10 @@ std::unique_ptr<VulkanDevice> VulkanDevice::CreateIfSupported(
       features_EXT_non_seamless_cube_map.Link(supported_features_2,
                                               device_create_info);
     }
+    if (ext_EXT_custom_border_color) {
+      features_EXT_custom_border_color.Link(supported_features_2,
+                                            device_create_info);
+    }
     ifn.vkGetPhysicalDeviceProperties2(physical_device, &properties_2);
     ifn.vkGetPhysicalDeviceFeatures2(physical_device, &supported_features_2);
   }
@@ -717,6 +728,15 @@ std::unique_ptr<VulkanDevice> VulkanDevice::CreateIfSupported(
     }
   }
 
+  if (ext_EXT_custom_border_color) {
+    if (with_gpu_emulation) {
+      XE_UI_VULKAN_FEATURE_2(features_EXT_custom_border_color,
+                             customBorderColors)
+      XE_UI_VULKAN_FEATURE_2(features_EXT_custom_border_color,
+                             customBorderColorWithoutFormat)
+    }
+  }
+
 #undef XE_UI_VULKAN_LIMIT
 #undef XE_UI_VULKAN_ENUM_LIMIT
 #undef XE_UI_VULKAN_FEATURE

src/xenia/ui/vulkan/vulkan_device.h

@@ -162,6 +162,11 @@ class VulkanDevice {
     // VK_EXT_non_seamless_cube_map (#423)
 
     bool nonSeamlessCubeMap = false;
+
+    // VK_EXT_custom_border_color (#288)
+
+    bool customBorderColors = false;
+    bool customBorderColorWithoutFormat = false;
   };
 
   // Properties of the core API and enabled extensions, and enabled features.