GPU/Vulkan: Use Metal layer instead NSView on macOS.

dep/vulkan-loader/include/vulkan_loader.h

@@ -38,9 +38,8 @@
 #endif
 
 #if defined(__APPLE__)
-// TODO: Switch to Metal
+// #define VK_USE_PLATFORM_MACOS_MVK
-#define VK_USE_PLATFORM_MACOS_MVK
+#define VK_USE_PLATFORM_METAL_EXT
-// #define VK_USE_PLATFORM_METAL_EXT
 #endif
 
 #include "vulkan/vulkan.h"

src/common/vulkan/context.cpp

@@ -336,7 +336,8 @@ bool Context::Create(std::string_view gpu_name, const WindowInfo* wi, std::uniqu
   }
 
   VkSurfaceKHR surface = VK_NULL_HANDLE;
-  if (enable_surface && (surface = SwapChain::CreateVulkanSurface(instance, *wi)) == VK_NULL_HANDLE)
+  WindowInfo wi_copy(*wi);
+  if (enable_surface && (surface = SwapChain::CreateVulkanSurface(instance, wi_copy)) == VK_NULL_HANDLE)
   {
     vkDestroyInstance(instance, nullptr);
     Vulkan::UnloadVulkanLibrary();
@@ -352,7 +353,7 @@ bool Context::Create(std::string_view gpu_name, const WindowInfo* wi, std::uniqu
   // Attempt to create the device.
   if (!g_vulkan_context->CreateDevice(surface, enable_validation_layer) ||
       !g_vulkan_context->CreateGlobalDescriptorPool() || !g_vulkan_context->CreateCommandBuffers() ||
-      (enable_surface && (*out_swap_chain = SwapChain::Create(*wi, surface, true)) == nullptr))
+      (enable_surface && (*out_swap_chain = SwapChain::Create(wi_copy, surface, true)) == nullptr))
   {
     // Since we are destroying the instance, we're also responsible for destroying the surface.
     if (surface != VK_NULL_HANDLE)

src/common/vulkan/swap_chain.cpp

@@ -16,6 +16,63 @@ Log_SetChannel(Vulkan::SwapChain);
 #include <X11/Xlib.h>
 #endif
 
+#if defined(__APPLE__)
+#include <objc/message.h>
+
+static bool CreateMetalLayer(WindowInfo& wi)
+{
+  id view = reinterpret_cast<id>(wi.window_handle);
+
+  Class clsCAMetalLayer = objc_getClass("CAMetalLayer");
+  if (!clsCAMetalLayer)
+  {
+    Log_ErrorPrint("Failed to get CAMetalLayer class.");
+    return false;
+  }
+
+  // [CAMetalLayer layer]
+  id layer = reinterpret_cast<id (*)(Class, SEL)>(objc_msgSend)(objc_getClass("CAMetalLayer"), sel_getUid("layer"));
+  if (!layer)
+  {
+    Log_ErrorPrint("Failed to create Metal layer.");
+    return false;
+  }
+
+  // [view setWantsLayer:YES]
+  reinterpret_cast<void (*)(id, SEL, BOOL)>(objc_msgSend)(view, sel_getUid("setWantsLayer:"), YES);
+
+  // [view setLayer:layer]
+  reinterpret_cast<void (*)(id, SEL, id)>(objc_msgSend)(view, sel_getUid("setLayer:"), layer);
+
+  // NSScreen* screen = [NSScreen mainScreen]
+  id screen = reinterpret_cast<id (*)(Class, SEL)>(objc_msgSend)(objc_getClass("NSScreen"), sel_getUid("mainScreen"));
+
+  // CGFloat factor = [screen backingScaleFactor]
+  double factor = reinterpret_cast<double (*)(id, SEL)>(objc_msgSend)(screen, sel_getUid("backingScaleFactor"));
+
+  // layer.contentsScale = factor
+  reinterpret_cast<void (*)(id, SEL, double)>(objc_msgSend)(layer, sel_getUid("setContentsScale:"), factor);
+
+  // Store the layer pointer, that way MoltenVK doesn't call [NSView layer] outside the main thread.
+  wi.surface_handle = layer;
+  return true;
+}
+
+static void DestroyMetalLayer(WindowInfo& wi)
+{
+  id view = reinterpret_cast<id>(wi.window_handle);
+  id layer = reinterpret_cast<id>(wi.surface_handle);
+  if (layer == nil)
+    return;
+
+  reinterpret_cast<void (*)(id, SEL, id)>(objc_msgSend)(view, sel_getUid("setLayer:"), nil);
+  reinterpret_cast<void (*)(id, SEL, BOOL)>(objc_msgSend)(view, sel_getUid("setWantsLayer:"), NO);
+  reinterpret_cast<void (*)(id, SEL)>(objc_msgSend)(layer, sel_getUid("release"));
+  wi.surface_handle = nullptr;
+}
+
+#endif
+
 namespace Vulkan {
 SwapChain::SwapChain(const WindowInfo& wi, VkSurfaceKHR surface, bool vsync)
   : m_wi(wi), m_surface(surface), m_vsync_enabled(vsync)
@@ -30,7 +87,7 @@ SwapChain::~SwapChain()
   DestroySurface();
 }
 
-VkSurfaceKHR SwapChain::CreateVulkanSurface(VkInstance instance, const WindowInfo& wi)
+VkSurfaceKHR SwapChain::CreateVulkanSurface(VkInstance instance, WindowInfo& wi)
 {
 #if defined(VK_USE_PLATFORM_WIN32_KHR)
   if (wi.type == WindowInfo::Type::Win32)
@@ -103,9 +160,11 @@ VkSurfaceKHR SwapChain::CreateVulkanSurface(VkInstance instance, const WindowInf
 #if defined(VK_USE_PLATFORM_METAL_EXT)
   if (wi.type == WindowInfo::Type::MacOS)
   {
-    // TODO: Create Metal layer
+    if (!wi.surface_handle && !CreateMetalLayer(wi))
+      return VK_NULL_HANDLE;
+
     VkMetalSurfaceCreateInfoEXT surface_create_info = {VK_STRUCTURE_TYPE_METAL_SURFACE_CREATE_INFO_EXT, nullptr, 0,
-                                                       static_cast<const CAMetalLayer*>(wi.window_handle)};
+                                                       static_cast<const CAMetalLayer*>(wi.surface_handle)};
 
     VkSurfaceKHR surface;
     VkResult res = vkCreateMetalSurfaceEXT(instance, &surface_create_info, nullptr, &surface);
@@ -138,6 +197,16 @@ VkSurfaceKHR SwapChain::CreateVulkanSurface(VkInstance instance, const WindowInf
   return VK_NULL_HANDLE;
 }
 
+void SwapChain::DestroyVulkanSurface(VkInstance instance, WindowInfo& wi, VkSurfaceKHR surface)
+{
+  vkDestroySurfaceKHR(g_vulkan_context->GetVulkanInstance(), surface, nullptr);
+
+#if defined(__APPLE__)
+  if (wi.type == WindowInfo::Type::MacOS && wi.surface_handle)
+    DestroyMetalLayer(wi);
+#endif
+}
+
 std::unique_ptr<SwapChain> SwapChain::Create(const WindowInfo& wi, VkSurfaceKHR surface, bool vsync)
 {
   std::unique_ptr<SwapChain> swap_chain = std::make_unique<SwapChain>(wi, surface, vsync);
@@ -495,7 +564,7 @@ bool SwapChain::RecreateSurface(const WindowInfo& new_wi)
 
 void SwapChain::DestroySurface()
 {
-  vkDestroySurfaceKHR(g_vulkan_context->GetVulkanInstance(), m_surface, nullptr);
+  DestroyVulkanSurface(g_vulkan_context->GetVulkanInstance(), m_wi, m_surface);
   m_surface = VK_NULL_HANDLE;
 }
 

src/common/vulkan/swap_chain.h

@@ -21,7 +21,10 @@ public:
   ~SwapChain();
 
   // Creates a vulkan-renderable surface for the specified window handle.
-  static VkSurfaceKHR CreateVulkanSurface(VkInstance instance, const WindowInfo& wi);
+  static VkSurfaceKHR CreateVulkanSurface(VkInstance instance, WindowInfo& wi);
+
+  // Destroys a previously-created surface.
+  static void DestroyVulkanSurface(VkInstance instance, WindowInfo& wi, VkSurfaceKHR surface);
 
   // Create a new swap chain from a pre-existing surface.
   static std::unique_ptr<SwapChain> Create(const WindowInfo& wi, VkSurfaceKHR surface, bool vsync);

src/common/window_info.h

@@ -29,4 +29,9 @@ struct WindowInfo
   u32 surface_width = 0;
   u32 surface_height = 0;
   SurfaceFormat surface_format = SurfaceFormat::RGB8;
+
+  // Needed for macOS.
+#ifdef __APPLE__
+  void* surface_handle = nullptr;
+#endif
 };

src/frontend-common/vulkan_host_display.cpp

@@ -100,14 +100,15 @@ bool VulkanHostDisplay::RecreateSwapChain(const WindowInfo& new_wi)
 {
   Assert(!m_swap_chain);
 
-  VkSurfaceKHR surface = Vulkan::SwapChain::CreateVulkanSurface(g_vulkan_context->GetVulkanInstance(), new_wi);
+  WindowInfo wi_copy(new_wi);
+  VkSurfaceKHR surface = Vulkan::SwapChain::CreateVulkanSurface(g_vulkan_context->GetVulkanInstance(), wi_copy);
   if (surface == VK_NULL_HANDLE)
   {
     Log_ErrorPrintf("Failed to create new surface for swap chain");
     return false;
   }
 
-  m_swap_chain = Vulkan::SwapChain::Create(new_wi, surface, false);
+  m_swap_chain = Vulkan::SwapChain::Create(wi_copy, surface, false);
   if (!m_swap_chain)
   {
     Log_ErrorPrintf("Failed to create swap chain");