vulkan-capture.c 43 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610
  1. #include <windows.h>
  2. #include "graphics-hook.h"
  3. #define VK_USE_PLATFORM_WIN32_KHR
  4. #include <malloc.h>
  5. #include <vulkan/vulkan.h>
  6. #include <vulkan/vk_layer.h>
  7. #undef VK_LAYER_EXPORT
  8. #if defined(WIN32)
  9. #define VK_LAYER_EXPORT __declspec(dllexport)
  10. #else
  11. #define VK_LAYER_EXPORT
  12. #endif
  13. #include <vulkan/vulkan_win32.h>
  14. #define COBJMACROS
  15. #include <dxgi.h>
  16. #include <d3d11.h>
  17. #include "vulkan-capture.h"
  18. /* ======================================================================== */
  19. /* defs/statics */
  20. /* shorten stuff because dear GOD is vulkan unclean. */
  21. #define VKAPI VKAPI_CALL
  22. #define VkFunc PFN_vkVoidFunction
  23. #define EXPORT VK_LAYER_EXPORT
  24. #define OBJ_MAX 16
  25. /* use the loader's dispatch table pointer as a key for internal data maps */
  26. #define GET_LDT(x) (*(void **)x)
  27. static bool vulkan_seen = false;
  28. static SRWLOCK mutex = SRWLOCK_INIT; // Faster CRITICAL_SECTION
  29. /* ======================================================================== */
  30. /* hook data */
  31. struct vk_swap_data {
  32. VkSwapchainKHR sc;
  33. VkExtent2D image_extent;
  34. VkFormat format;
  35. HWND hwnd;
  36. VkImage export_image;
  37. bool layout_initialized;
  38. VkDeviceMemory export_mem;
  39. VkImage swap_images[OBJ_MAX];
  40. uint32_t image_count;
  41. HANDLE handle;
  42. struct shtex_data *shtex_info;
  43. ID3D11Texture2D *d3d11_tex;
  44. bool captured;
  45. };
  46. struct vk_queue_data {
  47. VkQueue queue;
  48. uint32_t fam_idx;
  49. };
  50. struct vk_frame_data {
  51. VkCommandPool cmd_pool;
  52. VkCommandBuffer cmd_buffer;
  53. VkFence fence;
  54. bool cmd_buffer_busy;
  55. };
  56. struct vk_family_data {
  57. struct vk_frame_data frames[OBJ_MAX];
  58. uint32_t image_count;
  59. };
  60. struct vk_data {
  61. bool valid;
  62. struct vk_device_funcs funcs;
  63. VkPhysicalDevice phy_device;
  64. VkDevice device;
  65. struct vk_swap_data swaps[OBJ_MAX];
  66. struct vk_swap_data *cur_swap;
  67. uint32_t swap_idx;
  68. struct vk_queue_data queues[OBJ_MAX];
  69. uint32_t queue_count;
  70. struct vk_family_data families[OBJ_MAX];
  71. VkExternalMemoryProperties external_mem_props;
  72. struct vk_inst_data *inst_data;
  73. VkAllocationCallbacks ac_storage;
  74. const VkAllocationCallbacks *ac;
  75. ID3D11Device *d3d11_device;
  76. ID3D11DeviceContext *d3d11_context;
  77. };
  78. static struct vk_swap_data *get_swap_data(struct vk_data *data,
  79. VkSwapchainKHR sc)
  80. {
  81. for (int i = 0; i < OBJ_MAX; i++) {
  82. if (data->swaps[i].sc == sc) {
  83. return &data->swaps[i];
  84. }
  85. }
  86. debug("get_swap_data failed, swapchain not found");
  87. return NULL;
  88. }
  89. static struct vk_swap_data *get_new_swap_data(struct vk_data *data)
  90. {
  91. for (int i = 0; i < OBJ_MAX; i++) {
  92. if (data->swaps[i].sc == VK_NULL_HANDLE) {
  93. return &data->swaps[i];
  94. }
  95. }
  96. debug("get_new_swap_data failed, no more free slot");
  97. return NULL;
  98. }
  99. /* ------------------------------------------------------------------------- */
  100. static inline size_t find_obj_idx(void *objs[], void *obj)
  101. {
  102. size_t idx = SIZE_MAX;
  103. AcquireSRWLockExclusive(&mutex);
  104. for (size_t i = 0; i < OBJ_MAX; i++) {
  105. if (objs[i] == obj) {
  106. idx = i;
  107. break;
  108. }
  109. }
  110. ReleaseSRWLockExclusive(&mutex);
  111. return idx;
  112. }
  113. static size_t get_obj_idx(void *objs[], void *obj)
  114. {
  115. size_t idx = SIZE_MAX;
  116. AcquireSRWLockExclusive(&mutex);
  117. for (size_t i = 0; i < OBJ_MAX; i++) {
  118. if (objs[i] == obj) {
  119. idx = i;
  120. break;
  121. }
  122. if (!objs[i] && idx == SIZE_MAX) {
  123. idx = i;
  124. }
  125. }
  126. ReleaseSRWLockExclusive(&mutex);
  127. return idx;
  128. }
  129. /* ------------------------------------------------------------------------- */
  130. static struct vk_data device_data[OBJ_MAX] = {0};
  131. static void *devices[OBJ_MAX] = {0};
  132. static inline struct vk_data *get_device_data(void *dev)
  133. {
  134. size_t idx = get_obj_idx(devices, GET_LDT(dev));
  135. if (idx == SIZE_MAX) {
  136. debug("out of device slots");
  137. return NULL;
  138. }
  139. return &device_data[idx];
  140. }
  141. static void vk_shtex_clear_fence(const struct vk_data *data,
  142. struct vk_frame_data *frame_data)
  143. {
  144. const VkFence fence = frame_data->fence;
  145. if (frame_data->cmd_buffer_busy) {
  146. VkDevice device = data->device;
  147. const struct vk_device_funcs *funcs = &data->funcs;
  148. funcs->WaitForFences(device, 1, &fence, VK_TRUE, ~0ull);
  149. funcs->ResetFences(device, 1, &fence);
  150. frame_data->cmd_buffer_busy = false;
  151. }
  152. }
  153. static void vk_shtex_wait_until_pool_idle(struct vk_data *data,
  154. struct vk_family_data *family_data)
  155. {
  156. for (uint32_t image_idx = 0; image_idx < family_data->image_count;
  157. image_idx++) {
  158. struct vk_frame_data *frame_data =
  159. &family_data->frames[image_idx];
  160. if (frame_data->cmd_pool != VK_NULL_HANDLE)
  161. vk_shtex_clear_fence(data, frame_data);
  162. }
  163. }
  164. static void vk_shtex_wait_until_idle(struct vk_data *data)
  165. {
  166. for (uint32_t fam_idx = 0; fam_idx < _countof(data->families);
  167. fam_idx++) {
  168. struct vk_family_data *family_data = &data->families[fam_idx];
  169. vk_shtex_wait_until_pool_idle(data, family_data);
  170. }
  171. }
  172. static void vk_shtex_free(struct vk_data *data)
  173. {
  174. capture_free();
  175. vk_shtex_wait_until_idle(data);
  176. for (int swap_idx = 0; swap_idx < OBJ_MAX; swap_idx++) {
  177. struct vk_swap_data *swap = &data->swaps[swap_idx];
  178. if (swap->export_image)
  179. data->funcs.DestroyImage(data->device,
  180. swap->export_image, data->ac);
  181. if (swap->export_mem)
  182. data->funcs.FreeMemory(data->device, swap->export_mem,
  183. NULL);
  184. if (swap->d3d11_tex) {
  185. ID3D11Texture2D_Release(swap->d3d11_tex);
  186. }
  187. swap->handle = INVALID_HANDLE_VALUE;
  188. swap->d3d11_tex = NULL;
  189. swap->export_mem = VK_NULL_HANDLE;
  190. swap->export_image = VK_NULL_HANDLE;
  191. swap->captured = false;
  192. }
  193. if (data->d3d11_context) {
  194. ID3D11DeviceContext_Release(data->d3d11_context);
  195. data->d3d11_context = NULL;
  196. }
  197. if (data->d3d11_device) {
  198. ID3D11Device_Release(data->d3d11_device);
  199. data->d3d11_device = NULL;
  200. }
  201. data->cur_swap = NULL;
  202. hlog("------------------ vulkan capture freed ------------------");
  203. }
  204. static void vk_remove_device(void *dev)
  205. {
  206. size_t idx = find_obj_idx(devices, GET_LDT(dev));
  207. if (idx == SIZE_MAX) {
  208. return;
  209. }
  210. struct vk_data *data = &device_data[idx];
  211. memset(data, 0, sizeof(*data));
  212. AcquireSRWLockExclusive(&mutex);
  213. devices[idx] = NULL;
  214. ReleaseSRWLockExclusive(&mutex);
  215. }
  216. /* ------------------------------------------------------------------------- */
  217. struct vk_surf_data {
  218. VkSurfaceKHR surf;
  219. HWND hwnd;
  220. struct vk_surf_data *next;
  221. };
  222. struct vk_inst_data {
  223. bool valid;
  224. struct vk_inst_funcs funcs;
  225. struct vk_surf_data *surfaces;
  226. };
  227. static void *object_malloc(const VkAllocationCallbacks *ac, size_t size,
  228. size_t alignment)
  229. {
  230. return ac ? ac->pfnAllocation(ac->pUserData, size, alignment,
  231. VK_SYSTEM_ALLOCATION_SCOPE_OBJECT)
  232. : _aligned_malloc(size, alignment);
  233. }
  234. static void object_free(const VkAllocationCallbacks *ac, void *memory)
  235. {
  236. if (ac)
  237. ac->pfnFree(ac->pUserData, memory);
  238. else
  239. _aligned_free(memory);
  240. }
  241. static void insert_surf_data(struct vk_inst_data *data, VkSurfaceKHR surf,
  242. HWND hwnd, const VkAllocationCallbacks *ac)
  243. {
  244. struct vk_surf_data *surf_data = object_malloc(
  245. ac, sizeof(struct vk_surf_data), _Alignof(struct vk_surf_data));
  246. if (surf_data) {
  247. surf_data->surf = surf;
  248. surf_data->hwnd = hwnd;
  249. AcquireSRWLockExclusive(&mutex);
  250. struct vk_surf_data *next = data->surfaces;
  251. surf_data->next = next;
  252. data->surfaces = surf_data;
  253. ReleaseSRWLockExclusive(&mutex);
  254. }
  255. }
  256. static HWND find_surf_hwnd(struct vk_inst_data *data, VkSurfaceKHR surf)
  257. {
  258. HWND hwnd = NULL;
  259. AcquireSRWLockExclusive(&mutex);
  260. struct vk_surf_data *surf_data = data->surfaces;
  261. while (surf_data) {
  262. if (surf_data->surf == surf) {
  263. hwnd = surf_data->hwnd;
  264. break;
  265. }
  266. surf_data = surf_data->next;
  267. }
  268. ReleaseSRWLockExclusive(&mutex);
  269. return hwnd;
  270. }
  271. static void erase_surf_data(struct vk_inst_data *data, VkSurfaceKHR surf,
  272. const VkAllocationCallbacks *ac)
  273. {
  274. AcquireSRWLockExclusive(&mutex);
  275. struct vk_surf_data *current = data->surfaces;
  276. if (current->surf == surf) {
  277. data->surfaces = current->next;
  278. } else {
  279. struct vk_surf_data *previous;
  280. do {
  281. previous = current;
  282. current = current->next;
  283. } while (current && current->surf != surf);
  284. if (current)
  285. previous->next = current->next;
  286. }
  287. ReleaseSRWLockExclusive(&mutex);
  288. object_free(ac, current);
  289. }
  290. /* ------------------------------------------------------------------------- */
  291. static struct vk_inst_data inst_data[OBJ_MAX] = {0};
  292. static void *instances[OBJ_MAX] = {0};
  293. static struct vk_inst_data *get_inst_data(void *inst)
  294. {
  295. size_t idx = get_obj_idx(instances, GET_LDT(inst));
  296. if (idx == SIZE_MAX) {
  297. debug("out of instance slots");
  298. return NULL;
  299. }
  300. vulkan_seen = true;
  301. return &inst_data[idx];
  302. }
  303. static inline struct vk_inst_funcs *get_inst_funcs(void *inst)
  304. {
  305. struct vk_inst_data *data = get_inst_data(inst);
  306. return &data->funcs;
  307. }
  308. static void remove_instance(void *inst)
  309. {
  310. size_t idx = find_obj_idx(instances, inst);
  311. if (idx == SIZE_MAX) {
  312. return;
  313. }
  314. struct vk_inst_data *data = &inst_data[idx];
  315. memset(data, 0, sizeof(*data));
  316. AcquireSRWLockExclusive(&mutex);
  317. instances[idx] = NULL;
  318. ReleaseSRWLockExclusive(&mutex);
  319. }
  320. /* ======================================================================== */
  321. /* capture */
  322. static inline bool vk_shtex_init_d3d11(struct vk_data *data)
  323. {
  324. D3D_FEATURE_LEVEL level_used;
  325. IDXGIFactory1 *factory;
  326. IDXGIAdapter1 *adapter;
  327. HRESULT hr;
  328. HMODULE d3d11 = load_system_library("d3d11.dll");
  329. if (!d3d11) {
  330. flog("failed to load d3d11: %d", GetLastError());
  331. return false;
  332. }
  333. HMODULE dxgi = load_system_library("dxgi.dll");
  334. if (!dxgi) {
  335. flog("failed to load dxgi: %d", GetLastError());
  336. return false;
  337. }
  338. HRESULT(WINAPI * create_factory)
  339. (REFIID, void **) = (void *)GetProcAddress(dxgi, "CreateDXGIFactory1");
  340. if (!create_factory) {
  341. flog("failed to get CreateDXGIFactory1 address: %d",
  342. GetLastError());
  343. return false;
  344. }
  345. PFN_D3D11_CREATE_DEVICE create =
  346. (void *)GetProcAddress(d3d11, "D3D11CreateDevice");
  347. if (!create) {
  348. flog("failed to get D3D11CreateDevice address: %d",
  349. GetLastError());
  350. return false;
  351. }
  352. hr = create_factory(&IID_IDXGIFactory1, &factory);
  353. if (FAILED(hr)) {
  354. flog_hr("failed to create factory", hr);
  355. return false;
  356. }
  357. hr = IDXGIFactory1_EnumAdapters1(factory, 0, &adapter);
  358. IDXGIFactory1_Release(factory);
  359. if (FAILED(hr)) {
  360. flog_hr("failed to create adapter", hr);
  361. return false;
  362. }
  363. static const D3D_FEATURE_LEVEL feature_levels[] = {
  364. D3D_FEATURE_LEVEL_11_0,
  365. D3D_FEATURE_LEVEL_10_1,
  366. D3D_FEATURE_LEVEL_10_0,
  367. D3D_FEATURE_LEVEL_9_3,
  368. };
  369. hr = create((IDXGIAdapter *)adapter, D3D_DRIVER_TYPE_UNKNOWN, NULL, 0,
  370. feature_levels,
  371. sizeof(feature_levels) / sizeof(D3D_FEATURE_LEVEL),
  372. D3D11_SDK_VERSION, &data->d3d11_device, &level_used,
  373. &data->d3d11_context);
  374. IDXGIAdapter1_Release(adapter);
  375. if (FAILED(hr)) {
  376. flog_hr("failed to create device", hr);
  377. return false;
  378. }
  379. return true;
  380. }
  381. static inline bool vk_shtex_init_d3d11_tex(struct vk_data *data,
  382. struct vk_swap_data *swap)
  383. {
  384. IDXGIResource *dxgi_res;
  385. HRESULT hr;
  386. const UINT width = swap->image_extent.width;
  387. const UINT height = swap->image_extent.height;
  388. flog("OBS requesting %s texture format. capture dimensions: %ux%u",
  389. vk_format_to_str(swap->format), width, height);
  390. const DXGI_FORMAT format = vk_format_to_dxgi(swap->format);
  391. if (format == DXGI_FORMAT_UNKNOWN) {
  392. flog("cannot convert to DXGI format");
  393. return false;
  394. }
  395. D3D11_TEXTURE2D_DESC desc = {0};
  396. desc.Width = width;
  397. desc.Height = height;
  398. desc.MipLevels = 1;
  399. desc.ArraySize = 1;
  400. desc.Format = format;
  401. desc.SampleDesc.Count = 1;
  402. desc.SampleDesc.Quality = 0;
  403. desc.Usage = D3D11_USAGE_DEFAULT;
  404. desc.MiscFlags = D3D11_RESOURCE_MISC_SHARED;
  405. desc.BindFlags = D3D11_BIND_RENDER_TARGET | D3D11_BIND_SHADER_RESOURCE;
  406. hr = ID3D11Device_CreateTexture2D(data->d3d11_device, &desc, NULL,
  407. &swap->d3d11_tex);
  408. if (FAILED(hr)) {
  409. flog_hr("failed to create texture", hr);
  410. return false;
  411. }
  412. hr = ID3D11Texture2D_QueryInterface(swap->d3d11_tex, &IID_IDXGIResource,
  413. &dxgi_res);
  414. if (FAILED(hr)) {
  415. flog_hr("failed to get IDXGIResource", hr);
  416. return false;
  417. }
  418. hr = IDXGIResource_GetSharedHandle(dxgi_res, &swap->handle);
  419. IDXGIResource_Release(dxgi_res);
  420. if (FAILED(hr)) {
  421. flog_hr("failed to get shared handle", hr);
  422. return false;
  423. }
  424. return true;
  425. }
  426. static inline bool vk_shtex_init_vulkan_tex(struct vk_data *data,
  427. struct vk_swap_data *swap)
  428. {
  429. struct vk_device_funcs *funcs = &data->funcs;
  430. VkExternalMemoryFeatureFlags f =
  431. data->external_mem_props.externalMemoryFeatures;
  432. /* -------------------------------------------------------- */
  433. /* create texture */
  434. VkExternalMemoryImageCreateInfo emici;
  435. emici.sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO;
  436. emici.pNext = NULL;
  437. emici.handleTypes =
  438. VK_EXTERNAL_MEMORY_HANDLE_TYPE_D3D11_TEXTURE_KMT_BIT;
  439. VkImageCreateInfo ici;
  440. ici.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO;
  441. ici.pNext = &emici;
  442. ici.flags = 0;
  443. ici.imageType = VK_IMAGE_TYPE_2D;
  444. ici.format = swap->format;
  445. ici.extent.width = swap->image_extent.width;
  446. ici.extent.height = swap->image_extent.height;
  447. ici.extent.depth = 1;
  448. ici.mipLevels = 1;
  449. ici.arrayLayers = 1;
  450. ici.samples = VK_SAMPLE_COUNT_1_BIT;
  451. ici.tiling = VK_IMAGE_TILING_OPTIMAL;
  452. ici.usage = VK_IMAGE_USAGE_TRANSFER_DST_BIT |
  453. VK_IMAGE_USAGE_SAMPLED_BIT;
  454. ici.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
  455. ici.queueFamilyIndexCount = 0;
  456. ici.pQueueFamilyIndices = 0;
  457. ici.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
  458. VkResult res;
  459. res = funcs->CreateImage(data->device, &ici, data->ac,
  460. &swap->export_image);
  461. if (VK_SUCCESS != res) {
  462. flog("failed to CreateImage: %s", result_to_str(res));
  463. swap->export_image = VK_NULL_HANDLE;
  464. return false;
  465. }
  466. swap->layout_initialized = false;
  467. /* -------------------------------------------------------- */
  468. /* get image memory requirements */
  469. VkMemoryRequirements mr;
  470. bool use_gimr2 = f & VK_EXTERNAL_MEMORY_FEATURE_DEDICATED_ONLY_BIT;
  471. if (use_gimr2) {
  472. VkMemoryDedicatedRequirements mdr = {0};
  473. mdr.sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS;
  474. mdr.pNext = NULL;
  475. VkMemoryRequirements2 mr2 = {0};
  476. mr2.sType = VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2;
  477. mr2.pNext = &mdr;
  478. VkImageMemoryRequirementsInfo2 imri2 = {0};
  479. imri2.sType =
  480. VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2;
  481. imri2.pNext = NULL;
  482. imri2.image = swap->export_image;
  483. funcs->GetImageMemoryRequirements2(data->device, &imri2, &mr2);
  484. mr = mr2.memoryRequirements;
  485. } else {
  486. funcs->GetImageMemoryRequirements(data->device,
  487. swap->export_image, &mr);
  488. }
  489. /* -------------------------------------------------------- */
  490. /* get memory type index */
  491. struct vk_inst_funcs *ifuncs = get_inst_funcs(data->phy_device);
  492. VkPhysicalDeviceMemoryProperties pdmp;
  493. ifuncs->GetPhysicalDeviceMemoryProperties(data->phy_device, &pdmp);
  494. uint32_t mem_type_idx = 0;
  495. for (; mem_type_idx < pdmp.memoryTypeCount; mem_type_idx++) {
  496. if ((mr.memoryTypeBits & (1 << mem_type_idx)) &&
  497. (pdmp.memoryTypes[mem_type_idx].propertyFlags &
  498. VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) ==
  499. VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) {
  500. break;
  501. }
  502. }
  503. if (mem_type_idx == pdmp.memoryTypeCount) {
  504. flog("failed to get memory type index");
  505. funcs->DestroyImage(data->device, swap->export_image, data->ac);
  506. swap->export_image = VK_NULL_HANDLE;
  507. return false;
  508. }
  509. /* -------------------------------------------------------- */
  510. /* allocate memory */
  511. VkImportMemoryWin32HandleInfoKHR imw32hi;
  512. imw32hi.sType = VK_STRUCTURE_TYPE_IMPORT_MEMORY_WIN32_HANDLE_INFO_KHR;
  513. imw32hi.pNext = NULL;
  514. imw32hi.name = NULL;
  515. imw32hi.handleType =
  516. VK_EXTERNAL_MEMORY_HANDLE_TYPE_D3D11_TEXTURE_KMT_BIT;
  517. imw32hi.handle = swap->handle;
  518. VkMemoryAllocateInfo mai;
  519. mai.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
  520. mai.pNext = &imw32hi;
  521. mai.allocationSize = mr.size;
  522. mai.memoryTypeIndex = mem_type_idx;
  523. VkMemoryDedicatedAllocateInfo mdai;
  524. mdai.sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO;
  525. mdai.pNext = NULL;
  526. mdai.buffer = VK_NULL_HANDLE;
  527. if (data->external_mem_props.externalMemoryFeatures &
  528. VK_EXTERNAL_MEMORY_FEATURE_DEDICATED_ONLY_BIT) {
  529. mdai.image = swap->export_image;
  530. imw32hi.pNext = &mdai;
  531. }
  532. res = funcs->AllocateMemory(data->device, &mai, NULL,
  533. &swap->export_mem);
  534. if (VK_SUCCESS != res) {
  535. flog("failed to AllocateMemory: %s", result_to_str(res));
  536. funcs->DestroyImage(data->device, swap->export_image, data->ac);
  537. swap->export_image = VK_NULL_HANDLE;
  538. return false;
  539. }
  540. /* -------------------------------------------------------- */
  541. /* bind image memory */
  542. bool use_bi2 = f & VK_EXTERNAL_MEMORY_FEATURE_DEDICATED_ONLY_BIT;
  543. if (use_bi2) {
  544. VkBindImageMemoryInfo bimi = {0};
  545. bimi.sType = VK_STRUCTURE_TYPE_BIND_IMAGE_MEMORY_INFO;
  546. bimi.image = swap->export_image;
  547. bimi.memory = swap->export_mem;
  548. bimi.memoryOffset = 0;
  549. res = funcs->BindImageMemory2(data->device, 1, &bimi);
  550. } else {
  551. res = funcs->BindImageMemory(data->device, swap->export_image,
  552. swap->export_mem, 0);
  553. }
  554. if (VK_SUCCESS != res) {
  555. flog("%s failed: %s",
  556. use_bi2 ? "BindImageMemory2" : "BindImageMemory",
  557. result_to_str(res));
  558. funcs->DestroyImage(data->device, swap->export_image, data->ac);
  559. swap->export_image = VK_NULL_HANDLE;
  560. return false;
  561. }
  562. return true;
  563. }
  564. static bool vk_shtex_init(struct vk_data *data, HWND window,
  565. struct vk_swap_data *swap)
  566. {
  567. if (!vk_shtex_init_d3d11(data)) {
  568. return false;
  569. }
  570. if (!vk_shtex_init_d3d11_tex(data, swap)) {
  571. return false;
  572. }
  573. if (!vk_shtex_init_vulkan_tex(data, swap)) {
  574. return false;
  575. }
  576. data->cur_swap = swap;
  577. swap->captured = capture_init_shtex(
  578. &swap->shtex_info, window, swap->image_extent.width,
  579. swap->image_extent.height, swap->image_extent.width,
  580. swap->image_extent.height, (uint32_t)swap->format, false,
  581. (uintptr_t)swap->handle);
  582. if (swap->captured) {
  583. if (global_hook_info->force_shmem) {
  584. flog("shared memory capture currently "
  585. "unsupported; ignoring");
  586. }
  587. hlog("vulkan shared texture capture successful");
  588. return true;
  589. }
  590. return false;
  591. }
  592. static void vk_shtex_create_family_objects(struct vk_data *data,
  593. uint32_t fam_idx,
  594. uint32_t image_count)
  595. {
  596. struct vk_family_data *family_data = &data->families[fam_idx];
  597. for (uint32_t image_index = 0; image_index < image_count;
  598. image_index++) {
  599. struct vk_frame_data *frame_data =
  600. &family_data->frames[image_index];
  601. VkCommandPoolCreateInfo cpci;
  602. cpci.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO;
  603. cpci.pNext = NULL;
  604. cpci.flags = 0;
  605. cpci.queueFamilyIndex = fam_idx;
  606. VkResult res = data->funcs.CreateCommandPool(
  607. data->device, &cpci, data->ac, &frame_data->cmd_pool);
  608. debug_res("CreateCommandPool", res);
  609. VkCommandBufferAllocateInfo cbai;
  610. cbai.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO;
  611. cbai.pNext = NULL;
  612. cbai.commandPool = frame_data->cmd_pool;
  613. cbai.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY;
  614. cbai.commandBufferCount = 1;
  615. res = data->funcs.AllocateCommandBuffers(
  616. data->device, &cbai, &frame_data->cmd_buffer);
  617. debug_res("AllocateCommandBuffers", res);
  618. *(void **)frame_data->cmd_buffer = *(void **)(data->device);
  619. VkFenceCreateInfo fci = {0};
  620. fci.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
  621. fci.pNext = NULL;
  622. fci.flags = 0;
  623. res = data->funcs.CreateFence(data->device, &fci, data->ac,
  624. &frame_data->fence);
  625. debug_res("CreateFence", res);
  626. }
  627. family_data->image_count = image_count;
  628. }
  629. static void vk_shtex_destroy_fence(struct vk_data *data, bool *cmd_buffer_busy,
  630. VkFence *fence)
  631. {
  632. VkDevice device = data->device;
  633. if (*cmd_buffer_busy) {
  634. data->funcs.WaitForFences(device, 1, fence, VK_TRUE, ~0ull);
  635. *cmd_buffer_busy = false;
  636. }
  637. data->funcs.DestroyFence(device, *fence, data->ac);
  638. *fence = VK_NULL_HANDLE;
  639. }
  640. static void vk_shtex_destroy_family_objects(struct vk_data *data,
  641. struct vk_family_data *family_data)
  642. {
  643. for (uint32_t image_idx = 0; image_idx < family_data->image_count;
  644. image_idx++) {
  645. struct vk_frame_data *frame_data =
  646. &family_data->frames[image_idx];
  647. bool *cmd_buffer_busy = &frame_data->cmd_buffer_busy;
  648. VkFence *fence = &frame_data->fence;
  649. vk_shtex_destroy_fence(data, cmd_buffer_busy, fence);
  650. data->funcs.DestroyCommandPool(data->device,
  651. frame_data->cmd_pool, data->ac);
  652. frame_data->cmd_pool = VK_NULL_HANDLE;
  653. }
  654. family_data->image_count = 0;
  655. }
  656. static void vk_shtex_capture(struct vk_data *data,
  657. struct vk_device_funcs *funcs,
  658. struct vk_swap_data *swap, uint32_t idx,
  659. VkQueue queue, const VkPresentInfoKHR *info)
  660. {
  661. VkResult res = VK_SUCCESS;
  662. VkCommandBufferBeginInfo begin_info;
  663. begin_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
  664. begin_info.pNext = NULL;
  665. begin_info.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
  666. begin_info.pInheritanceInfo = NULL;
  667. VkImageMemoryBarrier mb[2];
  668. VkImageMemoryBarrier *src_mb = &mb[0];
  669. VkImageMemoryBarrier *dst_mb = &mb[1];
  670. /* ------------------------------------------------------ */
  671. /* do image copy */
  672. const uint32_t image_index = info->pImageIndices[idx];
  673. VkImage cur_backbuffer = swap->swap_images[image_index];
  674. uint32_t fam_idx = 0;
  675. for (uint32_t i = 0; i < data->queue_count; i++) {
  676. if (data->queues[i].queue == queue)
  677. fam_idx = data->queues[i].fam_idx;
  678. }
  679. if (fam_idx >= _countof(data->families))
  680. return;
  681. struct vk_family_data *family_data = &data->families[fam_idx];
  682. const uint32_t image_count = swap->image_count;
  683. if (family_data->image_count < image_count) {
  684. if (family_data->image_count > 0)
  685. vk_shtex_destroy_family_objects(data, family_data);
  686. vk_shtex_create_family_objects(data, fam_idx, image_count);
  687. }
  688. struct vk_frame_data *frame_data = &family_data->frames[image_index];
  689. vk_shtex_clear_fence(data, frame_data);
  690. res = funcs->ResetCommandPool(data->device, frame_data->cmd_pool, 0);
  691. #ifdef MORE_DEBUGGING
  692. debug_res("ResetCommandPool", res);
  693. #endif
  694. const VkCommandBuffer cmd_buffer = frame_data->cmd_buffer;
  695. res = funcs->BeginCommandBuffer(cmd_buffer, &begin_info);
  696. #ifdef MORE_DEBUGGING
  697. debug_res("BeginCommandBuffer", res);
  698. #endif
  699. /* ------------------------------------------------------ */
  700. /* transition shared texture if necessary */
  701. if (!swap->layout_initialized) {
  702. VkImageMemoryBarrier imb;
  703. imb.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER;
  704. imb.pNext = NULL;
  705. imb.srcAccessMask = 0;
  706. imb.dstAccessMask = 0;
  707. imb.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED;
  708. imb.newLayout = VK_IMAGE_LAYOUT_GENERAL;
  709. imb.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
  710. imb.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
  711. imb.image = swap->export_image;
  712. imb.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
  713. imb.subresourceRange.baseMipLevel = 0;
  714. imb.subresourceRange.levelCount = 1;
  715. imb.subresourceRange.baseArrayLayer = 0;
  716. imb.subresourceRange.layerCount = 1;
  717. funcs->CmdPipelineBarrier(cmd_buffer,
  718. VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
  719. VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, 0,
  720. 0, NULL, 0, NULL, 1, &imb);
  721. swap->layout_initialized = true;
  722. }
  723. /* ------------------------------------------------------ */
  724. /* transition cur_backbuffer to transfer source state */
  725. src_mb->sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER;
  726. src_mb->pNext = NULL;
  727. src_mb->srcAccessMask = VK_ACCESS_MEMORY_READ_BIT;
  728. src_mb->dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
  729. src_mb->oldLayout = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR;
  730. src_mb->newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
  731. src_mb->srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
  732. src_mb->dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
  733. src_mb->image = cur_backbuffer;
  734. src_mb->subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
  735. src_mb->subresourceRange.baseMipLevel = 0;
  736. src_mb->subresourceRange.levelCount = 1;
  737. src_mb->subresourceRange.baseArrayLayer = 0;
  738. src_mb->subresourceRange.layerCount = 1;
  739. /* ------------------------------------------------------ */
  740. /* transition exportedTexture to transfer dest state */
  741. dst_mb->sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER;
  742. dst_mb->pNext = NULL;
  743. dst_mb->srcAccessMask = 0;
  744. dst_mb->dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
  745. dst_mb->oldLayout = VK_IMAGE_LAYOUT_GENERAL;
  746. dst_mb->newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
  747. dst_mb->srcQueueFamilyIndex = VK_QUEUE_FAMILY_EXTERNAL;
  748. dst_mb->dstQueueFamilyIndex = fam_idx;
  749. dst_mb->image = swap->export_image;
  750. dst_mb->subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
  751. dst_mb->subresourceRange.baseMipLevel = 0;
  752. dst_mb->subresourceRange.levelCount = 1;
  753. dst_mb->subresourceRange.baseArrayLayer = 0;
  754. dst_mb->subresourceRange.layerCount = 1;
  755. funcs->CmdPipelineBarrier(cmd_buffer,
  756. VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
  757. VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, NULL, 0,
  758. NULL, 2, mb);
  759. /* ------------------------------------------------------ */
  760. /* copy cur_backbuffer's content to our interop image */
  761. VkImageCopy cpy;
  762. cpy.srcSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
  763. cpy.srcSubresource.mipLevel = 0;
  764. cpy.srcSubresource.baseArrayLayer = 0;
  765. cpy.srcSubresource.layerCount = 1;
  766. cpy.srcOffset.x = 0;
  767. cpy.srcOffset.y = 0;
  768. cpy.srcOffset.z = 0;
  769. cpy.dstSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
  770. cpy.dstSubresource.mipLevel = 0;
  771. cpy.dstSubresource.baseArrayLayer = 0;
  772. cpy.dstSubresource.layerCount = 1;
  773. cpy.dstOffset.x = 0;
  774. cpy.dstOffset.y = 0;
  775. cpy.dstOffset.z = 0;
  776. cpy.extent.width = swap->image_extent.width;
  777. cpy.extent.height = swap->image_extent.height;
  778. cpy.extent.depth = 1;
  779. funcs->CmdCopyImage(cmd_buffer, cur_backbuffer,
  780. VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
  781. swap->export_image,
  782. VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &cpy);
  783. /* ------------------------------------------------------ */
  784. /* Restore the swap chain image layout to what it was
  785. * before. This may not be strictly needed, but it is
  786. * generally good to restore things to their original
  787. * state. */
  788. src_mb->srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
  789. src_mb->dstAccessMask = VK_ACCESS_MEMORY_READ_BIT;
  790. src_mb->oldLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
  791. src_mb->newLayout = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR;
  792. dst_mb->srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
  793. dst_mb->dstAccessMask = 0;
  794. dst_mb->oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
  795. dst_mb->newLayout = VK_IMAGE_LAYOUT_GENERAL;
  796. dst_mb->srcQueueFamilyIndex = fam_idx;
  797. dst_mb->dstQueueFamilyIndex = VK_QUEUE_FAMILY_EXTERNAL;
  798. funcs->CmdPipelineBarrier(cmd_buffer, VK_PIPELINE_STAGE_TRANSFER_BIT,
  799. VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT |
  800. VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
  801. 0, 0, NULL, 0, NULL, 2, mb);
  802. funcs->EndCommandBuffer(cmd_buffer);
  803. /* ------------------------------------------------------ */
  804. VkSubmitInfo submit_info;
  805. submit_info.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO;
  806. submit_info.pNext = NULL;
  807. submit_info.waitSemaphoreCount = 0;
  808. submit_info.pWaitSemaphores = NULL;
  809. submit_info.pWaitDstStageMask = NULL;
  810. submit_info.commandBufferCount = 1;
  811. submit_info.pCommandBuffers = &cmd_buffer;
  812. submit_info.signalSemaphoreCount = 0;
  813. submit_info.pSignalSemaphores = NULL;
  814. const VkFence fence = frame_data->fence;
  815. res = funcs->QueueSubmit(queue, 1, &submit_info, fence);
  816. #ifdef MORE_DEBUGGING
  817. debug_res("QueueSubmit", res);
  818. #endif
  819. if (res == VK_SUCCESS)
  820. frame_data->cmd_buffer_busy = true;
  821. }
  822. static inline bool valid_rect(struct vk_swap_data *swap)
  823. {
  824. return !!swap->image_extent.width && !!swap->image_extent.height;
  825. }
  826. static void vk_capture(struct vk_data *data, VkQueue queue,
  827. const VkPresentInfoKHR *info)
  828. {
  829. struct vk_swap_data *swap = NULL;
  830. HWND window = NULL;
  831. uint32_t idx = 0;
  832. #ifdef MORE_DEBUGGING
  833. debug("QueuePresentKHR called on "
  834. "devicekey %p, swapchain count %d",
  835. &data->funcs, info->swapchainCount);
  836. #endif
  837. /* use first swap chain associated with a window */
  838. for (; idx < info->swapchainCount; idx++) {
  839. struct vk_swap_data *cur_swap =
  840. get_swap_data(data, info->pSwapchains[idx]);
  841. window = cur_swap->hwnd;
  842. if (!!window) {
  843. swap = cur_swap;
  844. break;
  845. }
  846. }
  847. if (!window) {
  848. return;
  849. }
  850. if (capture_should_stop()) {
  851. vk_shtex_free(data);
  852. }
  853. if (capture_should_init()) {
  854. if (valid_rect(swap) && !vk_shtex_init(data, window, swap)) {
  855. vk_shtex_free(data);
  856. data->valid = false;
  857. flog("vk_shtex_init failed");
  858. }
  859. }
  860. if (capture_ready()) {
  861. if (swap != data->cur_swap) {
  862. vk_shtex_free(data);
  863. return;
  864. }
  865. vk_shtex_capture(data, &data->funcs, swap, idx, queue, info);
  866. }
  867. }
  868. static VkResult VKAPI OBS_QueuePresentKHR(VkQueue queue,
  869. const VkPresentInfoKHR *info)
  870. {
  871. struct vk_data *data = get_device_data(queue);
  872. struct vk_device_funcs *funcs = &data->funcs;
  873. if (data->valid) {
  874. vk_capture(data, queue, info);
  875. }
  876. return funcs->QueuePresentKHR(queue, info);
  877. }
  878. /* ======================================================================== */
  879. /* setup hooks */
  880. static inline bool is_inst_link_info(VkLayerInstanceCreateInfo *lici)
  881. {
  882. return lici->sType == VK_STRUCTURE_TYPE_LOADER_INSTANCE_CREATE_INFO &&
  883. lici->function == VK_LAYER_LINK_INFO;
  884. }
  885. static VkResult VKAPI OBS_CreateInstance(const VkInstanceCreateInfo *cinfo,
  886. const VkAllocationCallbacks *ac,
  887. VkInstance *p_inst)
  888. {
  889. VkInstanceCreateInfo info = *cinfo;
  890. bool funcs_not_found = false;
  891. /* -------------------------------------------------------- */
  892. /* step through chain until we get to the link info */
  893. VkLayerInstanceCreateInfo *lici = (void *)info.pNext;
  894. while (lici && !is_inst_link_info(lici)) {
  895. lici = (VkLayerInstanceCreateInfo *)lici->pNext;
  896. }
  897. if (lici == NULL) {
  898. return VK_ERROR_INITIALIZATION_FAILED;
  899. }
  900. PFN_vkGetInstanceProcAddr gpa =
  901. lici->u.pLayerInfo->pfnNextGetInstanceProcAddr;
  902. /* -------------------------------------------------------- */
  903. /* move chain on for next layer */
  904. lici->u.pLayerInfo = lici->u.pLayerInfo->pNext;
  905. /* -------------------------------------------------------- */
  906. /* (HACK) Set api version to 1.1 if set to 1.0 */
  907. /* We do this to get our extensions working properly */
  908. VkApplicationInfo ai;
  909. if (info.pApplicationInfo) {
  910. ai = *info.pApplicationInfo;
  911. if (ai.apiVersion < VK_API_VERSION_1_1)
  912. ai.apiVersion = VK_API_VERSION_1_1;
  913. } else {
  914. ai.sType = VK_STRUCTURE_TYPE_APPLICATION_INFO;
  915. ai.pNext = NULL;
  916. ai.pApplicationName = NULL;
  917. ai.applicationVersion = 0;
  918. ai.pEngineName = NULL;
  919. ai.engineVersion = 0;
  920. ai.apiVersion = VK_API_VERSION_1_1;
  921. }
  922. info.pApplicationInfo = &ai;
  923. /* -------------------------------------------------------- */
  924. /* create instance */
  925. PFN_vkCreateInstance create = (void *)gpa(NULL, "vkCreateInstance");
  926. VkResult res = create(&info, ac, p_inst);
  927. VkInstance inst = *p_inst;
  928. /* -------------------------------------------------------- */
  929. /* fetch the functions we need */
  930. struct vk_inst_data *data = get_inst_data(inst);
  931. struct vk_inst_funcs *funcs = &data->funcs;
  932. #define GETADDR(x) \
  933. do { \
  934. funcs->x = (void *)gpa(inst, "vk" #x); \
  935. if (!funcs->x) { \
  936. flog("could not get instance " \
  937. "address for %s", \
  938. #x); \
  939. funcs_not_found = true; \
  940. } \
  941. } while (false)
  942. GETADDR(GetInstanceProcAddr);
  943. GETADDR(DestroyInstance);
  944. GETADDR(CreateWin32SurfaceKHR);
  945. GETADDR(DestroySurfaceKHR);
  946. GETADDR(GetPhysicalDeviceMemoryProperties);
  947. GETADDR(GetPhysicalDeviceImageFormatProperties2);
  948. GETADDR(EnumerateDeviceExtensionProperties);
  949. #undef GETADDR
  950. data->valid = !funcs_not_found;
  951. return res;
  952. }
  953. static VkResult VKAPI OBS_DestroyInstance(VkInstance instance,
  954. const VkAllocationCallbacks *ac)
  955. {
  956. struct vk_inst_funcs *funcs = get_inst_funcs(instance);
  957. funcs->DestroyInstance(instance, ac);
  958. remove_instance(instance);
  959. return VK_SUCCESS;
  960. }
  961. static bool
  962. vk_shared_tex_supported(struct vk_inst_funcs *funcs,
  963. VkPhysicalDevice phy_device, VkFormat format,
  964. VkImageUsageFlags usage,
  965. VkExternalMemoryProperties *external_mem_props)
  966. {
  967. VkPhysicalDeviceImageFormatInfo2 info;
  968. VkPhysicalDeviceExternalImageFormatInfo external_info;
  969. external_info.sType =
  970. VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_IMAGE_FORMAT_INFO;
  971. external_info.pNext = NULL;
  972. external_info.handleType =
  973. VK_EXTERNAL_MEMORY_HANDLE_TYPE_D3D11_TEXTURE_KMT_BIT;
  974. info.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2;
  975. info.pNext = &external_info;
  976. info.format = format;
  977. info.type = VK_IMAGE_TYPE_2D;
  978. info.tiling = VK_IMAGE_TILING_OPTIMAL;
  979. info.flags = 0;
  980. info.usage = usage;
  981. VkExternalImageFormatProperties external_props = {0};
  982. external_props.sType =
  983. VK_STRUCTURE_TYPE_EXTERNAL_IMAGE_FORMAT_PROPERTIES;
  984. external_props.pNext = NULL;
  985. VkImageFormatProperties2 props = {0};
  986. props.sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_PROPERTIES_2;
  987. props.pNext = &external_props;
  988. VkResult result = funcs->GetPhysicalDeviceImageFormatProperties2(
  989. phy_device, &info, &props);
  990. *external_mem_props = external_props.externalMemoryProperties;
  991. const VkExternalMemoryFeatureFlags features =
  992. external_mem_props->externalMemoryFeatures;
  993. return ((VK_SUCCESS == result) &&
  994. (features & VK_EXTERNAL_MEMORY_FEATURE_IMPORTABLE_BIT));
  995. }
  996. static inline bool is_device_link_info(VkLayerDeviceCreateInfo *lici)
  997. {
  998. return lici->sType == VK_STRUCTURE_TYPE_LOADER_DEVICE_CREATE_INFO &&
  999. lici->function == VK_LAYER_LINK_INFO;
  1000. }
  1001. static VkResult VKAPI OBS_CreateDevice(VkPhysicalDevice phy_device,
  1002. const VkDeviceCreateInfo *info,
  1003. const VkAllocationCallbacks *ac,
  1004. VkDevice *p_device)
  1005. {
  1006. struct vk_inst_data *idata = get_inst_data(phy_device);
  1007. struct vk_inst_funcs *ifuncs = &idata->funcs;
  1008. struct vk_data *data = NULL;
  1009. VkResult ret = VK_ERROR_INITIALIZATION_FAILED;
  1010. VkLayerDeviceCreateInfo *ldci = (void *)info->pNext;
  1011. /* -------------------------------------------------------- */
  1012. /* step through chain until we get to the link info */
  1013. while (ldci && !is_device_link_info(ldci)) {
  1014. ldci = (VkLayerDeviceCreateInfo *)ldci->pNext;
  1015. }
  1016. if (!ldci) {
  1017. goto fail;
  1018. }
  1019. PFN_vkGetInstanceProcAddr gipa;
  1020. PFN_vkGetDeviceProcAddr gdpa;
  1021. gipa = ldci->u.pLayerInfo->pfnNextGetInstanceProcAddr;
  1022. gdpa = ldci->u.pLayerInfo->pfnNextGetDeviceProcAddr;
  1023. /* -------------------------------------------------------- */
  1024. /* move chain on for next layer */
  1025. ldci->u.pLayerInfo = ldci->u.pLayerInfo->pNext;
  1026. /* -------------------------------------------------------- */
  1027. /* create device and initialize hook data */
  1028. PFN_vkCreateDevice createFunc =
  1029. (PFN_vkCreateDevice)gipa(VK_NULL_HANDLE, "vkCreateDevice");
  1030. ret = createFunc(phy_device, info, ac, p_device);
  1031. if (ret != VK_SUCCESS) {
  1032. goto fail;
  1033. }
  1034. VkDevice device = *p_device;
  1035. data = get_device_data(*p_device);
  1036. struct vk_device_funcs *dfuncs = &data->funcs;
  1037. data->valid = false; /* set true below if it doesn't go to fail */
  1038. data->phy_device = phy_device;
  1039. data->device = device;
  1040. /* -------------------------------------------------------- */
  1041. /* fetch the functions we need */
  1042. bool funcs_not_found = false;
  1043. #define GETADDR(x) \
  1044. do { \
  1045. dfuncs->x = (void *)gdpa(device, "vk" #x); \
  1046. if (!dfuncs->x) { \
  1047. flog("could not get device " \
  1048. "address for %s", \
  1049. #x); \
  1050. funcs_not_found = true; \
  1051. } \
  1052. } while (false)
  1053. #define GETADDR_OPTIONAL(x) \
  1054. do { \
  1055. dfuncs->x = (void *)gdpa(device, "vk" #x); \
  1056. } while (false)
  1057. GETADDR(GetDeviceProcAddr);
  1058. GETADDR(DestroyDevice);
  1059. GETADDR(CreateSwapchainKHR);
  1060. GETADDR(DestroySwapchainKHR);
  1061. GETADDR(QueuePresentKHR);
  1062. GETADDR(AllocateMemory);
  1063. GETADDR(FreeMemory);
  1064. GETADDR(BindImageMemory);
  1065. GETADDR(BindImageMemory2);
  1066. GETADDR(GetSwapchainImagesKHR);
  1067. GETADDR(CreateImage);
  1068. GETADDR(DestroyImage);
  1069. GETADDR(GetImageMemoryRequirements);
  1070. GETADDR(GetImageMemoryRequirements2);
  1071. GETADDR(ResetCommandPool);
  1072. GETADDR(BeginCommandBuffer);
  1073. GETADDR(EndCommandBuffer);
  1074. GETADDR(CmdCopyImage);
  1075. GETADDR(CmdPipelineBarrier);
  1076. GETADDR(GetDeviceQueue);
  1077. GETADDR(QueueSubmit);
  1078. GETADDR(CreateCommandPool);
  1079. GETADDR(DestroyCommandPool);
  1080. GETADDR(AllocateCommandBuffers);
  1081. GETADDR(CreateFence);
  1082. GETADDR(DestroyFence);
  1083. GETADDR(WaitForFences);
  1084. GETADDR(ResetFences);
  1085. #undef GETADDR_OPTIONAL
  1086. #undef GETADDR
  1087. if (funcs_not_found) {
  1088. goto fail;
  1089. }
  1090. if (!idata->valid) {
  1091. flog("instance not valid");
  1092. goto fail;
  1093. }
  1094. const char *required_device_extensions[] = {
  1095. VK_KHR_EXTERNAL_MEMORY_WIN32_EXTENSION_NAME};
  1096. uint32_t device_extension_count = 0;
  1097. ret = ifuncs->EnumerateDeviceExtensionProperties(
  1098. phy_device, NULL, &device_extension_count, NULL);
  1099. if (ret != VK_SUCCESS)
  1100. goto fail;
  1101. VkExtensionProperties *device_extensions = _malloca(
  1102. sizeof(VkExtensionProperties) * device_extension_count);
  1103. ret = ifuncs->EnumerateDeviceExtensionProperties(
  1104. phy_device, NULL, &device_extension_count, device_extensions);
  1105. if (ret != VK_SUCCESS)
  1106. goto fail;
  1107. bool extensions_found = true;
  1108. for (uint32_t i = 0; i < _countof(required_device_extensions); i++) {
  1109. const char *const required_extension =
  1110. required_device_extensions[i];
  1111. bool found = false;
  1112. for (uint32_t j = 0; j < device_extension_count; j++) {
  1113. if (!strcmp(required_extension,
  1114. device_extensions[j].extensionName)) {
  1115. found = true;
  1116. break;
  1117. }
  1118. }
  1119. if (!found) {
  1120. flog("missing device extension: %s",
  1121. required_extension);
  1122. extensions_found = false;
  1123. }
  1124. }
  1125. if (!extensions_found)
  1126. goto fail;
  1127. VkFormat format = VK_FORMAT_R8G8B8A8_UNORM;
  1128. VkImageUsageFlags usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT |
  1129. VK_IMAGE_USAGE_TRANSFER_DST_BIT;
  1130. if (!vk_shared_tex_supported(ifuncs, phy_device, format, usage,
  1131. &data->external_mem_props)) {
  1132. flog("texture sharing is not supported");
  1133. goto fail;
  1134. }
  1135. data->inst_data = idata;
  1136. data->ac = NULL;
  1137. if (ac) {
  1138. data->ac_storage = *ac;
  1139. data->ac = &data->ac_storage;
  1140. }
  1141. data->valid = true;
  1142. fail:
  1143. return ret;
  1144. }
  1145. static void VKAPI OBS_DestroyDevice(VkDevice device,
  1146. const VkAllocationCallbacks *ac)
  1147. {
  1148. struct vk_data *data = get_device_data(device);
  1149. if (!data)
  1150. return;
  1151. if (data->valid) {
  1152. for (uint32_t fam_idx = 0; fam_idx < _countof(data->families);
  1153. fam_idx++) {
  1154. struct vk_family_data *family_data =
  1155. &data->families[fam_idx];
  1156. if (family_data->image_count > 0) {
  1157. vk_shtex_destroy_family_objects(data,
  1158. family_data);
  1159. }
  1160. }
  1161. }
  1162. data->queue_count = 0;
  1163. vk_remove_device(device);
  1164. data->funcs.DestroyDevice(device, ac);
  1165. }
  1166. static VkResult VKAPI
  1167. OBS_CreateSwapchainKHR(VkDevice device, const VkSwapchainCreateInfoKHR *cinfo,
  1168. const VkAllocationCallbacks *ac, VkSwapchainKHR *p_sc)
  1169. {
  1170. struct vk_data *data = get_device_data(device);
  1171. VkSwapchainCreateInfoKHR info = *cinfo;
  1172. if (data->valid)
  1173. info.imageUsage |= VK_IMAGE_USAGE_TRANSFER_SRC_BIT;
  1174. struct vk_device_funcs *funcs = &data->funcs;
  1175. VkResult res = funcs->CreateSwapchainKHR(device, &info, ac, p_sc);
  1176. debug_res("CreateSwapchainKHR", res);
  1177. if ((res != VK_SUCCESS) || !data->valid)
  1178. return res;
  1179. VkSwapchainKHR sc = *p_sc;
  1180. uint32_t count = 0;
  1181. res = funcs->GetSwapchainImagesKHR(data->device, sc, &count, NULL);
  1182. debug_res("GetSwapchainImagesKHR", res);
  1183. struct vk_swap_data *swap = get_new_swap_data(data);
  1184. if (count > 0) {
  1185. if (count > OBJ_MAX)
  1186. count = OBJ_MAX;
  1187. res = funcs->GetSwapchainImagesKHR(data->device, sc, &count,
  1188. swap->swap_images);
  1189. debug_res("GetSwapchainImagesKHR", res);
  1190. }
  1191. swap->sc = sc;
  1192. swap->image_extent = cinfo->imageExtent;
  1193. swap->format = cinfo->imageFormat;
  1194. swap->hwnd = find_surf_hwnd(data->inst_data, cinfo->surface);
  1195. swap->image_count = count;
  1196. swap->d3d11_tex = NULL;
  1197. return VK_SUCCESS;
  1198. }
  1199. static void VKAPI OBS_DestroySwapchainKHR(VkDevice device, VkSwapchainKHR sc,
  1200. const VkAllocationCallbacks *ac)
  1201. {
  1202. struct vk_data *data = get_device_data(device);
  1203. struct vk_device_funcs *funcs = &data->funcs;
  1204. if (data->valid) {
  1205. struct vk_swap_data *swap = get_swap_data(data, sc);
  1206. if (swap) {
  1207. if (data->cur_swap == swap) {
  1208. vk_shtex_free(data);
  1209. }
  1210. swap->sc = VK_NULL_HANDLE;
  1211. swap->hwnd = NULL;
  1212. }
  1213. }
  1214. funcs->DestroySwapchainKHR(device, sc, ac);
  1215. }
  1216. static void VKAPI OBS_GetDeviceQueue(VkDevice device, uint32_t queueFamilyIndex,
  1217. uint32_t queueIndex, VkQueue *pQueue)
  1218. {
  1219. struct vk_data *data = get_device_data(device);
  1220. struct vk_device_funcs *funcs = &data->funcs;
  1221. funcs->GetDeviceQueue(device, queueFamilyIndex, queueIndex, pQueue);
  1222. for (uint32_t i = 0; i < data->queue_count; ++i) {
  1223. if (data->queues[i].queue == *pQueue)
  1224. return;
  1225. }
  1226. if (data->queue_count < _countof(data->queues)) {
  1227. data->queues[data->queue_count].queue = *pQueue;
  1228. data->queues[data->queue_count].fam_idx = queueFamilyIndex;
  1229. ++data->queue_count;
  1230. }
  1231. }
  1232. static VkResult VKAPI OBS_CreateWin32SurfaceKHR(
  1233. VkInstance inst, const VkWin32SurfaceCreateInfoKHR *info,
  1234. const VkAllocationCallbacks *ac, VkSurfaceKHR *surf)
  1235. {
  1236. struct vk_inst_data *data = get_inst_data(inst);
  1237. struct vk_inst_funcs *funcs = &data->funcs;
  1238. VkResult res = funcs->CreateWin32SurfaceKHR(inst, info, ac, surf);
  1239. if (res == VK_SUCCESS)
  1240. insert_surf_data(data, *surf, info->hwnd, ac);
  1241. return res;
  1242. }
  1243. static void VKAPI OBS_DestroySurfaceKHR(VkInstance inst, VkSurfaceKHR surf,
  1244. const VkAllocationCallbacks *ac)
  1245. {
  1246. struct vk_inst_data *data = get_inst_data(inst);
  1247. struct vk_inst_funcs *funcs = &data->funcs;
  1248. erase_surf_data(data, surf, ac);
  1249. funcs->DestroySurfaceKHR(inst, surf, ac);
  1250. }
  1251. #define GETPROCADDR(func) \
  1252. if (!strcmp(name, "vk" #func)) \
  1253. return (VkFunc)&OBS_##func;
  1254. static VkFunc VKAPI OBS_GetDeviceProcAddr(VkDevice dev, const char *name)
  1255. {
  1256. struct vk_data *data = get_device_data(dev);
  1257. struct vk_device_funcs *funcs = &data->funcs;
  1258. debug_procaddr("vkGetDeviceProcAddr(%p, \"%s\")", dev, name);
  1259. GETPROCADDR(GetDeviceProcAddr);
  1260. GETPROCADDR(CreateDevice);
  1261. GETPROCADDR(DestroyDevice);
  1262. GETPROCADDR(CreateSwapchainKHR);
  1263. GETPROCADDR(DestroySwapchainKHR);
  1264. GETPROCADDR(QueuePresentKHR);
  1265. GETPROCADDR(GetDeviceQueue);
  1266. if (funcs->GetDeviceProcAddr == NULL)
  1267. return NULL;
  1268. return funcs->GetDeviceProcAddr(dev, name);
  1269. }
  1270. static VkFunc VKAPI OBS_GetInstanceProcAddr(VkInstance inst, const char *name)
  1271. {
  1272. debug_procaddr("vkGetInstanceProcAddr(%p, \"%s\")", inst, name);
  1273. /* instance chain functions we intercept */
  1274. GETPROCADDR(GetInstanceProcAddr);
  1275. GETPROCADDR(CreateInstance);
  1276. GETPROCADDR(DestroyInstance);
  1277. GETPROCADDR(CreateWin32SurfaceKHR);
  1278. GETPROCADDR(DestroySurfaceKHR);
  1279. /* device chain functions we intercept */
  1280. GETPROCADDR(GetDeviceProcAddr);
  1281. GETPROCADDR(CreateDevice);
  1282. GETPROCADDR(DestroyDevice);
  1283. struct vk_inst_funcs *funcs = get_inst_funcs(inst);
  1284. if (funcs->GetInstanceProcAddr == NULL)
  1285. return NULL;
  1286. return funcs->GetInstanceProcAddr(inst, name);
  1287. }
  1288. #undef GETPROCADDR
  1289. EXPORT VkResult VKAPI OBS_Negotiate(VkNegotiateLayerInterface *nli)
  1290. {
  1291. if (nli->loaderLayerInterfaceVersion >= 2) {
  1292. nli->sType = LAYER_NEGOTIATE_INTERFACE_STRUCT;
  1293. nli->pNext = NULL;
  1294. nli->pfnGetInstanceProcAddr = OBS_GetInstanceProcAddr;
  1295. nli->pfnGetDeviceProcAddr = OBS_GetDeviceProcAddr;
  1296. nli->pfnGetPhysicalDeviceProcAddr = NULL;
  1297. }
  1298. const uint32_t cur_ver = CURRENT_LOADER_LAYER_INTERFACE_VERSION;
  1299. if (nli->loaderLayerInterfaceVersion > cur_ver) {
  1300. nli->loaderLayerInterfaceVersion = cur_ver;
  1301. }
  1302. return VK_SUCCESS;
  1303. }
  1304. bool hook_vulkan(void)
  1305. {
  1306. static bool hooked = false;
  1307. if (!hooked && vulkan_seen) {
  1308. hlog("Hooked Vulkan");
  1309. hooked = true;
  1310. }
  1311. return hooked;
  1312. }