vulkan-capture.c 41 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513
  1. #include <windows.h>
  2. #include "graphics-hook.h"
  3. #define VK_USE_PLATFORM_WIN32_KHR
  4. #include <malloc.h>
  5. #include <vulkan/vulkan.h>
  6. #include <vulkan/vk_layer.h>
  7. #undef VK_LAYER_EXPORT
  8. #if defined(WIN32)
  9. #define VK_LAYER_EXPORT __declspec(dllexport)
  10. #else
  11. #define VK_LAYER_EXPORT
  12. #endif
  13. #include <vulkan/vulkan_win32.h>
  14. #define COBJMACROS
  15. #include <dxgi.h>
  16. #include <d3d11.h>
  17. #include "vulkan-capture.h"
  18. /* ======================================================================== */
  19. /* defs/statics */
  20. /* shorten stuff because dear GOD is vulkan unclean. */
  21. #define VKAPI VKAPI_CALL
  22. #define VkFunc PFN_vkVoidFunction
  23. #define EXPORT VK_LAYER_EXPORT
  24. #define OBJ_MAX 16
  25. /* use the loader's dispatch table pointer as a key for internal data maps */
  26. #define GET_LDT(x) (*(void **)x)
  27. /* clang-format off */
  28. static const GUID dxgi_factory1_guid =
  29. {0x770aae78, 0xf26f, 0x4dba, {0xa8, 0x29, 0x25, 0x3c, 0x83, 0xd1, 0xb3, 0x87}};
  30. static const GUID dxgi_resource_guid =
  31. {0x035f3ab4, 0x482e, 0x4e50, {0xb4, 0x1f, 0x8a, 0x7f, 0x8b, 0xd8, 0x96, 0x0b}};
  32. /* clang-format on */
  33. static bool vulkan_seen = false;
  34. static SRWLOCK mutex = SRWLOCK_INIT; // Faster CRITICAL_SECTION
  35. /* ======================================================================== */
  36. /* hook data */
  37. struct vk_swap_data {
  38. VkSwapchainKHR sc;
  39. VkExtent2D image_extent;
  40. VkFormat format;
  41. HWND hwnd;
  42. VkImage export_image;
  43. bool layout_initialized;
  44. VkDeviceMemory export_mem;
  45. VkImage swap_images[OBJ_MAX];
  46. uint32_t image_count;
  47. HANDLE handle;
  48. struct shtex_data *shtex_info;
  49. ID3D11Texture2D *d3d11_tex;
  50. bool captured;
  51. };
  52. struct vk_queue_data {
  53. VkQueue queue;
  54. uint32_t fam_idx;
  55. };
  56. struct vk_cmd_pool_data {
  57. VkCommandPool cmd_pool;
  58. VkCommandBuffer cmd_buffers[OBJ_MAX];
  59. VkFence fences[OBJ_MAX];
  60. bool cmd_buffer_busy[OBJ_MAX];
  61. uint32_t image_count;
  62. };
  63. struct vk_data {
  64. bool valid;
  65. struct vk_device_funcs funcs;
  66. VkPhysicalDevice phy_device;
  67. VkDevice device;
  68. struct vk_swap_data swaps[OBJ_MAX];
  69. struct vk_swap_data *cur_swap;
  70. uint32_t swap_idx;
  71. struct vk_queue_data queues[OBJ_MAX];
  72. uint32_t queue_count;
  73. struct vk_cmd_pool_data cmd_pools[OBJ_MAX];
  74. VkExternalMemoryProperties external_mem_props;
  75. struct vk_inst_data *inst_data;
  76. ID3D11Device *d3d11_device;
  77. ID3D11DeviceContext *d3d11_context;
  78. };
  79. static struct vk_swap_data *get_swap_data(struct vk_data *data,
  80. VkSwapchainKHR sc)
  81. {
  82. for (int i = 0; i < OBJ_MAX; i++) {
  83. if (data->swaps[i].sc == sc) {
  84. return &data->swaps[i];
  85. }
  86. }
  87. debug("get_swap_data failed, swapchain not found");
  88. return NULL;
  89. }
  90. static struct vk_swap_data *get_new_swap_data(struct vk_data *data)
  91. {
  92. for (int i = 0; i < OBJ_MAX; i++) {
  93. if (data->swaps[i].sc == VK_NULL_HANDLE) {
  94. return &data->swaps[i];
  95. }
  96. }
  97. debug("get_new_swap_data failed, no more free slot");
  98. return NULL;
  99. }
  100. /* ------------------------------------------------------------------------- */
  101. static inline size_t find_obj_idx(void *objs[], void *obj)
  102. {
  103. size_t idx = SIZE_MAX;
  104. AcquireSRWLockExclusive(&mutex);
  105. for (size_t i = 0; i < OBJ_MAX; i++) {
  106. if (objs[i] == obj) {
  107. idx = i;
  108. break;
  109. }
  110. }
  111. ReleaseSRWLockExclusive(&mutex);
  112. return idx;
  113. }
  114. static size_t get_obj_idx(void *objs[], void *obj)
  115. {
  116. size_t idx = SIZE_MAX;
  117. AcquireSRWLockExclusive(&mutex);
  118. for (size_t i = 0; i < OBJ_MAX; i++) {
  119. if (objs[i] == obj) {
  120. idx = i;
  121. break;
  122. }
  123. if (!objs[i] && idx == SIZE_MAX) {
  124. idx = i;
  125. }
  126. }
  127. ReleaseSRWLockExclusive(&mutex);
  128. return idx;
  129. }
  130. /* ------------------------------------------------------------------------- */
  131. static struct vk_data device_data[OBJ_MAX] = {0};
  132. static void *devices[OBJ_MAX] = {0};
  133. static inline struct vk_data *get_device_data(void *dev)
  134. {
  135. size_t idx = get_obj_idx(devices, GET_LDT(dev));
  136. if (idx == SIZE_MAX) {
  137. debug("out of device slots");
  138. return NULL;
  139. }
  140. return &device_data[idx];
  141. }
  142. static void vk_shtex_clear_fence(struct vk_data *data,
  143. struct vk_cmd_pool_data *pool_data,
  144. uint32_t image_idx)
  145. {
  146. VkFence fence = pool_data->fences[image_idx];
  147. if (pool_data->cmd_buffer_busy[image_idx]) {
  148. VkDevice device = data->device;
  149. struct vk_device_funcs *funcs = &data->funcs;
  150. funcs->WaitForFences(device, 1, &fence, VK_TRUE, ~0ull);
  151. funcs->ResetFences(device, 1, &fence);
  152. pool_data->cmd_buffer_busy[image_idx] = false;
  153. }
  154. }
  155. static void vk_shtex_wait_until_pool_idle(struct vk_data *data,
  156. struct vk_cmd_pool_data *pool_data)
  157. {
  158. for (uint32_t image_idx = 0; image_idx < pool_data->image_count;
  159. image_idx++) {
  160. vk_shtex_clear_fence(data, pool_data, image_idx);
  161. }
  162. }
  163. static void vk_shtex_wait_until_idle(struct vk_data *data)
  164. {
  165. for (uint32_t fam_idx = 0; fam_idx < _countof(data->cmd_pools);
  166. fam_idx++) {
  167. struct vk_cmd_pool_data *pool_data = &data->cmd_pools[fam_idx];
  168. if (pool_data->cmd_pool != VK_NULL_HANDLE)
  169. vk_shtex_wait_until_pool_idle(data, pool_data);
  170. }
  171. }
  172. static void vk_shtex_free(struct vk_data *data)
  173. {
  174. capture_free();
  175. vk_shtex_wait_until_idle(data);
  176. for (int swap_idx = 0; swap_idx < OBJ_MAX; swap_idx++) {
  177. struct vk_swap_data *swap = &data->swaps[swap_idx];
  178. if (swap->export_image)
  179. data->funcs.DestroyImage(data->device,
  180. swap->export_image, NULL);
  181. if (swap->export_mem)
  182. data->funcs.FreeMemory(data->device, swap->export_mem,
  183. NULL);
  184. if (swap->d3d11_tex) {
  185. ID3D11Resource_Release(swap->d3d11_tex);
  186. }
  187. swap->handle = INVALID_HANDLE_VALUE;
  188. swap->d3d11_tex = NULL;
  189. swap->export_mem = VK_NULL_HANDLE;
  190. swap->export_image = VK_NULL_HANDLE;
  191. swap->captured = false;
  192. }
  193. if (data->d3d11_context) {
  194. ID3D11DeviceContext_Release(data->d3d11_context);
  195. data->d3d11_context = NULL;
  196. }
  197. if (data->d3d11_device) {
  198. ID3D11Device_Release(data->d3d11_device);
  199. data->d3d11_device = NULL;
  200. }
  201. data->cur_swap = NULL;
  202. hlog("------------------ vulkan capture freed ------------------");
  203. }
  204. static void vk_remove_device(void *dev)
  205. {
  206. size_t idx = find_obj_idx(devices, GET_LDT(dev));
  207. if (idx == SIZE_MAX) {
  208. return;
  209. }
  210. struct vk_data *data = &device_data[idx];
  211. memset(data, 0, sizeof(*data));
  212. AcquireSRWLockExclusive(&mutex);
  213. devices[idx] = NULL;
  214. ReleaseSRWLockExclusive(&mutex);
  215. }
  216. /* ------------------------------------------------------------------------- */
  217. struct vk_surf_data {
  218. VkSurfaceKHR surf;
  219. HWND hwnd;
  220. struct vk_surf_data *next;
  221. };
  222. struct vk_inst_data {
  223. bool valid;
  224. struct vk_inst_funcs funcs;
  225. struct vk_surf_data *surfaces;
  226. };
  227. static void insert_surf_data(struct vk_inst_data *data, VkSurfaceKHR surf,
  228. HWND hwnd)
  229. {
  230. struct vk_surf_data *surf_data = malloc(sizeof(struct vk_surf_data));
  231. if (surf_data) {
  232. surf_data->surf = surf;
  233. surf_data->hwnd = hwnd;
  234. AcquireSRWLockExclusive(&mutex);
  235. struct vk_surf_data *next = data->surfaces;
  236. surf_data->next = next;
  237. data->surfaces = surf_data;
  238. ReleaseSRWLockExclusive(&mutex);
  239. }
  240. }
  241. static HWND find_surf_hwnd(struct vk_inst_data *data, VkSurfaceKHR surf)
  242. {
  243. HWND hwnd = NULL;
  244. AcquireSRWLockExclusive(&mutex);
  245. struct vk_surf_data *surf_data = data->surfaces;
  246. while (surf_data) {
  247. if (surf_data->surf == surf) {
  248. hwnd = surf_data->hwnd;
  249. break;
  250. }
  251. surf_data = surf_data->next;
  252. }
  253. ReleaseSRWLockExclusive(&mutex);
  254. return hwnd;
  255. }
  256. static void erase_surf_data(struct vk_inst_data *data, VkSurfaceKHR surf)
  257. {
  258. AcquireSRWLockExclusive(&mutex);
  259. struct vk_surf_data *current = data->surfaces;
  260. if (current->surf == surf) {
  261. data->surfaces = current->next;
  262. } else {
  263. struct vk_surf_data *previous;
  264. do {
  265. previous = current;
  266. current = current->next;
  267. } while (current && current->surf != surf);
  268. if (current)
  269. previous->next = current->next;
  270. }
  271. ReleaseSRWLockExclusive(&mutex);
  272. free(current);
  273. }
  274. /* ------------------------------------------------------------------------- */
  275. static struct vk_inst_data inst_data[OBJ_MAX] = {0};
  276. static void *instances[OBJ_MAX] = {0};
  277. static struct vk_inst_data *get_inst_data(void *inst)
  278. {
  279. size_t idx = get_obj_idx(instances, GET_LDT(inst));
  280. if (idx == SIZE_MAX) {
  281. debug("out of instance slots");
  282. return NULL;
  283. }
  284. vulkan_seen = true;
  285. return &inst_data[idx];
  286. }
  287. static inline struct vk_inst_funcs *get_inst_funcs(void *inst)
  288. {
  289. struct vk_inst_data *data = get_inst_data(inst);
  290. return &data->funcs;
  291. }
  292. static void remove_instance(void *inst)
  293. {
  294. size_t idx = find_obj_idx(instances, inst);
  295. if (idx == SIZE_MAX) {
  296. return;
  297. }
  298. struct vk_inst_data *data = &inst_data[idx];
  299. memset(data, 0, sizeof(*data));
  300. AcquireSRWLockExclusive(&mutex);
  301. instances[idx] = NULL;
  302. ReleaseSRWLockExclusive(&mutex);
  303. }
  304. /* ======================================================================== */
  305. /* capture */
  306. static inline bool vk_shtex_init_d3d11(struct vk_data *data)
  307. {
  308. D3D_FEATURE_LEVEL level_used;
  309. IDXGIFactory1 *factory;
  310. IDXGIAdapter *adapter;
  311. HRESULT hr;
  312. HMODULE d3d11 = load_system_library("d3d11.dll");
  313. if (!d3d11) {
  314. flog("failed to load d3d11: %d", GetLastError());
  315. return false;
  316. }
  317. HMODULE dxgi = load_system_library("dxgi.dll");
  318. if (!dxgi) {
  319. flog("failed to load dxgi: %d", GetLastError());
  320. return false;
  321. }
  322. HRESULT(WINAPI * create_factory)
  323. (REFIID, void **) = (void *)GetProcAddress(dxgi, "CreateDXGIFactory1");
  324. if (!create_factory) {
  325. flog("failed to get CreateDXGIFactory1 address: %d",
  326. GetLastError());
  327. return false;
  328. }
  329. PFN_D3D11_CREATE_DEVICE create =
  330. (void *)GetProcAddress(d3d11, "D3D11CreateDevice");
  331. if (!create) {
  332. flog("failed to get D3D11CreateDevice address: %d",
  333. GetLastError());
  334. return false;
  335. }
  336. hr = create_factory(&dxgi_factory1_guid, (void **)&factory);
  337. if (FAILED(hr)) {
  338. flog_hr("failed to create factory", hr);
  339. return false;
  340. }
  341. hr = IDXGIFactory1_EnumAdapters1(factory, 0,
  342. (IDXGIAdapter1 **)&adapter);
  343. IDXGIFactory1_Release(factory);
  344. if (FAILED(hr)) {
  345. flog_hr("failed to create adapter", hr);
  346. return false;
  347. }
  348. static const D3D_FEATURE_LEVEL feature_levels[] = {
  349. D3D_FEATURE_LEVEL_11_0,
  350. D3D_FEATURE_LEVEL_10_1,
  351. D3D_FEATURE_LEVEL_10_0,
  352. D3D_FEATURE_LEVEL_9_3,
  353. };
  354. hr = create(adapter, D3D_DRIVER_TYPE_UNKNOWN, NULL, 0, feature_levels,
  355. sizeof(feature_levels) / sizeof(D3D_FEATURE_LEVEL),
  356. D3D11_SDK_VERSION, &data->d3d11_device, &level_used,
  357. &data->d3d11_context);
  358. IDXGIAdapter_Release(adapter);
  359. if (FAILED(hr)) {
  360. flog_hr("failed to create device", hr);
  361. return false;
  362. }
  363. return true;
  364. }
  365. static inline bool vk_shtex_init_d3d11_tex(struct vk_data *data,
  366. struct vk_swap_data *swap)
  367. {
  368. IDXGIResource *dxgi_res;
  369. HRESULT hr;
  370. D3D11_TEXTURE2D_DESC desc = {0};
  371. desc.Width = swap->image_extent.width;
  372. desc.Height = swap->image_extent.height;
  373. desc.MipLevels = 1;
  374. desc.ArraySize = 1;
  375. flog("OBS requesting %s texture format. capture dimensions: %dx%d",
  376. vk_format_to_str(swap->format), (int)desc.Width, (int)desc.Height);
  377. desc.Format = vk_format_to_dxgi(swap->format);
  378. desc.SampleDesc.Count = 1;
  379. desc.SampleDesc.Quality = 0;
  380. desc.Usage = D3D11_USAGE_DEFAULT;
  381. desc.MiscFlags = D3D11_RESOURCE_MISC_SHARED;
  382. desc.BindFlags = D3D11_BIND_RENDER_TARGET | D3D11_BIND_SHADER_RESOURCE;
  383. hr = ID3D11Device_CreateTexture2D(data->d3d11_device, &desc, NULL,
  384. &swap->d3d11_tex);
  385. if (FAILED(hr)) {
  386. flog_hr("failed to create texture", hr);
  387. return false;
  388. }
  389. hr = ID3D11Device_QueryInterface(swap->d3d11_tex, &dxgi_resource_guid,
  390. (void **)&dxgi_res);
  391. if (FAILED(hr)) {
  392. flog_hr("failed to get IDXGIResource", hr);
  393. return false;
  394. }
  395. hr = IDXGIResource_GetSharedHandle(dxgi_res, &swap->handle);
  396. IDXGIResource_Release(dxgi_res);
  397. if (FAILED(hr)) {
  398. flog_hr("failed to get shared handle", hr);
  399. return false;
  400. }
  401. return true;
  402. }
  403. static inline bool vk_shtex_init_vulkan_tex(struct vk_data *data,
  404. struct vk_swap_data *swap)
  405. {
  406. struct vk_device_funcs *funcs = &data->funcs;
  407. VkExternalMemoryFeatureFlags f =
  408. data->external_mem_props.externalMemoryFeatures;
  409. /* -------------------------------------------------------- */
  410. /* create texture */
  411. VkExternalMemoryImageCreateInfo emici;
  412. emici.sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO;
  413. emici.pNext = NULL;
  414. emici.handleTypes =
  415. VK_EXTERNAL_MEMORY_HANDLE_TYPE_D3D11_TEXTURE_KMT_BIT;
  416. VkImageCreateInfo ici;
  417. ici.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO;
  418. ici.pNext = &emici;
  419. ici.flags = 0;
  420. ici.imageType = VK_IMAGE_TYPE_2D;
  421. ici.format = swap->format;
  422. ici.extent.width = swap->image_extent.width;
  423. ici.extent.height = swap->image_extent.height;
  424. ici.extent.depth = 1;
  425. ici.mipLevels = 1;
  426. ici.arrayLayers = 1;
  427. ici.samples = VK_SAMPLE_COUNT_1_BIT;
  428. ici.tiling = VK_IMAGE_TILING_OPTIMAL;
  429. ici.usage = VK_IMAGE_USAGE_TRANSFER_DST_BIT |
  430. VK_IMAGE_USAGE_SAMPLED_BIT;
  431. ici.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
  432. ici.queueFamilyIndexCount = 0;
  433. ici.pQueueFamilyIndices = 0;
  434. ici.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
  435. VkResult res;
  436. res = funcs->CreateImage(data->device, &ici, NULL, &swap->export_image);
  437. if (VK_SUCCESS != res) {
  438. flog("failed to CreateImage: %s", result_to_str(res));
  439. swap->export_image = VK_NULL_HANDLE;
  440. return false;
  441. }
  442. swap->layout_initialized = false;
  443. /* -------------------------------------------------------- */
  444. /* get image memory requirements */
  445. VkMemoryRequirements mr;
  446. bool use_gimr2 = f & VK_EXTERNAL_MEMORY_FEATURE_DEDICATED_ONLY_BIT;
  447. if (use_gimr2) {
  448. VkMemoryDedicatedRequirements mdr = {0};
  449. mdr.sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS;
  450. mdr.pNext = NULL;
  451. VkMemoryRequirements2 mr2 = {0};
  452. mr2.sType = VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2;
  453. mr2.pNext = &mdr;
  454. VkImageMemoryRequirementsInfo2 imri2 = {0};
  455. imri2.sType =
  456. VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2;
  457. imri2.pNext = NULL;
  458. imri2.image = swap->export_image;
  459. funcs->GetImageMemoryRequirements2(data->device, &imri2, &mr2);
  460. mr = mr2.memoryRequirements;
  461. } else {
  462. funcs->GetImageMemoryRequirements(data->device,
  463. swap->export_image, &mr);
  464. }
  465. /* -------------------------------------------------------- */
  466. /* get memory type index */
  467. struct vk_inst_funcs *ifuncs = get_inst_funcs(data->phy_device);
  468. VkPhysicalDeviceMemoryProperties pdmp;
  469. ifuncs->GetPhysicalDeviceMemoryProperties(data->phy_device, &pdmp);
  470. uint32_t mem_type_idx = 0;
  471. for (; mem_type_idx < pdmp.memoryTypeCount; mem_type_idx++) {
  472. if ((mr.memoryTypeBits & (1 << mem_type_idx)) &&
  473. (pdmp.memoryTypes[mem_type_idx].propertyFlags &
  474. VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) ==
  475. VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) {
  476. break;
  477. }
  478. }
  479. if (mem_type_idx == pdmp.memoryTypeCount) {
  480. flog("failed to get memory type index");
  481. funcs->DestroyImage(data->device, swap->export_image, NULL);
  482. swap->export_image = VK_NULL_HANDLE;
  483. return false;
  484. }
  485. /* -------------------------------------------------------- */
  486. /* allocate memory */
  487. VkImportMemoryWin32HandleInfoKHR imw32hi;
  488. imw32hi.sType = VK_STRUCTURE_TYPE_IMPORT_MEMORY_WIN32_HANDLE_INFO_KHR;
  489. imw32hi.pNext = NULL;
  490. imw32hi.name = NULL;
  491. imw32hi.handleType =
  492. VK_EXTERNAL_MEMORY_HANDLE_TYPE_D3D11_TEXTURE_KMT_BIT;
  493. imw32hi.handle = swap->handle;
  494. VkMemoryAllocateInfo mai;
  495. mai.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
  496. mai.pNext = &imw32hi;
  497. mai.allocationSize = mr.size;
  498. mai.memoryTypeIndex = mem_type_idx;
  499. VkMemoryDedicatedAllocateInfo mdai;
  500. mdai.sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO;
  501. mdai.pNext = NULL;
  502. mdai.buffer = VK_NULL_HANDLE;
  503. if (data->external_mem_props.externalMemoryFeatures &
  504. VK_EXTERNAL_MEMORY_FEATURE_DEDICATED_ONLY_BIT) {
  505. mdai.image = swap->export_image;
  506. imw32hi.pNext = &mdai;
  507. }
  508. res = funcs->AllocateMemory(data->device, &mai, NULL,
  509. &swap->export_mem);
  510. if (VK_SUCCESS != res) {
  511. flog("failed to AllocateMemory: %s", result_to_str(res));
  512. funcs->DestroyImage(data->device, swap->export_image, NULL);
  513. swap->export_image = VK_NULL_HANDLE;
  514. return false;
  515. }
  516. /* -------------------------------------------------------- */
  517. /* bind image memory */
  518. bool use_bi2 = f & VK_EXTERNAL_MEMORY_FEATURE_DEDICATED_ONLY_BIT;
  519. if (use_bi2) {
  520. VkBindImageMemoryInfo bimi = {0};
  521. bimi.sType = VK_STRUCTURE_TYPE_BIND_IMAGE_MEMORY_INFO;
  522. bimi.image = swap->export_image;
  523. bimi.memory = swap->export_mem;
  524. bimi.memoryOffset = 0;
  525. res = funcs->BindImageMemory2(data->device, 1, &bimi);
  526. } else {
  527. res = funcs->BindImageMemory(data->device, swap->export_image,
  528. swap->export_mem, 0);
  529. }
  530. if (VK_SUCCESS != res) {
  531. flog("%s failed: %s",
  532. use_bi2 ? "BindImageMemory2" : "BindImageMemory",
  533. result_to_str(res));
  534. funcs->DestroyImage(data->device, swap->export_image, NULL);
  535. swap->export_image = VK_NULL_HANDLE;
  536. return false;
  537. }
  538. return true;
  539. }
  540. static bool vk_shtex_init(struct vk_data *data, HWND window,
  541. struct vk_swap_data *swap)
  542. {
  543. if (!vk_shtex_init_d3d11(data)) {
  544. return false;
  545. }
  546. if (!vk_shtex_init_d3d11_tex(data, swap)) {
  547. return false;
  548. }
  549. if (!vk_shtex_init_vulkan_tex(data, swap)) {
  550. return false;
  551. }
  552. data->cur_swap = swap;
  553. swap->captured = capture_init_shtex(
  554. &swap->shtex_info, window, swap->image_extent.width,
  555. swap->image_extent.height, swap->image_extent.width,
  556. swap->image_extent.height, (uint32_t)swap->format, false,
  557. (uintptr_t)swap->handle);
  558. if (swap->captured) {
  559. if (global_hook_info->force_shmem) {
  560. flog("shared memory capture currently "
  561. "unsupported; ignoring");
  562. }
  563. hlog("vulkan shared texture capture successful");
  564. return true;
  565. }
  566. return false;
  567. }
  568. static void vk_shtex_create_cmd_pool_objects(struct vk_data *data,
  569. uint32_t fam_idx,
  570. uint32_t image_count)
  571. {
  572. struct vk_cmd_pool_data *pool_data = &data->cmd_pools[fam_idx];
  573. VkCommandPoolCreateInfo cpci;
  574. cpci.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO;
  575. cpci.pNext = NULL;
  576. cpci.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT;
  577. cpci.queueFamilyIndex = fam_idx;
  578. VkResult res = data->funcs.CreateCommandPool(data->device, &cpci, NULL,
  579. &pool_data->cmd_pool);
  580. debug_res("CreateCommandPool", res);
  581. VkCommandBufferAllocateInfo cbai;
  582. cbai.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO;
  583. cbai.pNext = NULL;
  584. cbai.commandPool = pool_data->cmd_pool;
  585. cbai.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY;
  586. cbai.commandBufferCount = image_count;
  587. res = data->funcs.AllocateCommandBuffers(data->device, &cbai,
  588. pool_data->cmd_buffers);
  589. debug_res("AllocateCommandBuffers", res);
  590. for (uint32_t image_index = 0; image_index < image_count;
  591. image_index++) {
  592. /* Dispatch table something or other. Well-designed API. */
  593. VkCommandBuffer cmd_buffer =
  594. pool_data->cmd_buffers[image_index];
  595. *(void **)cmd_buffer = *(void **)(data->device);
  596. VkFence *fence = &pool_data->fences[image_index];
  597. VkFenceCreateInfo fci = {0};
  598. fci.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
  599. fci.pNext = NULL;
  600. fci.flags = 0;
  601. res = data->funcs.CreateFence(data->device, &fci, NULL, fence);
  602. debug_res("CreateFence", res);
  603. }
  604. pool_data->image_count = image_count;
  605. }
  606. static void vk_shtex_destroy_fence(struct vk_data *data, bool *cmd_buffer_busy,
  607. VkFence *fence)
  608. {
  609. VkDevice device = data->device;
  610. if (*cmd_buffer_busy) {
  611. data->funcs.WaitForFences(device, 1, fence, VK_TRUE, ~0ull);
  612. *cmd_buffer_busy = false;
  613. }
  614. data->funcs.DestroyFence(device, *fence, NULL);
  615. *fence = VK_NULL_HANDLE;
  616. }
  617. static void
  618. vk_shtex_destroy_cmd_pool_objects(struct vk_data *data,
  619. struct vk_cmd_pool_data *pool_data)
  620. {
  621. for (uint32_t image_idx = 0; image_idx < pool_data->image_count;
  622. image_idx++) {
  623. bool *cmd_buffer_busy = &pool_data->cmd_buffer_busy[image_idx];
  624. VkFence *fence = &pool_data->fences[image_idx];
  625. vk_shtex_destroy_fence(data, cmd_buffer_busy, fence);
  626. }
  627. data->funcs.DestroyCommandPool(data->device, pool_data->cmd_pool, NULL);
  628. pool_data->cmd_pool = VK_NULL_HANDLE;
  629. pool_data->image_count = 0;
  630. }
  631. static void vk_shtex_capture(struct vk_data *data,
  632. struct vk_device_funcs *funcs,
  633. struct vk_swap_data *swap, uint32_t idx,
  634. VkQueue queue, const VkPresentInfoKHR *info)
  635. {
  636. VkResult res = VK_SUCCESS;
  637. VkCommandBufferBeginInfo begin_info;
  638. begin_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
  639. begin_info.pNext = NULL;
  640. begin_info.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
  641. begin_info.pInheritanceInfo = NULL;
  642. VkImageMemoryBarrier mb[2];
  643. VkImageMemoryBarrier *src_mb = &mb[0];
  644. VkImageMemoryBarrier *dst_mb = &mb[1];
  645. /* ------------------------------------------------------ */
  646. /* do image copy */
  647. const uint32_t image_index = info->pImageIndices[idx];
  648. VkImage cur_backbuffer = swap->swap_images[image_index];
  649. uint32_t fam_idx = 0;
  650. for (uint32_t i = 0; i < data->queue_count; i++) {
  651. if (data->queues[i].queue == queue)
  652. fam_idx = data->queues[i].fam_idx;
  653. }
  654. if (fam_idx >= _countof(data->cmd_pools))
  655. return;
  656. struct vk_cmd_pool_data *pool_data = &data->cmd_pools[fam_idx];
  657. VkCommandPool *pool = &pool_data->cmd_pool;
  658. const uint32_t image_count = swap->image_count;
  659. if (pool_data->image_count < image_count) {
  660. if (*pool != VK_NULL_HANDLE)
  661. vk_shtex_destroy_cmd_pool_objects(data, pool_data);
  662. vk_shtex_create_cmd_pool_objects(data, fam_idx, image_count);
  663. }
  664. vk_shtex_clear_fence(data, pool_data, image_index);
  665. VkCommandBuffer cmd_buffer = pool_data->cmd_buffers[image_index];
  666. res = funcs->BeginCommandBuffer(cmd_buffer, &begin_info);
  667. #ifdef MORE_DEBUGGING
  668. debug_res("BeginCommandBuffer", res);
  669. #endif
  670. /* ------------------------------------------------------ */
  671. /* transition shared texture if necessary */
  672. if (!swap->layout_initialized) {
  673. VkImageMemoryBarrier imb;
  674. imb.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER;
  675. imb.pNext = NULL;
  676. imb.srcAccessMask = 0;
  677. imb.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
  678. imb.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED;
  679. imb.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
  680. imb.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
  681. imb.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
  682. imb.image = swap->export_image;
  683. imb.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
  684. imb.subresourceRange.baseMipLevel = 0;
  685. imb.subresourceRange.levelCount = 1;
  686. imb.subresourceRange.baseArrayLayer = 0;
  687. imb.subresourceRange.layerCount = 1;
  688. funcs->CmdPipelineBarrier(cmd_buffer,
  689. VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
  690. VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0,
  691. NULL, 0, NULL, 1, &imb);
  692. swap->layout_initialized = true;
  693. }
  694. /* ------------------------------------------------------ */
  695. /* transition cur_backbuffer to transfer source state */
  696. src_mb->sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER;
  697. src_mb->pNext = NULL;
  698. src_mb->srcAccessMask = VK_ACCESS_MEMORY_READ_BIT;
  699. src_mb->dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
  700. src_mb->oldLayout = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR;
  701. src_mb->newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
  702. src_mb->srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
  703. src_mb->dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
  704. src_mb->image = cur_backbuffer;
  705. src_mb->subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
  706. src_mb->subresourceRange.baseMipLevel = 0;
  707. src_mb->subresourceRange.levelCount = 1;
  708. src_mb->subresourceRange.baseArrayLayer = 0;
  709. src_mb->subresourceRange.layerCount = 1;
  710. /* ------------------------------------------------------ */
  711. /* transition exportedTexture to transfer dest state */
  712. dst_mb->sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER;
  713. dst_mb->pNext = NULL;
  714. dst_mb->srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
  715. dst_mb->dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
  716. dst_mb->oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
  717. dst_mb->newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
  718. dst_mb->srcQueueFamilyIndex = VK_QUEUE_FAMILY_EXTERNAL;
  719. dst_mb->dstQueueFamilyIndex = fam_idx;
  720. dst_mb->image = swap->export_image;
  721. dst_mb->subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
  722. dst_mb->subresourceRange.baseMipLevel = 0;
  723. dst_mb->subresourceRange.levelCount = 1;
  724. dst_mb->subresourceRange.baseArrayLayer = 0;
  725. dst_mb->subresourceRange.layerCount = 1;
  726. funcs->CmdPipelineBarrier(cmd_buffer,
  727. VK_PIPELINE_STAGE_TRANSFER_BIT |
  728. VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
  729. VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, NULL, 0,
  730. NULL, 2, mb);
  731. /* ------------------------------------------------------ */
  732. /* copy cur_backbuffer's content to our interop image */
  733. VkImageCopy cpy;
  734. cpy.srcSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
  735. cpy.srcSubresource.mipLevel = 0;
  736. cpy.srcSubresource.baseArrayLayer = 0;
  737. cpy.srcSubresource.layerCount = 1;
  738. cpy.srcOffset.x = 0;
  739. cpy.srcOffset.y = 0;
  740. cpy.srcOffset.z = 0;
  741. cpy.dstSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
  742. cpy.dstSubresource.mipLevel = 0;
  743. cpy.dstSubresource.baseArrayLayer = 0;
  744. cpy.dstSubresource.layerCount = 1;
  745. cpy.dstOffset.x = 0;
  746. cpy.dstOffset.y = 0;
  747. cpy.dstOffset.z = 0;
  748. cpy.extent.width = swap->image_extent.width;
  749. cpy.extent.height = swap->image_extent.height;
  750. cpy.extent.depth = 1;
  751. funcs->CmdCopyImage(cmd_buffer, cur_backbuffer,
  752. VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
  753. swap->export_image,
  754. VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &cpy);
  755. /* ------------------------------------------------------ */
  756. /* Restore the swap chain image layout to what it was
  757. * before. This may not be strictly needed, but it is
  758. * generally good to restore things to their original
  759. * state. */
  760. src_mb->srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
  761. src_mb->dstAccessMask = VK_ACCESS_MEMORY_READ_BIT;
  762. src_mb->oldLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
  763. src_mb->newLayout = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR;
  764. dst_mb->srcQueueFamilyIndex = fam_idx;
  765. dst_mb->dstQueueFamilyIndex = VK_QUEUE_FAMILY_EXTERNAL;
  766. funcs->CmdPipelineBarrier(cmd_buffer, VK_PIPELINE_STAGE_TRANSFER_BIT,
  767. VK_PIPELINE_STAGE_TRANSFER_BIT |
  768. VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
  769. 0, 0, NULL, 0, NULL, 2, mb);
  770. funcs->EndCommandBuffer(cmd_buffer);
  771. /* ------------------------------------------------------ */
  772. VkSubmitInfo submit_info;
  773. submit_info.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO;
  774. submit_info.pNext = NULL;
  775. submit_info.waitSemaphoreCount = 0;
  776. submit_info.pWaitSemaphores = NULL;
  777. submit_info.pWaitDstStageMask = NULL;
  778. submit_info.commandBufferCount = 1;
  779. submit_info.pCommandBuffers = &cmd_buffer;
  780. submit_info.signalSemaphoreCount = 0;
  781. submit_info.pSignalSemaphores = NULL;
  782. VkFence fence = pool_data->fences[image_index];
  783. res = funcs->QueueSubmit(queue, 1, &submit_info, fence);
  784. #ifdef MORE_DEBUGGING
  785. debug_res("QueueSubmit", res);
  786. #endif
  787. if (res == VK_SUCCESS)
  788. pool_data->cmd_buffer_busy[image_index] = true;
  789. }
  790. static inline bool valid_rect(struct vk_swap_data *swap)
  791. {
  792. return !!swap->image_extent.width && !!swap->image_extent.height;
  793. }
  794. static void vk_capture(struct vk_data *data, VkQueue queue,
  795. const VkPresentInfoKHR *info)
  796. {
  797. struct vk_swap_data *swap = NULL;
  798. HWND window = NULL;
  799. uint32_t idx = 0;
  800. #ifdef MORE_DEBUGGING
  801. debug("QueuePresentKHR called on "
  802. "devicekey %p, swapchain count %d",
  803. &data->funcs, info->swapchainCount);
  804. #endif
  805. /* use first swap chain associated with a window */
  806. for (; idx < info->swapchainCount; idx++) {
  807. struct vk_swap_data *cur_swap =
  808. get_swap_data(data, info->pSwapchains[idx]);
  809. window = cur_swap->hwnd;
  810. if (!!window) {
  811. swap = cur_swap;
  812. break;
  813. }
  814. }
  815. if (!window) {
  816. return;
  817. }
  818. if (capture_should_stop()) {
  819. vk_shtex_free(data);
  820. }
  821. if (capture_should_init()) {
  822. if (valid_rect(swap) && !vk_shtex_init(data, window, swap)) {
  823. vk_shtex_free(data);
  824. data->valid = false;
  825. flog("vk_shtex_init failed");
  826. }
  827. }
  828. if (capture_ready()) {
  829. if (swap != data->cur_swap) {
  830. vk_shtex_free(data);
  831. return;
  832. }
  833. vk_shtex_capture(data, &data->funcs, swap, idx, queue, info);
  834. }
  835. }
  836. static VkResult VKAPI OBS_QueuePresentKHR(VkQueue queue,
  837. const VkPresentInfoKHR *info)
  838. {
  839. struct vk_data *data = get_device_data(queue);
  840. struct vk_device_funcs *funcs = &data->funcs;
  841. if (data->valid) {
  842. vk_capture(data, queue, info);
  843. }
  844. return funcs->QueuePresentKHR(queue, info);
  845. }
  846. /* ======================================================================== */
  847. /* setup hooks */
  848. static inline bool is_inst_link_info(VkLayerInstanceCreateInfo *lici)
  849. {
  850. return lici->sType == VK_STRUCTURE_TYPE_LOADER_INSTANCE_CREATE_INFO &&
  851. lici->function == VK_LAYER_LINK_INFO;
  852. }
  853. static VkResult VKAPI OBS_CreateInstance(const VkInstanceCreateInfo *cinfo,
  854. const VkAllocationCallbacks *ac,
  855. VkInstance *p_inst)
  856. {
  857. VkInstanceCreateInfo info = *cinfo;
  858. bool funcs_not_found = false;
  859. /* -------------------------------------------------------- */
  860. /* step through chain until we get to the link info */
  861. VkLayerInstanceCreateInfo *lici = (void *)info.pNext;
  862. while (lici && !is_inst_link_info(lici)) {
  863. lici = (VkLayerInstanceCreateInfo *)lici->pNext;
  864. }
  865. if (lici == NULL) {
  866. return VK_ERROR_INITIALIZATION_FAILED;
  867. }
  868. PFN_vkGetInstanceProcAddr gpa =
  869. lici->u.pLayerInfo->pfnNextGetInstanceProcAddr;
  870. /* -------------------------------------------------------- */
  871. /* move chain on for next layer */
  872. lici->u.pLayerInfo = lici->u.pLayerInfo->pNext;
  873. /* -------------------------------------------------------- */
  874. /* (HACK) Set api version to 1.1 if set to 1.0 */
  875. /* We do this to get our extensions working properly */
  876. VkApplicationInfo ai;
  877. if (info.pApplicationInfo) {
  878. ai = *info.pApplicationInfo;
  879. if (ai.apiVersion < VK_API_VERSION_1_1)
  880. ai.apiVersion = VK_API_VERSION_1_1;
  881. } else {
  882. ai.sType = VK_STRUCTURE_TYPE_APPLICATION_INFO;
  883. ai.pNext = NULL;
  884. ai.pApplicationName = NULL;
  885. ai.applicationVersion = 0;
  886. ai.pEngineName = NULL;
  887. ai.engineVersion = 0;
  888. ai.apiVersion = VK_API_VERSION_1_1;
  889. }
  890. info.pApplicationInfo = &ai;
  891. /* -------------------------------------------------------- */
  892. /* create instance */
  893. PFN_vkCreateInstance create = (void *)gpa(NULL, "vkCreateInstance");
  894. VkResult res = create(&info, ac, p_inst);
  895. VkInstance inst = *p_inst;
  896. /* -------------------------------------------------------- */
  897. /* fetch the functions we need */
  898. struct vk_inst_data *data = get_inst_data(inst);
  899. struct vk_inst_funcs *funcs = &data->funcs;
  900. #define GETADDR(x) \
  901. do { \
  902. funcs->x = (void *)gpa(inst, "vk" #x); \
  903. if (!funcs->x) { \
  904. flog("could not get instance " \
  905. "address for %s", \
  906. #x); \
  907. funcs_not_found = true; \
  908. } \
  909. } while (false)
  910. GETADDR(GetInstanceProcAddr);
  911. GETADDR(DestroyInstance);
  912. GETADDR(CreateWin32SurfaceKHR);
  913. GETADDR(DestroySurfaceKHR);
  914. GETADDR(GetPhysicalDeviceMemoryProperties);
  915. GETADDR(GetPhysicalDeviceImageFormatProperties2);
  916. #undef GETADDR
  917. data->valid = !funcs_not_found;
  918. return res;
  919. }
  920. static VkResult VKAPI OBS_DestroyInstance(VkInstance instance,
  921. const VkAllocationCallbacks *ac)
  922. {
  923. struct vk_inst_funcs *funcs = get_inst_funcs(instance);
  924. funcs->DestroyInstance(instance, ac);
  925. remove_instance(instance);
  926. return VK_SUCCESS;
  927. }
  928. static bool
  929. vk_shared_tex_supported(struct vk_inst_funcs *funcs,
  930. VkPhysicalDevice phy_device, VkFormat format,
  931. VkImageUsageFlags usage,
  932. VkExternalMemoryProperties *external_mem_props)
  933. {
  934. VkPhysicalDeviceImageFormatInfo2 info;
  935. VkPhysicalDeviceExternalImageFormatInfo external_info;
  936. external_info.sType =
  937. VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_IMAGE_FORMAT_INFO;
  938. external_info.pNext = NULL;
  939. external_info.handleType =
  940. VK_EXTERNAL_MEMORY_HANDLE_TYPE_D3D11_TEXTURE_KMT_BIT;
  941. info.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2;
  942. info.pNext = &external_info;
  943. info.format = format;
  944. info.type = VK_IMAGE_TYPE_2D;
  945. info.tiling = VK_IMAGE_TILING_OPTIMAL;
  946. info.flags = 0;
  947. info.usage = usage;
  948. VkExternalImageFormatProperties external_props = {0};
  949. external_props.sType =
  950. VK_STRUCTURE_TYPE_EXTERNAL_IMAGE_FORMAT_PROPERTIES;
  951. external_props.pNext = NULL;
  952. VkImageFormatProperties2 props = {0};
  953. props.sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_PROPERTIES_2;
  954. props.pNext = &external_props;
  955. VkResult result = funcs->GetPhysicalDeviceImageFormatProperties2(
  956. phy_device, &info, &props);
  957. *external_mem_props = external_props.externalMemoryProperties;
  958. const VkExternalMemoryFeatureFlags features =
  959. external_mem_props->externalMemoryFeatures;
  960. return ((VK_SUCCESS == result) &&
  961. (features & VK_EXTERNAL_MEMORY_FEATURE_IMPORTABLE_BIT));
  962. }
  963. static inline bool is_device_link_info(VkLayerDeviceCreateInfo *lici)
  964. {
  965. return lici->sType == VK_STRUCTURE_TYPE_LOADER_DEVICE_CREATE_INFO &&
  966. lici->function == VK_LAYER_LINK_INFO;
  967. }
  968. static VkResult VKAPI OBS_CreateDevice(VkPhysicalDevice phy_device,
  969. const VkDeviceCreateInfo *cinfo,
  970. const VkAllocationCallbacks *ac,
  971. VkDevice *p_device)
  972. {
  973. VkDeviceCreateInfo info = *cinfo;
  974. struct vk_inst_data *idata = get_inst_data(phy_device);
  975. struct vk_inst_funcs *ifuncs = &idata->funcs;
  976. struct vk_data *data = NULL;
  977. VkResult ret = VK_ERROR_INITIALIZATION_FAILED;
  978. VkLayerDeviceCreateInfo *ldci = (void *)info.pNext;
  979. /* -------------------------------------------------------- */
  980. /* step through chain until we get to the link info */
  981. while (ldci && !is_device_link_info(ldci)) {
  982. ldci = (VkLayerDeviceCreateInfo *)ldci->pNext;
  983. }
  984. if (!ldci) {
  985. goto fail;
  986. }
  987. PFN_vkGetInstanceProcAddr gipa;
  988. PFN_vkGetDeviceProcAddr gdpa;
  989. gipa = ldci->u.pLayerInfo->pfnNextGetInstanceProcAddr;
  990. gdpa = ldci->u.pLayerInfo->pfnNextGetDeviceProcAddr;
  991. /* -------------------------------------------------------- */
  992. /* move chain on for next layer */
  993. ldci->u.pLayerInfo = ldci->u.pLayerInfo->pNext;
  994. /* -------------------------------------------------------- */
  995. /* create device and initialize hook data */
  996. PFN_vkCreateDevice createFunc =
  997. (PFN_vkCreateDevice)gipa(VK_NULL_HANDLE, "vkCreateDevice");
  998. ret = createFunc(phy_device, idata->valid ? &info : cinfo, ac,
  999. p_device);
  1000. if (ret != VK_SUCCESS) {
  1001. goto fail;
  1002. }
  1003. VkDevice device = *p_device;
  1004. data = get_device_data(*p_device);
  1005. struct vk_device_funcs *dfuncs = &data->funcs;
  1006. data->valid = false; /* set true below if it doesn't go to fail */
  1007. data->phy_device = phy_device;
  1008. data->device = device;
  1009. /* -------------------------------------------------------- */
  1010. /* fetch the functions we need */
  1011. bool funcs_not_found = false;
  1012. #define GETADDR(x) \
  1013. do { \
  1014. dfuncs->x = (void *)gdpa(device, "vk" #x); \
  1015. if (!dfuncs->x) { \
  1016. flog("could not get device " \
  1017. "address for %s", \
  1018. #x); \
  1019. funcs_not_found = true; \
  1020. } \
  1021. } while (false)
  1022. #define GETADDR_OPTIONAL(x) \
  1023. do { \
  1024. dfuncs->x = (void *)gdpa(device, "vk" #x); \
  1025. } while (false)
  1026. GETADDR(GetDeviceProcAddr);
  1027. GETADDR(DestroyDevice);
  1028. GETADDR(CreateSwapchainKHR);
  1029. GETADDR(DestroySwapchainKHR);
  1030. GETADDR(QueuePresentKHR);
  1031. GETADDR(AllocateMemory);
  1032. GETADDR(FreeMemory);
  1033. GETADDR(BindImageMemory);
  1034. GETADDR(BindImageMemory2);
  1035. GETADDR(GetSwapchainImagesKHR);
  1036. GETADDR(CreateImage);
  1037. GETADDR(DestroyImage);
  1038. GETADDR(GetImageMemoryRequirements);
  1039. GETADDR(GetImageMemoryRequirements2);
  1040. GETADDR(BeginCommandBuffer);
  1041. GETADDR(EndCommandBuffer);
  1042. GETADDR(CmdCopyImage);
  1043. GETADDR(CmdPipelineBarrier);
  1044. GETADDR(GetDeviceQueue);
  1045. GETADDR(QueueSubmit);
  1046. GETADDR(CreateCommandPool);
  1047. GETADDR(DestroyCommandPool);
  1048. GETADDR(AllocateCommandBuffers);
  1049. GETADDR(CreateFence);
  1050. GETADDR(DestroyFence);
  1051. GETADDR(WaitForFences);
  1052. GETADDR(ResetFences);
  1053. #undef GETADDR_OPTIONAL
  1054. #undef GETADDR
  1055. if (funcs_not_found) {
  1056. goto fail;
  1057. }
  1058. if (!idata->valid) {
  1059. flog("instance not valid");
  1060. goto fail;
  1061. }
  1062. VkFormat format = VK_FORMAT_R8G8B8A8_UNORM;
  1063. VkImageUsageFlags usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT |
  1064. VK_IMAGE_USAGE_TRANSFER_DST_BIT;
  1065. if (!vk_shared_tex_supported(ifuncs, phy_device, format, usage,
  1066. &data->external_mem_props)) {
  1067. flog("texture sharing is not supported");
  1068. goto fail;
  1069. }
  1070. data->inst_data = idata;
  1071. data->valid = true;
  1072. fail:
  1073. return ret;
  1074. }
  1075. static void VKAPI OBS_DestroyDevice(VkDevice device,
  1076. const VkAllocationCallbacks *ac)
  1077. {
  1078. struct vk_data *data = get_device_data(device);
  1079. if (!data)
  1080. return;
  1081. for (uint32_t fam_idx = 0; fam_idx < _countof(data->cmd_pools);
  1082. fam_idx++) {
  1083. struct vk_cmd_pool_data *pool_data = &data->cmd_pools[fam_idx];
  1084. if (pool_data->cmd_pool != VK_NULL_HANDLE) {
  1085. vk_shtex_destroy_cmd_pool_objects(data, pool_data);
  1086. }
  1087. }
  1088. data->queue_count = 0;
  1089. vk_remove_device(device);
  1090. data->funcs.DestroyDevice(device, ac);
  1091. }
  1092. static VkResult VKAPI
  1093. OBS_CreateSwapchainKHR(VkDevice device, const VkSwapchainCreateInfoKHR *cinfo,
  1094. const VkAllocationCallbacks *ac, VkSwapchainKHR *p_sc)
  1095. {
  1096. struct vk_data *data = get_device_data(device);
  1097. struct vk_device_funcs *funcs = &data->funcs;
  1098. VkSwapchainCreateInfoKHR info = *cinfo;
  1099. info.imageUsage |= VK_IMAGE_USAGE_TRANSFER_SRC_BIT;
  1100. VkResult res = funcs->CreateSwapchainKHR(device, &info, ac, p_sc);
  1101. debug_res("CreateSwapchainKHR", res);
  1102. if (res != VK_SUCCESS)
  1103. return res;
  1104. VkSwapchainKHR sc = *p_sc;
  1105. uint32_t count = 0;
  1106. res = funcs->GetSwapchainImagesKHR(data->device, sc, &count, NULL);
  1107. debug_res("GetSwapchainImagesKHR", res);
  1108. struct vk_swap_data *swap = get_new_swap_data(data);
  1109. if (count > 0) {
  1110. if (count > OBJ_MAX)
  1111. count = OBJ_MAX;
  1112. res = funcs->GetSwapchainImagesKHR(data->device, sc, &count,
  1113. swap->swap_images);
  1114. debug_res("GetSwapchainImagesKHR", res);
  1115. }
  1116. swap->sc = sc;
  1117. swap->image_extent = cinfo->imageExtent;
  1118. swap->format = cinfo->imageFormat;
  1119. swap->hwnd = find_surf_hwnd(data->inst_data, cinfo->surface);
  1120. swap->image_count = count;
  1121. return VK_SUCCESS;
  1122. }
  1123. static void VKAPI OBS_DestroySwapchainKHR(VkDevice device, VkSwapchainKHR sc,
  1124. const VkAllocationCallbacks *ac)
  1125. {
  1126. struct vk_data *data = get_device_data(device);
  1127. struct vk_device_funcs *funcs = &data->funcs;
  1128. struct vk_swap_data *swap = get_swap_data(data, sc);
  1129. if (swap) {
  1130. if (data->cur_swap == swap) {
  1131. vk_shtex_free(data);
  1132. }
  1133. swap->sc = VK_NULL_HANDLE;
  1134. swap->hwnd = NULL;
  1135. }
  1136. funcs->DestroySwapchainKHR(device, sc, ac);
  1137. }
  1138. static void VKAPI OBS_GetDeviceQueue(VkDevice device, uint32_t queueFamilyIndex,
  1139. uint32_t queueIndex, VkQueue *pQueue)
  1140. {
  1141. struct vk_data *data = get_device_data(device);
  1142. struct vk_device_funcs *funcs = &data->funcs;
  1143. funcs->GetDeviceQueue(device, queueFamilyIndex, queueIndex, pQueue);
  1144. for (uint32_t i = 0; i < data->queue_count; ++i) {
  1145. if (data->queues[i].queue == *pQueue)
  1146. return;
  1147. }
  1148. if (data->queue_count < _countof(data->queues)) {
  1149. data->queues[data->queue_count].queue = *pQueue;
  1150. data->queues[data->queue_count].fam_idx = queueFamilyIndex;
  1151. ++data->queue_count;
  1152. }
  1153. }
  1154. static VkResult VKAPI OBS_CreateWin32SurfaceKHR(
  1155. VkInstance inst, const VkWin32SurfaceCreateInfoKHR *info,
  1156. const VkAllocationCallbacks *ac, VkSurfaceKHR *surf)
  1157. {
  1158. struct vk_inst_data *data = get_inst_data(inst);
  1159. struct vk_inst_funcs *funcs = &data->funcs;
  1160. VkResult res = funcs->CreateWin32SurfaceKHR(inst, info, ac, surf);
  1161. if (res == VK_SUCCESS)
  1162. insert_surf_data(data, *surf, info->hwnd);
  1163. return res;
  1164. }
  1165. static void VKAPI OBS_DestroySurfaceKHR(VkInstance inst, VkSurfaceKHR surf,
  1166. const VkAllocationCallbacks *ac)
  1167. {
  1168. struct vk_inst_data *data = get_inst_data(inst);
  1169. struct vk_inst_funcs *funcs = &data->funcs;
  1170. erase_surf_data(data, surf);
  1171. funcs->DestroySurfaceKHR(inst, surf, ac);
  1172. }
  1173. #define GETPROCADDR(func) \
  1174. if (!strcmp(name, "vk" #func)) \
  1175. return (VkFunc)&OBS_##func;
  1176. static VkFunc VKAPI OBS_GetDeviceProcAddr(VkDevice dev, const char *name)
  1177. {
  1178. struct vk_data *data = get_device_data(dev);
  1179. struct vk_device_funcs *funcs = &data->funcs;
  1180. debug_procaddr("vkGetDeviceProcAddr(%p, \"%s\")", dev, name);
  1181. GETPROCADDR(GetDeviceProcAddr);
  1182. GETPROCADDR(CreateDevice);
  1183. GETPROCADDR(DestroyDevice);
  1184. GETPROCADDR(CreateSwapchainKHR);
  1185. GETPROCADDR(DestroySwapchainKHR);
  1186. GETPROCADDR(QueuePresentKHR);
  1187. GETPROCADDR(GetDeviceQueue);
  1188. if (funcs->GetDeviceProcAddr == NULL)
  1189. return NULL;
  1190. return funcs->GetDeviceProcAddr(dev, name);
  1191. }
  1192. static VkFunc VKAPI OBS_GetInstanceProcAddr(VkInstance inst, const char *name)
  1193. {
  1194. debug_procaddr("vkGetInstanceProcAddr(%p, \"%s\")", inst, name);
  1195. /* instance chain functions we intercept */
  1196. GETPROCADDR(GetInstanceProcAddr);
  1197. GETPROCADDR(CreateInstance);
  1198. GETPROCADDR(DestroyInstance);
  1199. GETPROCADDR(CreateWin32SurfaceKHR);
  1200. GETPROCADDR(DestroySurfaceKHR);
  1201. /* device chain functions we intercept */
  1202. GETPROCADDR(GetDeviceProcAddr);
  1203. GETPROCADDR(CreateDevice);
  1204. GETPROCADDR(DestroyDevice);
  1205. struct vk_inst_funcs *funcs = get_inst_funcs(inst);
  1206. if (funcs->GetInstanceProcAddr == NULL)
  1207. return NULL;
  1208. return funcs->GetInstanceProcAddr(inst, name);
  1209. }
  1210. #undef GETPROCADDR
  1211. EXPORT VkResult VKAPI OBS_Negotiate(VkNegotiateLayerInterface *nli)
  1212. {
  1213. if (nli->loaderLayerInterfaceVersion >= 2) {
  1214. nli->sType = LAYER_NEGOTIATE_INTERFACE_STRUCT;
  1215. nli->pNext = NULL;
  1216. nli->pfnGetInstanceProcAddr = OBS_GetInstanceProcAddr;
  1217. nli->pfnGetDeviceProcAddr = OBS_GetDeviceProcAddr;
  1218. nli->pfnGetPhysicalDeviceProcAddr = NULL;
  1219. }
  1220. const uint32_t cur_ver = CURRENT_LOADER_LAYER_INTERFACE_VERSION;
  1221. if (nli->loaderLayerInterfaceVersion > cur_ver) {
  1222. nli->loaderLayerInterfaceVersion = cur_ver;
  1223. }
  1224. return VK_SUCCESS;
  1225. }
  1226. bool hook_vulkan(void)
  1227. {
  1228. static bool hooked = false;
  1229. if (!hooked && vulkan_seen) {
  1230. hlog("Hooked Vulkan");
  1231. hooked = true;
  1232. }
  1233. return hooked;
  1234. }