vulkan-capture.c 42 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542
  1. #include <windows.h>
  2. #include "graphics-hook.h"
  3. #define VK_USE_PLATFORM_WIN32_KHR
  4. #include <malloc.h>
  5. #include <vulkan/vulkan.h>
  6. #include <vulkan/vk_layer.h>
  7. #undef VK_LAYER_EXPORT
  8. #if defined(WIN32)
  9. #define VK_LAYER_EXPORT __declspec(dllexport)
  10. #else
  11. #define VK_LAYER_EXPORT
  12. #endif
  13. #include <vulkan/vulkan_win32.h>
  14. #define COBJMACROS
  15. #include <dxgi.h>
  16. #include <d3d11.h>
  17. #include "vulkan-capture.h"
  18. /* ======================================================================== */
  19. /* defs/statics */
  20. /* shorten stuff because dear GOD is vulkan unclean. */
  21. #define VKAPI VKAPI_CALL
  22. #define VkFunc PFN_vkVoidFunction
  23. #define EXPORT VK_LAYER_EXPORT
  24. #define OBJ_MAX 16
  25. /* use the loader's dispatch table pointer as a key for internal data maps */
  26. #define GET_LDT(x) (*(void **)x)
  27. #define DUMMY_WINDOW_CLASS_NAME L"graphics_hook_vk_dummy_window"
  28. /* clang-format off */
  29. static const GUID dxgi_factory1_guid =
  30. {0x770aae78, 0xf26f, 0x4dba, {0xa8, 0x29, 0x25, 0x3c, 0x83, 0xd1, 0xb3, 0x87}};
  31. static const GUID dxgi_resource_guid =
  32. {0x035f3ab4, 0x482e, 0x4e50, {0xb4, 0x1f, 0x8a, 0x7f, 0x8b, 0xd8, 0x96, 0x0b}};
  33. /* clang-format on */
  34. static bool vulkan_seen = false;
  35. static CRITICAL_SECTION mutex;
  36. /* ======================================================================== */
  37. /* hook data */
  38. struct vk_swap_data {
  39. VkSwapchainKHR sc;
  40. VkExtent2D image_extent;
  41. VkFormat format;
  42. VkSurfaceKHR surf;
  43. VkImage export_image;
  44. bool layout_initialized;
  45. VkDeviceMemory export_mem;
  46. VkImage swap_images[OBJ_MAX];
  47. uint32_t image_count;
  48. HANDLE handle;
  49. struct shtex_data *shtex_info;
  50. ID3D11Texture2D *d3d11_tex;
  51. bool captured;
  52. };
  53. struct vk_queue_data {
  54. VkQueue queue;
  55. uint32_t fam_idx;
  56. };
  57. struct vk_cmd_pool_data {
  58. VkCommandPool cmd_pool;
  59. VkCommandBuffer cmd_buffers[OBJ_MAX];
  60. VkFence fences[OBJ_MAX];
  61. bool cmd_buffer_busy[OBJ_MAX];
  62. uint32_t image_count;
  63. };
  64. struct vk_data {
  65. bool valid;
  66. struct vk_device_funcs funcs;
  67. VkPhysicalDevice phy_device;
  68. VkDevice device;
  69. struct vk_swap_data swaps[OBJ_MAX];
  70. struct vk_swap_data *cur_swap;
  71. uint32_t swap_idx;
  72. struct vk_queue_data queues[OBJ_MAX];
  73. uint32_t queue_count;
  74. struct vk_cmd_pool_data cmd_pools[OBJ_MAX];
  75. VkExternalMemoryProperties external_mem_props;
  76. ID3D11Device *d3d11_device;
  77. ID3D11DeviceContext *d3d11_context;
  78. IDXGISwapChain *dxgi_swap;
  79. HWND dummy_hwnd;
  80. };
  81. static struct vk_swap_data *get_swap_data(struct vk_data *data,
  82. VkSwapchainKHR sc)
  83. {
  84. for (int i = 0; i < OBJ_MAX; i++) {
  85. if (data->swaps[i].sc == sc) {
  86. return &data->swaps[i];
  87. }
  88. }
  89. debug("get_swap_data failed, swapchain not found");
  90. return NULL;
  91. }
  92. static struct vk_swap_data *get_new_swap_data(struct vk_data *data)
  93. {
  94. for (int i = 0; i < OBJ_MAX; i++) {
  95. if (data->swaps[i].surf == NULL && data->swaps[i].sc == NULL) {
  96. return &data->swaps[i];
  97. }
  98. }
  99. debug("get_new_swap_data failed, no more free slot");
  100. return NULL;
  101. }
  102. /* ------------------------------------------------------------------------- */
  103. static inline size_t find_obj_idx(void *objs[], void *obj)
  104. {
  105. size_t idx = SIZE_MAX;
  106. EnterCriticalSection(&mutex);
  107. for (size_t i = 0; i < OBJ_MAX; i++) {
  108. if (objs[i] == obj) {
  109. idx = i;
  110. break;
  111. }
  112. }
  113. LeaveCriticalSection(&mutex);
  114. return idx;
  115. }
  116. static size_t get_obj_idx(void *objs[], void *obj)
  117. {
  118. size_t idx = SIZE_MAX;
  119. EnterCriticalSection(&mutex);
  120. for (size_t i = 0; i < OBJ_MAX; i++) {
  121. if (objs[i] == obj) {
  122. idx = i;
  123. break;
  124. }
  125. if (!objs[i] && idx == SIZE_MAX) {
  126. idx = i;
  127. }
  128. }
  129. LeaveCriticalSection(&mutex);
  130. return idx;
  131. }
  132. /* ------------------------------------------------------------------------- */
  133. static struct vk_data device_data[OBJ_MAX] = {0};
  134. static void *devices[OBJ_MAX] = {0};
  135. static inline struct vk_data *get_device_data(void *dev)
  136. {
  137. size_t idx = get_obj_idx(devices, GET_LDT(dev));
  138. if (idx == SIZE_MAX) {
  139. debug("out of device slots");
  140. return NULL;
  141. }
  142. return &device_data[idx];
  143. }
  144. static void vk_shtex_clear_fence(struct vk_data *data,
  145. struct vk_cmd_pool_data *pool_data,
  146. uint32_t image_idx)
  147. {
  148. VkFence fence = pool_data->fences[image_idx];
  149. if (pool_data->cmd_buffer_busy[image_idx]) {
  150. VkDevice device = data->device;
  151. struct vk_device_funcs *funcs = &data->funcs;
  152. funcs->WaitForFences(device, 1, &fence, VK_TRUE, ~0ull);
  153. funcs->ResetFences(device, 1, &fence);
  154. pool_data->cmd_buffer_busy[image_idx] = false;
  155. }
  156. }
  157. static void vk_shtex_wait_until_pool_idle(struct vk_data *data,
  158. struct vk_cmd_pool_data *pool_data)
  159. {
  160. for (uint32_t image_idx = 0; image_idx < pool_data->image_count;
  161. image_idx++) {
  162. vk_shtex_clear_fence(data, pool_data, image_idx);
  163. }
  164. }
  165. static void vk_shtex_wait_until_idle(struct vk_data *data)
  166. {
  167. for (uint32_t fam_idx = 0; fam_idx < _countof(data->cmd_pools);
  168. fam_idx++) {
  169. struct vk_cmd_pool_data *pool_data = &data->cmd_pools[fam_idx];
  170. if (pool_data->cmd_pool != VK_NULL_HANDLE)
  171. vk_shtex_wait_until_pool_idle(data, pool_data);
  172. }
  173. }
  174. static void vk_shtex_free(struct vk_data *data)
  175. {
  176. capture_free();
  177. vk_shtex_wait_until_idle(data);
  178. for (int swap_idx = 0; swap_idx < OBJ_MAX; swap_idx++) {
  179. struct vk_swap_data *swap = &data->swaps[swap_idx];
  180. if (swap->export_image)
  181. data->funcs.DestroyImage(data->device,
  182. swap->export_image, NULL);
  183. if (swap->export_mem)
  184. data->funcs.FreeMemory(data->device, swap->export_mem,
  185. NULL);
  186. if (swap->d3d11_tex) {
  187. ID3D11Resource_Release(swap->d3d11_tex);
  188. }
  189. swap->handle = INVALID_HANDLE_VALUE;
  190. swap->d3d11_tex = NULL;
  191. swap->export_mem = NULL;
  192. swap->export_image = NULL;
  193. swap->captured = false;
  194. }
  195. if (data->d3d11_context) {
  196. ID3D11DeviceContext_Release(data->d3d11_context);
  197. data->d3d11_context = NULL;
  198. }
  199. if (data->d3d11_device) {
  200. ID3D11Device_Release(data->d3d11_device);
  201. data->d3d11_device = NULL;
  202. }
  203. if (data->dxgi_swap) {
  204. IDXGISwapChain_Release(data->dxgi_swap);
  205. data->dxgi_swap = NULL;
  206. }
  207. data->cur_swap = NULL;
  208. hlog("------------------ vulkan capture freed ------------------");
  209. }
  210. static void vk_remove_device(void *dev)
  211. {
  212. size_t idx = find_obj_idx(devices, GET_LDT(dev));
  213. if (idx == SIZE_MAX) {
  214. return;
  215. }
  216. struct vk_data *data = &device_data[idx];
  217. memset(data, 0, sizeof(*data));
  218. EnterCriticalSection(&mutex);
  219. devices[idx] = NULL;
  220. LeaveCriticalSection(&mutex);
  221. }
  222. /* ------------------------------------------------------------------------- */
  223. struct vk_surf_data {
  224. VkSurfaceKHR surf;
  225. HINSTANCE hinstance;
  226. HWND hwnd;
  227. };
  228. struct vk_inst_data {
  229. bool valid;
  230. struct vk_inst_funcs funcs;
  231. struct vk_surf_data surfaces[OBJ_MAX];
  232. };
  233. static struct vk_surf_data *find_surf_data(struct vk_inst_data *data,
  234. VkSurfaceKHR surf)
  235. {
  236. int idx = OBJ_MAX;
  237. for (int i = 0; i < OBJ_MAX; i++) {
  238. if (data->surfaces[i].surf == surf) {
  239. return &data->surfaces[i];
  240. } else if (data->surfaces[i].surf == NULL && idx == OBJ_MAX) {
  241. idx = i;
  242. }
  243. }
  244. if (idx != OBJ_MAX) {
  245. data->surfaces[idx].surf = surf;
  246. return &data->surfaces[idx];
  247. }
  248. debug("find_surf_data failed, no more free slots");
  249. return NULL;
  250. }
  251. /* ------------------------------------------------------------------------- */
  252. static struct vk_inst_data inst_data[OBJ_MAX] = {0};
  253. static void *instances[OBJ_MAX] = {0};
  254. static struct vk_inst_data *get_inst_data(void *inst)
  255. {
  256. size_t idx = get_obj_idx(instances, GET_LDT(inst));
  257. if (idx == SIZE_MAX) {
  258. debug("out of instance slots");
  259. return NULL;
  260. }
  261. vulkan_seen = true;
  262. return &inst_data[idx];
  263. }
  264. static inline struct vk_inst_funcs *get_inst_funcs(void *inst)
  265. {
  266. struct vk_inst_data *data = get_inst_data(inst);
  267. return &data->funcs;
  268. }
  269. static void remove_instance(void *inst)
  270. {
  271. size_t idx = find_obj_idx(instances, inst);
  272. if (idx == SIZE_MAX) {
  273. return;
  274. }
  275. struct vk_inst_data *data = &inst_data[idx];
  276. memset(data, 0, sizeof(*data));
  277. EnterCriticalSection(&mutex);
  278. instances[idx] = NULL;
  279. LeaveCriticalSection(&mutex);
  280. }
  281. /* ======================================================================== */
  282. /* capture */
  283. static bool vk_register_window(void)
  284. {
  285. WNDCLASSW wc = {0};
  286. wc.style = CS_OWNDC;
  287. wc.hInstance = GetModuleHandle(NULL);
  288. wc.lpfnWndProc = DefWindowProc;
  289. wc.lpszClassName = DUMMY_WINDOW_CLASS_NAME;
  290. if (!RegisterClassW(&wc)) {
  291. flog("failed to register window class: %d", GetLastError());
  292. return false;
  293. }
  294. return true;
  295. }
  296. static inline bool vk_shtex_init_window(struct vk_data *data)
  297. {
  298. static bool registered = false;
  299. if (!registered) {
  300. static bool failure = false;
  301. if (failure || !vk_register_window()) {
  302. failure = true;
  303. return false;
  304. }
  305. registered = true;
  306. }
  307. data->dummy_hwnd = CreateWindowExW(
  308. 0, DUMMY_WINDOW_CLASS_NAME, L"Dummy VK window, ignore",
  309. WS_POPUP | WS_CLIPCHILDREN | WS_CLIPSIBLINGS, 0, 0, 2, 2, NULL,
  310. NULL, GetModuleHandle(NULL), NULL);
  311. if (!data->dummy_hwnd) {
  312. flog("failed to create window: %d", GetLastError());
  313. return false;
  314. }
  315. return true;
  316. }
  317. static inline bool vk_shtex_init_d3d11(struct vk_data *data)
  318. {
  319. D3D_FEATURE_LEVEL level_used;
  320. IDXGIFactory1 *factory;
  321. IDXGIAdapter *adapter;
  322. HRESULT hr;
  323. HMODULE d3d11 = load_system_library("d3d11.dll");
  324. if (!d3d11) {
  325. flog("failed to load d3d11: %d", GetLastError());
  326. return false;
  327. }
  328. HMODULE dxgi = load_system_library("dxgi.dll");
  329. if (!dxgi) {
  330. flog("failed to load dxgi: %d", GetLastError());
  331. return false;
  332. }
  333. DXGI_SWAP_CHAIN_DESC desc = {0};
  334. desc.BufferCount = 2;
  335. desc.BufferDesc.Format = DXGI_FORMAT_B8G8R8A8_UNORM;
  336. desc.BufferDesc.Width = 2;
  337. desc.BufferDesc.Height = 2;
  338. desc.BufferUsage = DXGI_USAGE_RENDER_TARGET_OUTPUT;
  339. desc.SampleDesc.Count = 1;
  340. desc.Windowed = true;
  341. desc.OutputWindow = data->dummy_hwnd;
  342. HRESULT(WINAPI * create_factory)
  343. (REFIID, void **) = (void *)GetProcAddress(dxgi, "CreateDXGIFactory1");
  344. if (!create_factory) {
  345. flog("failed to get CreateDXGIFactory1 address: %d",
  346. GetLastError());
  347. return false;
  348. }
  349. PFN_D3D11_CREATE_DEVICE_AND_SWAP_CHAIN create =
  350. (void *)GetProcAddress(d3d11, "D3D11CreateDeviceAndSwapChain");
  351. if (!create) {
  352. flog("failed to get D3D11CreateDeviceAndSwapChain address: %d",
  353. GetLastError());
  354. return false;
  355. }
  356. hr = create_factory(&dxgi_factory1_guid, (void **)&factory);
  357. if (FAILED(hr)) {
  358. flog_hr("failed to create factory", hr);
  359. return false;
  360. }
  361. hr = IDXGIFactory1_EnumAdapters1(factory, 0,
  362. (IDXGIAdapter1 **)&adapter);
  363. IDXGIFactory1_Release(factory);
  364. if (FAILED(hr)) {
  365. flog_hr("failed to create adapter", hr);
  366. return false;
  367. }
  368. static const D3D_FEATURE_LEVEL feature_levels[] = {
  369. D3D_FEATURE_LEVEL_11_0,
  370. D3D_FEATURE_LEVEL_10_1,
  371. D3D_FEATURE_LEVEL_10_0,
  372. D3D_FEATURE_LEVEL_9_3,
  373. };
  374. hr = create(adapter, D3D_DRIVER_TYPE_UNKNOWN, NULL, 0, feature_levels,
  375. sizeof(feature_levels) / sizeof(D3D_FEATURE_LEVEL),
  376. D3D11_SDK_VERSION, &desc, &data->dxgi_swap,
  377. &data->d3d11_device, &level_used, &data->d3d11_context);
  378. IDXGIAdapter_Release(adapter);
  379. if (FAILED(hr)) {
  380. flog_hr("failed to create device", hr);
  381. return false;
  382. }
  383. return true;
  384. }
  385. static inline bool vk_shtex_init_d3d11_tex(struct vk_data *data,
  386. struct vk_swap_data *swap)
  387. {
  388. IDXGIResource *dxgi_res;
  389. HRESULT hr;
  390. D3D11_TEXTURE2D_DESC desc = {0};
  391. desc.Width = swap->image_extent.width;
  392. desc.Height = swap->image_extent.height;
  393. desc.MipLevels = 1;
  394. desc.ArraySize = 1;
  395. flog("OBS requesting %s texture format",
  396. vk_format_to_str(swap->format));
  397. desc.Format = vk_format_to_dxgi(swap->format);
  398. desc.SampleDesc.Count = 1;
  399. desc.SampleDesc.Quality = 0;
  400. desc.Usage = D3D11_USAGE_DEFAULT;
  401. desc.MiscFlags = D3D11_RESOURCE_MISC_SHARED;
  402. desc.BindFlags = D3D11_BIND_RENDER_TARGET | D3D11_BIND_SHADER_RESOURCE;
  403. hr = ID3D11Device_CreateTexture2D(data->d3d11_device, &desc, NULL,
  404. &swap->d3d11_tex);
  405. if (FAILED(hr)) {
  406. flog_hr("failed to create texture", hr);
  407. return false;
  408. }
  409. hr = ID3D11Device_QueryInterface(swap->d3d11_tex, &dxgi_resource_guid,
  410. (void **)&dxgi_res);
  411. if (FAILED(hr)) {
  412. flog_hr("failed to get IDXGIResource", hr);
  413. return false;
  414. }
  415. hr = IDXGIResource_GetSharedHandle(dxgi_res, &swap->handle);
  416. IDXGIResource_Release(dxgi_res);
  417. if (FAILED(hr)) {
  418. flog_hr("failed to get shared handle", hr);
  419. return false;
  420. }
  421. return true;
  422. }
  423. static inline bool vk_shtex_init_vulkan_tex(struct vk_data *data,
  424. struct vk_swap_data *swap)
  425. {
  426. struct vk_device_funcs *funcs = &data->funcs;
  427. VkExternalMemoryFeatureFlags f =
  428. data->external_mem_props.externalMemoryFeatures;
  429. /* -------------------------------------------------------- */
  430. /* create texture */
  431. VkExternalMemoryImageCreateInfo emici;
  432. emici.sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO;
  433. emici.pNext = NULL;
  434. emici.handleTypes =
  435. VK_EXTERNAL_MEMORY_HANDLE_TYPE_D3D11_TEXTURE_KMT_BIT;
  436. VkImageCreateInfo ici;
  437. ici.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO;
  438. ici.pNext = &emici;
  439. ici.flags = 0;
  440. ici.imageType = VK_IMAGE_TYPE_2D;
  441. ici.format = swap->format;
  442. ici.extent.width = swap->image_extent.width;
  443. ici.extent.height = swap->image_extent.height;
  444. ici.extent.depth = 1;
  445. ici.mipLevels = 1;
  446. ici.arrayLayers = 1;
  447. ici.samples = VK_SAMPLE_COUNT_1_BIT;
  448. ici.tiling = VK_IMAGE_TILING_OPTIMAL;
  449. ici.usage = VK_IMAGE_USAGE_TRANSFER_DST_BIT |
  450. VK_IMAGE_USAGE_SAMPLED_BIT;
  451. ici.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
  452. ici.queueFamilyIndexCount = 0;
  453. ici.pQueueFamilyIndices = 0;
  454. ici.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
  455. VkResult res;
  456. res = funcs->CreateImage(data->device, &ici, NULL, &swap->export_image);
  457. if (VK_SUCCESS != res) {
  458. flog("failed to CreateImage: %s", result_to_str(res));
  459. swap->export_image = NULL;
  460. return false;
  461. }
  462. swap->layout_initialized = false;
  463. /* -------------------------------------------------------- */
  464. /* get image memory requirements */
  465. VkMemoryRequirements mr;
  466. bool use_gimr2 = f & VK_EXTERNAL_MEMORY_FEATURE_DEDICATED_ONLY_BIT;
  467. if (use_gimr2) {
  468. VkMemoryDedicatedRequirements mdr = {0};
  469. mdr.sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS;
  470. mdr.pNext = NULL;
  471. VkMemoryRequirements2 mr2 = {0};
  472. mr2.sType = VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2;
  473. mr2.pNext = &mdr;
  474. VkImageMemoryRequirementsInfo2 imri2 = {0};
  475. imri2.sType =
  476. VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2;
  477. imri2.pNext = NULL;
  478. imri2.image = swap->export_image;
  479. funcs->GetImageMemoryRequirements2(data->device, &imri2, &mr2);
  480. mr = mr2.memoryRequirements;
  481. } else {
  482. funcs->GetImageMemoryRequirements(data->device,
  483. swap->export_image, &mr);
  484. }
  485. /* -------------------------------------------------------- */
  486. /* get memory type index */
  487. struct vk_inst_funcs *ifuncs = get_inst_funcs(data->phy_device);
  488. VkPhysicalDeviceMemoryProperties pdmp;
  489. ifuncs->GetPhysicalDeviceMemoryProperties(data->phy_device, &pdmp);
  490. uint32_t mem_type_idx = 0;
  491. for (; mem_type_idx < pdmp.memoryTypeCount; mem_type_idx++) {
  492. if ((mr.memoryTypeBits & (1 << mem_type_idx)) &&
  493. (pdmp.memoryTypes[mem_type_idx].propertyFlags &
  494. VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) ==
  495. VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) {
  496. break;
  497. }
  498. }
  499. if (mem_type_idx == pdmp.memoryTypeCount) {
  500. flog("failed to get memory type index");
  501. funcs->DestroyImage(data->device, swap->export_image, NULL);
  502. swap->export_image = NULL;
  503. return false;
  504. }
  505. /* -------------------------------------------------------- */
  506. /* allocate memory */
  507. VkImportMemoryWin32HandleInfoKHR imw32hi;
  508. imw32hi.sType = VK_STRUCTURE_TYPE_IMPORT_MEMORY_WIN32_HANDLE_INFO_KHR;
  509. imw32hi.pNext = NULL;
  510. imw32hi.name = NULL;
  511. imw32hi.handleType =
  512. VK_EXTERNAL_MEMORY_HANDLE_TYPE_D3D11_TEXTURE_KMT_BIT;
  513. imw32hi.handle = swap->handle;
  514. VkMemoryAllocateInfo mai;
  515. mai.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
  516. mai.pNext = &imw32hi;
  517. mai.allocationSize = mr.size;
  518. mai.memoryTypeIndex = mem_type_idx;
  519. VkMemoryDedicatedAllocateInfo mdai;
  520. mdai.sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO;
  521. mdai.pNext = NULL;
  522. mdai.buffer = VK_NULL_HANDLE;
  523. if (data->external_mem_props.externalMemoryFeatures &
  524. VK_EXTERNAL_MEMORY_FEATURE_DEDICATED_ONLY_BIT) {
  525. mdai.image = swap->export_image;
  526. imw32hi.pNext = &mdai;
  527. }
  528. res = funcs->AllocateMemory(data->device, &mai, NULL,
  529. &swap->export_mem);
  530. if (VK_SUCCESS != res) {
  531. flog("failed to AllocateMemory: %s", result_to_str(res));
  532. funcs->DestroyImage(data->device, swap->export_image, NULL);
  533. swap->export_image = NULL;
  534. return false;
  535. }
  536. /* -------------------------------------------------------- */
  537. /* bind image memory */
  538. bool use_bi2 = f & VK_EXTERNAL_MEMORY_FEATURE_DEDICATED_ONLY_BIT;
  539. if (use_bi2) {
  540. VkBindImageMemoryInfo bimi = {0};
  541. bimi.sType = VK_STRUCTURE_TYPE_BIND_IMAGE_MEMORY_INFO;
  542. bimi.image = swap->export_image;
  543. bimi.memory = swap->export_mem;
  544. bimi.memoryOffset = 0;
  545. res = funcs->BindImageMemory2(data->device, 1, &bimi);
  546. } else {
  547. res = funcs->BindImageMemory(data->device, swap->export_image,
  548. swap->export_mem, 0);
  549. }
  550. if (VK_SUCCESS != res) {
  551. flog("%s failed: %s",
  552. use_bi2 ? "BindImageMemory2" : "BindImageMemory",
  553. result_to_str(res));
  554. funcs->DestroyImage(data->device, swap->export_image, NULL);
  555. swap->export_image = NULL;
  556. return false;
  557. }
  558. return true;
  559. }
  560. static bool vk_shtex_init(struct vk_data *data, HWND window,
  561. struct vk_swap_data *swap)
  562. {
  563. if (!vk_shtex_init_window(data)) {
  564. return false;
  565. }
  566. if (!vk_shtex_init_d3d11(data)) {
  567. return false;
  568. }
  569. if (!vk_shtex_init_d3d11_tex(data, swap)) {
  570. return false;
  571. }
  572. if (!vk_shtex_init_vulkan_tex(data, swap)) {
  573. return false;
  574. }
  575. data->cur_swap = swap;
  576. swap->captured = capture_init_shtex(
  577. &swap->shtex_info, window, swap->image_extent.width,
  578. swap->image_extent.height, swap->image_extent.width,
  579. swap->image_extent.height, (uint32_t)swap->format, false,
  580. (uintptr_t)swap->handle);
  581. if (swap->captured) {
  582. if (global_hook_info->force_shmem) {
  583. flog("shared memory capture currently "
  584. "unsupported; ignoring");
  585. }
  586. hlog("vulkan shared texture capture successful");
  587. return true;
  588. }
  589. return false;
  590. }
  591. static void vk_shtex_create_cmd_pool_objects(struct vk_data *data,
  592. uint32_t fam_idx,
  593. uint32_t image_count)
  594. {
  595. struct vk_cmd_pool_data *pool_data = &data->cmd_pools[fam_idx];
  596. VkCommandPoolCreateInfo cpci;
  597. cpci.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO;
  598. cpci.pNext = NULL;
  599. cpci.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT;
  600. cpci.queueFamilyIndex = fam_idx;
  601. VkResult res = data->funcs.CreateCommandPool(data->device, &cpci, NULL,
  602. &pool_data->cmd_pool);
  603. debug_res("CreateCommandPool", res);
  604. VkCommandBufferAllocateInfo cbai;
  605. cbai.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO;
  606. cbai.pNext = NULL;
  607. cbai.commandPool = pool_data->cmd_pool;
  608. cbai.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY;
  609. cbai.commandBufferCount = image_count;
  610. res = data->funcs.AllocateCommandBuffers(data->device, &cbai,
  611. pool_data->cmd_buffers);
  612. debug_res("AllocateCommandBuffers", res);
  613. for (uint32_t image_index = 0; image_index < image_count;
  614. image_index++) {
  615. /* Dispatch table something or other. Well-designed API. */
  616. VkCommandBuffer cmd_buffer =
  617. pool_data->cmd_buffers[image_index];
  618. *(void **)cmd_buffer = *(void **)(data->device);
  619. VkFence *fence = &pool_data->fences[image_index];
  620. VkFenceCreateInfo fci = {0};
  621. fci.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
  622. fci.pNext = NULL;
  623. fci.flags = 0;
  624. res = data->funcs.CreateFence(data->device, &fci, NULL, fence);
  625. debug_res("CreateFence", res);
  626. }
  627. pool_data->image_count = image_count;
  628. }
  629. static void vk_shtex_destroy_fence(struct vk_data *data, bool *cmd_buffer_busy,
  630. VkFence *fence)
  631. {
  632. VkDevice device = data->device;
  633. if (*cmd_buffer_busy) {
  634. data->funcs.WaitForFences(device, 1, fence, VK_TRUE, ~0ull);
  635. *cmd_buffer_busy = false;
  636. }
  637. data->funcs.DestroyFence(device, *fence, NULL);
  638. *fence = VK_NULL_HANDLE;
  639. }
  640. static void
  641. vk_shtex_destroy_cmd_pool_objects(struct vk_data *data,
  642. struct vk_cmd_pool_data *pool_data)
  643. {
  644. for (uint32_t image_idx = 0; image_idx < pool_data->image_count;
  645. image_idx++) {
  646. bool *cmd_buffer_busy = &pool_data->cmd_buffer_busy[image_idx];
  647. VkFence *fence = &pool_data->fences[image_idx];
  648. vk_shtex_destroy_fence(data, cmd_buffer_busy, fence);
  649. }
  650. data->funcs.DestroyCommandPool(data->device, pool_data->cmd_pool, NULL);
  651. pool_data->cmd_pool = VK_NULL_HANDLE;
  652. pool_data->image_count = 0;
  653. }
  654. static void vk_shtex_capture(struct vk_data *data,
  655. struct vk_device_funcs *funcs,
  656. struct vk_swap_data *swap, uint32_t idx,
  657. VkQueue queue, const VkPresentInfoKHR *info)
  658. {
  659. VkResult res = VK_SUCCESS;
  660. VkCommandBufferBeginInfo begin_info;
  661. begin_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
  662. begin_info.pNext = NULL;
  663. begin_info.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
  664. begin_info.pInheritanceInfo = NULL;
  665. VkImageMemoryBarrier mb[2];
  666. VkImageMemoryBarrier *src_mb = &mb[0];
  667. VkImageMemoryBarrier *dst_mb = &mb[1];
  668. /* ------------------------------------------------------ */
  669. /* do image copy */
  670. const uint32_t image_index = info->pImageIndices[idx];
  671. VkImage cur_backbuffer = swap->swap_images[image_index];
  672. uint32_t fam_idx = 0;
  673. for (uint32_t i = 0; i < data->queue_count; i++) {
  674. if (data->queues[i].queue == queue)
  675. fam_idx = data->queues[i].fam_idx;
  676. }
  677. if (fam_idx >= _countof(data->cmd_pools))
  678. return;
  679. struct vk_cmd_pool_data *pool_data = &data->cmd_pools[fam_idx];
  680. VkCommandPool *pool = &pool_data->cmd_pool;
  681. const uint32_t image_count = swap->image_count;
  682. if (pool_data->image_count < image_count) {
  683. if (*pool != VK_NULL_HANDLE)
  684. vk_shtex_destroy_cmd_pool_objects(data, pool_data);
  685. vk_shtex_create_cmd_pool_objects(data, fam_idx, image_count);
  686. }
  687. vk_shtex_clear_fence(data, pool_data, image_index);
  688. VkCommandBuffer cmd_buffer = pool_data->cmd_buffers[image_index];
  689. res = funcs->BeginCommandBuffer(cmd_buffer, &begin_info);
  690. debug_res("BeginCommandBuffer", res);
  691. /* ------------------------------------------------------ */
  692. /* transition shared texture if necessary */
  693. if (!swap->layout_initialized) {
  694. VkImageMemoryBarrier imb;
  695. imb.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER;
  696. imb.pNext = NULL;
  697. imb.srcAccessMask = 0;
  698. imb.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
  699. imb.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED;
  700. imb.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
  701. imb.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
  702. imb.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
  703. imb.image = swap->export_image;
  704. imb.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
  705. imb.subresourceRange.baseMipLevel = 0;
  706. imb.subresourceRange.levelCount = 1;
  707. imb.subresourceRange.baseArrayLayer = 0;
  708. imb.subresourceRange.layerCount = 1;
  709. funcs->CmdPipelineBarrier(cmd_buffer,
  710. VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
  711. VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0,
  712. NULL, 0, NULL, 1, &imb);
  713. swap->layout_initialized = true;
  714. }
  715. /* ------------------------------------------------------ */
  716. /* transition cur_backbuffer to transfer source state */
  717. src_mb->sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER;
  718. src_mb->pNext = NULL;
  719. src_mb->srcAccessMask = VK_ACCESS_MEMORY_READ_BIT;
  720. src_mb->dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
  721. src_mb->oldLayout = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR;
  722. src_mb->newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
  723. src_mb->srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
  724. src_mb->dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
  725. src_mb->image = cur_backbuffer;
  726. src_mb->subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
  727. src_mb->subresourceRange.baseMipLevel = 0;
  728. src_mb->subresourceRange.levelCount = 1;
  729. src_mb->subresourceRange.baseArrayLayer = 0;
  730. src_mb->subresourceRange.layerCount = 1;
  731. /* ------------------------------------------------------ */
  732. /* transition exportedTexture to transfer dest state */
  733. dst_mb->sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER;
  734. dst_mb->pNext = NULL;
  735. dst_mb->srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
  736. dst_mb->dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
  737. dst_mb->oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
  738. dst_mb->newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
  739. dst_mb->srcQueueFamilyIndex = VK_QUEUE_FAMILY_EXTERNAL;
  740. dst_mb->dstQueueFamilyIndex = fam_idx;
  741. dst_mb->image = swap->export_image;
  742. dst_mb->subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
  743. dst_mb->subresourceRange.baseMipLevel = 0;
  744. dst_mb->subresourceRange.levelCount = 1;
  745. dst_mb->subresourceRange.baseArrayLayer = 0;
  746. dst_mb->subresourceRange.layerCount = 1;
  747. funcs->CmdPipelineBarrier(cmd_buffer,
  748. VK_PIPELINE_STAGE_TRANSFER_BIT |
  749. VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
  750. VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, NULL, 0,
  751. NULL, 2, mb);
  752. /* ------------------------------------------------------ */
  753. /* copy cur_backbuffer's content to our interop image */
  754. VkImageCopy cpy;
  755. cpy.srcSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
  756. cpy.srcSubresource.mipLevel = 0;
  757. cpy.srcSubresource.baseArrayLayer = 0;
  758. cpy.srcSubresource.layerCount = 1;
  759. cpy.srcOffset.x = 0;
  760. cpy.srcOffset.y = 0;
  761. cpy.srcOffset.z = 0;
  762. cpy.dstSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
  763. cpy.dstSubresource.mipLevel = 0;
  764. cpy.dstSubresource.baseArrayLayer = 0;
  765. cpy.dstSubresource.layerCount = 1;
  766. cpy.dstOffset.x = 0;
  767. cpy.dstOffset.y = 0;
  768. cpy.dstOffset.z = 0;
  769. cpy.extent.width = swap->image_extent.width;
  770. cpy.extent.height = swap->image_extent.height;
  771. cpy.extent.depth = 1;
  772. funcs->CmdCopyImage(cmd_buffer, cur_backbuffer,
  773. VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
  774. swap->export_image,
  775. VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &cpy);
  776. /* ------------------------------------------------------ */
  777. /* Restore the swap chain image layout to what it was
  778. * before. This may not be strictly needed, but it is
  779. * generally good to restore things to their original
  780. * state. */
  781. src_mb->srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
  782. src_mb->dstAccessMask = VK_ACCESS_MEMORY_READ_BIT;
  783. src_mb->oldLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
  784. src_mb->newLayout = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR;
  785. dst_mb->srcQueueFamilyIndex = fam_idx;
  786. dst_mb->dstQueueFamilyIndex = VK_QUEUE_FAMILY_EXTERNAL;
  787. funcs->CmdPipelineBarrier(cmd_buffer, VK_PIPELINE_STAGE_TRANSFER_BIT,
  788. VK_PIPELINE_STAGE_TRANSFER_BIT |
  789. VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
  790. 0, 0, NULL, 0, NULL, 2, mb);
  791. funcs->EndCommandBuffer(cmd_buffer);
  792. /* ------------------------------------------------------ */
  793. VkSubmitInfo submit_info;
  794. submit_info.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO;
  795. submit_info.pNext = NULL;
  796. submit_info.waitSemaphoreCount = 0;
  797. submit_info.pWaitSemaphores = NULL;
  798. submit_info.pWaitDstStageMask = NULL;
  799. submit_info.commandBufferCount = 1;
  800. submit_info.pCommandBuffers = &cmd_buffer;
  801. submit_info.signalSemaphoreCount = 0;
  802. submit_info.pSignalSemaphores = NULL;
  803. VkFence fence = pool_data->fences[image_index];
  804. res = funcs->QueueSubmit(queue, 1, &submit_info, fence);
  805. debug_res("QueueSubmit", res);
  806. if (res == VK_SUCCESS)
  807. pool_data->cmd_buffer_busy[image_index] = true;
  808. }
  809. static inline HWND get_swap_window(struct vk_swap_data *swap)
  810. {
  811. for (size_t i = 0; i < OBJ_MAX; i++) {
  812. struct vk_surf_data *surf_data =
  813. find_surf_data(&inst_data[i], swap->surf);
  814. if (!!surf_data && surf_data->surf == swap->surf) {
  815. return surf_data->hwnd;
  816. }
  817. }
  818. return NULL;
  819. }
  820. static void vk_capture(struct vk_data *data, VkQueue queue,
  821. const VkPresentInfoKHR *info)
  822. {
  823. struct vk_swap_data *swap = NULL;
  824. HWND window = NULL;
  825. uint32_t idx = 0;
  826. debug("QueuePresentKHR called on "
  827. "devicekey %p, swapchain count %d",
  828. &data->funcs, info->swapchainCount);
  829. /* use first swap chain associated with a window */
  830. for (; idx < info->swapchainCount; idx++) {
  831. struct vk_swap_data *cur_swap =
  832. get_swap_data(data, info->pSwapchains[idx]);
  833. window = get_swap_window(cur_swap);
  834. if (!!window) {
  835. swap = cur_swap;
  836. break;
  837. }
  838. }
  839. if (!window) {
  840. return;
  841. }
  842. if (capture_should_stop()) {
  843. vk_shtex_free(data);
  844. }
  845. if (capture_should_init()) {
  846. if (!vk_shtex_init(data, window, swap)) {
  847. vk_shtex_free(data);
  848. data->valid = false;
  849. }
  850. }
  851. if (capture_ready()) {
  852. if (swap != data->cur_swap) {
  853. vk_shtex_free(data);
  854. return;
  855. }
  856. vk_shtex_capture(data, &data->funcs, swap, idx, queue, info);
  857. }
  858. }
  859. static VkResult VKAPI OBS_QueuePresentKHR(VkQueue queue,
  860. const VkPresentInfoKHR *info)
  861. {
  862. struct vk_data *data = get_device_data(queue);
  863. struct vk_device_funcs *funcs = &data->funcs;
  864. if (data->valid) {
  865. vk_capture(data, queue, info);
  866. }
  867. return funcs->QueuePresentKHR(queue, info);
  868. }
  869. /* ======================================================================== */
  870. /* setup hooks */
  871. static inline bool is_inst_link_info(VkLayerInstanceCreateInfo *lici)
  872. {
  873. return lici->sType == VK_STRUCTURE_TYPE_LOADER_INSTANCE_CREATE_INFO &&
  874. lici->function == VK_LAYER_LINK_INFO;
  875. }
  876. static VkResult VKAPI OBS_CreateInstance(const VkInstanceCreateInfo *cinfo,
  877. const VkAllocationCallbacks *ac,
  878. VkInstance *p_inst)
  879. {
  880. VkInstanceCreateInfo info = *cinfo;
  881. bool funcs_not_found = false;
  882. /* -------------------------------------------------------- */
  883. /* step through chain until we get to the link info */
  884. VkLayerInstanceCreateInfo *lici = (void *)info.pNext;
  885. while (lici && !is_inst_link_info(lici)) {
  886. lici = (VkLayerInstanceCreateInfo *)lici->pNext;
  887. }
  888. if (lici == NULL) {
  889. return VK_ERROR_INITIALIZATION_FAILED;
  890. }
  891. PFN_vkGetInstanceProcAddr gpa =
  892. lici->u.pLayerInfo->pfnNextGetInstanceProcAddr;
  893. /* -------------------------------------------------------- */
  894. /* move chain on for next layer */
  895. lici->u.pLayerInfo = lici->u.pLayerInfo->pNext;
  896. /* -------------------------------------------------------- */
  897. /* (HACK) Set api version to 1.1 if set to 1.0 */
  898. /* We do this to get our extensions working properly */
  899. VkApplicationInfo ai;
  900. if (info.pApplicationInfo) {
  901. ai = *info.pApplicationInfo;
  902. if (ai.apiVersion < VK_API_VERSION_1_1)
  903. ai.apiVersion = VK_API_VERSION_1_1;
  904. } else {
  905. ai.sType = VK_STRUCTURE_TYPE_APPLICATION_INFO;
  906. ai.pNext = NULL;
  907. ai.pApplicationName = NULL;
  908. ai.applicationVersion = 0;
  909. ai.pEngineName = NULL;
  910. ai.engineVersion = 0;
  911. ai.apiVersion = VK_API_VERSION_1_1;
  912. }
  913. /* -------------------------------------------------------- */
  914. /* create instance */
  915. PFN_vkCreateInstance create = (void *)gpa(NULL, "vkCreateInstance");
  916. VkResult res = create(&info, ac, p_inst);
  917. VkInstance inst = *p_inst;
  918. /* -------------------------------------------------------- */
  919. /* fetch the functions we need */
  920. struct vk_inst_data *data = get_inst_data(inst);
  921. struct vk_inst_funcs *funcs = &data->funcs;
  922. #define GETADDR(x) \
  923. do { \
  924. funcs->x = (void *)gpa(inst, "vk" #x); \
  925. if (!funcs->x) { \
  926. flog("could not get instance " \
  927. "address for %s", \
  928. #x); \
  929. funcs_not_found = true; \
  930. } \
  931. } while (false)
  932. GETADDR(GetInstanceProcAddr);
  933. GETADDR(DestroyInstance);
  934. GETADDR(CreateWin32SurfaceKHR);
  935. GETADDR(GetPhysicalDeviceMemoryProperties);
  936. GETADDR(GetPhysicalDeviceImageFormatProperties2);
  937. #undef GETADDR
  938. data->valid = !funcs_not_found;
  939. return res;
  940. }
  941. static VkResult VKAPI OBS_DestroyInstance(VkInstance instance,
  942. const VkAllocationCallbacks *ac)
  943. {
  944. struct vk_inst_funcs *funcs = get_inst_funcs(instance);
  945. funcs->DestroyInstance(instance, ac);
  946. remove_instance(instance);
  947. return VK_SUCCESS;
  948. }
  949. static bool
  950. vk_shared_tex_supported(struct vk_inst_funcs *funcs,
  951. VkPhysicalDevice phy_device, VkFormat format,
  952. VkImageUsageFlags usage,
  953. VkExternalMemoryProperties *external_mem_props)
  954. {
  955. VkPhysicalDeviceImageFormatInfo2 info;
  956. VkPhysicalDeviceExternalImageFormatInfo external_info;
  957. external_info.sType =
  958. VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_IMAGE_FORMAT_INFO;
  959. external_info.pNext = NULL;
  960. external_info.handleType =
  961. VK_EXTERNAL_MEMORY_HANDLE_TYPE_D3D11_TEXTURE_KMT_BIT;
  962. info.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2;
  963. info.pNext = &external_info;
  964. info.format = format;
  965. info.type = VK_IMAGE_TYPE_2D;
  966. info.tiling = VK_IMAGE_TILING_OPTIMAL;
  967. info.flags = 0;
  968. info.usage = usage;
  969. VkExternalImageFormatProperties external_props = {0};
  970. external_props.sType =
  971. VK_STRUCTURE_TYPE_EXTERNAL_IMAGE_FORMAT_PROPERTIES;
  972. external_props.pNext = NULL;
  973. VkImageFormatProperties2 props = {0};
  974. props.sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_PROPERTIES_2;
  975. props.pNext = &external_props;
  976. VkResult result = funcs->GetPhysicalDeviceImageFormatProperties2(
  977. phy_device, &info, &props);
  978. *external_mem_props = external_props.externalMemoryProperties;
  979. const VkExternalMemoryFeatureFlags features =
  980. external_mem_props->externalMemoryFeatures;
  981. return ((VK_SUCCESS == result) &&
  982. (features & VK_EXTERNAL_MEMORY_FEATURE_IMPORTABLE_BIT));
  983. }
  984. static inline bool is_device_link_info(VkLayerDeviceCreateInfo *lici)
  985. {
  986. return lici->sType == VK_STRUCTURE_TYPE_LOADER_DEVICE_CREATE_INFO &&
  987. lici->function == VK_LAYER_LINK_INFO;
  988. }
  989. static VkResult VKAPI OBS_CreateDevice(VkPhysicalDevice phy_device,
  990. const VkDeviceCreateInfo *cinfo,
  991. const VkAllocationCallbacks *ac,
  992. VkDevice *p_device)
  993. {
  994. VkDeviceCreateInfo info = *cinfo;
  995. struct vk_inst_data *idata = get_inst_data(phy_device);
  996. struct vk_inst_funcs *ifuncs = &idata->funcs;
  997. struct vk_data *data = NULL;
  998. VkResult ret = VK_ERROR_INITIALIZATION_FAILED;
  999. VkLayerDeviceCreateInfo *ldci = (void *)info.pNext;
  1000. /* -------------------------------------------------------- */
  1001. /* step through chain until we get to the link info */
  1002. while (ldci && !is_device_link_info(ldci)) {
  1003. ldci = (VkLayerDeviceCreateInfo *)ldci->pNext;
  1004. }
  1005. if (!ldci) {
  1006. goto fail;
  1007. }
  1008. PFN_vkGetInstanceProcAddr gipa;
  1009. PFN_vkGetDeviceProcAddr gdpa;
  1010. gipa = ldci->u.pLayerInfo->pfnNextGetInstanceProcAddr;
  1011. gdpa = ldci->u.pLayerInfo->pfnNextGetDeviceProcAddr;
  1012. /* -------------------------------------------------------- */
  1013. /* move chain on for next layer */
  1014. ldci->u.pLayerInfo = ldci->u.pLayerInfo->pNext;
  1015. /* -------------------------------------------------------- */
  1016. /* create device and initialize hook data */
  1017. PFN_vkCreateDevice createFunc =
  1018. (PFN_vkCreateDevice)gipa(VK_NULL_HANDLE, "vkCreateDevice");
  1019. ret = createFunc(phy_device, idata->valid ? &info : cinfo, ac,
  1020. p_device);
  1021. if (ret != VK_SUCCESS) {
  1022. goto fail;
  1023. }
  1024. VkDevice device = *p_device;
  1025. data = get_device_data(*p_device);
  1026. struct vk_device_funcs *dfuncs = &data->funcs;
  1027. data->valid = false; /* set true below if it doesn't go to fail */
  1028. data->phy_device = phy_device;
  1029. data->device = device;
  1030. /* -------------------------------------------------------- */
  1031. /* fetch the functions we need */
  1032. bool funcs_not_found = false;
  1033. #define GETADDR(x) \
  1034. do { \
  1035. dfuncs->x = (void *)gdpa(device, "vk" #x); \
  1036. if (!dfuncs->x) { \
  1037. flog("could not get device " \
  1038. "address for %s", \
  1039. #x); \
  1040. funcs_not_found = true; \
  1041. } \
  1042. } while (false)
  1043. #define GETADDR_OPTIONAL(x) \
  1044. do { \
  1045. dfuncs->x = (void *)gdpa(device, "vk" #x); \
  1046. } while (false)
  1047. GETADDR(GetDeviceProcAddr);
  1048. GETADDR(DestroyDevice);
  1049. GETADDR(CreateSwapchainKHR);
  1050. GETADDR(DestroySwapchainKHR);
  1051. GETADDR(QueuePresentKHR);
  1052. GETADDR(AllocateMemory);
  1053. GETADDR(FreeMemory);
  1054. GETADDR(BindImageMemory);
  1055. GETADDR(BindImageMemory2);
  1056. GETADDR(GetSwapchainImagesKHR);
  1057. GETADDR(CreateImage);
  1058. GETADDR(DestroyImage);
  1059. GETADDR(GetImageMemoryRequirements);
  1060. GETADDR(GetImageMemoryRequirements2);
  1061. GETADDR(BeginCommandBuffer);
  1062. GETADDR(EndCommandBuffer);
  1063. GETADDR(CmdCopyImage);
  1064. GETADDR(CmdPipelineBarrier);
  1065. GETADDR(GetDeviceQueue);
  1066. GETADDR(QueueSubmit);
  1067. GETADDR(CreateCommandPool);
  1068. GETADDR(DestroyCommandPool);
  1069. GETADDR(AllocateCommandBuffers);
  1070. GETADDR(CreateFence);
  1071. GETADDR(DestroyFence);
  1072. GETADDR(WaitForFences);
  1073. GETADDR(ResetFences);
  1074. #undef GETADDR_OPTIONAL
  1075. #undef GETADDR
  1076. if (funcs_not_found) {
  1077. goto fail;
  1078. }
  1079. if (!idata->valid) {
  1080. goto fail;
  1081. }
  1082. VkFormat format = VK_FORMAT_R8G8B8A8_UNORM;
  1083. VkImageUsageFlags usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT |
  1084. VK_IMAGE_USAGE_TRANSFER_DST_BIT;
  1085. if (!vk_shared_tex_supported(ifuncs, phy_device, format, usage,
  1086. &data->external_mem_props)) {
  1087. flog("texture sharing is not supported");
  1088. goto fail;
  1089. }
  1090. data->valid = true;
  1091. fail:
  1092. return ret;
  1093. }
  1094. static void VKAPI OBS_DestroyDevice(VkDevice device,
  1095. const VkAllocationCallbacks *ac)
  1096. {
  1097. struct vk_data *data = get_device_data(device);
  1098. if (!data)
  1099. return;
  1100. for (uint32_t fam_idx = 0; fam_idx < _countof(data->cmd_pools);
  1101. fam_idx++) {
  1102. struct vk_cmd_pool_data *pool_data = &data->cmd_pools[fam_idx];
  1103. if (pool_data->cmd_pool != VK_NULL_HANDLE) {
  1104. vk_shtex_destroy_cmd_pool_objects(data, pool_data);
  1105. }
  1106. }
  1107. data->queue_count = 0;
  1108. vk_remove_device(device);
  1109. data->funcs.DestroyDevice(device, ac);
  1110. }
  1111. static VkResult VKAPI
  1112. OBS_CreateSwapchainKHR(VkDevice device, const VkSwapchainCreateInfoKHR *cinfo,
  1113. const VkAllocationCallbacks *ac, VkSwapchainKHR *p_sc)
  1114. {
  1115. struct vk_data *data = get_device_data(device);
  1116. struct vk_device_funcs *funcs = &data->funcs;
  1117. struct vk_swap_data *swap = get_new_swap_data(data);
  1118. swap->surf = cinfo->surface;
  1119. swap->image_extent = cinfo->imageExtent;
  1120. swap->format = cinfo->imageFormat;
  1121. VkSwapchainCreateInfoKHR info = *cinfo;
  1122. info.imageUsage |= VK_IMAGE_USAGE_TRANSFER_SRC_BIT;
  1123. VkResult res = funcs->CreateSwapchainKHR(device, &info, ac, p_sc);
  1124. VkSwapchainKHR sc = *p_sc;
  1125. uint32_t count = 0;
  1126. res = funcs->GetSwapchainImagesKHR(data->device, sc, &count, NULL);
  1127. debug_res("GetSwapchainImagesKHR", res);
  1128. if (count > 0) {
  1129. if (count > OBJ_MAX)
  1130. count = OBJ_MAX;
  1131. res = funcs->GetSwapchainImagesKHR(data->device, sc, &count,
  1132. swap->swap_images);
  1133. debug_res("GetSwapchainImagesKHR", res);
  1134. swap->image_count = count;
  1135. }
  1136. swap->sc = sc;
  1137. return res;
  1138. }
  1139. static void VKAPI OBS_DestroySwapchainKHR(VkDevice device, VkSwapchainKHR sc,
  1140. const VkAllocationCallbacks *ac)
  1141. {
  1142. struct vk_data *data = get_device_data(device);
  1143. struct vk_device_funcs *funcs = &data->funcs;
  1144. struct vk_swap_data *swap = get_swap_data(data, sc);
  1145. if (swap) {
  1146. if (data->cur_swap == swap) {
  1147. vk_shtex_free(data);
  1148. }
  1149. swap->sc = VK_NULL_HANDLE;
  1150. swap->surf = NULL;
  1151. }
  1152. funcs->DestroySwapchainKHR(device, sc, ac);
  1153. }
  1154. static void VKAPI OBS_GetDeviceQueue(VkDevice device, uint32_t queueFamilyIndex,
  1155. uint32_t queueIndex, VkQueue *pQueue)
  1156. {
  1157. struct vk_data *data = get_device_data(device);
  1158. struct vk_device_funcs *funcs = &data->funcs;
  1159. funcs->GetDeviceQueue(device, queueFamilyIndex, queueIndex, pQueue);
  1160. for (uint32_t i = 0; i < data->queue_count; ++i) {
  1161. if (data->queues[i].queue == *pQueue)
  1162. return;
  1163. }
  1164. if (data->queue_count < _countof(data->queues)) {
  1165. data->queues[data->queue_count].queue = *pQueue;
  1166. data->queues[data->queue_count].fam_idx = queueFamilyIndex;
  1167. ++data->queue_count;
  1168. }
  1169. }
  1170. static VkResult VKAPI OBS_CreateWin32SurfaceKHR(
  1171. VkInstance inst, const VkWin32SurfaceCreateInfoKHR *info,
  1172. const VkAllocationCallbacks *ac, VkSurfaceKHR *surf)
  1173. {
  1174. struct vk_inst_data *data = get_inst_data(inst);
  1175. struct vk_inst_funcs *funcs = &data->funcs;
  1176. VkResult res = funcs->CreateWin32SurfaceKHR(inst, info, ac, surf);
  1177. if (NULL != surf && VK_NULL_HANDLE != *surf) {
  1178. struct vk_surf_data *surf_data = find_surf_data(data, *surf);
  1179. surf_data->hinstance = info->hinstance;
  1180. surf_data->hwnd = info->hwnd;
  1181. }
  1182. return res;
  1183. }
  1184. #define GETPROCADDR(func) \
  1185. if (!strcmp(name, "vk" #func)) \
  1186. return (VkFunc)&OBS_##func;
  1187. static VkFunc VKAPI OBS_GetDeviceProcAddr(VkDevice dev, const char *name)
  1188. {
  1189. struct vk_data *data = get_device_data(dev);
  1190. struct vk_device_funcs *funcs = &data->funcs;
  1191. debug_procaddr("vkGetDeviceProcAddr(%p, \"%s\")", dev, name);
  1192. GETPROCADDR(GetDeviceProcAddr);
  1193. GETPROCADDR(CreateDevice);
  1194. GETPROCADDR(DestroyDevice);
  1195. GETPROCADDR(CreateSwapchainKHR);
  1196. GETPROCADDR(DestroySwapchainKHR);
  1197. GETPROCADDR(QueuePresentKHR);
  1198. GETPROCADDR(GetDeviceQueue);
  1199. if (funcs->GetDeviceProcAddr == NULL)
  1200. return NULL;
  1201. return funcs->GetDeviceProcAddr(dev, name);
  1202. }
  1203. static VkFunc VKAPI OBS_GetInstanceProcAddr(VkInstance inst, const char *name)
  1204. {
  1205. debug_procaddr("vkGetInstanceProcAddr(%p, \"%s\")", inst, name);
  1206. /* instance chain functions we intercept */
  1207. GETPROCADDR(GetInstanceProcAddr);
  1208. GETPROCADDR(CreateInstance);
  1209. GETPROCADDR(DestroyInstance);
  1210. GETPROCADDR(CreateWin32SurfaceKHR);
  1211. /* device chain functions we intercept */
  1212. GETPROCADDR(GetDeviceProcAddr);
  1213. GETPROCADDR(CreateDevice);
  1214. GETPROCADDR(DestroyDevice);
  1215. struct vk_inst_funcs *funcs = get_inst_funcs(inst);
  1216. if (funcs->GetInstanceProcAddr == NULL)
  1217. return NULL;
  1218. return funcs->GetInstanceProcAddr(inst, name);
  1219. }
  1220. #undef GETPROCADDR
  1221. EXPORT VkResult VKAPI OBS_Negotiate(VkNegotiateLayerInterface *nli)
  1222. {
  1223. if (nli->loaderLayerInterfaceVersion >= 2) {
  1224. nli->sType = LAYER_NEGOTIATE_INTERFACE_STRUCT;
  1225. nli->pNext = NULL;
  1226. nli->pfnGetInstanceProcAddr = OBS_GetInstanceProcAddr;
  1227. nli->pfnGetDeviceProcAddr = OBS_GetDeviceProcAddr;
  1228. nli->pfnGetPhysicalDeviceProcAddr = NULL;
  1229. }
  1230. const uint32_t cur_ver = CURRENT_LOADER_LAYER_INTERFACE_VERSION;
  1231. if (nli->loaderLayerInterfaceVersion > cur_ver) {
  1232. nli->loaderLayerInterfaceVersion = cur_ver;
  1233. }
  1234. return VK_SUCCESS;
  1235. }
  1236. bool hook_vulkan(void)
  1237. {
  1238. static bool hooked = false;
  1239. if (!hooked && vulkan_seen) {
  1240. hlog("Hooked Vulkan");
  1241. hooked = true;
  1242. }
  1243. return hooked;
  1244. }
  1245. static bool vulkan_initialized = false;
  1246. bool init_vk_layer()
  1247. {
  1248. if (!vulkan_initialized) {
  1249. InitializeCriticalSection(&mutex);
  1250. vulkan_initialized = true;
  1251. }
  1252. return true;
  1253. }
  1254. bool shutdown_vk_layer()
  1255. {
  1256. if (vulkan_initialized) {
  1257. DeleteCriticalSection(&mutex);
  1258. }
  1259. return true;
  1260. }