vulkan-capture.c 43 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614
  1. #include <windows.h>
  2. #include "graphics-hook.h"
  3. #define VK_USE_PLATFORM_WIN32_KHR
  4. #include <malloc.h>
  5. #include <vulkan/vulkan.h>
  6. #include <vulkan/vk_layer.h>
  7. #undef VK_LAYER_EXPORT
  8. #if defined(WIN32)
  9. #define VK_LAYER_EXPORT __declspec(dllexport)
  10. #else
  11. #define VK_LAYER_EXPORT
  12. #endif
  13. #include <vulkan/vulkan_win32.h>
  14. #define COBJMACROS
  15. #include <dxgi.h>
  16. #include <d3d11.h>
  17. #include "vulkan-capture.h"
  18. /* ======================================================================== */
  19. /* defs/statics */
  20. /* shorten stuff because dear GOD is vulkan unclean. */
  21. #define VKAPI VKAPI_CALL
  22. #define VkFunc PFN_vkVoidFunction
  23. #define EXPORT VK_LAYER_EXPORT
  24. #define OBJ_MAX 16
  25. /* use the loader's dispatch table pointer as a key for internal data maps */
  26. #define GET_LDT(x) (*(void **)x)
  27. static bool vulkan_seen = false;
  28. static SRWLOCK mutex = SRWLOCK_INIT; // Faster CRITICAL_SECTION
  29. /* ======================================================================== */
  30. /* hook data */
  31. struct vk_swap_data {
  32. VkSwapchainKHR sc;
  33. VkExtent2D image_extent;
  34. VkFormat format;
  35. HWND hwnd;
  36. VkImage export_image;
  37. bool layout_initialized;
  38. VkDeviceMemory export_mem;
  39. VkImage swap_images[OBJ_MAX];
  40. uint32_t image_count;
  41. HANDLE handle;
  42. struct shtex_data *shtex_info;
  43. ID3D11Texture2D *d3d11_tex;
  44. bool captured;
  45. };
  46. struct vk_queue_data {
  47. VkQueue queue;
  48. uint32_t fam_idx;
  49. };
  50. struct vk_frame_data {
  51. VkCommandPool cmd_pool;
  52. VkCommandBuffer cmd_buffer;
  53. VkFence fence;
  54. bool cmd_buffer_busy;
  55. };
  56. struct vk_family_data {
  57. struct vk_frame_data frames[OBJ_MAX];
  58. uint32_t frame_index;
  59. uint32_t frame_count;
  60. };
  61. struct vk_data {
  62. bool valid;
  63. struct vk_device_funcs funcs;
  64. VkPhysicalDevice phy_device;
  65. VkDevice device;
  66. struct vk_swap_data swaps[OBJ_MAX];
  67. struct vk_swap_data *cur_swap;
  68. uint32_t swap_idx;
  69. struct vk_queue_data queues[OBJ_MAX];
  70. uint32_t queue_count;
  71. struct vk_family_data families[OBJ_MAX];
  72. VkExternalMemoryProperties external_mem_props;
  73. struct vk_inst_data *inst_data;
  74. VkAllocationCallbacks ac_storage;
  75. const VkAllocationCallbacks *ac;
  76. ID3D11Device *d3d11_device;
  77. ID3D11DeviceContext *d3d11_context;
  78. };
  79. static struct vk_swap_data *get_swap_data(struct vk_data *data,
  80. VkSwapchainKHR sc)
  81. {
  82. for (int i = 0; i < OBJ_MAX; i++) {
  83. if (data->swaps[i].sc == sc) {
  84. return &data->swaps[i];
  85. }
  86. }
  87. debug("get_swap_data failed, swapchain not found");
  88. return NULL;
  89. }
  90. static struct vk_swap_data *get_new_swap_data(struct vk_data *data)
  91. {
  92. for (int i = 0; i < OBJ_MAX; i++) {
  93. if (data->swaps[i].sc == VK_NULL_HANDLE) {
  94. return &data->swaps[i];
  95. }
  96. }
  97. debug("get_new_swap_data failed, no more free slot");
  98. return NULL;
  99. }
  100. /* ------------------------------------------------------------------------- */
  101. static inline size_t find_obj_idx(void *objs[], void *obj)
  102. {
  103. size_t idx = SIZE_MAX;
  104. AcquireSRWLockExclusive(&mutex);
  105. for (size_t i = 0; i < OBJ_MAX; i++) {
  106. if (objs[i] == obj) {
  107. idx = i;
  108. break;
  109. }
  110. }
  111. ReleaseSRWLockExclusive(&mutex);
  112. return idx;
  113. }
  114. static size_t get_obj_idx(void *objs[], void *obj)
  115. {
  116. size_t idx = SIZE_MAX;
  117. AcquireSRWLockExclusive(&mutex);
  118. for (size_t i = 0; i < OBJ_MAX; i++) {
  119. if (objs[i] == obj) {
  120. idx = i;
  121. break;
  122. }
  123. if (!objs[i] && idx == SIZE_MAX) {
  124. idx = i;
  125. }
  126. }
  127. ReleaseSRWLockExclusive(&mutex);
  128. return idx;
  129. }
  130. /* ------------------------------------------------------------------------- */
  131. static struct vk_data device_data[OBJ_MAX] = {0};
  132. static void *devices[OBJ_MAX] = {0};
  133. static inline struct vk_data *get_device_data(void *dev)
  134. {
  135. size_t idx = get_obj_idx(devices, GET_LDT(dev));
  136. if (idx == SIZE_MAX) {
  137. debug("out of device slots");
  138. return NULL;
  139. }
  140. return &device_data[idx];
  141. }
  142. static void vk_shtex_clear_fence(const struct vk_data *data,
  143. struct vk_frame_data *frame_data)
  144. {
  145. const VkFence fence = frame_data->fence;
  146. if (frame_data->cmd_buffer_busy) {
  147. VkDevice device = data->device;
  148. const struct vk_device_funcs *funcs = &data->funcs;
  149. funcs->WaitForFences(device, 1, &fence, VK_TRUE, ~0ull);
  150. funcs->ResetFences(device, 1, &fence);
  151. frame_data->cmd_buffer_busy = false;
  152. }
  153. }
  154. static void vk_shtex_wait_until_pool_idle(struct vk_data *data,
  155. struct vk_family_data *family_data)
  156. {
  157. for (uint32_t frame_idx = 0; frame_idx < family_data->frame_count;
  158. frame_idx++) {
  159. struct vk_frame_data *frame_data =
  160. &family_data->frames[frame_idx];
  161. if (frame_data->cmd_pool != VK_NULL_HANDLE)
  162. vk_shtex_clear_fence(data, frame_data);
  163. }
  164. }
  165. static void vk_shtex_wait_until_idle(struct vk_data *data)
  166. {
  167. for (uint32_t fam_idx = 0; fam_idx < _countof(data->families);
  168. fam_idx++) {
  169. struct vk_family_data *family_data = &data->families[fam_idx];
  170. vk_shtex_wait_until_pool_idle(data, family_data);
  171. }
  172. }
  173. static void vk_shtex_free(struct vk_data *data)
  174. {
  175. capture_free();
  176. vk_shtex_wait_until_idle(data);
  177. for (int swap_idx = 0; swap_idx < OBJ_MAX; swap_idx++) {
  178. struct vk_swap_data *swap = &data->swaps[swap_idx];
  179. if (swap->export_image)
  180. data->funcs.DestroyImage(data->device,
  181. swap->export_image, data->ac);
  182. if (swap->export_mem)
  183. data->funcs.FreeMemory(data->device, swap->export_mem,
  184. NULL);
  185. if (swap->d3d11_tex) {
  186. ID3D11Texture2D_Release(swap->d3d11_tex);
  187. }
  188. swap->handle = INVALID_HANDLE_VALUE;
  189. swap->d3d11_tex = NULL;
  190. swap->export_mem = VK_NULL_HANDLE;
  191. swap->export_image = VK_NULL_HANDLE;
  192. swap->captured = false;
  193. }
  194. if (data->d3d11_context) {
  195. ID3D11DeviceContext_Release(data->d3d11_context);
  196. data->d3d11_context = NULL;
  197. }
  198. if (data->d3d11_device) {
  199. ID3D11Device_Release(data->d3d11_device);
  200. data->d3d11_device = NULL;
  201. }
  202. data->cur_swap = NULL;
  203. hlog("------------------ vulkan capture freed ------------------");
  204. }
  205. static void vk_remove_device(void *dev)
  206. {
  207. size_t idx = find_obj_idx(devices, GET_LDT(dev));
  208. if (idx == SIZE_MAX) {
  209. return;
  210. }
  211. struct vk_data *data = &device_data[idx];
  212. memset(data, 0, sizeof(*data));
  213. AcquireSRWLockExclusive(&mutex);
  214. devices[idx] = NULL;
  215. ReleaseSRWLockExclusive(&mutex);
  216. }
  217. /* ------------------------------------------------------------------------- */
  218. struct vk_surf_data {
  219. VkSurfaceKHR surf;
  220. HWND hwnd;
  221. struct vk_surf_data *next;
  222. };
  223. struct vk_inst_data {
  224. bool valid;
  225. struct vk_inst_funcs funcs;
  226. struct vk_surf_data *surfaces;
  227. };
  228. static void *object_malloc(const VkAllocationCallbacks *ac, size_t size,
  229. size_t alignment)
  230. {
  231. return ac ? ac->pfnAllocation(ac->pUserData, size, alignment,
  232. VK_SYSTEM_ALLOCATION_SCOPE_OBJECT)
  233. : _aligned_malloc(size, alignment);
  234. }
  235. static void object_free(const VkAllocationCallbacks *ac, void *memory)
  236. {
  237. if (ac)
  238. ac->pfnFree(ac->pUserData, memory);
  239. else
  240. _aligned_free(memory);
  241. }
  242. static void insert_surf_data(struct vk_inst_data *data, VkSurfaceKHR surf,
  243. HWND hwnd, const VkAllocationCallbacks *ac)
  244. {
  245. struct vk_surf_data *surf_data = object_malloc(
  246. ac, sizeof(struct vk_surf_data), _Alignof(struct vk_surf_data));
  247. if (surf_data) {
  248. surf_data->surf = surf;
  249. surf_data->hwnd = hwnd;
  250. AcquireSRWLockExclusive(&mutex);
  251. struct vk_surf_data *next = data->surfaces;
  252. surf_data->next = next;
  253. data->surfaces = surf_data;
  254. ReleaseSRWLockExclusive(&mutex);
  255. }
  256. }
  257. static HWND find_surf_hwnd(struct vk_inst_data *data, VkSurfaceKHR surf)
  258. {
  259. HWND hwnd = NULL;
  260. AcquireSRWLockExclusive(&mutex);
  261. struct vk_surf_data *surf_data = data->surfaces;
  262. while (surf_data) {
  263. if (surf_data->surf == surf) {
  264. hwnd = surf_data->hwnd;
  265. break;
  266. }
  267. surf_data = surf_data->next;
  268. }
  269. ReleaseSRWLockExclusive(&mutex);
  270. return hwnd;
  271. }
  272. static void erase_surf_data(struct vk_inst_data *data, VkSurfaceKHR surf,
  273. const VkAllocationCallbacks *ac)
  274. {
  275. AcquireSRWLockExclusive(&mutex);
  276. struct vk_surf_data *current = data->surfaces;
  277. if (current->surf == surf) {
  278. data->surfaces = current->next;
  279. } else {
  280. struct vk_surf_data *previous;
  281. do {
  282. previous = current;
  283. current = current->next;
  284. } while (current && current->surf != surf);
  285. if (current)
  286. previous->next = current->next;
  287. }
  288. ReleaseSRWLockExclusive(&mutex);
  289. object_free(ac, current);
  290. }
  291. /* ------------------------------------------------------------------------- */
  292. static struct vk_inst_data inst_data[OBJ_MAX] = {0};
  293. static void *instances[OBJ_MAX] = {0};
  294. static struct vk_inst_data *get_inst_data(void *inst)
  295. {
  296. size_t idx = get_obj_idx(instances, GET_LDT(inst));
  297. if (idx == SIZE_MAX) {
  298. debug("out of instance slots");
  299. return NULL;
  300. }
  301. vulkan_seen = true;
  302. return &inst_data[idx];
  303. }
  304. static inline struct vk_inst_funcs *get_inst_funcs(void *inst)
  305. {
  306. struct vk_inst_data *data = get_inst_data(inst);
  307. return &data->funcs;
  308. }
  309. static void remove_instance(void *inst)
  310. {
  311. size_t idx = find_obj_idx(instances, inst);
  312. if (idx == SIZE_MAX) {
  313. return;
  314. }
  315. struct vk_inst_data *data = &inst_data[idx];
  316. memset(data, 0, sizeof(*data));
  317. AcquireSRWLockExclusive(&mutex);
  318. instances[idx] = NULL;
  319. ReleaseSRWLockExclusive(&mutex);
  320. }
  321. /* ======================================================================== */
  322. /* capture */
  323. static inline bool vk_shtex_init_d3d11(struct vk_data *data)
  324. {
  325. D3D_FEATURE_LEVEL level_used;
  326. IDXGIFactory1 *factory;
  327. IDXGIAdapter1 *adapter;
  328. HRESULT hr;
  329. HMODULE d3d11 = load_system_library("d3d11.dll");
  330. if (!d3d11) {
  331. flog("failed to load d3d11: %d", GetLastError());
  332. return false;
  333. }
  334. HMODULE dxgi = load_system_library("dxgi.dll");
  335. if (!dxgi) {
  336. flog("failed to load dxgi: %d", GetLastError());
  337. return false;
  338. }
  339. HRESULT(WINAPI * create_factory)
  340. (REFIID, void **) = (void *)GetProcAddress(dxgi, "CreateDXGIFactory1");
  341. if (!create_factory) {
  342. flog("failed to get CreateDXGIFactory1 address: %d",
  343. GetLastError());
  344. return false;
  345. }
  346. PFN_D3D11_CREATE_DEVICE create =
  347. (void *)GetProcAddress(d3d11, "D3D11CreateDevice");
  348. if (!create) {
  349. flog("failed to get D3D11CreateDevice address: %d",
  350. GetLastError());
  351. return false;
  352. }
  353. hr = create_factory(&IID_IDXGIFactory1, &factory);
  354. if (FAILED(hr)) {
  355. flog_hr("failed to create factory", hr);
  356. return false;
  357. }
  358. hr = IDXGIFactory1_EnumAdapters1(factory, 0, &adapter);
  359. IDXGIFactory1_Release(factory);
  360. if (FAILED(hr)) {
  361. flog_hr("failed to create adapter", hr);
  362. return false;
  363. }
  364. static const D3D_FEATURE_LEVEL feature_levels[] = {
  365. D3D_FEATURE_LEVEL_11_0,
  366. D3D_FEATURE_LEVEL_10_1,
  367. D3D_FEATURE_LEVEL_10_0,
  368. D3D_FEATURE_LEVEL_9_3,
  369. };
  370. hr = create((IDXGIAdapter *)adapter, D3D_DRIVER_TYPE_UNKNOWN, NULL, 0,
  371. feature_levels,
  372. sizeof(feature_levels) / sizeof(D3D_FEATURE_LEVEL),
  373. D3D11_SDK_VERSION, &data->d3d11_device, &level_used,
  374. &data->d3d11_context);
  375. IDXGIAdapter1_Release(adapter);
  376. if (FAILED(hr)) {
  377. flog_hr("failed to create device", hr);
  378. return false;
  379. }
  380. return true;
  381. }
  382. static inline bool vk_shtex_init_d3d11_tex(struct vk_data *data,
  383. struct vk_swap_data *swap)
  384. {
  385. IDXGIResource *dxgi_res;
  386. HRESULT hr;
  387. const UINT width = swap->image_extent.width;
  388. const UINT height = swap->image_extent.height;
  389. flog("OBS requesting %s texture format. capture dimensions: %ux%u",
  390. vk_format_to_str(swap->format), width, height);
  391. const DXGI_FORMAT format = vk_format_to_dxgi(swap->format);
  392. if (format == DXGI_FORMAT_UNKNOWN) {
  393. flog("cannot convert to DXGI format");
  394. return false;
  395. }
  396. D3D11_TEXTURE2D_DESC desc = {0};
  397. desc.Width = width;
  398. desc.Height = height;
  399. desc.MipLevels = 1;
  400. desc.ArraySize = 1;
  401. desc.Format = format;
  402. desc.SampleDesc.Count = 1;
  403. desc.SampleDesc.Quality = 0;
  404. desc.Usage = D3D11_USAGE_DEFAULT;
  405. desc.MiscFlags = D3D11_RESOURCE_MISC_SHARED;
  406. desc.BindFlags = D3D11_BIND_RENDER_TARGET | D3D11_BIND_SHADER_RESOURCE;
  407. hr = ID3D11Device_CreateTexture2D(data->d3d11_device, &desc, NULL,
  408. &swap->d3d11_tex);
  409. if (FAILED(hr)) {
  410. flog_hr("failed to create texture", hr);
  411. return false;
  412. }
  413. hr = ID3D11Texture2D_QueryInterface(swap->d3d11_tex, &IID_IDXGIResource,
  414. &dxgi_res);
  415. if (FAILED(hr)) {
  416. flog_hr("failed to get IDXGIResource", hr);
  417. return false;
  418. }
  419. hr = IDXGIResource_GetSharedHandle(dxgi_res, &swap->handle);
  420. IDXGIResource_Release(dxgi_res);
  421. if (FAILED(hr)) {
  422. flog_hr("failed to get shared handle", hr);
  423. return false;
  424. }
  425. return true;
  426. }
  427. static inline bool vk_shtex_init_vulkan_tex(struct vk_data *data,
  428. struct vk_swap_data *swap)
  429. {
  430. struct vk_device_funcs *funcs = &data->funcs;
  431. VkExternalMemoryFeatureFlags f =
  432. data->external_mem_props.externalMemoryFeatures;
  433. /* -------------------------------------------------------- */
  434. /* create texture */
  435. VkExternalMemoryImageCreateInfo emici;
  436. emici.sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO;
  437. emici.pNext = NULL;
  438. emici.handleTypes =
  439. VK_EXTERNAL_MEMORY_HANDLE_TYPE_D3D11_TEXTURE_KMT_BIT;
  440. VkImageCreateInfo ici;
  441. ici.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO;
  442. ici.pNext = &emici;
  443. ici.flags = 0;
  444. ici.imageType = VK_IMAGE_TYPE_2D;
  445. ici.format = swap->format;
  446. ici.extent.width = swap->image_extent.width;
  447. ici.extent.height = swap->image_extent.height;
  448. ici.extent.depth = 1;
  449. ici.mipLevels = 1;
  450. ici.arrayLayers = 1;
  451. ici.samples = VK_SAMPLE_COUNT_1_BIT;
  452. ici.tiling = VK_IMAGE_TILING_OPTIMAL;
  453. ici.usage = VK_IMAGE_USAGE_TRANSFER_DST_BIT |
  454. VK_IMAGE_USAGE_SAMPLED_BIT;
  455. ici.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
  456. ici.queueFamilyIndexCount = 0;
  457. ici.pQueueFamilyIndices = 0;
  458. ici.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
  459. VkResult res;
  460. res = funcs->CreateImage(data->device, &ici, data->ac,
  461. &swap->export_image);
  462. if (VK_SUCCESS != res) {
  463. flog("failed to CreateImage: %s", result_to_str(res));
  464. swap->export_image = VK_NULL_HANDLE;
  465. return false;
  466. }
  467. swap->layout_initialized = false;
  468. /* -------------------------------------------------------- */
  469. /* get image memory requirements */
  470. VkMemoryRequirements mr;
  471. bool use_gimr2 = f & VK_EXTERNAL_MEMORY_FEATURE_DEDICATED_ONLY_BIT;
  472. if (use_gimr2) {
  473. VkMemoryDedicatedRequirements mdr = {0};
  474. mdr.sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS;
  475. mdr.pNext = NULL;
  476. VkMemoryRequirements2 mr2 = {0};
  477. mr2.sType = VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2;
  478. mr2.pNext = &mdr;
  479. VkImageMemoryRequirementsInfo2 imri2 = {0};
  480. imri2.sType =
  481. VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2;
  482. imri2.pNext = NULL;
  483. imri2.image = swap->export_image;
  484. funcs->GetImageMemoryRequirements2(data->device, &imri2, &mr2);
  485. mr = mr2.memoryRequirements;
  486. } else {
  487. funcs->GetImageMemoryRequirements(data->device,
  488. swap->export_image, &mr);
  489. }
  490. /* -------------------------------------------------------- */
  491. /* get memory type index */
  492. struct vk_inst_funcs *ifuncs = get_inst_funcs(data->phy_device);
  493. VkPhysicalDeviceMemoryProperties pdmp;
  494. ifuncs->GetPhysicalDeviceMemoryProperties(data->phy_device, &pdmp);
  495. uint32_t mem_type_idx = 0;
  496. for (; mem_type_idx < pdmp.memoryTypeCount; mem_type_idx++) {
  497. if ((mr.memoryTypeBits & (1 << mem_type_idx)) &&
  498. (pdmp.memoryTypes[mem_type_idx].propertyFlags &
  499. VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) ==
  500. VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) {
  501. break;
  502. }
  503. }
  504. if (mem_type_idx == pdmp.memoryTypeCount) {
  505. flog("failed to get memory type index");
  506. funcs->DestroyImage(data->device, swap->export_image, data->ac);
  507. swap->export_image = VK_NULL_HANDLE;
  508. return false;
  509. }
  510. /* -------------------------------------------------------- */
  511. /* allocate memory */
  512. VkImportMemoryWin32HandleInfoKHR imw32hi;
  513. imw32hi.sType = VK_STRUCTURE_TYPE_IMPORT_MEMORY_WIN32_HANDLE_INFO_KHR;
  514. imw32hi.pNext = NULL;
  515. imw32hi.name = NULL;
  516. imw32hi.handleType =
  517. VK_EXTERNAL_MEMORY_HANDLE_TYPE_D3D11_TEXTURE_KMT_BIT;
  518. imw32hi.handle = swap->handle;
  519. VkMemoryAllocateInfo mai;
  520. mai.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
  521. mai.pNext = &imw32hi;
  522. mai.allocationSize = mr.size;
  523. mai.memoryTypeIndex = mem_type_idx;
  524. VkMemoryDedicatedAllocateInfo mdai;
  525. mdai.sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO;
  526. mdai.pNext = NULL;
  527. mdai.buffer = VK_NULL_HANDLE;
  528. if (data->external_mem_props.externalMemoryFeatures &
  529. VK_EXTERNAL_MEMORY_FEATURE_DEDICATED_ONLY_BIT) {
  530. mdai.image = swap->export_image;
  531. imw32hi.pNext = &mdai;
  532. }
  533. res = funcs->AllocateMemory(data->device, &mai, NULL,
  534. &swap->export_mem);
  535. if (VK_SUCCESS != res) {
  536. flog("failed to AllocateMemory: %s", result_to_str(res));
  537. funcs->DestroyImage(data->device, swap->export_image, data->ac);
  538. swap->export_image = VK_NULL_HANDLE;
  539. return false;
  540. }
  541. /* -------------------------------------------------------- */
  542. /* bind image memory */
  543. bool use_bi2 = f & VK_EXTERNAL_MEMORY_FEATURE_DEDICATED_ONLY_BIT;
  544. if (use_bi2) {
  545. VkBindImageMemoryInfo bimi = {0};
  546. bimi.sType = VK_STRUCTURE_TYPE_BIND_IMAGE_MEMORY_INFO;
  547. bimi.image = swap->export_image;
  548. bimi.memory = swap->export_mem;
  549. bimi.memoryOffset = 0;
  550. res = funcs->BindImageMemory2(data->device, 1, &bimi);
  551. } else {
  552. res = funcs->BindImageMemory(data->device, swap->export_image,
  553. swap->export_mem, 0);
  554. }
  555. if (VK_SUCCESS != res) {
  556. flog("%s failed: %s",
  557. use_bi2 ? "BindImageMemory2" : "BindImageMemory",
  558. result_to_str(res));
  559. funcs->DestroyImage(data->device, swap->export_image, data->ac);
  560. swap->export_image = VK_NULL_HANDLE;
  561. return false;
  562. }
  563. return true;
  564. }
  565. static bool vk_shtex_init(struct vk_data *data, HWND window,
  566. struct vk_swap_data *swap)
  567. {
  568. if (!vk_shtex_init_d3d11(data)) {
  569. return false;
  570. }
  571. if (!vk_shtex_init_d3d11_tex(data, swap)) {
  572. return false;
  573. }
  574. if (!vk_shtex_init_vulkan_tex(data, swap)) {
  575. return false;
  576. }
  577. data->cur_swap = swap;
  578. swap->captured = capture_init_shtex(&swap->shtex_info, window,
  579. swap->image_extent.width,
  580. swap->image_extent.height,
  581. (uint32_t)swap->format, false,
  582. (uintptr_t)swap->handle);
  583. if (swap->captured) {
  584. if (global_hook_info->force_shmem) {
  585. flog("shared memory capture currently "
  586. "unsupported; ignoring");
  587. }
  588. hlog("vulkan shared texture capture successful");
  589. return true;
  590. }
  591. return false;
  592. }
  593. static void vk_shtex_create_family_objects(struct vk_data *data,
  594. uint32_t fam_idx,
  595. uint32_t image_count)
  596. {
  597. struct vk_family_data *family_data = &data->families[fam_idx];
  598. for (uint32_t image_index = 0; image_index < image_count;
  599. image_index++) {
  600. struct vk_frame_data *frame_data =
  601. &family_data->frames[image_index];
  602. VkCommandPoolCreateInfo cpci;
  603. cpci.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO;
  604. cpci.pNext = NULL;
  605. cpci.flags = 0;
  606. cpci.queueFamilyIndex = fam_idx;
  607. VkResult res = data->funcs.CreateCommandPool(
  608. data->device, &cpci, data->ac, &frame_data->cmd_pool);
  609. debug_res("CreateCommandPool", res);
  610. VkCommandBufferAllocateInfo cbai;
  611. cbai.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO;
  612. cbai.pNext = NULL;
  613. cbai.commandPool = frame_data->cmd_pool;
  614. cbai.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY;
  615. cbai.commandBufferCount = 1;
  616. res = data->funcs.AllocateCommandBuffers(
  617. data->device, &cbai, &frame_data->cmd_buffer);
  618. debug_res("AllocateCommandBuffers", res);
  619. *(void **)frame_data->cmd_buffer = *(void **)(data->device);
  620. VkFenceCreateInfo fci = {0};
  621. fci.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
  622. fci.pNext = NULL;
  623. fci.flags = 0;
  624. res = data->funcs.CreateFence(data->device, &fci, data->ac,
  625. &frame_data->fence);
  626. debug_res("CreateFence", res);
  627. }
  628. family_data->frame_index = 0;
  629. family_data->frame_count = image_count;
  630. }
  631. static void vk_shtex_destroy_fence(struct vk_data *data, bool *cmd_buffer_busy,
  632. VkFence *fence)
  633. {
  634. VkDevice device = data->device;
  635. if (*cmd_buffer_busy) {
  636. data->funcs.WaitForFences(device, 1, fence, VK_TRUE, ~0ull);
  637. *cmd_buffer_busy = false;
  638. }
  639. data->funcs.DestroyFence(device, *fence, data->ac);
  640. *fence = VK_NULL_HANDLE;
  641. }
  642. static void vk_shtex_destroy_family_objects(struct vk_data *data,
  643. struct vk_family_data *family_data)
  644. {
  645. for (uint32_t frame_idx = 0; frame_idx < family_data->frame_count;
  646. frame_idx++) {
  647. struct vk_frame_data *frame_data =
  648. &family_data->frames[frame_idx];
  649. bool *cmd_buffer_busy = &frame_data->cmd_buffer_busy;
  650. VkFence *fence = &frame_data->fence;
  651. vk_shtex_destroy_fence(data, cmd_buffer_busy, fence);
  652. data->funcs.DestroyCommandPool(data->device,
  653. frame_data->cmd_pool, data->ac);
  654. frame_data->cmd_pool = VK_NULL_HANDLE;
  655. }
  656. family_data->frame_count = 0;
  657. }
  658. static void vk_shtex_capture(struct vk_data *data,
  659. struct vk_device_funcs *funcs,
  660. struct vk_swap_data *swap, uint32_t idx,
  661. VkQueue queue, const VkPresentInfoKHR *info)
  662. {
  663. VkResult res = VK_SUCCESS;
  664. VkCommandBufferBeginInfo begin_info;
  665. begin_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
  666. begin_info.pNext = NULL;
  667. begin_info.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
  668. begin_info.pInheritanceInfo = NULL;
  669. VkImageMemoryBarrier mb[2];
  670. VkImageMemoryBarrier *src_mb = &mb[0];
  671. VkImageMemoryBarrier *dst_mb = &mb[1];
  672. /* ------------------------------------------------------ */
  673. /* do image copy */
  674. const uint32_t image_index = info->pImageIndices[idx];
  675. VkImage cur_backbuffer = swap->swap_images[image_index];
  676. uint32_t fam_idx = 0;
  677. for (uint32_t i = 0; i < data->queue_count; i++) {
  678. if (data->queues[i].queue == queue)
  679. fam_idx = data->queues[i].fam_idx;
  680. }
  681. if (fam_idx >= _countof(data->families))
  682. return;
  683. struct vk_family_data *family_data = &data->families[fam_idx];
  684. const uint32_t image_count = swap->image_count;
  685. if (family_data->frame_count < image_count) {
  686. if (family_data->frame_count > 0)
  687. vk_shtex_destroy_family_objects(data, family_data);
  688. vk_shtex_create_family_objects(data, fam_idx, image_count);
  689. }
  690. const uint32_t frame_index = family_data->frame_index;
  691. struct vk_frame_data *frame_data = &family_data->frames[frame_index];
  692. family_data->frame_index = (frame_index + 1) % family_data->frame_count;
  693. vk_shtex_clear_fence(data, frame_data);
  694. res = funcs->ResetCommandPool(data->device, frame_data->cmd_pool, 0);
  695. #ifdef MORE_DEBUGGING
  696. debug_res("ResetCommandPool", res);
  697. #endif
  698. const VkCommandBuffer cmd_buffer = frame_data->cmd_buffer;
  699. res = funcs->BeginCommandBuffer(cmd_buffer, &begin_info);
  700. #ifdef MORE_DEBUGGING
  701. debug_res("BeginCommandBuffer", res);
  702. #endif
  703. /* ------------------------------------------------------ */
  704. /* transition shared texture if necessary */
  705. if (!swap->layout_initialized) {
  706. VkImageMemoryBarrier imb;
  707. imb.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER;
  708. imb.pNext = NULL;
  709. imb.srcAccessMask = 0;
  710. imb.dstAccessMask = 0;
  711. imb.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED;
  712. imb.newLayout = VK_IMAGE_LAYOUT_GENERAL;
  713. imb.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
  714. imb.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
  715. imb.image = swap->export_image;
  716. imb.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
  717. imb.subresourceRange.baseMipLevel = 0;
  718. imb.subresourceRange.levelCount = 1;
  719. imb.subresourceRange.baseArrayLayer = 0;
  720. imb.subresourceRange.layerCount = 1;
  721. funcs->CmdPipelineBarrier(cmd_buffer,
  722. VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
  723. VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, 0,
  724. 0, NULL, 0, NULL, 1, &imb);
  725. swap->layout_initialized = true;
  726. }
  727. /* ------------------------------------------------------ */
  728. /* transition cur_backbuffer to transfer source state */
  729. src_mb->sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER;
  730. src_mb->pNext = NULL;
  731. src_mb->srcAccessMask = VK_ACCESS_MEMORY_READ_BIT;
  732. src_mb->dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
  733. src_mb->oldLayout = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR;
  734. src_mb->newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
  735. src_mb->srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
  736. src_mb->dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
  737. src_mb->image = cur_backbuffer;
  738. src_mb->subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
  739. src_mb->subresourceRange.baseMipLevel = 0;
  740. src_mb->subresourceRange.levelCount = 1;
  741. src_mb->subresourceRange.baseArrayLayer = 0;
  742. src_mb->subresourceRange.layerCount = 1;
  743. /* ------------------------------------------------------ */
  744. /* transition exportedTexture to transfer dest state */
  745. dst_mb->sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER;
  746. dst_mb->pNext = NULL;
  747. dst_mb->srcAccessMask = 0;
  748. dst_mb->dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
  749. dst_mb->oldLayout = VK_IMAGE_LAYOUT_GENERAL;
  750. dst_mb->newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
  751. dst_mb->srcQueueFamilyIndex = VK_QUEUE_FAMILY_EXTERNAL;
  752. dst_mb->dstQueueFamilyIndex = fam_idx;
  753. dst_mb->image = swap->export_image;
  754. dst_mb->subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
  755. dst_mb->subresourceRange.baseMipLevel = 0;
  756. dst_mb->subresourceRange.levelCount = 1;
  757. dst_mb->subresourceRange.baseArrayLayer = 0;
  758. dst_mb->subresourceRange.layerCount = 1;
  759. funcs->CmdPipelineBarrier(cmd_buffer,
  760. VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
  761. VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, NULL, 0,
  762. NULL, 2, mb);
  763. /* ------------------------------------------------------ */
  764. /* copy cur_backbuffer's content to our interop image */
  765. VkImageCopy cpy;
  766. cpy.srcSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
  767. cpy.srcSubresource.mipLevel = 0;
  768. cpy.srcSubresource.baseArrayLayer = 0;
  769. cpy.srcSubresource.layerCount = 1;
  770. cpy.srcOffset.x = 0;
  771. cpy.srcOffset.y = 0;
  772. cpy.srcOffset.z = 0;
  773. cpy.dstSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
  774. cpy.dstSubresource.mipLevel = 0;
  775. cpy.dstSubresource.baseArrayLayer = 0;
  776. cpy.dstSubresource.layerCount = 1;
  777. cpy.dstOffset.x = 0;
  778. cpy.dstOffset.y = 0;
  779. cpy.dstOffset.z = 0;
  780. cpy.extent.width = swap->image_extent.width;
  781. cpy.extent.height = swap->image_extent.height;
  782. cpy.extent.depth = 1;
  783. funcs->CmdCopyImage(cmd_buffer, cur_backbuffer,
  784. VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
  785. swap->export_image,
  786. VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &cpy);
  787. /* ------------------------------------------------------ */
  788. /* Restore the swap chain image layout to what it was
  789. * before. This may not be strictly needed, but it is
  790. * generally good to restore things to their original
  791. * state. */
  792. src_mb->srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
  793. src_mb->dstAccessMask = VK_ACCESS_MEMORY_READ_BIT;
  794. src_mb->oldLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
  795. src_mb->newLayout = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR;
  796. dst_mb->srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
  797. dst_mb->dstAccessMask = 0;
  798. dst_mb->oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
  799. dst_mb->newLayout = VK_IMAGE_LAYOUT_GENERAL;
  800. dst_mb->srcQueueFamilyIndex = fam_idx;
  801. dst_mb->dstQueueFamilyIndex = VK_QUEUE_FAMILY_EXTERNAL;
  802. funcs->CmdPipelineBarrier(cmd_buffer, VK_PIPELINE_STAGE_TRANSFER_BIT,
  803. VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT |
  804. VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
  805. 0, 0, NULL, 0, NULL, 2, mb);
  806. funcs->EndCommandBuffer(cmd_buffer);
  807. /* ------------------------------------------------------ */
  808. VkSubmitInfo submit_info;
  809. submit_info.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO;
  810. submit_info.pNext = NULL;
  811. submit_info.waitSemaphoreCount = 0;
  812. submit_info.pWaitSemaphores = NULL;
  813. submit_info.pWaitDstStageMask = NULL;
  814. submit_info.commandBufferCount = 1;
  815. submit_info.pCommandBuffers = &cmd_buffer;
  816. submit_info.signalSemaphoreCount = 0;
  817. submit_info.pSignalSemaphores = NULL;
  818. const VkFence fence = frame_data->fence;
  819. res = funcs->QueueSubmit(queue, 1, &submit_info, fence);
  820. #ifdef MORE_DEBUGGING
  821. debug_res("QueueSubmit", res);
  822. #endif
  823. if (res == VK_SUCCESS)
  824. frame_data->cmd_buffer_busy = true;
  825. }
  826. static inline bool valid_rect(struct vk_swap_data *swap)
  827. {
  828. return !!swap->image_extent.width && !!swap->image_extent.height;
  829. }
  830. static void vk_capture(struct vk_data *data, VkQueue queue,
  831. const VkPresentInfoKHR *info)
  832. {
  833. struct vk_swap_data *swap = NULL;
  834. HWND window = NULL;
  835. uint32_t idx = 0;
  836. #ifdef MORE_DEBUGGING
  837. debug("QueuePresentKHR called on "
  838. "devicekey %p, swapchain count %d",
  839. &data->funcs, info->swapchainCount);
  840. #endif
  841. /* use first swap chain associated with a window */
  842. for (; idx < info->swapchainCount; idx++) {
  843. struct vk_swap_data *cur_swap =
  844. get_swap_data(data, info->pSwapchains[idx]);
  845. window = cur_swap->hwnd;
  846. if (!!window) {
  847. swap = cur_swap;
  848. break;
  849. }
  850. }
  851. if (!window) {
  852. return;
  853. }
  854. if (capture_should_stop()) {
  855. vk_shtex_free(data);
  856. }
  857. if (capture_should_init()) {
  858. if (valid_rect(swap) && !vk_shtex_init(data, window, swap)) {
  859. vk_shtex_free(data);
  860. data->valid = false;
  861. flog("vk_shtex_init failed");
  862. }
  863. }
  864. if (capture_ready()) {
  865. if (swap != data->cur_swap) {
  866. vk_shtex_free(data);
  867. return;
  868. }
  869. vk_shtex_capture(data, &data->funcs, swap, idx, queue, info);
  870. }
  871. }
  872. static VkResult VKAPI OBS_QueuePresentKHR(VkQueue queue,
  873. const VkPresentInfoKHR *info)
  874. {
  875. struct vk_data *data = get_device_data(queue);
  876. struct vk_device_funcs *funcs = &data->funcs;
  877. if (data->valid) {
  878. vk_capture(data, queue, info);
  879. }
  880. return funcs->QueuePresentKHR(queue, info);
  881. }
  882. /* ======================================================================== */
  883. /* setup hooks */
  884. static inline bool is_inst_link_info(VkLayerInstanceCreateInfo *lici)
  885. {
  886. return lici->sType == VK_STRUCTURE_TYPE_LOADER_INSTANCE_CREATE_INFO &&
  887. lici->function == VK_LAYER_LINK_INFO;
  888. }
  889. static VkResult VKAPI OBS_CreateInstance(const VkInstanceCreateInfo *cinfo,
  890. const VkAllocationCallbacks *ac,
  891. VkInstance *p_inst)
  892. {
  893. VkInstanceCreateInfo info = *cinfo;
  894. bool funcs_not_found = false;
  895. /* -------------------------------------------------------- */
  896. /* step through chain until we get to the link info */
  897. VkLayerInstanceCreateInfo *lici = (void *)info.pNext;
  898. while (lici && !is_inst_link_info(lici)) {
  899. lici = (VkLayerInstanceCreateInfo *)lici->pNext;
  900. }
  901. if (lici == NULL) {
  902. return VK_ERROR_INITIALIZATION_FAILED;
  903. }
  904. PFN_vkGetInstanceProcAddr gpa =
  905. lici->u.pLayerInfo->pfnNextGetInstanceProcAddr;
  906. /* -------------------------------------------------------- */
  907. /* move chain on for next layer */
  908. lici->u.pLayerInfo = lici->u.pLayerInfo->pNext;
  909. /* -------------------------------------------------------- */
  910. /* (HACK) Set api version to 1.1 if set to 1.0 */
  911. /* We do this to get our extensions working properly */
  912. VkApplicationInfo ai;
  913. if (info.pApplicationInfo) {
  914. ai = *info.pApplicationInfo;
  915. if (ai.apiVersion < VK_API_VERSION_1_1)
  916. ai.apiVersion = VK_API_VERSION_1_1;
  917. } else {
  918. ai.sType = VK_STRUCTURE_TYPE_APPLICATION_INFO;
  919. ai.pNext = NULL;
  920. ai.pApplicationName = NULL;
  921. ai.applicationVersion = 0;
  922. ai.pEngineName = NULL;
  923. ai.engineVersion = 0;
  924. ai.apiVersion = VK_API_VERSION_1_1;
  925. }
  926. info.pApplicationInfo = &ai;
  927. /* -------------------------------------------------------- */
  928. /* create instance */
  929. PFN_vkCreateInstance create = (void *)gpa(NULL, "vkCreateInstance");
  930. VkResult res = create(&info, ac, p_inst);
  931. VkInstance inst = *p_inst;
  932. /* -------------------------------------------------------- */
  933. /* fetch the functions we need */
  934. struct vk_inst_data *data = get_inst_data(inst);
  935. struct vk_inst_funcs *funcs = &data->funcs;
  936. #define GETADDR(x) \
  937. do { \
  938. funcs->x = (void *)gpa(inst, "vk" #x); \
  939. if (!funcs->x) { \
  940. flog("could not get instance " \
  941. "address for %s", \
  942. #x); \
  943. funcs_not_found = true; \
  944. } \
  945. } while (false)
  946. GETADDR(GetInstanceProcAddr);
  947. GETADDR(DestroyInstance);
  948. GETADDR(CreateWin32SurfaceKHR);
  949. GETADDR(DestroySurfaceKHR);
  950. GETADDR(GetPhysicalDeviceMemoryProperties);
  951. GETADDR(GetPhysicalDeviceImageFormatProperties2);
  952. GETADDR(EnumerateDeviceExtensionProperties);
  953. #undef GETADDR
  954. data->valid = !funcs_not_found;
  955. return res;
  956. }
  957. static VkResult VKAPI OBS_DestroyInstance(VkInstance instance,
  958. const VkAllocationCallbacks *ac)
  959. {
  960. struct vk_inst_funcs *funcs = get_inst_funcs(instance);
  961. funcs->DestroyInstance(instance, ac);
  962. remove_instance(instance);
  963. return VK_SUCCESS;
  964. }
  965. static bool
  966. vk_shared_tex_supported(struct vk_inst_funcs *funcs,
  967. VkPhysicalDevice phy_device, VkFormat format,
  968. VkImageUsageFlags usage,
  969. VkExternalMemoryProperties *external_mem_props)
  970. {
  971. VkPhysicalDeviceImageFormatInfo2 info;
  972. VkPhysicalDeviceExternalImageFormatInfo external_info;
  973. external_info.sType =
  974. VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_IMAGE_FORMAT_INFO;
  975. external_info.pNext = NULL;
  976. external_info.handleType =
  977. VK_EXTERNAL_MEMORY_HANDLE_TYPE_D3D11_TEXTURE_KMT_BIT;
  978. info.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2;
  979. info.pNext = &external_info;
  980. info.format = format;
  981. info.type = VK_IMAGE_TYPE_2D;
  982. info.tiling = VK_IMAGE_TILING_OPTIMAL;
  983. info.flags = 0;
  984. info.usage = usage;
  985. VkExternalImageFormatProperties external_props = {0};
  986. external_props.sType =
  987. VK_STRUCTURE_TYPE_EXTERNAL_IMAGE_FORMAT_PROPERTIES;
  988. external_props.pNext = NULL;
  989. VkImageFormatProperties2 props = {0};
  990. props.sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_PROPERTIES_2;
  991. props.pNext = &external_props;
  992. VkResult result = funcs->GetPhysicalDeviceImageFormatProperties2(
  993. phy_device, &info, &props);
  994. *external_mem_props = external_props.externalMemoryProperties;
  995. const VkExternalMemoryFeatureFlags features =
  996. external_mem_props->externalMemoryFeatures;
  997. return ((VK_SUCCESS == result) &&
  998. (features & VK_EXTERNAL_MEMORY_FEATURE_IMPORTABLE_BIT));
  999. }
  1000. static inline bool is_device_link_info(VkLayerDeviceCreateInfo *lici)
  1001. {
  1002. return lici->sType == VK_STRUCTURE_TYPE_LOADER_DEVICE_CREATE_INFO &&
  1003. lici->function == VK_LAYER_LINK_INFO;
  1004. }
  1005. static VkResult VKAPI OBS_CreateDevice(VkPhysicalDevice phy_device,
  1006. const VkDeviceCreateInfo *info,
  1007. const VkAllocationCallbacks *ac,
  1008. VkDevice *p_device)
  1009. {
  1010. struct vk_inst_data *idata = get_inst_data(phy_device);
  1011. struct vk_inst_funcs *ifuncs = &idata->funcs;
  1012. struct vk_data *data = NULL;
  1013. VkResult ret = VK_ERROR_INITIALIZATION_FAILED;
  1014. VkLayerDeviceCreateInfo *ldci = (void *)info->pNext;
  1015. /* -------------------------------------------------------- */
  1016. /* step through chain until we get to the link info */
  1017. while (ldci && !is_device_link_info(ldci)) {
  1018. ldci = (VkLayerDeviceCreateInfo *)ldci->pNext;
  1019. }
  1020. if (!ldci) {
  1021. goto fail;
  1022. }
  1023. PFN_vkGetInstanceProcAddr gipa;
  1024. PFN_vkGetDeviceProcAddr gdpa;
  1025. gipa = ldci->u.pLayerInfo->pfnNextGetInstanceProcAddr;
  1026. gdpa = ldci->u.pLayerInfo->pfnNextGetDeviceProcAddr;
  1027. /* -------------------------------------------------------- */
  1028. /* move chain on for next layer */
  1029. ldci->u.pLayerInfo = ldci->u.pLayerInfo->pNext;
  1030. /* -------------------------------------------------------- */
  1031. /* create device and initialize hook data */
  1032. PFN_vkCreateDevice createFunc =
  1033. (PFN_vkCreateDevice)gipa(VK_NULL_HANDLE, "vkCreateDevice");
  1034. ret = createFunc(phy_device, info, ac, p_device);
  1035. if (ret != VK_SUCCESS) {
  1036. goto fail;
  1037. }
  1038. VkDevice device = *p_device;
  1039. data = get_device_data(*p_device);
  1040. struct vk_device_funcs *dfuncs = &data->funcs;
  1041. data->valid = false; /* set true below if it doesn't go to fail */
  1042. data->phy_device = phy_device;
  1043. data->device = device;
  1044. /* -------------------------------------------------------- */
  1045. /* fetch the functions we need */
  1046. bool funcs_not_found = false;
  1047. #define GETADDR(x) \
  1048. do { \
  1049. dfuncs->x = (void *)gdpa(device, "vk" #x); \
  1050. if (!dfuncs->x) { \
  1051. flog("could not get device " \
  1052. "address for %s", \
  1053. #x); \
  1054. funcs_not_found = true; \
  1055. } \
  1056. } while (false)
  1057. #define GETADDR_OPTIONAL(x) \
  1058. do { \
  1059. dfuncs->x = (void *)gdpa(device, "vk" #x); \
  1060. } while (false)
  1061. GETADDR(GetDeviceProcAddr);
  1062. GETADDR(DestroyDevice);
  1063. GETADDR(CreateSwapchainKHR);
  1064. GETADDR(DestroySwapchainKHR);
  1065. GETADDR(QueuePresentKHR);
  1066. GETADDR(AllocateMemory);
  1067. GETADDR(FreeMemory);
  1068. GETADDR(BindImageMemory);
  1069. GETADDR(BindImageMemory2);
  1070. GETADDR(GetSwapchainImagesKHR);
  1071. GETADDR(CreateImage);
  1072. GETADDR(DestroyImage);
  1073. GETADDR(GetImageMemoryRequirements);
  1074. GETADDR(GetImageMemoryRequirements2);
  1075. GETADDR(ResetCommandPool);
  1076. GETADDR(BeginCommandBuffer);
  1077. GETADDR(EndCommandBuffer);
  1078. GETADDR(CmdCopyImage);
  1079. GETADDR(CmdPipelineBarrier);
  1080. GETADDR(GetDeviceQueue);
  1081. GETADDR(QueueSubmit);
  1082. GETADDR(CreateCommandPool);
  1083. GETADDR(DestroyCommandPool);
  1084. GETADDR(AllocateCommandBuffers);
  1085. GETADDR(CreateFence);
  1086. GETADDR(DestroyFence);
  1087. GETADDR(WaitForFences);
  1088. GETADDR(ResetFences);
  1089. #undef GETADDR_OPTIONAL
  1090. #undef GETADDR
  1091. if (funcs_not_found) {
  1092. goto fail;
  1093. }
  1094. if (!idata->valid) {
  1095. flog("instance not valid");
  1096. goto fail;
  1097. }
  1098. const char *required_device_extensions[] = {
  1099. VK_KHR_EXTERNAL_MEMORY_WIN32_EXTENSION_NAME};
  1100. uint32_t device_extension_count = 0;
  1101. ret = ifuncs->EnumerateDeviceExtensionProperties(
  1102. phy_device, NULL, &device_extension_count, NULL);
  1103. if (ret != VK_SUCCESS)
  1104. goto fail;
  1105. VkExtensionProperties *device_extensions = _malloca(
  1106. sizeof(VkExtensionProperties) * device_extension_count);
  1107. ret = ifuncs->EnumerateDeviceExtensionProperties(
  1108. phy_device, NULL, &device_extension_count, device_extensions);
  1109. if (ret != VK_SUCCESS)
  1110. goto fail;
  1111. bool extensions_found = true;
  1112. for (uint32_t i = 0; i < _countof(required_device_extensions); i++) {
  1113. const char *const required_extension =
  1114. required_device_extensions[i];
  1115. bool found = false;
  1116. for (uint32_t j = 0; j < device_extension_count; j++) {
  1117. if (!strcmp(required_extension,
  1118. device_extensions[j].extensionName)) {
  1119. found = true;
  1120. break;
  1121. }
  1122. }
  1123. if (!found) {
  1124. flog("missing device extension: %s",
  1125. required_extension);
  1126. extensions_found = false;
  1127. }
  1128. }
  1129. if (!extensions_found)
  1130. goto fail;
  1131. VkFormat format = VK_FORMAT_R8G8B8A8_UNORM;
  1132. VkImageUsageFlags usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT |
  1133. VK_IMAGE_USAGE_TRANSFER_DST_BIT;
  1134. if (!vk_shared_tex_supported(ifuncs, phy_device, format, usage,
  1135. &data->external_mem_props)) {
  1136. flog("texture sharing is not supported");
  1137. goto fail;
  1138. }
  1139. data->inst_data = idata;
  1140. data->ac = NULL;
  1141. if (ac) {
  1142. data->ac_storage = *ac;
  1143. data->ac = &data->ac_storage;
  1144. }
  1145. data->valid = true;
  1146. fail:
  1147. return ret;
  1148. }
  1149. static void VKAPI OBS_DestroyDevice(VkDevice device,
  1150. const VkAllocationCallbacks *ac)
  1151. {
  1152. struct vk_data *data = get_device_data(device);
  1153. if (!data)
  1154. return;
  1155. if (data->valid) {
  1156. for (uint32_t fam_idx = 0; fam_idx < _countof(data->families);
  1157. fam_idx++) {
  1158. struct vk_family_data *family_data =
  1159. &data->families[fam_idx];
  1160. if (family_data->frame_count > 0) {
  1161. vk_shtex_destroy_family_objects(data,
  1162. family_data);
  1163. }
  1164. }
  1165. }
  1166. data->queue_count = 0;
  1167. vk_remove_device(device);
  1168. data->funcs.DestroyDevice(device, ac);
  1169. }
  1170. static VkResult VKAPI
  1171. OBS_CreateSwapchainKHR(VkDevice device, const VkSwapchainCreateInfoKHR *cinfo,
  1172. const VkAllocationCallbacks *ac, VkSwapchainKHR *p_sc)
  1173. {
  1174. struct vk_data *data = get_device_data(device);
  1175. VkSwapchainCreateInfoKHR info = *cinfo;
  1176. if (data->valid)
  1177. info.imageUsage |= VK_IMAGE_USAGE_TRANSFER_SRC_BIT;
  1178. struct vk_device_funcs *funcs = &data->funcs;
  1179. VkResult res = funcs->CreateSwapchainKHR(device, &info, ac, p_sc);
  1180. debug_res("CreateSwapchainKHR", res);
  1181. if ((res != VK_SUCCESS) || !data->valid)
  1182. return res;
  1183. VkSwapchainKHR sc = *p_sc;
  1184. uint32_t count = 0;
  1185. res = funcs->GetSwapchainImagesKHR(data->device, sc, &count, NULL);
  1186. debug_res("GetSwapchainImagesKHR", res);
  1187. struct vk_swap_data *swap = get_new_swap_data(data);
  1188. if (count > 0) {
  1189. if (count > OBJ_MAX)
  1190. count = OBJ_MAX;
  1191. res = funcs->GetSwapchainImagesKHR(data->device, sc, &count,
  1192. swap->swap_images);
  1193. debug_res("GetSwapchainImagesKHR", res);
  1194. }
  1195. swap->sc = sc;
  1196. swap->image_extent = cinfo->imageExtent;
  1197. swap->format = cinfo->imageFormat;
  1198. swap->hwnd = find_surf_hwnd(data->inst_data, cinfo->surface);
  1199. swap->image_count = count;
  1200. swap->d3d11_tex = NULL;
  1201. return VK_SUCCESS;
  1202. }
  1203. static void VKAPI OBS_DestroySwapchainKHR(VkDevice device, VkSwapchainKHR sc,
  1204. const VkAllocationCallbacks *ac)
  1205. {
  1206. struct vk_data *data = get_device_data(device);
  1207. struct vk_device_funcs *funcs = &data->funcs;
  1208. if (data->valid) {
  1209. struct vk_swap_data *swap = get_swap_data(data, sc);
  1210. if (swap) {
  1211. if (data->cur_swap == swap) {
  1212. vk_shtex_free(data);
  1213. }
  1214. swap->sc = VK_NULL_HANDLE;
  1215. swap->hwnd = NULL;
  1216. }
  1217. }
  1218. funcs->DestroySwapchainKHR(device, sc, ac);
  1219. }
  1220. static void VKAPI OBS_GetDeviceQueue(VkDevice device, uint32_t queueFamilyIndex,
  1221. uint32_t queueIndex, VkQueue *pQueue)
  1222. {
  1223. struct vk_data *data = get_device_data(device);
  1224. struct vk_device_funcs *funcs = &data->funcs;
  1225. funcs->GetDeviceQueue(device, queueFamilyIndex, queueIndex, pQueue);
  1226. for (uint32_t i = 0; i < data->queue_count; ++i) {
  1227. if (data->queues[i].queue == *pQueue)
  1228. return;
  1229. }
  1230. if (data->queue_count < _countof(data->queues)) {
  1231. data->queues[data->queue_count].queue = *pQueue;
  1232. data->queues[data->queue_count].fam_idx = queueFamilyIndex;
  1233. ++data->queue_count;
  1234. }
  1235. }
  1236. static VkResult VKAPI OBS_CreateWin32SurfaceKHR(
  1237. VkInstance inst, const VkWin32SurfaceCreateInfoKHR *info,
  1238. const VkAllocationCallbacks *ac, VkSurfaceKHR *surf)
  1239. {
  1240. struct vk_inst_data *data = get_inst_data(inst);
  1241. struct vk_inst_funcs *funcs = &data->funcs;
  1242. VkResult res = funcs->CreateWin32SurfaceKHR(inst, info, ac, surf);
  1243. if (res == VK_SUCCESS)
  1244. insert_surf_data(data, *surf, info->hwnd, ac);
  1245. return res;
  1246. }
  1247. static void VKAPI OBS_DestroySurfaceKHR(VkInstance inst, VkSurfaceKHR surf,
  1248. const VkAllocationCallbacks *ac)
  1249. {
  1250. struct vk_inst_data *data = get_inst_data(inst);
  1251. struct vk_inst_funcs *funcs = &data->funcs;
  1252. erase_surf_data(data, surf, ac);
  1253. funcs->DestroySurfaceKHR(inst, surf, ac);
  1254. }
  1255. #define GETPROCADDR(func) \
  1256. if (!strcmp(name, "vk" #func)) \
  1257. return (VkFunc)&OBS_##func;
  1258. static VkFunc VKAPI OBS_GetDeviceProcAddr(VkDevice dev, const char *name)
  1259. {
  1260. struct vk_data *data = get_device_data(dev);
  1261. struct vk_device_funcs *funcs = &data->funcs;
  1262. debug_procaddr("vkGetDeviceProcAddr(%p, \"%s\")", dev, name);
  1263. GETPROCADDR(GetDeviceProcAddr);
  1264. GETPROCADDR(CreateDevice);
  1265. GETPROCADDR(DestroyDevice);
  1266. GETPROCADDR(CreateSwapchainKHR);
  1267. GETPROCADDR(DestroySwapchainKHR);
  1268. GETPROCADDR(QueuePresentKHR);
  1269. GETPROCADDR(GetDeviceQueue);
  1270. if (funcs->GetDeviceProcAddr == NULL)
  1271. return NULL;
  1272. return funcs->GetDeviceProcAddr(dev, name);
  1273. }
  1274. static VkFunc VKAPI OBS_GetInstanceProcAddr(VkInstance inst, const char *name)
  1275. {
  1276. debug_procaddr("vkGetInstanceProcAddr(%p, \"%s\")", inst, name);
  1277. /* instance chain functions we intercept */
  1278. GETPROCADDR(GetInstanceProcAddr);
  1279. GETPROCADDR(CreateInstance);
  1280. GETPROCADDR(DestroyInstance);
  1281. GETPROCADDR(CreateWin32SurfaceKHR);
  1282. GETPROCADDR(DestroySurfaceKHR);
  1283. /* device chain functions we intercept */
  1284. GETPROCADDR(GetDeviceProcAddr);
  1285. GETPROCADDR(CreateDevice);
  1286. GETPROCADDR(DestroyDevice);
  1287. struct vk_inst_funcs *funcs = get_inst_funcs(inst);
  1288. if (funcs->GetInstanceProcAddr == NULL)
  1289. return NULL;
  1290. return funcs->GetInstanceProcAddr(inst, name);
  1291. }
  1292. #undef GETPROCADDR
  1293. EXPORT VkResult VKAPI OBS_Negotiate(VkNegotiateLayerInterface *nli)
  1294. {
  1295. if (nli->loaderLayerInterfaceVersion >= 2) {
  1296. nli->sType = LAYER_NEGOTIATE_INTERFACE_STRUCT;
  1297. nli->pNext = NULL;
  1298. nli->pfnGetInstanceProcAddr = OBS_GetInstanceProcAddr;
  1299. nli->pfnGetDeviceProcAddr = OBS_GetDeviceProcAddr;
  1300. nli->pfnGetPhysicalDeviceProcAddr = NULL;
  1301. }
  1302. const uint32_t cur_ver = CURRENT_LOADER_LAYER_INTERFACE_VERSION;
  1303. if (nli->loaderLayerInterfaceVersion > cur_ver) {
  1304. nli->loaderLayerInterfaceVersion = cur_ver;
  1305. }
  1306. return VK_SUCCESS;
  1307. }
  1308. bool hook_vulkan(void)
  1309. {
  1310. static bool hooked = false;
  1311. if (!hooked && vulkan_seen) {
  1312. hlog("Hooked Vulkan");
  1313. hooked = true;
  1314. }
  1315. return hooked;
  1316. }