vulkan-capture.c 63 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103
  1. #include <windows.h>
  2. #include "graphics-hook.h"
  3. #define VK_USE_PLATFORM_WIN32_KHR
  4. #include <malloc.h>
  5. #include <vulkan/vulkan.h>
  6. #include <vulkan/vk_layer.h>
  7. #include <vulkan/vulkan_win32.h>
  8. #define COBJMACROS
  9. #include <dxgi.h>
  10. #include <d3d11.h>
  11. #include "dxgi-helpers.hpp"
  12. #include "vulkan-capture.h"
  13. /* ======================================================================== */
  14. /* defs/statics */
  15. /* use the loader's dispatch table pointer as a key for internal data maps */
  16. #define GET_LDT(x) (*(void **)x)
  17. static bool vulkan_seen = false;
  18. /* ======================================================================== */
  19. /* hook data */
  20. struct vk_obj_node {
  21. uint64_t obj;
  22. struct vk_obj_node *next;
  23. };
  24. struct vk_obj_list {
  25. struct vk_obj_node *root;
  26. SRWLOCK mutex;
  27. };
  28. struct vk_swap_data {
  29. struct vk_obj_node node;
  30. VkExtent2D image_extent;
  31. VkFormat format;
  32. HWND hwnd;
  33. VkImage export_image;
  34. bool layout_initialized;
  35. VkDeviceMemory export_mem;
  36. VkImage *swap_images;
  37. uint32_t image_count;
  38. HANDLE handle;
  39. struct shtex_data *shtex_info;
  40. ID3D11Texture2D *d3d11_tex;
  41. bool captured;
  42. };
  43. struct vk_queue_data {
  44. struct vk_obj_node node;
  45. uint32_t fam_idx;
  46. bool supports_transfer;
  47. struct vk_frame_data *frames;
  48. uint32_t frame_index;
  49. uint32_t frame_count;
  50. };
  51. struct vk_swap_view_data {
  52. struct vk_obj_node node;
  53. };
  54. #define OBS_COLOR_ATTACHMENT_LIMIT 8
  55. struct vk_framebuffer_data {
  56. struct vk_obj_node node;
  57. VkFramebuffer alternates[1 << OBS_COLOR_ATTACHMENT_LIMIT];
  58. uint32_t color_attachment_mask;
  59. };
  60. struct vk_frame_data {
  61. VkCommandPool cmd_pool;
  62. VkCommandBuffer cmd_buffer;
  63. VkFence fence;
  64. bool cmd_buffer_busy;
  65. };
  66. struct vk_surf_data {
  67. struct vk_obj_node node;
  68. HWND hwnd;
  69. };
  70. struct vk_inst_data {
  71. struct vk_obj_node node;
  72. VkInstance instance;
  73. bool valid;
  74. struct vk_inst_funcs funcs;
  75. struct vk_obj_list surfaces;
  76. };
  77. struct vk_data {
  78. struct vk_obj_node node;
  79. VkDevice device;
  80. bool valid;
  81. struct vk_device_funcs funcs;
  82. VkPhysicalDevice phy_device;
  83. struct vk_obj_list swaps;
  84. struct vk_swap_data *cur_swap;
  85. struct vk_obj_list queues;
  86. struct vk_obj_list swap_views;
  87. struct vk_obj_list framebuffers;
  88. VkExternalMemoryProperties external_mem_props;
  89. struct vk_inst_data *inst_data;
  90. VkAllocationCallbacks ac_storage;
  91. const VkAllocationCallbacks *ac;
  92. ID3D11Device *d3d11_device;
  93. ID3D11DeviceContext *d3d11_context;
  94. };
  95. __declspec(thread) int vk_presenting = 0;
  96. /* ------------------------------------------------------------------------- */
  97. static void *vk_alloc(const VkAllocationCallbacks *ac, size_t size, size_t alignment,
  98. enum VkSystemAllocationScope scope)
  99. {
  100. return ac ? ac->pfnAllocation(ac->pUserData, size, alignment, scope) : _aligned_malloc(size, alignment);
  101. }
  102. static void vk_free(const VkAllocationCallbacks *ac, void *memory)
  103. {
  104. if (ac)
  105. ac->pfnFree(ac->pUserData, memory);
  106. else
  107. _aligned_free(memory);
  108. }
  109. static void add_obj_data(struct vk_obj_list *list, uint64_t obj, void *data)
  110. {
  111. AcquireSRWLockExclusive(&list->mutex);
  112. struct vk_obj_node *const node = data;
  113. node->obj = obj;
  114. node->next = list->root;
  115. list->root = node;
  116. ReleaseSRWLockExclusive(&list->mutex);
  117. }
  118. static struct vk_obj_node *get_obj_data(struct vk_obj_list *list, uint64_t obj)
  119. {
  120. struct vk_obj_node *data = NULL;
  121. AcquireSRWLockExclusive(&list->mutex);
  122. struct vk_obj_node *node = list->root;
  123. while (node) {
  124. if (node->obj == obj) {
  125. data = node;
  126. break;
  127. }
  128. node = node->next;
  129. }
  130. ReleaseSRWLockExclusive(&list->mutex);
  131. return data;
  132. }
  133. static struct vk_obj_node *remove_obj_data(struct vk_obj_list *list, uint64_t obj)
  134. {
  135. struct vk_obj_node *data = NULL;
  136. AcquireSRWLockExclusive(&list->mutex);
  137. struct vk_obj_node *prev = NULL;
  138. struct vk_obj_node *node = list->root;
  139. while (node) {
  140. if (node->obj == obj) {
  141. data = node;
  142. if (prev)
  143. prev->next = node->next;
  144. else
  145. list->root = node->next;
  146. break;
  147. }
  148. prev = node;
  149. node = node->next;
  150. }
  151. ReleaseSRWLockExclusive(&list->mutex);
  152. return data;
  153. }
  154. static void init_obj_list(struct vk_obj_list *list)
  155. {
  156. list->root = NULL;
  157. InitializeSRWLock(&list->mutex);
  158. }
  159. static struct vk_obj_node *obj_walk_begin(struct vk_obj_list *list)
  160. {
  161. AcquireSRWLockExclusive(&list->mutex);
  162. return list->root;
  163. }
  164. static struct vk_obj_node *obj_walk_next(struct vk_obj_node *node)
  165. {
  166. return node->next;
  167. }
  168. static void obj_walk_end(struct vk_obj_list *list)
  169. {
  170. ReleaseSRWLockExclusive(&list->mutex);
  171. }
  172. /* ------------------------------------------------------------------------- */
  173. static struct vk_obj_list devices;
  174. static struct vk_data *alloc_device_data(const VkAllocationCallbacks *ac)
  175. {
  176. struct vk_data *data =
  177. vk_alloc(ac, sizeof(struct vk_data), _Alignof(struct vk_data), VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
  178. return data;
  179. }
  180. static void init_device_data(struct vk_data *data, VkDevice device)
  181. {
  182. add_obj_data(&devices, (uint64_t)GET_LDT(device), data);
  183. data->device = device;
  184. }
  185. static struct vk_data *get_device_data(VkDevice device)
  186. {
  187. return (struct vk_data *)get_obj_data(&devices, (uint64_t)GET_LDT(device));
  188. }
  189. static struct vk_data *get_device_data_by_queue(VkQueue queue)
  190. {
  191. return (struct vk_data *)get_obj_data(&devices, (uint64_t)GET_LDT(queue));
  192. }
  193. static struct vk_data *get_device_data_by_command_buffer(VkCommandBuffer commandBuffer)
  194. {
  195. return (struct vk_data *)get_obj_data(&devices, (uint64_t)GET_LDT(commandBuffer));
  196. }
  197. static struct vk_data *remove_device_data(VkDevice device)
  198. {
  199. return (struct vk_data *)remove_obj_data(&devices, (uint64_t)GET_LDT(device));
  200. }
  201. static void free_device_data(struct vk_data *data, const VkAllocationCallbacks *ac)
  202. {
  203. vk_free(ac, data);
  204. }
  205. /* ------------------------------------------------------------------------- */
  206. static struct vk_queue_data *add_queue_data(struct vk_data *data, VkQueue queue, uint32_t fam_idx,
  207. bool supports_transfer, const VkAllocationCallbacks *ac)
  208. {
  209. struct vk_queue_data *const queue_data = vk_alloc(
  210. ac, sizeof(struct vk_queue_data), _Alignof(struct vk_queue_data), VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
  211. add_obj_data(&data->queues, (uint64_t)queue, queue_data);
  212. queue_data->fam_idx = fam_idx;
  213. queue_data->supports_transfer = supports_transfer;
  214. queue_data->frames = NULL;
  215. queue_data->frame_index = 0;
  216. queue_data->frame_count = 0;
  217. return queue_data;
  218. }
  219. static struct vk_queue_data *get_queue_data(struct vk_data *data, VkQueue queue)
  220. {
  221. return (struct vk_queue_data *)get_obj_data(&data->queues, (uint64_t)queue);
  222. }
  223. static void remove_free_queue_all(struct vk_data *data, const VkAllocationCallbacks *ac)
  224. {
  225. struct vk_queue_data *queue_data = (struct vk_queue_data *)data->queues.root;
  226. while (data->queues.root) {
  227. remove_obj_data(&data->queues, queue_data->node.obj);
  228. vk_free(ac, queue_data);
  229. queue_data = (struct vk_queue_data *)data->queues.root;
  230. }
  231. }
  232. static struct vk_queue_data *queue_walk_begin(struct vk_data *data)
  233. {
  234. return (struct vk_queue_data *)obj_walk_begin(&data->queues);
  235. }
  236. static struct vk_queue_data *queue_walk_next(struct vk_queue_data *queue_data)
  237. {
  238. return (struct vk_queue_data *)obj_walk_next((struct vk_obj_node *)queue_data);
  239. }
  240. static void queue_walk_end(struct vk_data *data)
  241. {
  242. obj_walk_end(&data->queues);
  243. }
  244. /* ------------------------------------------------------------------------- */
  245. static struct vk_swap_view_data *add_swap_view_data(struct vk_data *data, VkImageView imageView,
  246. const VkAllocationCallbacks *ac)
  247. {
  248. struct vk_swap_view_data *const swap_view_data = vk_alloc(ac, sizeof(struct vk_swap_view_data),
  249. _Alignof(struct vk_swap_view_data),
  250. VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
  251. add_obj_data(&data->swap_views, (uint64_t)imageView, swap_view_data);
  252. return swap_view_data;
  253. }
  254. static struct vk_swap_view_data *get_swap_view_data(struct vk_data *data, VkImageView imageView)
  255. {
  256. return (struct vk_swap_view_data *)get_obj_data(&data->swap_views, (uint64_t)imageView);
  257. }
  258. static void remove_free_swap_view_data(struct vk_data *data, VkImageView imageView, const VkAllocationCallbacks *ac)
  259. {
  260. struct vk_swap_data *const swap_view_data =
  261. (struct vk_swap_data *)remove_obj_data(&data->swap_views, (uint64_t)imageView);
  262. vk_free(ac, swap_view_data);
  263. }
  264. /* ------------------------------------------------------------------------- */
  265. static struct vk_framebuffer_data *add_framebuffer_data(struct vk_data *data, VkFramebuffer framebuffer,
  266. const VkAllocationCallbacks *ac)
  267. {
  268. struct vk_framebuffer_data *const framebuffer_data = vk_alloc(ac, sizeof(struct vk_framebuffer_data),
  269. _Alignof(struct vk_framebuffer_data),
  270. VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
  271. add_obj_data(&data->framebuffers, (uint64_t)framebuffer, framebuffer_data);
  272. for (size_t i = 0; i < _countof(framebuffer_data->alternates); ++i) {
  273. framebuffer_data->alternates[i] = VK_NULL_HANDLE;
  274. }
  275. framebuffer_data->color_attachment_mask = 0;
  276. return framebuffer_data;
  277. }
  278. static struct vk_framebuffer_data *get_framebuffer_data(struct vk_data *data, VkFramebuffer framebuffer)
  279. {
  280. return (struct vk_framebuffer_data *)get_obj_data(&data->framebuffers, (uint64_t)framebuffer);
  281. }
  282. static void remove_free_framebuffer_data(struct vk_data *data, VkFramebuffer framebuffer,
  283. const VkAllocationCallbacks *ac)
  284. {
  285. struct vk_swap_data *const framebuffer_data =
  286. (struct vk_swap_data *)remove_obj_data(&data->framebuffers, (uint64_t)framebuffer);
  287. vk_free(ac, framebuffer_data);
  288. }
  289. /* ------------------------------------------------------------------------- */
  290. static struct vk_swap_data *alloc_swap_data(const VkAllocationCallbacks *ac)
  291. {
  292. struct vk_swap_data *const swap_data = vk_alloc(ac, sizeof(struct vk_swap_data), _Alignof(struct vk_swap_data),
  293. VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
  294. return swap_data;
  295. }
  296. static void init_swap_data(struct vk_swap_data *swap_data, struct vk_data *data, VkSwapchainKHR sc)
  297. {
  298. add_obj_data(&data->swaps, (uint64_t)sc, swap_data);
  299. }
  300. static struct vk_swap_data *get_swap_data(struct vk_data *data, VkSwapchainKHR sc)
  301. {
  302. return (struct vk_swap_data *)get_obj_data(&data->swaps, (uint64_t)sc);
  303. }
  304. static void remove_free_swap_data(struct vk_data *data, VkSwapchainKHR sc, const VkAllocationCallbacks *ac)
  305. {
  306. struct vk_swap_data *const swap_data = (struct vk_swap_data *)remove_obj_data(&data->swaps, (uint64_t)sc);
  307. vk_free(ac, swap_data);
  308. }
  309. static struct vk_swap_data *swap_walk_begin(struct vk_data *data)
  310. {
  311. return (struct vk_swap_data *)obj_walk_begin(&data->swaps);
  312. }
  313. static struct vk_swap_data *swap_walk_next(struct vk_swap_data *swap_data)
  314. {
  315. return (struct vk_swap_data *)obj_walk_next((struct vk_obj_node *)swap_data);
  316. }
  317. static void swap_walk_end(struct vk_data *data)
  318. {
  319. obj_walk_end(&data->swaps);
  320. }
  321. /* ------------------------------------------------------------------------- */
  322. static void vk_shtex_clear_fence(const struct vk_data *data, struct vk_frame_data *frame_data)
  323. {
  324. const VkFence fence = frame_data->fence;
  325. if (frame_data->cmd_buffer_busy) {
  326. VkDevice device = data->device;
  327. const struct vk_device_funcs *funcs = &data->funcs;
  328. funcs->WaitForFences(device, 1, &fence, VK_TRUE, ~0ull);
  329. funcs->ResetFences(device, 1, &fence);
  330. frame_data->cmd_buffer_busy = false;
  331. }
  332. }
  333. static void vk_shtex_wait_until_pool_idle(struct vk_data *data, struct vk_queue_data *queue_data)
  334. {
  335. for (uint32_t frame_idx = 0; frame_idx < queue_data->frame_count; frame_idx++) {
  336. struct vk_frame_data *frame_data = &queue_data->frames[frame_idx];
  337. if (frame_data->cmd_pool != VK_NULL_HANDLE)
  338. vk_shtex_clear_fence(data, frame_data);
  339. }
  340. }
  341. static void vk_shtex_wait_until_idle(struct vk_data *data)
  342. {
  343. struct vk_queue_data *queue_data = queue_walk_begin(data);
  344. while (queue_data) {
  345. vk_shtex_wait_until_pool_idle(data, queue_data);
  346. queue_data = queue_walk_next(queue_data);
  347. }
  348. queue_walk_end(data);
  349. }
  350. static void vk_shtex_free(struct vk_data *data)
  351. {
  352. capture_free();
  353. vk_shtex_wait_until_idle(data);
  354. struct vk_swap_data *swap = swap_walk_begin(data);
  355. while (swap) {
  356. VkDevice device = data->device;
  357. if (swap->export_image)
  358. data->funcs.DestroyImage(device, swap->export_image, data->ac);
  359. if (swap->export_mem)
  360. data->funcs.FreeMemory(device, swap->export_mem, NULL);
  361. if (swap->d3d11_tex) {
  362. ID3D11Texture2D_Release(swap->d3d11_tex);
  363. }
  364. swap->handle = INVALID_HANDLE_VALUE;
  365. swap->d3d11_tex = NULL;
  366. swap->export_mem = VK_NULL_HANDLE;
  367. swap->export_image = VK_NULL_HANDLE;
  368. swap->captured = false;
  369. swap = swap_walk_next(swap);
  370. }
  371. swap_walk_end(data);
  372. if (data->d3d11_context) {
  373. ID3D11DeviceContext_Release(data->d3d11_context);
  374. data->d3d11_context = NULL;
  375. }
  376. if (data->d3d11_device) {
  377. ID3D11Device_Release(data->d3d11_device);
  378. data->d3d11_device = NULL;
  379. }
  380. data->cur_swap = NULL;
  381. hlog("------------------ vulkan capture freed ------------------");
  382. }
  383. /* ------------------------------------------------------------------------- */
  384. static void add_surf_data(struct vk_inst_data *idata, VkSurfaceKHR surf, HWND hwnd, const VkAllocationCallbacks *ac)
  385. {
  386. struct vk_surf_data *surf_data = vk_alloc(ac, sizeof(struct vk_surf_data), _Alignof(struct vk_surf_data),
  387. VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
  388. if (surf_data) {
  389. surf_data->hwnd = hwnd;
  390. add_obj_data(&idata->surfaces, (uint64_t)surf, surf_data);
  391. }
  392. }
  393. static HWND find_surf_hwnd(struct vk_inst_data *idata, VkSurfaceKHR surf)
  394. {
  395. struct vk_surf_data *surf_data = (struct vk_surf_data *)get_obj_data(&idata->surfaces, (uint64_t)surf);
  396. if (surf_data)
  397. return surf_data->hwnd;
  398. return 0;
  399. }
  400. static void remove_free_surf_data(struct vk_inst_data *idata, VkSurfaceKHR surf, const VkAllocationCallbacks *ac)
  401. {
  402. struct vk_surf_data *surf_data = (struct vk_surf_data *)remove_obj_data(&idata->surfaces, (uint64_t)surf);
  403. vk_free(ac, surf_data);
  404. }
  405. /* ------------------------------------------------------------------------- */
  406. static struct vk_obj_list instances;
  407. static struct vk_inst_data *alloc_inst_data(const VkAllocationCallbacks *ac)
  408. {
  409. struct vk_inst_data *idata = vk_alloc(ac, sizeof(struct vk_inst_data), _Alignof(struct vk_inst_data),
  410. VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE);
  411. return idata;
  412. }
  413. static void init_inst_data(struct vk_inst_data *idata, VkInstance instance)
  414. {
  415. add_obj_data(&instances, (uint64_t)GET_LDT(instance), idata);
  416. idata->instance = instance;
  417. }
  418. static struct vk_inst_data *get_inst_data(VkInstance instance)
  419. {
  420. return (struct vk_inst_data *)get_obj_data(&instances, (uint64_t)GET_LDT(instance));
  421. }
  422. static struct vk_inst_funcs *get_inst_funcs(VkInstance instance)
  423. {
  424. struct vk_inst_data *idata = (struct vk_inst_data *)get_inst_data(instance);
  425. return &idata->funcs;
  426. }
  427. static struct vk_inst_data *get_inst_data_by_physical_device(VkPhysicalDevice physicalDevice)
  428. {
  429. return (struct vk_inst_data *)get_obj_data(&instances, (uint64_t)GET_LDT(physicalDevice));
  430. }
  431. static struct vk_inst_funcs *get_inst_funcs_by_physical_device(VkPhysicalDevice physicalDevice)
  432. {
  433. struct vk_inst_data *idata = (struct vk_inst_data *)get_inst_data_by_physical_device(physicalDevice);
  434. return &idata->funcs;
  435. }
  436. static void remove_free_inst_data(VkInstance inst, const VkAllocationCallbacks *ac)
  437. {
  438. struct vk_inst_data *idata = (struct vk_inst_data *)remove_obj_data(&instances, (uint64_t)GET_LDT(inst));
  439. vk_free(ac, idata);
  440. }
  441. /* ======================================================================== */
  442. /* capture */
  443. static inline bool vk_shtex_init_d3d11(struct vk_data *data)
  444. {
  445. D3D_FEATURE_LEVEL level_used;
  446. IDXGIFactory1 *factory;
  447. IDXGIAdapter1 *adapter;
  448. HRESULT hr;
  449. HMODULE d3d11 = load_system_library("d3d11.dll");
  450. if (!d3d11) {
  451. flog("failed to load d3d11: %d", GetLastError());
  452. return false;
  453. }
  454. HMODULE dxgi = load_system_library("dxgi.dll");
  455. if (!dxgi) {
  456. flog("failed to load dxgi: %d", GetLastError());
  457. return false;
  458. }
  459. HRESULT(WINAPI * create_factory)
  460. (REFIID, void **) = (void *)GetProcAddress(dxgi, "CreateDXGIFactory1");
  461. if (!create_factory) {
  462. flog("failed to get CreateDXGIFactory1 address: %d", GetLastError());
  463. return false;
  464. }
  465. PFN_D3D11_CREATE_DEVICE create = (void *)GetProcAddress(d3d11, "D3D11CreateDevice");
  466. if (!create) {
  467. flog("failed to get D3D11CreateDevice address: %d", GetLastError());
  468. return false;
  469. }
  470. hr = create_factory(&IID_IDXGIFactory1, &factory);
  471. if (FAILED(hr)) {
  472. flog_hr("failed to create factory", hr);
  473. return false;
  474. }
  475. hr = IDXGIFactory1_EnumAdapters1(factory, 0, &adapter);
  476. IDXGIFactory1_Release(factory);
  477. if (FAILED(hr)) {
  478. flog_hr("failed to create adapter", hr);
  479. return false;
  480. }
  481. static const D3D_FEATURE_LEVEL feature_levels[] = {
  482. D3D_FEATURE_LEVEL_11_0,
  483. D3D_FEATURE_LEVEL_10_1,
  484. D3D_FEATURE_LEVEL_10_0,
  485. D3D_FEATURE_LEVEL_9_3,
  486. };
  487. hr = create((IDXGIAdapter *)adapter, D3D_DRIVER_TYPE_UNKNOWN, NULL, 0, feature_levels,
  488. sizeof(feature_levels) / sizeof(D3D_FEATURE_LEVEL), D3D11_SDK_VERSION, &data->d3d11_device,
  489. &level_used, &data->d3d11_context);
  490. IDXGIAdapter1_Release(adapter);
  491. if (FAILED(hr)) {
  492. flog_hr("failed to create device", hr);
  493. return false;
  494. }
  495. return true;
  496. }
  497. static inline bool vk_shtex_init_d3d11_tex(struct vk_data *data, struct vk_swap_data *swap)
  498. {
  499. IDXGIResource *dxgi_res;
  500. HRESULT hr;
  501. const UINT width = swap->image_extent.width;
  502. const UINT height = swap->image_extent.height;
  503. flog("OBS requesting %s texture format. capture dimensions: %ux%u", vk_format_to_str(swap->format), width,
  504. height);
  505. const DXGI_FORMAT format = vk_format_to_dxgi(swap->format);
  506. if (format == DXGI_FORMAT_UNKNOWN) {
  507. flog("cannot convert to DXGI format");
  508. return false;
  509. }
  510. D3D11_TEXTURE2D_DESC desc = {0};
  511. desc.Width = width;
  512. desc.Height = height;
  513. desc.MipLevels = 1;
  514. desc.ArraySize = 1;
  515. desc.Format = apply_dxgi_format_typeless(format, global_hook_info->allow_srgb_alias);
  516. desc.SampleDesc.Count = 1;
  517. desc.SampleDesc.Quality = 0;
  518. desc.Usage = D3D11_USAGE_DEFAULT;
  519. desc.MiscFlags = D3D11_RESOURCE_MISC_SHARED;
  520. desc.BindFlags = D3D11_BIND_SHADER_RESOURCE;
  521. hr = ID3D11Device_CreateTexture2D(data->d3d11_device, &desc, NULL, &swap->d3d11_tex);
  522. if (FAILED(hr)) {
  523. flog_hr("failed to create texture", hr);
  524. return false;
  525. }
  526. hr = ID3D11Texture2D_QueryInterface(swap->d3d11_tex, &IID_IDXGIResource, &dxgi_res);
  527. if (FAILED(hr)) {
  528. flog_hr("failed to get IDXGIResource", hr);
  529. return false;
  530. }
  531. hr = IDXGIResource_GetSharedHandle(dxgi_res, &swap->handle);
  532. IDXGIResource_Release(dxgi_res);
  533. if (FAILED(hr)) {
  534. flog_hr("failed to get shared handle", hr);
  535. return false;
  536. }
  537. return true;
  538. }
  539. static inline bool vk_shtex_init_vulkan_tex(struct vk_data *data, struct vk_swap_data *swap)
  540. {
  541. struct vk_device_funcs *funcs = &data->funcs;
  542. VkExternalMemoryFeatureFlags f = data->external_mem_props.externalMemoryFeatures;
  543. /* -------------------------------------------------------- */
  544. /* create texture */
  545. VkExternalMemoryImageCreateInfo emici;
  546. emici.sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO;
  547. emici.pNext = NULL;
  548. emici.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_D3D11_TEXTURE_KMT_BIT;
  549. VkImageCreateInfo ici;
  550. ici.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO;
  551. ici.pNext = &emici;
  552. ici.flags = 0;
  553. ici.imageType = VK_IMAGE_TYPE_2D;
  554. ici.format = swap->format;
  555. ici.extent.width = swap->image_extent.width;
  556. ici.extent.height = swap->image_extent.height;
  557. ici.extent.depth = 1;
  558. ici.mipLevels = 1;
  559. ici.arrayLayers = 1;
  560. ici.samples = VK_SAMPLE_COUNT_1_BIT;
  561. ici.tiling = VK_IMAGE_TILING_OPTIMAL;
  562. ici.usage = VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_SAMPLED_BIT;
  563. ici.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
  564. ici.queueFamilyIndexCount = 0;
  565. ici.pQueueFamilyIndices = 0;
  566. ici.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
  567. VkDevice device = data->device;
  568. VkResult res;
  569. res = funcs->CreateImage(device, &ici, data->ac, &swap->export_image);
  570. if (VK_SUCCESS != res) {
  571. flog("failed to CreateImage: %s", result_to_str(res));
  572. swap->export_image = VK_NULL_HANDLE;
  573. return false;
  574. }
  575. swap->layout_initialized = false;
  576. /* -------------------------------------------------------- */
  577. /* get image memory requirements */
  578. VkMemoryRequirements mr;
  579. bool use_gimr2 = f & VK_EXTERNAL_MEMORY_FEATURE_DEDICATED_ONLY_BIT;
  580. if (use_gimr2) {
  581. VkMemoryDedicatedRequirements mdr = {0};
  582. mdr.sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS;
  583. mdr.pNext = NULL;
  584. VkMemoryRequirements2 mr2 = {0};
  585. mr2.sType = VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2;
  586. mr2.pNext = &mdr;
  587. VkImageMemoryRequirementsInfo2 imri2 = {0};
  588. imri2.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2;
  589. imri2.pNext = NULL;
  590. imri2.image = swap->export_image;
  591. funcs->GetImageMemoryRequirements2(device, &imri2, &mr2);
  592. mr = mr2.memoryRequirements;
  593. } else {
  594. funcs->GetImageMemoryRequirements(device, swap->export_image, &mr);
  595. }
  596. /* -------------------------------------------------------- */
  597. /* get memory type index */
  598. struct vk_inst_funcs *ifuncs = get_inst_funcs_by_physical_device(data->phy_device);
  599. VkPhysicalDeviceMemoryProperties pdmp;
  600. ifuncs->GetPhysicalDeviceMemoryProperties(data->phy_device, &pdmp);
  601. /* -------------------------------------------------------- */
  602. /* allocate memory */
  603. VkImportMemoryWin32HandleInfoKHR imw32hi;
  604. imw32hi.sType = VK_STRUCTURE_TYPE_IMPORT_MEMORY_WIN32_HANDLE_INFO_KHR;
  605. imw32hi.pNext = NULL;
  606. imw32hi.name = NULL;
  607. imw32hi.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_D3D11_TEXTURE_KMT_BIT;
  608. imw32hi.handle = swap->handle;
  609. VkMemoryAllocateInfo mai;
  610. mai.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
  611. mai.pNext = &imw32hi;
  612. mai.allocationSize = mr.size;
  613. VkMemoryDedicatedAllocateInfo mdai;
  614. mdai.sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO;
  615. mdai.pNext = NULL;
  616. mdai.buffer = VK_NULL_HANDLE;
  617. if (data->external_mem_props.externalMemoryFeatures & VK_EXTERNAL_MEMORY_FEATURE_DEDICATED_ONLY_BIT) {
  618. mdai.image = swap->export_image;
  619. imw32hi.pNext = &mdai;
  620. }
  621. bool allocated = false;
  622. for (uint32_t i = 0; i < pdmp.memoryTypeCount; ++i) {
  623. if ((mr.memoryTypeBits & (1 << i)) &&
  624. (pdmp.memoryTypes[i].propertyFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) ==
  625. VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) {
  626. mai.memoryTypeIndex = i;
  627. res = funcs->AllocateMemory(device, &mai, NULL, &swap->export_mem);
  628. allocated = res == VK_SUCCESS;
  629. if (allocated)
  630. break;
  631. flog("failed to AllocateMemory (DEVICE_LOCAL): %s (%d)", result_to_str(res), (int)res);
  632. }
  633. }
  634. if (!allocated) {
  635. /* Try again without DEVICE_LOCAL */
  636. for (uint32_t i = 0; i < pdmp.memoryTypeCount; ++i) {
  637. if ((mr.memoryTypeBits & (1 << i)) &&
  638. (pdmp.memoryTypes[i].propertyFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) !=
  639. VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) {
  640. mai.memoryTypeIndex = i;
  641. res = funcs->AllocateMemory(device, &mai, NULL, &swap->export_mem);
  642. allocated = res == VK_SUCCESS;
  643. if (allocated)
  644. break;
  645. flog("failed to AllocateMemory (not DEVICE_LOCAL): %s (%d)", result_to_str(res),
  646. (int)res);
  647. }
  648. }
  649. }
  650. if (!allocated) {
  651. flog("failed to allocate memory of any type");
  652. funcs->DestroyImage(device, swap->export_image, data->ac);
  653. swap->export_image = VK_NULL_HANDLE;
  654. return false;
  655. }
  656. /* -------------------------------------------------------- */
  657. /* bind image memory */
  658. bool use_bi2 = f & VK_EXTERNAL_MEMORY_FEATURE_DEDICATED_ONLY_BIT;
  659. if (use_bi2) {
  660. VkBindImageMemoryInfo bimi = {0};
  661. bimi.sType = VK_STRUCTURE_TYPE_BIND_IMAGE_MEMORY_INFO;
  662. bimi.image = swap->export_image;
  663. bimi.memory = swap->export_mem;
  664. bimi.memoryOffset = 0;
  665. res = funcs->BindImageMemory2(device, 1, &bimi);
  666. } else {
  667. res = funcs->BindImageMemory(device, swap->export_image, swap->export_mem, 0);
  668. }
  669. if (VK_SUCCESS != res) {
  670. flog("%s failed: %s", use_bi2 ? "BindImageMemory2" : "BindImageMemory", result_to_str(res));
  671. funcs->DestroyImage(device, swap->export_image, data->ac);
  672. swap->export_image = VK_NULL_HANDLE;
  673. return false;
  674. }
  675. return true;
  676. }
  677. static bool vk_shtex_init(struct vk_data *data, HWND window, struct vk_swap_data *swap)
  678. {
  679. if (!vk_shtex_init_d3d11(data)) {
  680. return false;
  681. }
  682. if (!vk_shtex_init_d3d11_tex(data, swap)) {
  683. return false;
  684. }
  685. if (!vk_shtex_init_vulkan_tex(data, swap)) {
  686. return false;
  687. }
  688. data->cur_swap = swap;
  689. swap->captured = capture_init_shtex(&swap->shtex_info, window, swap->image_extent.width,
  690. swap->image_extent.height, (uint32_t)swap->format, false,
  691. (uintptr_t)swap->handle);
  692. if (!swap->captured)
  693. return false;
  694. if (global_hook_info->force_shmem) {
  695. flog("shared memory capture currently "
  696. "unsupported; ignoring");
  697. }
  698. hlog("vulkan shared texture capture successful");
  699. return true;
  700. }
  701. static void vk_shtex_create_frame_objects(struct vk_data *data, struct vk_queue_data *queue_data, uint32_t image_count)
  702. {
  703. queue_data->frames = vk_alloc(data->ac, image_count * sizeof(struct vk_frame_data),
  704. _Alignof(struct vk_frame_data), VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
  705. memset(queue_data->frames, 0, image_count * sizeof(struct vk_frame_data));
  706. queue_data->frame_index = 0;
  707. queue_data->frame_count = image_count;
  708. VkDevice device = data->device;
  709. for (uint32_t image_index = 0; image_index < image_count; image_index++) {
  710. struct vk_frame_data *frame_data = &queue_data->frames[image_index];
  711. VkCommandPoolCreateInfo cpci;
  712. cpci.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO;
  713. cpci.pNext = NULL;
  714. cpci.flags = 0;
  715. cpci.queueFamilyIndex = queue_data->fam_idx;
  716. VkResult res = data->funcs.CreateCommandPool(device, &cpci, data->ac, &frame_data->cmd_pool);
  717. debug_res("CreateCommandPool", res);
  718. VkCommandBufferAllocateInfo cbai;
  719. cbai.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO;
  720. cbai.pNext = NULL;
  721. cbai.commandPool = frame_data->cmd_pool;
  722. cbai.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY;
  723. cbai.commandBufferCount = 1;
  724. res = data->funcs.AllocateCommandBuffers(device, &cbai, &frame_data->cmd_buffer);
  725. debug_res("AllocateCommandBuffers", res);
  726. GET_LDT(frame_data->cmd_buffer) = GET_LDT(device);
  727. VkFenceCreateInfo fci = {0};
  728. fci.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
  729. fci.pNext = NULL;
  730. fci.flags = 0;
  731. res = data->funcs.CreateFence(device, &fci, data->ac, &frame_data->fence);
  732. debug_res("CreateFence", res);
  733. }
  734. }
  735. static void vk_shtex_destroy_fence(struct vk_data *data, bool *cmd_buffer_busy, VkFence *fence)
  736. {
  737. VkDevice device = data->device;
  738. if (*cmd_buffer_busy) {
  739. data->funcs.WaitForFences(device, 1, fence, VK_TRUE, ~0ull);
  740. *cmd_buffer_busy = false;
  741. }
  742. data->funcs.DestroyFence(device, *fence, data->ac);
  743. *fence = VK_NULL_HANDLE;
  744. }
  745. static void vk_shtex_destroy_frame_objects(struct vk_data *data, struct vk_queue_data *queue_data)
  746. {
  747. VkDevice device = data->device;
  748. for (uint32_t frame_idx = 0; frame_idx < queue_data->frame_count; frame_idx++) {
  749. struct vk_frame_data *frame_data = &queue_data->frames[frame_idx];
  750. bool *cmd_buffer_busy = &frame_data->cmd_buffer_busy;
  751. VkFence *fence = &frame_data->fence;
  752. vk_shtex_destroy_fence(data, cmd_buffer_busy, fence);
  753. data->funcs.DestroyCommandPool(device, frame_data->cmd_pool, data->ac);
  754. frame_data->cmd_pool = VK_NULL_HANDLE;
  755. }
  756. vk_free(data->ac, queue_data->frames);
  757. queue_data->frames = NULL;
  758. queue_data->frame_count = 0;
  759. }
  760. static void vk_shtex_capture(struct vk_data *data, struct vk_device_funcs *funcs, struct vk_swap_data *swap,
  761. uint32_t idx, VkQueue queue, const VkPresentInfoKHR *info)
  762. {
  763. VkResult res = VK_SUCCESS;
  764. VkCommandBufferBeginInfo begin_info;
  765. begin_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
  766. begin_info.pNext = NULL;
  767. begin_info.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
  768. begin_info.pInheritanceInfo = NULL;
  769. VkImageMemoryBarrier mb[2];
  770. VkImageMemoryBarrier *src_mb = &mb[0];
  771. VkImageMemoryBarrier *dst_mb = &mb[1];
  772. /* ------------------------------------------------------ */
  773. /* do image copy */
  774. const uint32_t image_index = info->pImageIndices[idx];
  775. VkImage cur_backbuffer = swap->swap_images[image_index];
  776. struct vk_queue_data *queue_data = get_queue_data(data, queue);
  777. uint32_t fam_idx = queue_data->fam_idx;
  778. const uint32_t image_count = swap->image_count;
  779. if (queue_data->frame_count < image_count) {
  780. if (queue_data->frame_count > 0)
  781. vk_shtex_destroy_frame_objects(data, queue_data);
  782. vk_shtex_create_frame_objects(data, queue_data, image_count);
  783. }
  784. const uint32_t frame_index = queue_data->frame_index;
  785. struct vk_frame_data *frame_data = &queue_data->frames[frame_index];
  786. queue_data->frame_index = (frame_index + 1) % queue_data->frame_count;
  787. vk_shtex_clear_fence(data, frame_data);
  788. VkDevice device = data->device;
  789. res = funcs->ResetCommandPool(device, frame_data->cmd_pool, 0);
  790. #ifdef MORE_DEBUGGING
  791. debug_res("ResetCommandPool", res);
  792. #endif
  793. const VkCommandBuffer cmd_buffer = frame_data->cmd_buffer;
  794. res = funcs->BeginCommandBuffer(cmd_buffer, &begin_info);
  795. #ifdef MORE_DEBUGGING
  796. debug_res("BeginCommandBuffer", res);
  797. #endif
  798. /* ------------------------------------------------------ */
  799. /* transition shared texture if necessary */
  800. if (!swap->layout_initialized) {
  801. VkImageMemoryBarrier imb;
  802. imb.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER;
  803. imb.pNext = NULL;
  804. imb.srcAccessMask = 0;
  805. imb.dstAccessMask = 0;
  806. imb.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED;
  807. imb.newLayout = VK_IMAGE_LAYOUT_GENERAL;
  808. imb.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
  809. imb.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
  810. imb.image = swap->export_image;
  811. imb.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
  812. imb.subresourceRange.baseMipLevel = 0;
  813. imb.subresourceRange.levelCount = 1;
  814. imb.subresourceRange.baseArrayLayer = 0;
  815. imb.subresourceRange.layerCount = 1;
  816. funcs->CmdPipelineBarrier(cmd_buffer, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
  817. VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, 0, 0, NULL, 0, NULL, 1, &imb);
  818. swap->layout_initialized = true;
  819. }
  820. /* ------------------------------------------------------ */
  821. /* transition cur_backbuffer to transfer source state */
  822. src_mb->sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER;
  823. src_mb->pNext = NULL;
  824. src_mb->srcAccessMask = VK_ACCESS_MEMORY_READ_BIT;
  825. src_mb->dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
  826. src_mb->oldLayout = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR;
  827. src_mb->newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
  828. src_mb->srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
  829. src_mb->dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
  830. src_mb->image = cur_backbuffer;
  831. src_mb->subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
  832. src_mb->subresourceRange.baseMipLevel = 0;
  833. src_mb->subresourceRange.levelCount = 1;
  834. src_mb->subresourceRange.baseArrayLayer = 0;
  835. src_mb->subresourceRange.layerCount = 1;
  836. /* ------------------------------------------------------ */
  837. /* transition exportedTexture to transfer dest state */
  838. dst_mb->sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER;
  839. dst_mb->pNext = NULL;
  840. dst_mb->srcAccessMask = 0;
  841. dst_mb->dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
  842. dst_mb->oldLayout = VK_IMAGE_LAYOUT_GENERAL;
  843. dst_mb->newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
  844. dst_mb->srcQueueFamilyIndex = VK_QUEUE_FAMILY_EXTERNAL;
  845. dst_mb->dstQueueFamilyIndex = fam_idx;
  846. dst_mb->image = swap->export_image;
  847. dst_mb->subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
  848. dst_mb->subresourceRange.baseMipLevel = 0;
  849. dst_mb->subresourceRange.levelCount = 1;
  850. dst_mb->subresourceRange.baseArrayLayer = 0;
  851. dst_mb->subresourceRange.layerCount = 1;
  852. funcs->CmdPipelineBarrier(cmd_buffer, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0,
  853. 0, NULL, 0, NULL, 2, mb);
  854. /* ------------------------------------------------------ */
  855. /* copy cur_backbuffer's content to our interop image */
  856. VkImageCopy cpy;
  857. cpy.srcSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
  858. cpy.srcSubresource.mipLevel = 0;
  859. cpy.srcSubresource.baseArrayLayer = 0;
  860. cpy.srcSubresource.layerCount = 1;
  861. cpy.srcOffset.x = 0;
  862. cpy.srcOffset.y = 0;
  863. cpy.srcOffset.z = 0;
  864. cpy.dstSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
  865. cpy.dstSubresource.mipLevel = 0;
  866. cpy.dstSubresource.baseArrayLayer = 0;
  867. cpy.dstSubresource.layerCount = 1;
  868. cpy.dstOffset.x = 0;
  869. cpy.dstOffset.y = 0;
  870. cpy.dstOffset.z = 0;
  871. cpy.extent.width = swap->image_extent.width;
  872. cpy.extent.height = swap->image_extent.height;
  873. cpy.extent.depth = 1;
  874. funcs->CmdCopyImage(cmd_buffer, cur_backbuffer, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, swap->export_image,
  875. VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &cpy);
  876. /* ------------------------------------------------------ */
  877. /* Restore the swap chain image layout to what it was
  878. * before. This may not be strictly needed, but it is
  879. * generally good to restore things to their original
  880. * state. */
  881. src_mb->srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
  882. src_mb->dstAccessMask = VK_ACCESS_MEMORY_READ_BIT;
  883. src_mb->oldLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
  884. src_mb->newLayout = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR;
  885. dst_mb->srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
  886. dst_mb->dstAccessMask = 0;
  887. dst_mb->oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
  888. dst_mb->newLayout = VK_IMAGE_LAYOUT_GENERAL;
  889. dst_mb->srcQueueFamilyIndex = fam_idx;
  890. dst_mb->dstQueueFamilyIndex = VK_QUEUE_FAMILY_EXTERNAL;
  891. funcs->CmdPipelineBarrier(cmd_buffer, VK_PIPELINE_STAGE_TRANSFER_BIT,
  892. VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT | VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, 0, 0, NULL,
  893. 0, NULL, 2, mb);
  894. funcs->EndCommandBuffer(cmd_buffer);
  895. /* ------------------------------------------------------ */
  896. VkSubmitInfo submit_info;
  897. submit_info.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO;
  898. submit_info.pNext = NULL;
  899. submit_info.waitSemaphoreCount = 0;
  900. submit_info.pWaitSemaphores = NULL;
  901. submit_info.pWaitDstStageMask = NULL;
  902. submit_info.commandBufferCount = 1;
  903. submit_info.pCommandBuffers = &cmd_buffer;
  904. submit_info.signalSemaphoreCount = 0;
  905. submit_info.pSignalSemaphores = NULL;
  906. const VkFence fence = frame_data->fence;
  907. res = funcs->QueueSubmit(queue, 1, &submit_info, fence);
  908. #ifdef MORE_DEBUGGING
  909. debug_res("QueueSubmit", res);
  910. #endif
  911. if (res == VK_SUCCESS)
  912. frame_data->cmd_buffer_busy = true;
  913. }
  914. static inline bool valid_rect(struct vk_swap_data *swap)
  915. {
  916. return !!swap->image_extent.width && !!swap->image_extent.height;
  917. }
  918. static void vk_capture(struct vk_data *data, VkQueue queue, const VkPresentInfoKHR *info)
  919. {
  920. struct vk_swap_data *swap = NULL;
  921. HWND window = NULL;
  922. uint32_t idx = 0;
  923. #ifdef MORE_DEBUGGING
  924. debug("QueuePresentKHR called on "
  925. "devicekey %p, swapchain count %d",
  926. &data->funcs, info->swapchainCount);
  927. #endif
  928. /* use first swap chain associated with a window */
  929. for (; idx < info->swapchainCount; idx++) {
  930. struct vk_swap_data *cur_swap = get_swap_data(data, info->pSwapchains[idx]);
  931. if (cur_swap) {
  932. window = cur_swap->hwnd;
  933. if (window != NULL) {
  934. swap = cur_swap;
  935. break;
  936. }
  937. }
  938. }
  939. if (!window) {
  940. return;
  941. }
  942. if (capture_should_stop()) {
  943. vk_shtex_free(data);
  944. }
  945. if (capture_should_init()) {
  946. if (valid_rect(swap) && !vk_shtex_init(data, window, swap)) {
  947. vk_shtex_free(data);
  948. data->valid = false;
  949. flog("vk_shtex_init failed");
  950. }
  951. }
  952. if (capture_ready()) {
  953. if (swap != data->cur_swap) {
  954. vk_shtex_free(data);
  955. return;
  956. }
  957. vk_shtex_capture(data, &data->funcs, swap, idx, queue, info);
  958. }
  959. }
  960. static VkResult VKAPI_CALL OBS_QueuePresentKHR(VkQueue queue, const VkPresentInfoKHR *info)
  961. {
  962. struct vk_data *const data = get_device_data_by_queue(queue);
  963. struct vk_queue_data *const queue_data = get_queue_data(data, queue);
  964. struct vk_device_funcs *const funcs = &data->funcs;
  965. if (data->valid && queue_data->supports_transfer) {
  966. vk_capture(data, queue, info);
  967. }
  968. if (vk_presenting != 0) {
  969. flog("non-zero vk_presenting: %d", vk_presenting);
  970. }
  971. vk_presenting++;
  972. VkResult res = funcs->QueuePresentKHR(queue, info);
  973. vk_presenting--;
  974. return res;
  975. }
  976. /* ======================================================================== */
  977. /* setup hooks */
  978. static inline bool is_inst_link_info(VkLayerInstanceCreateInfo *lici)
  979. {
  980. return lici->sType == VK_STRUCTURE_TYPE_LOADER_INSTANCE_CREATE_INFO && lici->function == VK_LAYER_LINK_INFO;
  981. }
  982. static VkResult VKAPI_CALL OBS_CreateInstance(const VkInstanceCreateInfo *cinfo, const VkAllocationCallbacks *ac,
  983. VkInstance *p_inst)
  984. {
  985. VkInstanceCreateInfo info = *cinfo;
  986. /* -------------------------------------------------------- */
  987. /* step through chain until we get to the link info */
  988. VkLayerInstanceCreateInfo *lici = (void *)info.pNext;
  989. while (lici && !is_inst_link_info(lici)) {
  990. lici = (VkLayerInstanceCreateInfo *)lici->pNext;
  991. }
  992. if (lici == NULL) {
  993. return VK_ERROR_INITIALIZATION_FAILED;
  994. }
  995. PFN_vkGetInstanceProcAddr gpa = lici->u.pLayerInfo->pfnNextGetInstanceProcAddr;
  996. /* -------------------------------------------------------- */
  997. /* move chain on for next layer */
  998. lici->u.pLayerInfo = lici->u.pLayerInfo->pNext;
  999. /* -------------------------------------------------------- */
  1000. /* (HACK) Set api version to 1.1 if set to 1.0 */
  1001. /* We do this to get our extensions working properly */
  1002. VkApplicationInfo ai;
  1003. if (info.pApplicationInfo) {
  1004. ai = *info.pApplicationInfo;
  1005. if (ai.apiVersion < VK_API_VERSION_1_1)
  1006. ai.apiVersion = VK_API_VERSION_1_1;
  1007. } else {
  1008. ai.sType = VK_STRUCTURE_TYPE_APPLICATION_INFO;
  1009. ai.pNext = NULL;
  1010. ai.pApplicationName = NULL;
  1011. ai.applicationVersion = 0;
  1012. ai.pEngineName = NULL;
  1013. ai.engineVersion = 0;
  1014. ai.apiVersion = VK_API_VERSION_1_1;
  1015. }
  1016. info.pApplicationInfo = &ai;
  1017. /* -------------------------------------------------------- */
  1018. /* allocate data node */
  1019. struct vk_inst_data *idata = alloc_inst_data(ac);
  1020. if (!idata)
  1021. return VK_ERROR_OUT_OF_HOST_MEMORY;
  1022. /* -------------------------------------------------------- */
  1023. /* create instance */
  1024. PFN_vkCreateInstance create = (void *)gpa(NULL, "vkCreateInstance");
  1025. VkResult res = create(&info, ac, p_inst);
  1026. bool valid = res == VK_SUCCESS;
  1027. if (!valid) {
  1028. /* try again with original arguments */
  1029. res = create(cinfo, ac, p_inst);
  1030. if (res != VK_SUCCESS) {
  1031. vk_free(ac, idata);
  1032. return res;
  1033. }
  1034. }
  1035. VkInstance inst = *p_inst;
  1036. init_inst_data(idata, inst);
  1037. /* -------------------------------------------------------- */
  1038. /* fetch the functions we need */
  1039. struct vk_inst_funcs *ifuncs = &idata->funcs;
  1040. #define GETADDR(x) \
  1041. do { \
  1042. ifuncs->x = (void *)gpa(inst, "vk" #x); \
  1043. if (!ifuncs->x) { \
  1044. flog("could not get instance " \
  1045. "address for vk" #x); \
  1046. funcs_found = false; \
  1047. } \
  1048. } while (false)
  1049. bool funcs_found = true;
  1050. GETADDR(GetInstanceProcAddr);
  1051. GETADDR(DestroyInstance);
  1052. GETADDR(CreateWin32SurfaceKHR);
  1053. GETADDR(DestroySurfaceKHR);
  1054. GETADDR(GetPhysicalDeviceQueueFamilyProperties);
  1055. GETADDR(GetPhysicalDeviceMemoryProperties);
  1056. GETADDR(GetPhysicalDeviceImageFormatProperties2);
  1057. GETADDR(EnumerateDeviceExtensionProperties);
  1058. #undef GETADDR
  1059. valid = valid && funcs_found;
  1060. idata->valid = valid;
  1061. if (valid)
  1062. init_obj_list(&idata->surfaces);
  1063. return res;
  1064. }
  1065. static void VKAPI_CALL OBS_DestroyInstance(VkInstance instance, const VkAllocationCallbacks *ac)
  1066. {
  1067. struct vk_inst_funcs *ifuncs = get_inst_funcs(instance);
  1068. PFN_vkDestroyInstance destroy_instance = ifuncs->DestroyInstance;
  1069. remove_free_inst_data(instance, ac);
  1070. destroy_instance(instance, ac);
  1071. }
  1072. static bool vk_shared_tex_supported(struct vk_inst_funcs *funcs, VkPhysicalDevice phy_device, VkFormat format,
  1073. VkImageUsageFlags usage, VkExternalMemoryProperties *external_mem_props)
  1074. {
  1075. VkPhysicalDeviceImageFormatInfo2 info;
  1076. VkPhysicalDeviceExternalImageFormatInfo external_info;
  1077. external_info.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_IMAGE_FORMAT_INFO;
  1078. external_info.pNext = NULL;
  1079. external_info.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_D3D11_TEXTURE_KMT_BIT;
  1080. info.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2;
  1081. info.pNext = &external_info;
  1082. info.format = format;
  1083. info.type = VK_IMAGE_TYPE_2D;
  1084. info.tiling = VK_IMAGE_TILING_OPTIMAL;
  1085. info.flags = 0;
  1086. info.usage = usage;
  1087. VkExternalImageFormatProperties external_props = {0};
  1088. external_props.sType = VK_STRUCTURE_TYPE_EXTERNAL_IMAGE_FORMAT_PROPERTIES;
  1089. external_props.pNext = NULL;
  1090. VkImageFormatProperties2 props = {0};
  1091. props.sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_PROPERTIES_2;
  1092. props.pNext = &external_props;
  1093. VkResult result = funcs->GetPhysicalDeviceImageFormatProperties2(phy_device, &info, &props);
  1094. *external_mem_props = external_props.externalMemoryProperties;
  1095. const VkExternalMemoryFeatureFlags features = external_mem_props->externalMemoryFeatures;
  1096. return ((VK_SUCCESS == result) && (features & VK_EXTERNAL_MEMORY_FEATURE_IMPORTABLE_BIT));
  1097. }
  1098. static inline bool is_device_link_info(VkLayerDeviceCreateInfo *lici)
  1099. {
  1100. return lici->sType == VK_STRUCTURE_TYPE_LOADER_DEVICE_CREATE_INFO && lici->function == VK_LAYER_LINK_INFO;
  1101. }
  1102. static VkResult VKAPI_CALL OBS_CreateDevice(VkPhysicalDevice phy_device, const VkDeviceCreateInfo *info,
  1103. const VkAllocationCallbacks *ac, VkDevice *p_device)
  1104. {
  1105. struct vk_inst_data *idata = get_inst_data_by_physical_device(phy_device);
  1106. struct vk_inst_funcs *ifuncs = &idata->funcs;
  1107. struct vk_data *data = NULL;
  1108. VkResult ret = VK_ERROR_INITIALIZATION_FAILED;
  1109. VkLayerDeviceCreateInfo *ldci = (void *)info->pNext;
  1110. /* -------------------------------------------------------- */
  1111. /* step through chain until we get to the link info */
  1112. while (ldci && !is_device_link_info(ldci)) {
  1113. ldci = (VkLayerDeviceCreateInfo *)ldci->pNext;
  1114. }
  1115. if (!ldci) {
  1116. goto fail;
  1117. }
  1118. PFN_vkGetInstanceProcAddr gipa;
  1119. PFN_vkGetDeviceProcAddr gdpa;
  1120. gipa = ldci->u.pLayerInfo->pfnNextGetInstanceProcAddr;
  1121. gdpa = ldci->u.pLayerInfo->pfnNextGetDeviceProcAddr;
  1122. /* -------------------------------------------------------- */
  1123. /* move chain on for next layer */
  1124. ldci->u.pLayerInfo = ldci->u.pLayerInfo->pNext;
  1125. /* -------------------------------------------------------- */
  1126. /* allocate data node */
  1127. data = alloc_device_data(ac);
  1128. if (!data)
  1129. return VK_ERROR_OUT_OF_HOST_MEMORY;
  1130. init_obj_list(&data->queues);
  1131. init_obj_list(&data->swap_views);
  1132. init_obj_list(&data->framebuffers);
  1133. /* -------------------------------------------------------- */
  1134. /* create device and initialize hook data */
  1135. PFN_vkCreateDevice createFunc = (PFN_vkCreateDevice)gipa(idata->instance, "vkCreateDevice");
  1136. ret = createFunc(phy_device, info, ac, p_device);
  1137. if (ret != VK_SUCCESS) {
  1138. vk_free(ac, data);
  1139. return ret;
  1140. }
  1141. VkDevice device = *p_device;
  1142. init_device_data(data, device);
  1143. data->valid = false; /* set true below if it doesn't go to fail */
  1144. data->phy_device = phy_device;
  1145. /* -------------------------------------------------------- */
  1146. /* fetch the functions we need */
  1147. struct vk_device_funcs *dfuncs = &data->funcs;
  1148. bool funcs_found = true;
  1149. #define GETADDR(x) \
  1150. do { \
  1151. dfuncs->x = (void *)gdpa(device, "vk" #x); \
  1152. if (!dfuncs->x) { \
  1153. flog("could not get device " \
  1154. "address for vk" #x); \
  1155. funcs_found = false; \
  1156. } \
  1157. } while (false)
  1158. #define GETADDR_OPTIONAL(x) \
  1159. do { \
  1160. dfuncs->x = (void *)gdpa(device, "vk" #x); \
  1161. } while (false)
  1162. GETADDR(GetDeviceProcAddr);
  1163. GETADDR(DestroyDevice);
  1164. GETADDR(CreateSwapchainKHR);
  1165. GETADDR(DestroySwapchainKHR);
  1166. GETADDR(QueuePresentKHR);
  1167. GETADDR(AllocateMemory);
  1168. GETADDR(FreeMemory);
  1169. GETADDR(BindImageMemory);
  1170. GETADDR(BindImageMemory2);
  1171. GETADDR(GetSwapchainImagesKHR);
  1172. GETADDR(CreateImage);
  1173. GETADDR(DestroyImage);
  1174. GETADDR(GetImageMemoryRequirements);
  1175. GETADDR(GetImageMemoryRequirements2);
  1176. GETADDR(ResetCommandPool);
  1177. GETADDR(BeginCommandBuffer);
  1178. GETADDR(EndCommandBuffer);
  1179. GETADDR(CmdCopyImage);
  1180. GETADDR(CmdPipelineBarrier);
  1181. GETADDR(GetDeviceQueue);
  1182. GETADDR(QueueSubmit);
  1183. GETADDR(CreateCommandPool);
  1184. GETADDR(DestroyCommandPool);
  1185. GETADDR(AllocateCommandBuffers);
  1186. GETADDR(CreateFence);
  1187. GETADDR(DestroyFence);
  1188. GETADDR(WaitForFences);
  1189. GETADDR(ResetFences);
  1190. GETADDR(CreateImageView);
  1191. GETADDR(DestroyImageView);
  1192. GETADDR(CreateFramebuffer);
  1193. GETADDR(DestroyFramebuffer);
  1194. GETADDR(CmdBeginRenderPass);
  1195. GETADDR_OPTIONAL(CmdBeginRenderPass2KHR);
  1196. GETADDR_OPTIONAL(CmdBeginRenderPass2);
  1197. #undef GETADDR_OPTIONAL
  1198. #undef GETADDR
  1199. if (!funcs_found) {
  1200. goto fail;
  1201. }
  1202. if (!idata->valid) {
  1203. flog("instance not valid");
  1204. goto fail;
  1205. }
  1206. const char *required_device_extensions[] = {VK_KHR_EXTERNAL_MEMORY_WIN32_EXTENSION_NAME};
  1207. uint32_t device_extension_count = 0;
  1208. ret = ifuncs->EnumerateDeviceExtensionProperties(phy_device, NULL, &device_extension_count, NULL);
  1209. if (ret != VK_SUCCESS)
  1210. goto fail;
  1211. VkExtensionProperties *device_extensions = _malloca(sizeof(VkExtensionProperties) * device_extension_count);
  1212. ret = ifuncs->EnumerateDeviceExtensionProperties(phy_device, NULL, &device_extension_count, device_extensions);
  1213. if (ret != VK_SUCCESS) {
  1214. _freea(device_extensions);
  1215. goto fail;
  1216. }
  1217. bool extensions_found = true;
  1218. for (uint32_t i = 0; i < _countof(required_device_extensions); i++) {
  1219. const char *const required_extension = required_device_extensions[i];
  1220. bool found = false;
  1221. for (uint32_t j = 0; j < device_extension_count; j++) {
  1222. if (!strcmp(required_extension, device_extensions[j].extensionName)) {
  1223. found = true;
  1224. break;
  1225. }
  1226. }
  1227. if (!found) {
  1228. flog("missing device extension: %s", required_extension);
  1229. extensions_found = false;
  1230. }
  1231. }
  1232. _freea(device_extensions);
  1233. if (!extensions_found)
  1234. goto fail;
  1235. VkFormat format = VK_FORMAT_R8G8B8A8_UNORM;
  1236. VkImageUsageFlags usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT;
  1237. if (!vk_shared_tex_supported(ifuncs, phy_device, format, usage, &data->external_mem_props)) {
  1238. flog("texture sharing is not supported");
  1239. goto fail;
  1240. }
  1241. data->inst_data = idata;
  1242. data->ac = NULL;
  1243. if (ac) {
  1244. data->ac_storage = *ac;
  1245. data->ac = &data->ac_storage;
  1246. }
  1247. uint32_t queue_family_property_count = 0;
  1248. ifuncs->GetPhysicalDeviceQueueFamilyProperties(phy_device, &queue_family_property_count, NULL);
  1249. VkQueueFamilyProperties *queue_family_properties =
  1250. _malloca(sizeof(VkQueueFamilyProperties) * queue_family_property_count);
  1251. ifuncs->GetPhysicalDeviceQueueFamilyProperties(phy_device, &queue_family_property_count,
  1252. queue_family_properties);
  1253. for (uint32_t info_index = 0, info_count = info->queueCreateInfoCount; info_index < info_count; ++info_index) {
  1254. const VkDeviceQueueCreateInfo *queue_info = &info->pQueueCreateInfos[info_index];
  1255. for (uint32_t queue_index = 0, queue_count = queue_info->queueCount; queue_index < queue_count;
  1256. ++queue_index) {
  1257. const uint32_t family_index = queue_info->queueFamilyIndex;
  1258. VkQueue queue;
  1259. data->funcs.GetDeviceQueue(device, family_index, queue_index, &queue);
  1260. const bool supports_transfer =
  1261. (queue_family_properties[family_index].queueFlags &
  1262. (VK_QUEUE_GRAPHICS_BIT | VK_QUEUE_COMPUTE_BIT | VK_QUEUE_TRANSFER_BIT)) != 0;
  1263. add_queue_data(data, queue, family_index, supports_transfer, ac);
  1264. }
  1265. }
  1266. _freea(queue_family_properties);
  1267. init_obj_list(&data->swaps);
  1268. data->cur_swap = NULL;
  1269. data->d3d11_device = NULL;
  1270. data->d3d11_context = NULL;
  1271. data->valid = true;
  1272. fail:
  1273. return ret;
  1274. }
  1275. static void VKAPI_CALL OBS_DestroyDevice(VkDevice device, const VkAllocationCallbacks *ac)
  1276. {
  1277. struct vk_data *data = remove_device_data(device);
  1278. if (data->valid) {
  1279. struct vk_queue_data *queue_data = queue_walk_begin(data);
  1280. while (queue_data) {
  1281. vk_shtex_destroy_frame_objects(data, queue_data);
  1282. queue_data = queue_walk_next(queue_data);
  1283. }
  1284. queue_walk_end(data);
  1285. remove_free_queue_all(data, ac);
  1286. }
  1287. PFN_vkDestroyDevice destroy_device = data->funcs.DestroyDevice;
  1288. vk_free(ac, data);
  1289. destroy_device(device, ac);
  1290. }
  1291. static VkResult VKAPI_CALL OBS_CreateSwapchainKHR(VkDevice device, const VkSwapchainCreateInfoKHR *cinfo,
  1292. const VkAllocationCallbacks *ac, VkSwapchainKHR *p_sc)
  1293. {
  1294. struct vk_data *data = get_device_data(device);
  1295. struct vk_device_funcs *funcs = &data->funcs;
  1296. if (!data->valid)
  1297. return funcs->CreateSwapchainKHR(device, cinfo, ac, p_sc);
  1298. VkSwapchainCreateInfoKHR info = *cinfo;
  1299. info.imageUsage |= VK_IMAGE_USAGE_TRANSFER_SRC_BIT;
  1300. VkResult res = funcs->CreateSwapchainKHR(device, &info, ac, p_sc);
  1301. debug_res("CreateSwapchainKHR", res);
  1302. if (res != VK_SUCCESS) {
  1303. /* try again with original imageUsage flags */
  1304. return funcs->CreateSwapchainKHR(device, cinfo, ac, p_sc);
  1305. }
  1306. VkSwapchainKHR sc = *p_sc;
  1307. uint32_t count = 0;
  1308. res = funcs->GetSwapchainImagesKHR(device, sc, &count, NULL);
  1309. debug_res("GetSwapchainImagesKHR", res);
  1310. if ((res == VK_SUCCESS) && (count > 0)) {
  1311. HWND hwnd = find_surf_hwnd(data->inst_data, cinfo->surface);
  1312. if (hwnd) {
  1313. struct vk_swap_data *swap_data = alloc_swap_data(ac);
  1314. if (swap_data) {
  1315. swap_data->swap_images = vk_alloc(ac, count * sizeof(VkImage), _Alignof(VkImage),
  1316. VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
  1317. res = funcs->GetSwapchainImagesKHR(device, sc, &count, swap_data->swap_images);
  1318. debug_res("GetSwapchainImagesKHR", res);
  1319. swap_data->image_extent = cinfo->imageExtent;
  1320. swap_data->format = cinfo->imageFormat;
  1321. swap_data->hwnd = hwnd;
  1322. swap_data->export_image = VK_NULL_HANDLE;
  1323. swap_data->layout_initialized = false;
  1324. swap_data->export_mem = VK_NULL_HANDLE;
  1325. swap_data->image_count = count;
  1326. swap_data->handle = INVALID_HANDLE_VALUE;
  1327. swap_data->shtex_info = NULL;
  1328. swap_data->d3d11_tex = NULL;
  1329. swap_data->captured = false;
  1330. init_swap_data(swap_data, data, sc);
  1331. }
  1332. }
  1333. }
  1334. return VK_SUCCESS;
  1335. }
  1336. static VkResult VKAPI_CALL OBS_CreateImageView(VkDevice device, const VkImageViewCreateInfo *pCreateInfo,
  1337. const VkAllocationCallbacks *pAllocator, VkImageView *pView)
  1338. {
  1339. bool from_swap_chain = false;
  1340. struct vk_data *const data = get_device_data(device);
  1341. if (data->valid) {
  1342. struct vk_swap_data *swap = swap_walk_begin(data);
  1343. while (swap) {
  1344. for (uint32_t i = 0, count = swap->image_count; i < count; ++i) {
  1345. from_swap_chain = swap->swap_images[i] == pCreateInfo->image;
  1346. if (from_swap_chain)
  1347. break;
  1348. }
  1349. if (from_swap_chain)
  1350. break;
  1351. swap = swap_walk_next(swap);
  1352. }
  1353. swap_walk_end(data);
  1354. }
  1355. if (from_swap_chain) {
  1356. const void *pCurrent = pCreateInfo->pNext;
  1357. while (pCurrent) {
  1358. VkBaseInStructure baseIn;
  1359. memcpy(&baseIn, pCurrent, sizeof(baseIn));
  1360. if (baseIn.sType == VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO) {
  1361. ((VkImageViewUsageCreateInfo *)pCurrent)->usage |= VK_IMAGE_USAGE_TRANSFER_SRC_BIT;
  1362. break;
  1363. }
  1364. pCurrent = baseIn.pNext;
  1365. }
  1366. }
  1367. const PFN_vkCreateImageView func = data->funcs.CreateImageView;
  1368. VkResult result = func(device, pCreateInfo, pAllocator, pView);
  1369. if ((result == VK_SUCCESS) && from_swap_chain)
  1370. add_swap_view_data(data, *pView, pAllocator);
  1371. return result;
  1372. }
  1373. static void VKAPI_CALL OBS_DestroyImageView(VkDevice device, VkImageView imageView,
  1374. const VkAllocationCallbacks *pAllocator)
  1375. {
  1376. struct vk_data *const data = get_device_data(device);
  1377. if (data->valid && (imageView != VK_NULL_HANDLE)) {
  1378. struct vk_swap_view_data *swap_view_data = get_swap_view_data(data, imageView);
  1379. if (swap_view_data)
  1380. remove_free_swap_view_data(data, imageView, pAllocator);
  1381. }
  1382. data->funcs.DestroyImageView(device, imageView, pAllocator);
  1383. }
  1384. static void
  1385. generate_framebuffer_variants(PFN_vkCreateFramebuffer func, VkDevice device, const VkFramebufferCreateInfo *pCreateInfo,
  1386. const VkAllocationCallbacks *pAllocator, struct vk_framebuffer_data *framebuffer_data,
  1387. const VkFramebufferAttachmentsCreateInfo *pAttachmentsCreateInfo, uint32_t colorCount)
  1388. {
  1389. const uint32_t variantCount = 1 << colorCount;
  1390. for (uint32_t colorMask = 0; colorMask < variantCount; ++colorMask) {
  1391. VkImageUsageFlags pPreviousUsage[OBS_COLOR_ATTACHMENT_LIMIT];
  1392. uint32_t colorIndex = 0;
  1393. for (uint32_t infoIndex = 0, count = pAttachmentsCreateInfo->attachmentImageInfoCount;
  1394. infoIndex < count; ++infoIndex) {
  1395. const VkFramebufferAttachmentImageInfo *const pInfo =
  1396. &pAttachmentsCreateInfo->pAttachmentImageInfos[infoIndex];
  1397. const VkImageUsageFlags usage = pInfo->usage;
  1398. if (usage & VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT) {
  1399. pPreviousUsage[colorIndex] = usage;
  1400. if (colorMask & (1 << colorIndex)) {
  1401. ((VkFramebufferAttachmentImageInfo *)pInfo)->usage |=
  1402. VK_IMAGE_USAGE_TRANSFER_SRC_BIT;
  1403. }
  1404. ++colorIndex;
  1405. if (colorIndex == OBS_COLOR_ATTACHMENT_LIMIT)
  1406. break;
  1407. }
  1408. }
  1409. VkFramebuffer framebuffer;
  1410. const VkResult result = func(device, pCreateInfo, pAllocator, &framebuffer);
  1411. if (result == VK_SUCCESS)
  1412. framebuffer_data->alternates[colorMask] = framebuffer;
  1413. colorIndex = 0;
  1414. for (uint32_t infoIndex = 0, count = pAttachmentsCreateInfo->attachmentImageInfoCount;
  1415. infoIndex < count; ++infoIndex) {
  1416. const VkFramebufferAttachmentImageInfo *const pInfo =
  1417. &pAttachmentsCreateInfo->pAttachmentImageInfos[infoIndex];
  1418. if (pInfo->usage & VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT) {
  1419. if (colorMask & (1 << colorIndex)) {
  1420. ((VkFramebufferAttachmentImageInfo *)pInfo)->usage = pPreviousUsage[colorIndex];
  1421. }
  1422. ++colorIndex;
  1423. if (colorIndex == OBS_COLOR_ATTACHMENT_LIMIT)
  1424. break;
  1425. }
  1426. }
  1427. }
  1428. }
  1429. static VkResult VKAPI_CALL OBS_CreateFramebuffer(VkDevice device, const VkFramebufferCreateInfo *pCreateInfo,
  1430. const VkAllocationCallbacks *pAllocator, VkFramebuffer *pFramebuffer)
  1431. {
  1432. struct vk_data *const data = get_device_data(device);
  1433. const PFN_vkCreateFramebuffer func = data->funcs.CreateFramebuffer;
  1434. VkResult result = func(device, pCreateInfo, pAllocator, pFramebuffer);
  1435. if (data->valid && (result == VK_SUCCESS) && (pCreateInfo->flags & VK_FRAMEBUFFER_CREATE_IMAGELESS_BIT)) {
  1436. struct vk_framebuffer_data *const framebuffer_data =
  1437. add_framebuffer_data(data, *pFramebuffer, pAllocator);
  1438. const void *pCurrent = pCreateInfo->pNext;
  1439. while (pCurrent) {
  1440. VkBaseInStructure baseIn;
  1441. memcpy(&baseIn, pCurrent, sizeof(baseIn));
  1442. if (baseIn.sType == VK_STRUCTURE_TYPE_FRAMEBUFFER_ATTACHMENTS_CREATE_INFO) {
  1443. uint32_t attachmentMask = 0;
  1444. uint32_t colorCount = 0;
  1445. const VkFramebufferAttachmentsCreateInfo *const pAttachmentsCreateInfo = pCurrent;
  1446. for (uint32_t infoIndex = 0,
  1447. count = min(sizeof(attachmentMask) * 8,
  1448. pAttachmentsCreateInfo->attachmentImageInfoCount);
  1449. infoIndex < count; ++infoIndex) {
  1450. if (pAttachmentsCreateInfo->pAttachmentImageInfos[infoIndex].usage &
  1451. VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT) {
  1452. attachmentMask |= 1 << infoIndex;
  1453. ++colorCount;
  1454. if (colorCount == OBS_COLOR_ATTACHMENT_LIMIT)
  1455. break;
  1456. }
  1457. }
  1458. generate_framebuffer_variants(func, device, pCreateInfo, pAllocator, framebuffer_data,
  1459. pAttachmentsCreateInfo, colorCount);
  1460. framebuffer_data->color_attachment_mask = attachmentMask;
  1461. break;
  1462. }
  1463. pCurrent = baseIn.pNext;
  1464. }
  1465. }
  1466. return result;
  1467. }
  1468. static void VKAPI_CALL OBS_DestroyFramebuffer(VkDevice device, VkFramebuffer framebuffer,
  1469. const VkAllocationCallbacks *pAllocator)
  1470. {
  1471. struct vk_data *const data = get_device_data(device);
  1472. struct vk_device_funcs *const funcs = &data->funcs;
  1473. if (data->valid && (framebuffer != VK_NULL_HANDLE)) {
  1474. struct vk_framebuffer_data *framebuffer_data = get_framebuffer_data(data, framebuffer);
  1475. if (framebuffer_data) {
  1476. for (size_t i = 0; i < _countof(framebuffer_data->alternates); ++i) {
  1477. if (framebuffer_data->alternates[i] != VK_NULL_HANDLE) {
  1478. funcs->DestroyFramebuffer(device, framebuffer_data->alternates[i], pAllocator);
  1479. framebuffer_data->alternates[i] = VK_NULL_HANDLE;
  1480. }
  1481. }
  1482. remove_free_framebuffer_data(data, framebuffer, pAllocator);
  1483. }
  1484. }
  1485. funcs->DestroyFramebuffer(device, framebuffer, pAllocator);
  1486. }
  1487. static const VkRenderPassBeginInfo *process_render_pass_begin_info(const VkRenderPassBeginInfo *pRenderPassBegin,
  1488. VkRenderPassBeginInfo *pAlternateBegin,
  1489. struct vk_data *data)
  1490. {
  1491. const void *pCurrent = pRenderPassBegin->pNext;
  1492. while (pCurrent) {
  1493. VkBaseInStructure baseIn;
  1494. memcpy(&baseIn, pCurrent, sizeof(baseIn));
  1495. if (baseIn.sType == VK_STRUCTURE_TYPE_RENDER_PASS_ATTACHMENT_BEGIN_INFO) {
  1496. struct vk_framebuffer_data *const framebuffer_data =
  1497. get_framebuffer_data(data, pRenderPassBegin->framebuffer);
  1498. if (framebuffer_data) {
  1499. const VkRenderPassAttachmentBeginInfo *const pAttachmentInfo = pCurrent;
  1500. uint32_t swapMask = 0;
  1501. uint32_t colorIndex = 0;
  1502. for (uint32_t infoIndex = 0, count = pAttachmentInfo->attachmentCount;
  1503. infoIndex < count; ++infoIndex) {
  1504. if (framebuffer_data->color_attachment_mask & (1 << infoIndex)) {
  1505. if (get_swap_view_data(data,
  1506. pAttachmentInfo->pAttachments[infoIndex])) {
  1507. swapMask |= 1 << colorIndex;
  1508. }
  1509. ++colorIndex;
  1510. if (colorIndex == OBS_COLOR_ATTACHMENT_LIMIT)
  1511. break;
  1512. }
  1513. }
  1514. if (swapMask > 0) {
  1515. VkFramebuffer alternate = framebuffer_data->alternates[swapMask];
  1516. if (alternate != VK_NULL_HANDLE) {
  1517. *pAlternateBegin = *pRenderPassBegin;
  1518. pAlternateBegin->framebuffer = framebuffer_data->alternates[swapMask];
  1519. pRenderPassBegin = pAlternateBegin;
  1520. }
  1521. }
  1522. }
  1523. break;
  1524. }
  1525. pCurrent = baseIn.pNext;
  1526. }
  1527. return pRenderPassBegin;
  1528. }
  1529. static void VKAPI_CALL OBS_CmdBeginRenderPass(VkCommandBuffer commandBuffer,
  1530. const VkRenderPassBeginInfo *pRenderPassBegin, VkSubpassContents contents)
  1531. {
  1532. struct vk_data *const data = get_device_data_by_command_buffer(commandBuffer);
  1533. VkRenderPassBeginInfo alternateBegin;
  1534. if (data->valid) {
  1535. pRenderPassBegin = process_render_pass_begin_info(pRenderPassBegin, &alternateBegin, data);
  1536. }
  1537. data->funcs.CmdBeginRenderPass(commandBuffer, pRenderPassBegin, contents);
  1538. }
  1539. static void VKAPI_CALL OBS_CmdBeginRenderPass2KHR(VkCommandBuffer commandBuffer,
  1540. const VkRenderPassBeginInfo *pRenderPassBegin,
  1541. const VkSubpassBeginInfo *pSubpassBeginInfo)
  1542. {
  1543. struct vk_data *const data = get_device_data_by_command_buffer(commandBuffer);
  1544. VkRenderPassBeginInfo alternateBegin;
  1545. if (data->valid) {
  1546. pRenderPassBegin = process_render_pass_begin_info(pRenderPassBegin, &alternateBegin, data);
  1547. }
  1548. data->funcs.CmdBeginRenderPass2KHR(commandBuffer, pRenderPassBegin, pSubpassBeginInfo);
  1549. }
  1550. static void VKAPI_CALL OBS_CmdBeginRenderPass2(VkCommandBuffer commandBuffer,
  1551. const VkRenderPassBeginInfo *pRenderPassBegin,
  1552. const VkSubpassBeginInfo *pSubpassBeginInfo)
  1553. {
  1554. struct vk_data *const data = get_device_data_by_command_buffer(commandBuffer);
  1555. VkRenderPassBeginInfo alternateBegin;
  1556. if (data->valid) {
  1557. pRenderPassBegin = process_render_pass_begin_info(pRenderPassBegin, &alternateBegin, data);
  1558. }
  1559. data->funcs.CmdBeginRenderPass2(commandBuffer, pRenderPassBegin, pSubpassBeginInfo);
  1560. }
  1561. static void VKAPI_CALL OBS_DestroySwapchainKHR(VkDevice device, VkSwapchainKHR sc, const VkAllocationCallbacks *ac)
  1562. {
  1563. struct vk_data *data = get_device_data(device);
  1564. struct vk_device_funcs *funcs = &data->funcs;
  1565. PFN_vkDestroySwapchainKHR destroy_swapchain = funcs->DestroySwapchainKHR;
  1566. if ((sc != VK_NULL_HANDLE) && data->valid) {
  1567. struct vk_swap_data *swap = get_swap_data(data, sc);
  1568. if (swap) {
  1569. if (data->cur_swap == swap) {
  1570. vk_shtex_free(data);
  1571. }
  1572. vk_free(ac, swap->swap_images);
  1573. remove_free_swap_data(data, sc, ac);
  1574. }
  1575. }
  1576. destroy_swapchain(device, sc, ac);
  1577. }
  1578. static VkResult VKAPI_CALL OBS_CreateWin32SurfaceKHR(VkInstance inst, const VkWin32SurfaceCreateInfoKHR *info,
  1579. const VkAllocationCallbacks *ac, VkSurfaceKHR *surf)
  1580. {
  1581. struct vk_inst_data *idata = get_inst_data(inst);
  1582. struct vk_inst_funcs *ifuncs = &idata->funcs;
  1583. VkResult res = ifuncs->CreateWin32SurfaceKHR(inst, info, ac, surf);
  1584. if ((res == VK_SUCCESS) && idata->valid)
  1585. add_surf_data(idata, *surf, info->hwnd, ac);
  1586. return res;
  1587. }
  1588. static void VKAPI_CALL OBS_DestroySurfaceKHR(VkInstance inst, VkSurfaceKHR surf, const VkAllocationCallbacks *ac)
  1589. {
  1590. struct vk_inst_data *idata = get_inst_data(inst);
  1591. struct vk_inst_funcs *ifuncs = &idata->funcs;
  1592. PFN_vkDestroySurfaceKHR destroy_surface = ifuncs->DestroySurfaceKHR;
  1593. if ((surf != VK_NULL_HANDLE) && idata->valid)
  1594. remove_free_surf_data(idata, surf, ac);
  1595. destroy_surface(inst, surf, ac);
  1596. }
  1597. /* clang-format off */
  1598. #define GETPROCADDR(func) \
  1599. if (!strcmp(pName, "vk" #func)) \
  1600. return (PFN_vkVoidFunction)&OBS_##func;
  1601. #define GETPROCADDR_IF_SUPPORTED(func) \
  1602. if (!strcmp(pName, "vk" #func)) \
  1603. return funcs->func ? (PFN_vkVoidFunction)&OBS_##func : NULL;
  1604. /* clang-format on */
  1605. static PFN_vkVoidFunction VKAPI_CALL OBS_GetDeviceProcAddr(VkDevice device, const char *pName)
  1606. {
  1607. struct vk_data *data = get_device_data(device);
  1608. struct vk_device_funcs *funcs = &data->funcs;
  1609. debug_procaddr("vkGetDeviceProcAddr(%p, \"%s\")", device, pName);
  1610. GETPROCADDR(GetDeviceProcAddr);
  1611. GETPROCADDR(DestroyDevice);
  1612. GETPROCADDR_IF_SUPPORTED(CreateSwapchainKHR);
  1613. GETPROCADDR_IF_SUPPORTED(DestroySwapchainKHR);
  1614. GETPROCADDR_IF_SUPPORTED(QueuePresentKHR);
  1615. GETPROCADDR(CreateImageView);
  1616. GETPROCADDR(DestroyImageView);
  1617. GETPROCADDR(CreateFramebuffer);
  1618. GETPROCADDR(DestroyFramebuffer);
  1619. GETPROCADDR(CmdBeginRenderPass);
  1620. GETPROCADDR_IF_SUPPORTED(CmdBeginRenderPass2KHR);
  1621. GETPROCADDR_IF_SUPPORTED(CmdBeginRenderPass2);
  1622. if (funcs->GetDeviceProcAddr == NULL)
  1623. return NULL;
  1624. return funcs->GetDeviceProcAddr(device, pName);
  1625. }
  1626. /* bad layers require spec violation */
  1627. #define RETURN_FP_FOR_NULL_INSTANCE 1
  1628. static PFN_vkVoidFunction VKAPI_CALL OBS_GetInstanceProcAddr(VkInstance instance, const char *pName)
  1629. {
  1630. debug_procaddr("vkGetInstanceProcAddr(%p, \"%s\")", instance, pName);
  1631. /* instance chain functions we intercept */
  1632. GETPROCADDR(GetInstanceProcAddr);
  1633. GETPROCADDR(CreateInstance);
  1634. #if RETURN_FP_FOR_NULL_INSTANCE
  1635. /* other instance chain functions we intercept */
  1636. GETPROCADDR(DestroyInstance);
  1637. GETPROCADDR(CreateWin32SurfaceKHR);
  1638. GETPROCADDR(DestroySurfaceKHR);
  1639. /* device chain functions we intercept */
  1640. GETPROCADDR(GetDeviceProcAddr);
  1641. GETPROCADDR(CreateDevice);
  1642. GETPROCADDR(DestroyDevice);
  1643. if (instance == NULL)
  1644. return NULL;
  1645. struct vk_inst_funcs *const funcs = get_inst_funcs(instance);
  1646. #else
  1647. if (instance == NULL)
  1648. return NULL;
  1649. struct vk_inst_funcs *const funcs = get_inst_funcs(instance);
  1650. /* other instance chain functions we intercept */
  1651. GETPROCADDR(DestroyInstance);
  1652. GETPROCADDR_IF_SUPPORTED(CreateWin32SurfaceKHR);
  1653. GETPROCADDR_IF_SUPPORTED(DestroySurfaceKHR);
  1654. /* device chain functions we intercept */
  1655. GETPROCADDR(GetDeviceProcAddr);
  1656. GETPROCADDR(CreateDevice);
  1657. GETPROCADDR(DestroyDevice);
  1658. #endif
  1659. const PFN_vkGetInstanceProcAddr gipa = funcs->GetInstanceProcAddr;
  1660. return gipa ? gipa(instance, pName) : NULL;
  1661. }
  1662. #undef GETPROCADDR
  1663. #ifndef _WIN64
  1664. #pragma comment(linker, "/EXPORT:OBS_Negotiate=_OBS_Negotiate@4")
  1665. #endif
  1666. __declspec(dllexport) VkResult VKAPI_CALL OBS_Negotiate(VkNegotiateLayerInterface *nli)
  1667. {
  1668. if (nli->loaderLayerInterfaceVersion >= 2) {
  1669. nli->sType = LAYER_NEGOTIATE_INTERFACE_STRUCT;
  1670. nli->pNext = NULL;
  1671. nli->pfnGetInstanceProcAddr = OBS_GetInstanceProcAddr;
  1672. nli->pfnGetDeviceProcAddr = OBS_GetDeviceProcAddr;
  1673. nli->pfnGetPhysicalDeviceProcAddr = NULL;
  1674. }
  1675. const uint32_t cur_ver = CURRENT_LOADER_LAYER_INTERFACE_VERSION;
  1676. if (nli->loaderLayerInterfaceVersion > cur_ver) {
  1677. nli->loaderLayerInterfaceVersion = cur_ver;
  1678. }
  1679. if (!vulkan_seen) {
  1680. init_obj_list(&instances);
  1681. init_obj_list(&devices);
  1682. vulkan_seen = true;
  1683. }
  1684. return VK_SUCCESS;
  1685. }
  1686. bool hook_vulkan(void)
  1687. {
  1688. static bool hooked = false;
  1689. if (!hooked && vulkan_seen) {
  1690. hlog("Hooked Vulkan");
  1691. hooked = true;
  1692. }
  1693. return hooked;
  1694. }