| 1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098 |
- #include <windows.h>
- #include "graphics-hook.h"
- #define VK_USE_PLATFORM_WIN32_KHR
- #include <malloc.h>
- #include <vulkan/vulkan.h>
- #include <vulkan/vk_layer.h>
- #include <vulkan/vulkan_win32.h>
- #define COBJMACROS
- #include <dxgi.h>
- #include <d3d11.h>
- #include "dxgi-helpers.hpp"
- #include "vulkan-capture.h"
- /* ======================================================================== */
- /* defs/statics */
- /* use the loader's dispatch table pointer as a key for internal data maps */
- #define GET_LDT(x) (*(void **)x)
- static bool vulkan_seen = false;
- /* ======================================================================== */
- /* hook data */
- struct vk_obj_node {
- uint64_t obj;
- struct vk_obj_node *next;
- };
- struct vk_obj_list {
- struct vk_obj_node *root;
- SRWLOCK mutex;
- };
- struct vk_swap_data {
- struct vk_obj_node node;
- VkExtent2D image_extent;
- VkFormat format;
- HWND hwnd;
- VkImage export_image;
- bool layout_initialized;
- VkDeviceMemory export_mem;
- VkImage *swap_images;
- uint32_t image_count;
- HANDLE handle;
- struct shtex_data *shtex_info;
- ID3D11Texture2D *d3d11_tex;
- bool captured;
- };
- struct vk_queue_data {
- struct vk_obj_node node;
- uint32_t fam_idx;
- bool supports_transfer;
- struct vk_frame_data *frames;
- uint32_t frame_index;
- uint32_t frame_count;
- };
- struct vk_swap_view_data {
- struct vk_obj_node node;
- };
- #define OBS_COLOR_ATTACHMENT_LIMIT 8
- struct vk_framebuffer_data {
- struct vk_obj_node node;
- VkFramebuffer alternates[1 << OBS_COLOR_ATTACHMENT_LIMIT];
- uint32_t color_attachment_mask;
- };
- struct vk_frame_data {
- VkCommandPool cmd_pool;
- VkCommandBuffer cmd_buffer;
- VkFence fence;
- bool cmd_buffer_busy;
- };
- struct vk_surf_data {
- struct vk_obj_node node;
- HWND hwnd;
- };
- struct vk_inst_data {
- struct vk_obj_node node;
- VkInstance instance;
- bool valid;
- struct vk_inst_funcs funcs;
- struct vk_obj_list surfaces;
- };
- struct vk_data {
- struct vk_obj_node node;
- VkDevice device;
- bool valid;
- struct vk_device_funcs funcs;
- VkPhysicalDevice phy_device;
- struct vk_obj_list swaps;
- struct vk_swap_data *cur_swap;
- struct vk_obj_list queues;
- struct vk_obj_list swap_views;
- struct vk_obj_list framebuffers;
- VkExternalMemoryProperties external_mem_props;
- struct vk_inst_data *inst_data;
- VkAllocationCallbacks ac_storage;
- const VkAllocationCallbacks *ac;
- ID3D11Device *d3d11_device;
- ID3D11DeviceContext *d3d11_context;
- };
- __declspec(thread) int vk_presenting = 0;
- /* ------------------------------------------------------------------------- */
- static void *vk_alloc(const VkAllocationCallbacks *ac, size_t size, size_t alignment,
- enum VkSystemAllocationScope scope)
- {
- return ac ? ac->pfnAllocation(ac->pUserData, size, alignment, scope) : _aligned_malloc(size, alignment);
- }
- static void vk_free(const VkAllocationCallbacks *ac, void *memory)
- {
- if (ac)
- ac->pfnFree(ac->pUserData, memory);
- else
- _aligned_free(memory);
- }
- static void add_obj_data(struct vk_obj_list *list, uint64_t obj, void *data)
- {
- AcquireSRWLockExclusive(&list->mutex);
- struct vk_obj_node *const node = data;
- node->obj = obj;
- node->next = list->root;
- list->root = node;
- ReleaseSRWLockExclusive(&list->mutex);
- }
- static struct vk_obj_node *get_obj_data(struct vk_obj_list *list, uint64_t obj)
- {
- struct vk_obj_node *data = NULL;
- AcquireSRWLockExclusive(&list->mutex);
- struct vk_obj_node *node = list->root;
- while (node) {
- if (node->obj == obj) {
- data = node;
- break;
- }
- node = node->next;
- }
- ReleaseSRWLockExclusive(&list->mutex);
- return data;
- }
- static struct vk_obj_node *remove_obj_data(struct vk_obj_list *list, uint64_t obj)
- {
- struct vk_obj_node *data = NULL;
- AcquireSRWLockExclusive(&list->mutex);
- struct vk_obj_node *prev = NULL;
- struct vk_obj_node *node = list->root;
- while (node) {
- if (node->obj == obj) {
- data = node;
- if (prev)
- prev->next = node->next;
- else
- list->root = node->next;
- break;
- }
- prev = node;
- node = node->next;
- }
- ReleaseSRWLockExclusive(&list->mutex);
- return data;
- }
- static void init_obj_list(struct vk_obj_list *list)
- {
- list->root = NULL;
- InitializeSRWLock(&list->mutex);
- }
- static struct vk_obj_node *obj_walk_begin(struct vk_obj_list *list)
- {
- AcquireSRWLockExclusive(&list->mutex);
- return list->root;
- }
- static struct vk_obj_node *obj_walk_next(struct vk_obj_node *node)
- {
- return node->next;
- }
- static void obj_walk_end(struct vk_obj_list *list)
- {
- ReleaseSRWLockExclusive(&list->mutex);
- }
- /* ------------------------------------------------------------------------- */
- static struct vk_obj_list devices;
- static struct vk_data *alloc_device_data(const VkAllocationCallbacks *ac)
- {
- struct vk_data *data =
- vk_alloc(ac, sizeof(struct vk_data), _Alignof(struct vk_data), VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
- return data;
- }
- static void init_device_data(struct vk_data *data, VkDevice device)
- {
- add_obj_data(&devices, (uint64_t)GET_LDT(device), data);
- data->device = device;
- }
- static struct vk_data *get_device_data(VkDevice device)
- {
- return (struct vk_data *)get_obj_data(&devices, (uint64_t)GET_LDT(device));
- }
- static struct vk_data *get_device_data_by_queue(VkQueue queue)
- {
- return (struct vk_data *)get_obj_data(&devices, (uint64_t)GET_LDT(queue));
- }
- static struct vk_data *get_device_data_by_command_buffer(VkCommandBuffer commandBuffer)
- {
- return (struct vk_data *)get_obj_data(&devices, (uint64_t)GET_LDT(commandBuffer));
- }
- static struct vk_data *remove_device_data(VkDevice device)
- {
- return (struct vk_data *)remove_obj_data(&devices, (uint64_t)GET_LDT(device));
- }
- static void free_device_data(struct vk_data *data, const VkAllocationCallbacks *ac)
- {
- vk_free(ac, data);
- }
- /* ------------------------------------------------------------------------- */
- static struct vk_queue_data *add_queue_data(struct vk_data *data, VkQueue queue, uint32_t fam_idx,
- bool supports_transfer, const VkAllocationCallbacks *ac)
- {
- struct vk_queue_data *const queue_data = vk_alloc(
- ac, sizeof(struct vk_queue_data), _Alignof(struct vk_queue_data), VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
- add_obj_data(&data->queues, (uint64_t)queue, queue_data);
- queue_data->fam_idx = fam_idx;
- queue_data->supports_transfer = supports_transfer;
- queue_data->frames = NULL;
- queue_data->frame_index = 0;
- queue_data->frame_count = 0;
- return queue_data;
- }
- static struct vk_queue_data *get_queue_data(struct vk_data *data, VkQueue queue)
- {
- return (struct vk_queue_data *)get_obj_data(&data->queues, (uint64_t)queue);
- }
- static void remove_free_queue_all(struct vk_data *data, const VkAllocationCallbacks *ac)
- {
- struct vk_queue_data *queue_data = (struct vk_queue_data *)data->queues.root;
- while (data->queues.root) {
- remove_obj_data(&data->queues, queue_data->node.obj);
- vk_free(ac, queue_data);
- queue_data = (struct vk_queue_data *)data->queues.root;
- }
- }
- static struct vk_queue_data *queue_walk_begin(struct vk_data *data)
- {
- return (struct vk_queue_data *)obj_walk_begin(&data->queues);
- }
- static struct vk_queue_data *queue_walk_next(struct vk_queue_data *queue_data)
- {
- return (struct vk_queue_data *)obj_walk_next((struct vk_obj_node *)queue_data);
- }
- static void queue_walk_end(struct vk_data *data)
- {
- obj_walk_end(&data->queues);
- }
- /* ------------------------------------------------------------------------- */
- static struct vk_swap_view_data *add_swap_view_data(struct vk_data *data, VkImageView imageView,
- const VkAllocationCallbacks *ac)
- {
- struct vk_swap_view_data *const swap_view_data = vk_alloc(ac, sizeof(struct vk_swap_view_data),
- _Alignof(struct vk_swap_view_data),
- VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
- add_obj_data(&data->swap_views, (uint64_t)imageView, swap_view_data);
- return swap_view_data;
- }
- static struct vk_swap_view_data *get_swap_view_data(struct vk_data *data, VkImageView imageView)
- {
- return (struct vk_swap_view_data *)get_obj_data(&data->swap_views, (uint64_t)imageView);
- }
- static void remove_free_swap_view_data(struct vk_data *data, VkImageView imageView, const VkAllocationCallbacks *ac)
- {
- struct vk_swap_data *const swap_view_data =
- (struct vk_swap_data *)remove_obj_data(&data->swap_views, (uint64_t)imageView);
- vk_free(ac, swap_view_data);
- }
- /* ------------------------------------------------------------------------- */
- static struct vk_framebuffer_data *add_framebuffer_data(struct vk_data *data, VkFramebuffer framebuffer,
- const VkAllocationCallbacks *ac)
- {
- struct vk_framebuffer_data *const framebuffer_data = vk_alloc(ac, sizeof(struct vk_framebuffer_data),
- _Alignof(struct vk_framebuffer_data),
- VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
- add_obj_data(&data->framebuffers, (uint64_t)framebuffer, framebuffer_data);
- for (size_t i = 0; i < _countof(framebuffer_data->alternates); ++i) {
- framebuffer_data->alternates[i] = VK_NULL_HANDLE;
- }
- framebuffer_data->color_attachment_mask = 0;
- return framebuffer_data;
- }
- static struct vk_framebuffer_data *get_framebuffer_data(struct vk_data *data, VkFramebuffer framebuffer)
- {
- return (struct vk_framebuffer_data *)get_obj_data(&data->framebuffers, (uint64_t)framebuffer);
- }
- static void remove_free_framebuffer_data(struct vk_data *data, VkFramebuffer framebuffer,
- const VkAllocationCallbacks *ac)
- {
- struct vk_swap_data *const framebuffer_data =
- (struct vk_swap_data *)remove_obj_data(&data->framebuffers, (uint64_t)framebuffer);
- vk_free(ac, framebuffer_data);
- }
- /* ------------------------------------------------------------------------- */
- static struct vk_swap_data *alloc_swap_data(const VkAllocationCallbacks *ac)
- {
- struct vk_swap_data *const swap_data = vk_alloc(ac, sizeof(struct vk_swap_data), _Alignof(struct vk_swap_data),
- VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
- return swap_data;
- }
- static void init_swap_data(struct vk_swap_data *swap_data, struct vk_data *data, VkSwapchainKHR sc)
- {
- add_obj_data(&data->swaps, (uint64_t)sc, swap_data);
- }
- static struct vk_swap_data *get_swap_data(struct vk_data *data, VkSwapchainKHR sc)
- {
- return (struct vk_swap_data *)get_obj_data(&data->swaps, (uint64_t)sc);
- }
- static void remove_free_swap_data(struct vk_data *data, VkSwapchainKHR sc, const VkAllocationCallbacks *ac)
- {
- struct vk_swap_data *const swap_data = (struct vk_swap_data *)remove_obj_data(&data->swaps, (uint64_t)sc);
- vk_free(ac, swap_data);
- }
- static struct vk_swap_data *swap_walk_begin(struct vk_data *data)
- {
- return (struct vk_swap_data *)obj_walk_begin(&data->swaps);
- }
- static struct vk_swap_data *swap_walk_next(struct vk_swap_data *swap_data)
- {
- return (struct vk_swap_data *)obj_walk_next((struct vk_obj_node *)swap_data);
- }
- static void swap_walk_end(struct vk_data *data)
- {
- obj_walk_end(&data->swaps);
- }
- /* ------------------------------------------------------------------------- */
- static void vk_shtex_clear_fence(const struct vk_data *data, struct vk_frame_data *frame_data)
- {
- const VkFence fence = frame_data->fence;
- if (frame_data->cmd_buffer_busy) {
- VkDevice device = data->device;
- const struct vk_device_funcs *funcs = &data->funcs;
- funcs->WaitForFences(device, 1, &fence, VK_TRUE, ~0ull);
- funcs->ResetFences(device, 1, &fence);
- frame_data->cmd_buffer_busy = false;
- }
- }
- static void vk_shtex_wait_until_pool_idle(struct vk_data *data, struct vk_queue_data *queue_data)
- {
- for (uint32_t frame_idx = 0; frame_idx < queue_data->frame_count; frame_idx++) {
- struct vk_frame_data *frame_data = &queue_data->frames[frame_idx];
- if (frame_data->cmd_pool != VK_NULL_HANDLE)
- vk_shtex_clear_fence(data, frame_data);
- }
- }
- static void vk_shtex_wait_until_idle(struct vk_data *data)
- {
- struct vk_queue_data *queue_data = queue_walk_begin(data);
- while (queue_data) {
- vk_shtex_wait_until_pool_idle(data, queue_data);
- queue_data = queue_walk_next(queue_data);
- }
- queue_walk_end(data);
- }
- static void vk_shtex_free(struct vk_data *data)
- {
- capture_free();
- vk_shtex_wait_until_idle(data);
- struct vk_swap_data *swap = swap_walk_begin(data);
- while (swap) {
- VkDevice device = data->device;
- if (swap->export_image)
- data->funcs.DestroyImage(device, swap->export_image, data->ac);
- if (swap->export_mem)
- data->funcs.FreeMemory(device, swap->export_mem, NULL);
- if (swap->d3d11_tex) {
- ID3D11Texture2D_Release(swap->d3d11_tex);
- }
- swap->handle = INVALID_HANDLE_VALUE;
- swap->d3d11_tex = NULL;
- swap->export_mem = VK_NULL_HANDLE;
- swap->export_image = VK_NULL_HANDLE;
- swap->captured = false;
- swap = swap_walk_next(swap);
- }
- swap_walk_end(data);
- if (data->d3d11_context) {
- ID3D11DeviceContext_Release(data->d3d11_context);
- data->d3d11_context = NULL;
- }
- if (data->d3d11_device) {
- ID3D11Device_Release(data->d3d11_device);
- data->d3d11_device = NULL;
- }
- data->cur_swap = NULL;
- hlog("------------------ vulkan capture freed ------------------");
- }
- /* ------------------------------------------------------------------------- */
- static void add_surf_data(struct vk_inst_data *idata, VkSurfaceKHR surf, HWND hwnd, const VkAllocationCallbacks *ac)
- {
- struct vk_surf_data *surf_data = vk_alloc(ac, sizeof(struct vk_surf_data), _Alignof(struct vk_surf_data),
- VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
- if (surf_data) {
- surf_data->hwnd = hwnd;
- add_obj_data(&idata->surfaces, (uint64_t)surf, surf_data);
- }
- }
- static HWND find_surf_hwnd(struct vk_inst_data *idata, VkSurfaceKHR surf)
- {
- struct vk_surf_data *surf_data = (struct vk_surf_data *)get_obj_data(&idata->surfaces, (uint64_t)surf);
- return surf_data->hwnd;
- }
- static void remove_free_surf_data(struct vk_inst_data *idata, VkSurfaceKHR surf, const VkAllocationCallbacks *ac)
- {
- struct vk_surf_data *surf_data = (struct vk_surf_data *)remove_obj_data(&idata->surfaces, (uint64_t)surf);
- vk_free(ac, surf_data);
- }
- /* ------------------------------------------------------------------------- */
- static struct vk_obj_list instances;
- static struct vk_inst_data *alloc_inst_data(const VkAllocationCallbacks *ac)
- {
- struct vk_inst_data *idata = vk_alloc(ac, sizeof(struct vk_inst_data), _Alignof(struct vk_inst_data),
- VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE);
- return idata;
- }
- static void init_inst_data(struct vk_inst_data *idata, VkInstance instance)
- {
- add_obj_data(&instances, (uint64_t)GET_LDT(instance), idata);
- idata->instance = instance;
- }
- static struct vk_inst_data *get_inst_data(VkInstance instance)
- {
- return (struct vk_inst_data *)get_obj_data(&instances, (uint64_t)GET_LDT(instance));
- }
- static struct vk_inst_funcs *get_inst_funcs(VkInstance instance)
- {
- struct vk_inst_data *idata = (struct vk_inst_data *)get_inst_data(instance);
- return &idata->funcs;
- }
- static struct vk_inst_data *get_inst_data_by_physical_device(VkPhysicalDevice physicalDevice)
- {
- return (struct vk_inst_data *)get_obj_data(&instances, (uint64_t)GET_LDT(physicalDevice));
- }
- static struct vk_inst_funcs *get_inst_funcs_by_physical_device(VkPhysicalDevice physicalDevice)
- {
- struct vk_inst_data *idata = (struct vk_inst_data *)get_inst_data_by_physical_device(physicalDevice);
- return &idata->funcs;
- }
- static void remove_free_inst_data(VkInstance inst, const VkAllocationCallbacks *ac)
- {
- struct vk_inst_data *idata = (struct vk_inst_data *)remove_obj_data(&instances, (uint64_t)GET_LDT(inst));
- vk_free(ac, idata);
- }
- /* ======================================================================== */
- /* capture */
- static inline bool vk_shtex_init_d3d11(struct vk_data *data)
- {
- D3D_FEATURE_LEVEL level_used;
- IDXGIFactory1 *factory;
- IDXGIAdapter1 *adapter;
- HRESULT hr;
- HMODULE d3d11 = load_system_library("d3d11.dll");
- if (!d3d11) {
- flog("failed to load d3d11: %d", GetLastError());
- return false;
- }
- HMODULE dxgi = load_system_library("dxgi.dll");
- if (!dxgi) {
- flog("failed to load dxgi: %d", GetLastError());
- return false;
- }
- HRESULT(WINAPI * create_factory)
- (REFIID, void **) = (void *)GetProcAddress(dxgi, "CreateDXGIFactory1");
- if (!create_factory) {
- flog("failed to get CreateDXGIFactory1 address: %d", GetLastError());
- return false;
- }
- PFN_D3D11_CREATE_DEVICE create = (void *)GetProcAddress(d3d11, "D3D11CreateDevice");
- if (!create) {
- flog("failed to get D3D11CreateDevice address: %d", GetLastError());
- return false;
- }
- hr = create_factory(&IID_IDXGIFactory1, &factory);
- if (FAILED(hr)) {
- flog_hr("failed to create factory", hr);
- return false;
- }
- hr = IDXGIFactory1_EnumAdapters1(factory, 0, &adapter);
- IDXGIFactory1_Release(factory);
- if (FAILED(hr)) {
- flog_hr("failed to create adapter", hr);
- return false;
- }
- static const D3D_FEATURE_LEVEL feature_levels[] = {
- D3D_FEATURE_LEVEL_11_0,
- D3D_FEATURE_LEVEL_10_1,
- D3D_FEATURE_LEVEL_10_0,
- D3D_FEATURE_LEVEL_9_3,
- };
- hr = create((IDXGIAdapter *)adapter, D3D_DRIVER_TYPE_UNKNOWN, NULL, 0, feature_levels,
- sizeof(feature_levels) / sizeof(D3D_FEATURE_LEVEL), D3D11_SDK_VERSION, &data->d3d11_device,
- &level_used, &data->d3d11_context);
- IDXGIAdapter1_Release(adapter);
- if (FAILED(hr)) {
- flog_hr("failed to create device", hr);
- return false;
- }
- return true;
- }
- static inline bool vk_shtex_init_d3d11_tex(struct vk_data *data, struct vk_swap_data *swap)
- {
- IDXGIResource *dxgi_res;
- HRESULT hr;
- const UINT width = swap->image_extent.width;
- const UINT height = swap->image_extent.height;
- flog("OBS requesting %s texture format. capture dimensions: %ux%u", vk_format_to_str(swap->format), width,
- height);
- const DXGI_FORMAT format = vk_format_to_dxgi(swap->format);
- if (format == DXGI_FORMAT_UNKNOWN) {
- flog("cannot convert to DXGI format");
- return false;
- }
- D3D11_TEXTURE2D_DESC desc = {0};
- desc.Width = width;
- desc.Height = height;
- desc.MipLevels = 1;
- desc.ArraySize = 1;
- desc.Format = apply_dxgi_format_typeless(format, global_hook_info->allow_srgb_alias);
- desc.SampleDesc.Count = 1;
- desc.SampleDesc.Quality = 0;
- desc.Usage = D3D11_USAGE_DEFAULT;
- desc.MiscFlags = D3D11_RESOURCE_MISC_SHARED;
- desc.BindFlags = D3D11_BIND_SHADER_RESOURCE;
- hr = ID3D11Device_CreateTexture2D(data->d3d11_device, &desc, NULL, &swap->d3d11_tex);
- if (FAILED(hr)) {
- flog_hr("failed to create texture", hr);
- return false;
- }
- hr = ID3D11Texture2D_QueryInterface(swap->d3d11_tex, &IID_IDXGIResource, &dxgi_res);
- if (FAILED(hr)) {
- flog_hr("failed to get IDXGIResource", hr);
- return false;
- }
- hr = IDXGIResource_GetSharedHandle(dxgi_res, &swap->handle);
- IDXGIResource_Release(dxgi_res);
- if (FAILED(hr)) {
- flog_hr("failed to get shared handle", hr);
- return false;
- }
- return true;
- }
- static inline bool vk_shtex_init_vulkan_tex(struct vk_data *data, struct vk_swap_data *swap)
- {
- struct vk_device_funcs *funcs = &data->funcs;
- VkExternalMemoryFeatureFlags f = data->external_mem_props.externalMemoryFeatures;
- /* -------------------------------------------------------- */
- /* create texture */
- VkExternalMemoryImageCreateInfo emici;
- emici.sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO;
- emici.pNext = NULL;
- emici.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_D3D11_TEXTURE_KMT_BIT;
- VkImageCreateInfo ici;
- ici.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO;
- ici.pNext = &emici;
- ici.flags = 0;
- ici.imageType = VK_IMAGE_TYPE_2D;
- ici.format = swap->format;
- ici.extent.width = swap->image_extent.width;
- ici.extent.height = swap->image_extent.height;
- ici.extent.depth = 1;
- ici.mipLevels = 1;
- ici.arrayLayers = 1;
- ici.samples = VK_SAMPLE_COUNT_1_BIT;
- ici.tiling = VK_IMAGE_TILING_OPTIMAL;
- ici.usage = VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_SAMPLED_BIT;
- ici.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
- ici.queueFamilyIndexCount = 0;
- ici.pQueueFamilyIndices = 0;
- ici.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
- VkDevice device = data->device;
- VkResult res;
- res = funcs->CreateImage(device, &ici, data->ac, &swap->export_image);
- if (VK_SUCCESS != res) {
- flog("failed to CreateImage: %s", result_to_str(res));
- swap->export_image = VK_NULL_HANDLE;
- return false;
- }
- swap->layout_initialized = false;
- /* -------------------------------------------------------- */
- /* get image memory requirements */
- VkMemoryRequirements mr;
- bool use_gimr2 = f & VK_EXTERNAL_MEMORY_FEATURE_DEDICATED_ONLY_BIT;
- if (use_gimr2) {
- VkMemoryDedicatedRequirements mdr = {0};
- mdr.sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS;
- mdr.pNext = NULL;
- VkMemoryRequirements2 mr2 = {0};
- mr2.sType = VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2;
- mr2.pNext = &mdr;
- VkImageMemoryRequirementsInfo2 imri2 = {0};
- imri2.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2;
- imri2.pNext = NULL;
- imri2.image = swap->export_image;
- funcs->GetImageMemoryRequirements2(device, &imri2, &mr2);
- mr = mr2.memoryRequirements;
- } else {
- funcs->GetImageMemoryRequirements(device, swap->export_image, &mr);
- }
- /* -------------------------------------------------------- */
- /* get memory type index */
- struct vk_inst_funcs *ifuncs = get_inst_funcs_by_physical_device(data->phy_device);
- VkPhysicalDeviceMemoryProperties pdmp;
- ifuncs->GetPhysicalDeviceMemoryProperties(data->phy_device, &pdmp);
- /* -------------------------------------------------------- */
- /* allocate memory */
- VkImportMemoryWin32HandleInfoKHR imw32hi;
- imw32hi.sType = VK_STRUCTURE_TYPE_IMPORT_MEMORY_WIN32_HANDLE_INFO_KHR;
- imw32hi.pNext = NULL;
- imw32hi.name = NULL;
- imw32hi.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_D3D11_TEXTURE_KMT_BIT;
- imw32hi.handle = swap->handle;
- VkMemoryAllocateInfo mai;
- mai.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
- mai.pNext = &imw32hi;
- mai.allocationSize = mr.size;
- VkMemoryDedicatedAllocateInfo mdai;
- mdai.sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO;
- mdai.pNext = NULL;
- mdai.buffer = VK_NULL_HANDLE;
- if (data->external_mem_props.externalMemoryFeatures & VK_EXTERNAL_MEMORY_FEATURE_DEDICATED_ONLY_BIT) {
- mdai.image = swap->export_image;
- imw32hi.pNext = &mdai;
- }
- bool allocated = false;
- for (uint32_t i = 0; i < pdmp.memoryTypeCount; ++i) {
- if ((mr.memoryTypeBits & (1 << i)) &&
- (pdmp.memoryTypes[i].propertyFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) ==
- VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) {
- mai.memoryTypeIndex = i;
- res = funcs->AllocateMemory(device, &mai, NULL, &swap->export_mem);
- allocated = res == VK_SUCCESS;
- if (allocated)
- break;
- flog("failed to AllocateMemory (DEVICE_LOCAL): %s (%d)", result_to_str(res), (int)res);
- }
- }
- if (!allocated) {
- /* Try again without DEVICE_LOCAL */
- for (uint32_t i = 0; i < pdmp.memoryTypeCount; ++i) {
- if ((mr.memoryTypeBits & (1 << i)) &&
- (pdmp.memoryTypes[i].propertyFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) !=
- VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) {
- mai.memoryTypeIndex = i;
- res = funcs->AllocateMemory(device, &mai, NULL, &swap->export_mem);
- allocated = res == VK_SUCCESS;
- if (allocated)
- break;
- flog("failed to AllocateMemory (not DEVICE_LOCAL): %s (%d)", result_to_str(res),
- (int)res);
- }
- }
- }
- if (!allocated) {
- flog("failed to allocate memory of any type");
- funcs->DestroyImage(device, swap->export_image, data->ac);
- swap->export_image = VK_NULL_HANDLE;
- return false;
- }
- /* -------------------------------------------------------- */
- /* bind image memory */
- bool use_bi2 = f & VK_EXTERNAL_MEMORY_FEATURE_DEDICATED_ONLY_BIT;
- if (use_bi2) {
- VkBindImageMemoryInfo bimi = {0};
- bimi.sType = VK_STRUCTURE_TYPE_BIND_IMAGE_MEMORY_INFO;
- bimi.image = swap->export_image;
- bimi.memory = swap->export_mem;
- bimi.memoryOffset = 0;
- res = funcs->BindImageMemory2(device, 1, &bimi);
- } else {
- res = funcs->BindImageMemory(device, swap->export_image, swap->export_mem, 0);
- }
- if (VK_SUCCESS != res) {
- flog("%s failed: %s", use_bi2 ? "BindImageMemory2" : "BindImageMemory", result_to_str(res));
- funcs->DestroyImage(device, swap->export_image, data->ac);
- swap->export_image = VK_NULL_HANDLE;
- return false;
- }
- return true;
- }
- static bool vk_shtex_init(struct vk_data *data, HWND window, struct vk_swap_data *swap)
- {
- if (!vk_shtex_init_d3d11(data)) {
- return false;
- }
- if (!vk_shtex_init_d3d11_tex(data, swap)) {
- return false;
- }
- if (!vk_shtex_init_vulkan_tex(data, swap)) {
- return false;
- }
- data->cur_swap = swap;
- swap->captured = capture_init_shtex(&swap->shtex_info, window, swap->image_extent.width,
- swap->image_extent.height, (uint32_t)swap->format, false,
- (uintptr_t)swap->handle);
- if (!swap->captured)
- return false;
- if (global_hook_info->force_shmem) {
- flog("shared memory capture currently "
- "unsupported; ignoring");
- }
- hlog("vulkan shared texture capture successful");
- return true;
- }
- static void vk_shtex_create_frame_objects(struct vk_data *data, struct vk_queue_data *queue_data, uint32_t image_count)
- {
- queue_data->frames = vk_alloc(data->ac, image_count * sizeof(struct vk_frame_data),
- _Alignof(struct vk_frame_data), VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
- memset(queue_data->frames, 0, image_count * sizeof(struct vk_frame_data));
- queue_data->frame_index = 0;
- queue_data->frame_count = image_count;
- VkDevice device = data->device;
- for (uint32_t image_index = 0; image_index < image_count; image_index++) {
- struct vk_frame_data *frame_data = &queue_data->frames[image_index];
- VkCommandPoolCreateInfo cpci;
- cpci.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO;
- cpci.pNext = NULL;
- cpci.flags = 0;
- cpci.queueFamilyIndex = queue_data->fam_idx;
- VkResult res = data->funcs.CreateCommandPool(device, &cpci, data->ac, &frame_data->cmd_pool);
- debug_res("CreateCommandPool", res);
- VkCommandBufferAllocateInfo cbai;
- cbai.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO;
- cbai.pNext = NULL;
- cbai.commandPool = frame_data->cmd_pool;
- cbai.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY;
- cbai.commandBufferCount = 1;
- res = data->funcs.AllocateCommandBuffers(device, &cbai, &frame_data->cmd_buffer);
- debug_res("AllocateCommandBuffers", res);
- GET_LDT(frame_data->cmd_buffer) = GET_LDT(device);
- VkFenceCreateInfo fci = {0};
- fci.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
- fci.pNext = NULL;
- fci.flags = 0;
- res = data->funcs.CreateFence(device, &fci, data->ac, &frame_data->fence);
- debug_res("CreateFence", res);
- }
- }
- static void vk_shtex_destroy_fence(struct vk_data *data, bool *cmd_buffer_busy, VkFence *fence)
- {
- VkDevice device = data->device;
- if (*cmd_buffer_busy) {
- data->funcs.WaitForFences(device, 1, fence, VK_TRUE, ~0ull);
- *cmd_buffer_busy = false;
- }
- data->funcs.DestroyFence(device, *fence, data->ac);
- *fence = VK_NULL_HANDLE;
- }
- static void vk_shtex_destroy_frame_objects(struct vk_data *data, struct vk_queue_data *queue_data)
- {
- VkDevice device = data->device;
- for (uint32_t frame_idx = 0; frame_idx < queue_data->frame_count; frame_idx++) {
- struct vk_frame_data *frame_data = &queue_data->frames[frame_idx];
- bool *cmd_buffer_busy = &frame_data->cmd_buffer_busy;
- VkFence *fence = &frame_data->fence;
- vk_shtex_destroy_fence(data, cmd_buffer_busy, fence);
- data->funcs.DestroyCommandPool(device, frame_data->cmd_pool, data->ac);
- frame_data->cmd_pool = VK_NULL_HANDLE;
- }
- vk_free(data->ac, queue_data->frames);
- queue_data->frames = NULL;
- queue_data->frame_count = 0;
- }
- static void vk_shtex_capture(struct vk_data *data, struct vk_device_funcs *funcs, struct vk_swap_data *swap,
- uint32_t idx, VkQueue queue, const VkPresentInfoKHR *info)
- {
- VkResult res = VK_SUCCESS;
- VkCommandBufferBeginInfo begin_info;
- begin_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
- begin_info.pNext = NULL;
- begin_info.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
- begin_info.pInheritanceInfo = NULL;
- VkImageMemoryBarrier mb[2];
- VkImageMemoryBarrier *src_mb = &mb[0];
- VkImageMemoryBarrier *dst_mb = &mb[1];
- /* ------------------------------------------------------ */
- /* do image copy */
- const uint32_t image_index = info->pImageIndices[idx];
- VkImage cur_backbuffer = swap->swap_images[image_index];
- struct vk_queue_data *queue_data = get_queue_data(data, queue);
- uint32_t fam_idx = queue_data->fam_idx;
- const uint32_t image_count = swap->image_count;
- if (queue_data->frame_count < image_count) {
- if (queue_data->frame_count > 0)
- vk_shtex_destroy_frame_objects(data, queue_data);
- vk_shtex_create_frame_objects(data, queue_data, image_count);
- }
- const uint32_t frame_index = queue_data->frame_index;
- struct vk_frame_data *frame_data = &queue_data->frames[frame_index];
- queue_data->frame_index = (frame_index + 1) % queue_data->frame_count;
- vk_shtex_clear_fence(data, frame_data);
- VkDevice device = data->device;
- res = funcs->ResetCommandPool(device, frame_data->cmd_pool, 0);
- #ifdef MORE_DEBUGGING
- debug_res("ResetCommandPool", res);
- #endif
- const VkCommandBuffer cmd_buffer = frame_data->cmd_buffer;
- res = funcs->BeginCommandBuffer(cmd_buffer, &begin_info);
- #ifdef MORE_DEBUGGING
- debug_res("BeginCommandBuffer", res);
- #endif
- /* ------------------------------------------------------ */
- /* transition shared texture if necessary */
- if (!swap->layout_initialized) {
- VkImageMemoryBarrier imb;
- imb.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER;
- imb.pNext = NULL;
- imb.srcAccessMask = 0;
- imb.dstAccessMask = 0;
- imb.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED;
- imb.newLayout = VK_IMAGE_LAYOUT_GENERAL;
- imb.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
- imb.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
- imb.image = swap->export_image;
- imb.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
- imb.subresourceRange.baseMipLevel = 0;
- imb.subresourceRange.levelCount = 1;
- imb.subresourceRange.baseArrayLayer = 0;
- imb.subresourceRange.layerCount = 1;
- funcs->CmdPipelineBarrier(cmd_buffer, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
- VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, 0, 0, NULL, 0, NULL, 1, &imb);
- swap->layout_initialized = true;
- }
- /* ------------------------------------------------------ */
- /* transition cur_backbuffer to transfer source state */
- src_mb->sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER;
- src_mb->pNext = NULL;
- src_mb->srcAccessMask = VK_ACCESS_MEMORY_READ_BIT;
- src_mb->dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
- src_mb->oldLayout = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR;
- src_mb->newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
- src_mb->srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
- src_mb->dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
- src_mb->image = cur_backbuffer;
- src_mb->subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
- src_mb->subresourceRange.baseMipLevel = 0;
- src_mb->subresourceRange.levelCount = 1;
- src_mb->subresourceRange.baseArrayLayer = 0;
- src_mb->subresourceRange.layerCount = 1;
- /* ------------------------------------------------------ */
- /* transition exportedTexture to transfer dest state */
- dst_mb->sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER;
- dst_mb->pNext = NULL;
- dst_mb->srcAccessMask = 0;
- dst_mb->dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
- dst_mb->oldLayout = VK_IMAGE_LAYOUT_GENERAL;
- dst_mb->newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
- dst_mb->srcQueueFamilyIndex = VK_QUEUE_FAMILY_EXTERNAL;
- dst_mb->dstQueueFamilyIndex = fam_idx;
- dst_mb->image = swap->export_image;
- dst_mb->subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
- dst_mb->subresourceRange.baseMipLevel = 0;
- dst_mb->subresourceRange.levelCount = 1;
- dst_mb->subresourceRange.baseArrayLayer = 0;
- dst_mb->subresourceRange.layerCount = 1;
- funcs->CmdPipelineBarrier(cmd_buffer, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0,
- 0, NULL, 0, NULL, 2, mb);
- /* ------------------------------------------------------ */
- /* copy cur_backbuffer's content to our interop image */
- VkImageCopy cpy;
- cpy.srcSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
- cpy.srcSubresource.mipLevel = 0;
- cpy.srcSubresource.baseArrayLayer = 0;
- cpy.srcSubresource.layerCount = 1;
- cpy.srcOffset.x = 0;
- cpy.srcOffset.y = 0;
- cpy.srcOffset.z = 0;
- cpy.dstSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
- cpy.dstSubresource.mipLevel = 0;
- cpy.dstSubresource.baseArrayLayer = 0;
- cpy.dstSubresource.layerCount = 1;
- cpy.dstOffset.x = 0;
- cpy.dstOffset.y = 0;
- cpy.dstOffset.z = 0;
- cpy.extent.width = swap->image_extent.width;
- cpy.extent.height = swap->image_extent.height;
- cpy.extent.depth = 1;
- funcs->CmdCopyImage(cmd_buffer, cur_backbuffer, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, swap->export_image,
- VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &cpy);
- /* ------------------------------------------------------ */
- /* Restore the swap chain image layout to what it was
- * before. This may not be strictly needed, but it is
- * generally good to restore things to their original
- * state. */
- src_mb->srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
- src_mb->dstAccessMask = VK_ACCESS_MEMORY_READ_BIT;
- src_mb->oldLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
- src_mb->newLayout = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR;
- dst_mb->srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
- dst_mb->dstAccessMask = 0;
- dst_mb->oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
- dst_mb->newLayout = VK_IMAGE_LAYOUT_GENERAL;
- dst_mb->srcQueueFamilyIndex = fam_idx;
- dst_mb->dstQueueFamilyIndex = VK_QUEUE_FAMILY_EXTERNAL;
- funcs->CmdPipelineBarrier(cmd_buffer, VK_PIPELINE_STAGE_TRANSFER_BIT,
- VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT | VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, 0, 0, NULL,
- 0, NULL, 2, mb);
- funcs->EndCommandBuffer(cmd_buffer);
- /* ------------------------------------------------------ */
- VkSubmitInfo submit_info;
- submit_info.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO;
- submit_info.pNext = NULL;
- submit_info.waitSemaphoreCount = 0;
- submit_info.pWaitSemaphores = NULL;
- submit_info.pWaitDstStageMask = NULL;
- submit_info.commandBufferCount = 1;
- submit_info.pCommandBuffers = &cmd_buffer;
- submit_info.signalSemaphoreCount = 0;
- submit_info.pSignalSemaphores = NULL;
- const VkFence fence = frame_data->fence;
- res = funcs->QueueSubmit(queue, 1, &submit_info, fence);
- #ifdef MORE_DEBUGGING
- debug_res("QueueSubmit", res);
- #endif
- if (res == VK_SUCCESS)
- frame_data->cmd_buffer_busy = true;
- }
- static inline bool valid_rect(struct vk_swap_data *swap)
- {
- return !!swap->image_extent.width && !!swap->image_extent.height;
- }
- static void vk_capture(struct vk_data *data, VkQueue queue, const VkPresentInfoKHR *info)
- {
- struct vk_swap_data *swap = NULL;
- HWND window = NULL;
- uint32_t idx = 0;
- #ifdef MORE_DEBUGGING
- debug("QueuePresentKHR called on "
- "devicekey %p, swapchain count %d",
- &data->funcs, info->swapchainCount);
- #endif
- /* use first swap chain associated with a window */
- for (; idx < info->swapchainCount; idx++) {
- struct vk_swap_data *cur_swap = get_swap_data(data, info->pSwapchains[idx]);
- if (cur_swap) {
- window = cur_swap->hwnd;
- if (window != NULL) {
- swap = cur_swap;
- break;
- }
- }
- }
- if (!window) {
- return;
- }
- if (capture_should_stop()) {
- vk_shtex_free(data);
- }
- if (capture_should_init()) {
- if (valid_rect(swap) && !vk_shtex_init(data, window, swap)) {
- vk_shtex_free(data);
- data->valid = false;
- flog("vk_shtex_init failed");
- }
- }
- if (capture_ready()) {
- if (swap != data->cur_swap) {
- vk_shtex_free(data);
- return;
- }
- vk_shtex_capture(data, &data->funcs, swap, idx, queue, info);
- }
- }
- static VkResult VKAPI_CALL OBS_QueuePresentKHR(VkQueue queue, const VkPresentInfoKHR *info)
- {
- struct vk_data *const data = get_device_data_by_queue(queue);
- struct vk_queue_data *const queue_data = get_queue_data(data, queue);
- struct vk_device_funcs *const funcs = &data->funcs;
- if (data->valid && queue_data->supports_transfer) {
- vk_capture(data, queue, info);
- }
- if (vk_presenting != 0) {
- flog("non-zero vk_presenting: %d", vk_presenting);
- }
- vk_presenting++;
- VkResult res = funcs->QueuePresentKHR(queue, info);
- vk_presenting--;
- return res;
- }
- /* ======================================================================== */
- /* setup hooks */
- static inline bool is_inst_link_info(VkLayerInstanceCreateInfo *lici)
- {
- return lici->sType == VK_STRUCTURE_TYPE_LOADER_INSTANCE_CREATE_INFO && lici->function == VK_LAYER_LINK_INFO;
- }
- static VkResult VKAPI_CALL OBS_CreateInstance(const VkInstanceCreateInfo *cinfo, const VkAllocationCallbacks *ac,
- VkInstance *p_inst)
- {
- VkInstanceCreateInfo info = *cinfo;
- /* -------------------------------------------------------- */
- /* step through chain until we get to the link info */
- VkLayerInstanceCreateInfo *lici = (void *)info.pNext;
- while (lici && !is_inst_link_info(lici)) {
- lici = (VkLayerInstanceCreateInfo *)lici->pNext;
- }
- if (lici == NULL) {
- return VK_ERROR_INITIALIZATION_FAILED;
- }
- PFN_vkGetInstanceProcAddr gpa = lici->u.pLayerInfo->pfnNextGetInstanceProcAddr;
- /* -------------------------------------------------------- */
- /* move chain on for next layer */
- lici->u.pLayerInfo = lici->u.pLayerInfo->pNext;
- /* -------------------------------------------------------- */
- /* (HACK) Set api version to 1.1 if set to 1.0 */
- /* We do this to get our extensions working properly */
- VkApplicationInfo ai;
- if (info.pApplicationInfo) {
- ai = *info.pApplicationInfo;
- if (ai.apiVersion < VK_API_VERSION_1_1)
- ai.apiVersion = VK_API_VERSION_1_1;
- } else {
- ai.sType = VK_STRUCTURE_TYPE_APPLICATION_INFO;
- ai.pNext = NULL;
- ai.pApplicationName = NULL;
- ai.applicationVersion = 0;
- ai.pEngineName = NULL;
- ai.engineVersion = 0;
- ai.apiVersion = VK_API_VERSION_1_1;
- }
- info.pApplicationInfo = &ai;
- /* -------------------------------------------------------- */
- /* allocate data node */
- struct vk_inst_data *idata = alloc_inst_data(ac);
- if (!idata)
- return VK_ERROR_OUT_OF_HOST_MEMORY;
- /* -------------------------------------------------------- */
- /* create instance */
- PFN_vkCreateInstance create = (void *)gpa(NULL, "vkCreateInstance");
- VkResult res = create(&info, ac, p_inst);
- bool valid = res == VK_SUCCESS;
- if (!valid) {
- /* try again with original arguments */
- res = create(cinfo, ac, p_inst);
- if (res != VK_SUCCESS) {
- vk_free(ac, idata);
- return res;
- }
- }
- VkInstance inst = *p_inst;
- init_inst_data(idata, inst);
- /* -------------------------------------------------------- */
- /* fetch the functions we need */
- struct vk_inst_funcs *ifuncs = &idata->funcs;
- #define GETADDR(x) \
- do { \
- ifuncs->x = (void *)gpa(inst, "vk" #x); \
- if (!ifuncs->x) { \
- flog("could not get instance " \
- "address for vk" #x); \
- funcs_found = false; \
- } \
- } while (false)
- bool funcs_found = true;
- GETADDR(GetInstanceProcAddr);
- GETADDR(DestroyInstance);
- GETADDR(CreateWin32SurfaceKHR);
- GETADDR(DestroySurfaceKHR);
- GETADDR(GetPhysicalDeviceQueueFamilyProperties);
- GETADDR(GetPhysicalDeviceMemoryProperties);
- GETADDR(GetPhysicalDeviceImageFormatProperties2);
- GETADDR(EnumerateDeviceExtensionProperties);
- #undef GETADDR
- valid = valid && funcs_found;
- idata->valid = valid;
- if (valid)
- init_obj_list(&idata->surfaces);
- return res;
- }
- static void VKAPI_CALL OBS_DestroyInstance(VkInstance instance, const VkAllocationCallbacks *ac)
- {
- struct vk_inst_funcs *ifuncs = get_inst_funcs(instance);
- PFN_vkDestroyInstance destroy_instance = ifuncs->DestroyInstance;
- remove_free_inst_data(instance, ac);
- destroy_instance(instance, ac);
- }
- static bool vk_shared_tex_supported(struct vk_inst_funcs *funcs, VkPhysicalDevice phy_device, VkFormat format,
- VkImageUsageFlags usage, VkExternalMemoryProperties *external_mem_props)
- {
- VkPhysicalDeviceImageFormatInfo2 info;
- VkPhysicalDeviceExternalImageFormatInfo external_info;
- external_info.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_IMAGE_FORMAT_INFO;
- external_info.pNext = NULL;
- external_info.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_D3D11_TEXTURE_KMT_BIT;
- info.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2;
- info.pNext = &external_info;
- info.format = format;
- info.type = VK_IMAGE_TYPE_2D;
- info.tiling = VK_IMAGE_TILING_OPTIMAL;
- info.flags = 0;
- info.usage = usage;
- VkExternalImageFormatProperties external_props = {0};
- external_props.sType = VK_STRUCTURE_TYPE_EXTERNAL_IMAGE_FORMAT_PROPERTIES;
- external_props.pNext = NULL;
- VkImageFormatProperties2 props = {0};
- props.sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_PROPERTIES_2;
- props.pNext = &external_props;
- VkResult result = funcs->GetPhysicalDeviceImageFormatProperties2(phy_device, &info, &props);
- *external_mem_props = external_props.externalMemoryProperties;
- const VkExternalMemoryFeatureFlags features = external_mem_props->externalMemoryFeatures;
- return ((VK_SUCCESS == result) && (features & VK_EXTERNAL_MEMORY_FEATURE_IMPORTABLE_BIT));
- }
- static inline bool is_device_link_info(VkLayerDeviceCreateInfo *lici)
- {
- return lici->sType == VK_STRUCTURE_TYPE_LOADER_DEVICE_CREATE_INFO && lici->function == VK_LAYER_LINK_INFO;
- }
- static VkResult VKAPI_CALL OBS_CreateDevice(VkPhysicalDevice phy_device, const VkDeviceCreateInfo *info,
- const VkAllocationCallbacks *ac, VkDevice *p_device)
- {
- struct vk_inst_data *idata = get_inst_data_by_physical_device(phy_device);
- struct vk_inst_funcs *ifuncs = &idata->funcs;
- struct vk_data *data = NULL;
- VkResult ret = VK_ERROR_INITIALIZATION_FAILED;
- VkLayerDeviceCreateInfo *ldci = (void *)info->pNext;
- /* -------------------------------------------------------- */
- /* step through chain until we get to the link info */
- while (ldci && !is_device_link_info(ldci)) {
- ldci = (VkLayerDeviceCreateInfo *)ldci->pNext;
- }
- if (!ldci) {
- goto fail;
- }
- PFN_vkGetInstanceProcAddr gipa;
- PFN_vkGetDeviceProcAddr gdpa;
- gipa = ldci->u.pLayerInfo->pfnNextGetInstanceProcAddr;
- gdpa = ldci->u.pLayerInfo->pfnNextGetDeviceProcAddr;
- /* -------------------------------------------------------- */
- /* move chain on for next layer */
- ldci->u.pLayerInfo = ldci->u.pLayerInfo->pNext;
- /* -------------------------------------------------------- */
- /* allocate data node */
- data = alloc_device_data(ac);
- if (!data)
- return VK_ERROR_OUT_OF_HOST_MEMORY;
- init_obj_list(&data->queues);
- init_obj_list(&data->swap_views);
- init_obj_list(&data->framebuffers);
- /* -------------------------------------------------------- */
- /* create device and initialize hook data */
- PFN_vkCreateDevice createFunc = (PFN_vkCreateDevice)gipa(idata->instance, "vkCreateDevice");
- ret = createFunc(phy_device, info, ac, p_device);
- if (ret != VK_SUCCESS) {
- vk_free(ac, data);
- return ret;
- }
- VkDevice device = *p_device;
- init_device_data(data, device);
- data->valid = false; /* set true below if it doesn't go to fail */
- data->phy_device = phy_device;
- /* -------------------------------------------------------- */
- /* fetch the functions we need */
- struct vk_device_funcs *dfuncs = &data->funcs;
- bool funcs_found = true;
- #define GETADDR(x) \
- do { \
- dfuncs->x = (void *)gdpa(device, "vk" #x); \
- if (!dfuncs->x) { \
- flog("could not get device " \
- "address for vk" #x); \
- funcs_found = false; \
- } \
- } while (false)
- #define GETADDR_OPTIONAL(x) \
- do { \
- dfuncs->x = (void *)gdpa(device, "vk" #x); \
- } while (false)
- GETADDR(GetDeviceProcAddr);
- GETADDR(DestroyDevice);
- GETADDR(CreateSwapchainKHR);
- GETADDR(DestroySwapchainKHR);
- GETADDR(QueuePresentKHR);
- GETADDR(AllocateMemory);
- GETADDR(FreeMemory);
- GETADDR(BindImageMemory);
- GETADDR(BindImageMemory2);
- GETADDR(GetSwapchainImagesKHR);
- GETADDR(CreateImage);
- GETADDR(DestroyImage);
- GETADDR(GetImageMemoryRequirements);
- GETADDR(GetImageMemoryRequirements2);
- GETADDR(ResetCommandPool);
- GETADDR(BeginCommandBuffer);
- GETADDR(EndCommandBuffer);
- GETADDR(CmdCopyImage);
- GETADDR(CmdPipelineBarrier);
- GETADDR(GetDeviceQueue);
- GETADDR(QueueSubmit);
- GETADDR(CreateCommandPool);
- GETADDR(DestroyCommandPool);
- GETADDR(AllocateCommandBuffers);
- GETADDR(CreateFence);
- GETADDR(DestroyFence);
- GETADDR(WaitForFences);
- GETADDR(ResetFences);
- GETADDR(CreateImageView);
- GETADDR(DestroyImageView);
- GETADDR(CreateFramebuffer);
- GETADDR(DestroyFramebuffer);
- GETADDR(CmdBeginRenderPass);
- GETADDR_OPTIONAL(CmdBeginRenderPass2KHR);
- GETADDR_OPTIONAL(CmdBeginRenderPass2);
- #undef GETADDR_OPTIONAL
- #undef GETADDR
- if (!funcs_found) {
- goto fail;
- }
- if (!idata->valid) {
- flog("instance not valid");
- goto fail;
- }
- const char *required_device_extensions[] = {VK_KHR_EXTERNAL_MEMORY_WIN32_EXTENSION_NAME};
- uint32_t device_extension_count = 0;
- ret = ifuncs->EnumerateDeviceExtensionProperties(phy_device, NULL, &device_extension_count, NULL);
- if (ret != VK_SUCCESS)
- goto fail;
- VkExtensionProperties *device_extensions = _malloca(sizeof(VkExtensionProperties) * device_extension_count);
- ret = ifuncs->EnumerateDeviceExtensionProperties(phy_device, NULL, &device_extension_count, device_extensions);
- if (ret != VK_SUCCESS) {
- _freea(device_extensions);
- goto fail;
- }
- bool extensions_found = true;
- for (uint32_t i = 0; i < _countof(required_device_extensions); i++) {
- const char *const required_extension = required_device_extensions[i];
- bool found = false;
- for (uint32_t j = 0; j < device_extension_count; j++) {
- if (!strcmp(required_extension, device_extensions[j].extensionName)) {
- found = true;
- break;
- }
- }
- if (!found) {
- flog("missing device extension: %s", required_extension);
- extensions_found = false;
- }
- }
- _freea(device_extensions);
- if (!extensions_found)
- goto fail;
- VkFormat format = VK_FORMAT_R8G8B8A8_UNORM;
- VkImageUsageFlags usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT;
- if (!vk_shared_tex_supported(ifuncs, phy_device, format, usage, &data->external_mem_props)) {
- flog("texture sharing is not supported");
- goto fail;
- }
- data->inst_data = idata;
- data->ac = NULL;
- if (ac) {
- data->ac_storage = *ac;
- data->ac = &data->ac_storage;
- }
- uint32_t queue_family_property_count = 0;
- ifuncs->GetPhysicalDeviceQueueFamilyProperties(phy_device, &queue_family_property_count, NULL);
- VkQueueFamilyProperties *queue_family_properties =
- _malloca(sizeof(VkQueueFamilyProperties) * queue_family_property_count);
- ifuncs->GetPhysicalDeviceQueueFamilyProperties(phy_device, &queue_family_property_count,
- queue_family_properties);
- for (uint32_t info_index = 0, info_count = info->queueCreateInfoCount; info_index < info_count; ++info_index) {
- const VkDeviceQueueCreateInfo *queue_info = &info->pQueueCreateInfos[info_index];
- for (uint32_t queue_index = 0, queue_count = queue_info->queueCount; queue_index < queue_count;
- ++queue_index) {
- const uint32_t family_index = queue_info->queueFamilyIndex;
- VkQueue queue;
- data->funcs.GetDeviceQueue(device, family_index, queue_index, &queue);
- const bool supports_transfer =
- (queue_family_properties[family_index].queueFlags &
- (VK_QUEUE_GRAPHICS_BIT | VK_QUEUE_COMPUTE_BIT | VK_QUEUE_TRANSFER_BIT)) != 0;
- add_queue_data(data, queue, family_index, supports_transfer, ac);
- }
- }
- _freea(queue_family_properties);
- init_obj_list(&data->swaps);
- data->cur_swap = NULL;
- data->d3d11_device = NULL;
- data->d3d11_context = NULL;
- data->valid = true;
- fail:
- return ret;
- }
- static void VKAPI_CALL OBS_DestroyDevice(VkDevice device, const VkAllocationCallbacks *ac)
- {
- struct vk_data *data = remove_device_data(device);
- if (data->valid) {
- struct vk_queue_data *queue_data = queue_walk_begin(data);
- while (queue_data) {
- vk_shtex_destroy_frame_objects(data, queue_data);
- queue_data = queue_walk_next(queue_data);
- }
- queue_walk_end(data);
- remove_free_queue_all(data, ac);
- }
- PFN_vkDestroyDevice destroy_device = data->funcs.DestroyDevice;
- vk_free(ac, data);
- destroy_device(device, ac);
- }
- static VkResult VKAPI_CALL OBS_CreateSwapchainKHR(VkDevice device, const VkSwapchainCreateInfoKHR *cinfo,
- const VkAllocationCallbacks *ac, VkSwapchainKHR *p_sc)
- {
- struct vk_data *data = get_device_data(device);
- struct vk_device_funcs *funcs = &data->funcs;
- if (!data->valid)
- return funcs->CreateSwapchainKHR(device, cinfo, ac, p_sc);
- VkSwapchainCreateInfoKHR info = *cinfo;
- info.imageUsage |= VK_IMAGE_USAGE_TRANSFER_SRC_BIT;
- VkResult res = funcs->CreateSwapchainKHR(device, &info, ac, p_sc);
- debug_res("CreateSwapchainKHR", res);
- if (res != VK_SUCCESS) {
- /* try again with original imageUsage flags */
- return funcs->CreateSwapchainKHR(device, cinfo, ac, p_sc);
- }
- VkSwapchainKHR sc = *p_sc;
- uint32_t count = 0;
- res = funcs->GetSwapchainImagesKHR(device, sc, &count, NULL);
- debug_res("GetSwapchainImagesKHR", res);
- if ((res == VK_SUCCESS) && (count > 0)) {
- struct vk_swap_data *swap_data = alloc_swap_data(ac);
- if (swap_data) {
- swap_data->swap_images = vk_alloc(ac, count * sizeof(VkImage), _Alignof(VkImage),
- VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
- res = funcs->GetSwapchainImagesKHR(device, sc, &count, swap_data->swap_images);
- debug_res("GetSwapchainImagesKHR", res);
- swap_data->image_extent = cinfo->imageExtent;
- swap_data->format = cinfo->imageFormat;
- swap_data->hwnd = find_surf_hwnd(data->inst_data, cinfo->surface);
- swap_data->export_image = VK_NULL_HANDLE;
- swap_data->layout_initialized = false;
- swap_data->export_mem = VK_NULL_HANDLE;
- swap_data->image_count = count;
- swap_data->handle = INVALID_HANDLE_VALUE;
- swap_data->shtex_info = NULL;
- swap_data->d3d11_tex = NULL;
- swap_data->captured = false;
- init_swap_data(swap_data, data, sc);
- }
- }
- return VK_SUCCESS;
- }
- static VkResult VKAPI_CALL OBS_CreateImageView(VkDevice device, const VkImageViewCreateInfo *pCreateInfo,
- const VkAllocationCallbacks *pAllocator, VkImageView *pView)
- {
- bool from_swap_chain = false;
- struct vk_data *const data = get_device_data(device);
- if (data->valid) {
- struct vk_swap_data *swap = swap_walk_begin(data);
- while (swap) {
- for (uint32_t i = 0, count = swap->image_count; i < count; ++i) {
- from_swap_chain = swap->swap_images[i] == pCreateInfo->image;
- if (from_swap_chain)
- break;
- }
- if (from_swap_chain)
- break;
- swap = swap_walk_next(swap);
- }
- swap_walk_end(data);
- }
- if (from_swap_chain) {
- const void *pCurrent = pCreateInfo->pNext;
- while (pCurrent) {
- VkBaseInStructure baseIn;
- memcpy(&baseIn, pCurrent, sizeof(baseIn));
- if (baseIn.sType == VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO) {
- ((VkImageViewUsageCreateInfo *)pCurrent)->usage |= VK_IMAGE_USAGE_TRANSFER_SRC_BIT;
- break;
- }
- pCurrent = baseIn.pNext;
- }
- }
- const PFN_vkCreateImageView func = data->funcs.CreateImageView;
- VkResult result = func(device, pCreateInfo, pAllocator, pView);
- if ((result == VK_SUCCESS) && from_swap_chain)
- add_swap_view_data(data, *pView, pAllocator);
- return result;
- }
- static void VKAPI_CALL OBS_DestroyImageView(VkDevice device, VkImageView imageView,
- const VkAllocationCallbacks *pAllocator)
- {
- struct vk_data *const data = get_device_data(device);
- if (data->valid && (imageView != VK_NULL_HANDLE)) {
- struct vk_swap_view_data *swap_view_data = get_swap_view_data(data, imageView);
- if (swap_view_data)
- remove_free_swap_view_data(data, imageView, pAllocator);
- }
- data->funcs.DestroyImageView(device, imageView, pAllocator);
- }
- static void
- generate_framebuffer_variants(PFN_vkCreateFramebuffer func, VkDevice device, const VkFramebufferCreateInfo *pCreateInfo,
- const VkAllocationCallbacks *pAllocator, struct vk_framebuffer_data *framebuffer_data,
- const VkFramebufferAttachmentsCreateInfo *pAttachmentsCreateInfo, uint32_t colorCount)
- {
- const uint32_t variantCount = 1 << colorCount;
- for (uint32_t colorMask = 0; colorMask < variantCount; ++colorMask) {
- VkImageUsageFlags pPreviousUsage[OBS_COLOR_ATTACHMENT_LIMIT];
- uint32_t colorIndex = 0;
- for (uint32_t infoIndex = 0, count = pAttachmentsCreateInfo->attachmentImageInfoCount;
- infoIndex < count; ++infoIndex) {
- const VkFramebufferAttachmentImageInfo *const pInfo =
- &pAttachmentsCreateInfo->pAttachmentImageInfos[infoIndex];
- const VkImageUsageFlags usage = pInfo->usage;
- if (usage & VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT) {
- pPreviousUsage[colorIndex] = usage;
- if (colorMask & (1 << colorIndex)) {
- ((VkFramebufferAttachmentImageInfo *)pInfo)->usage |=
- VK_IMAGE_USAGE_TRANSFER_SRC_BIT;
- }
- ++colorIndex;
- if (colorIndex == OBS_COLOR_ATTACHMENT_LIMIT)
- break;
- }
- }
- VkFramebuffer framebuffer;
- const VkResult result = func(device, pCreateInfo, pAllocator, &framebuffer);
- if (result == VK_SUCCESS)
- framebuffer_data->alternates[colorMask] = framebuffer;
- colorIndex = 0;
- for (uint32_t infoIndex = 0, count = pAttachmentsCreateInfo->attachmentImageInfoCount;
- infoIndex < count; ++infoIndex) {
- const VkFramebufferAttachmentImageInfo *const pInfo =
- &pAttachmentsCreateInfo->pAttachmentImageInfos[infoIndex];
- if (pInfo->usage & VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT) {
- if (colorMask & (1 << colorIndex)) {
- ((VkFramebufferAttachmentImageInfo *)pInfo)->usage = pPreviousUsage[colorIndex];
- }
- ++colorIndex;
- if (colorIndex == OBS_COLOR_ATTACHMENT_LIMIT)
- break;
- }
- }
- }
- }
- static VkResult VKAPI_CALL OBS_CreateFramebuffer(VkDevice device, const VkFramebufferCreateInfo *pCreateInfo,
- const VkAllocationCallbacks *pAllocator, VkFramebuffer *pFramebuffer)
- {
- struct vk_data *const data = get_device_data(device);
- const PFN_vkCreateFramebuffer func = data->funcs.CreateFramebuffer;
- VkResult result = func(device, pCreateInfo, pAllocator, pFramebuffer);
- if (data->valid && (result == VK_SUCCESS) && (pCreateInfo->flags & VK_FRAMEBUFFER_CREATE_IMAGELESS_BIT)) {
- struct vk_framebuffer_data *const framebuffer_data =
- add_framebuffer_data(data, *pFramebuffer, pAllocator);
- const void *pCurrent = pCreateInfo->pNext;
- while (pCurrent) {
- VkBaseInStructure baseIn;
- memcpy(&baseIn, pCurrent, sizeof(baseIn));
- if (baseIn.sType == VK_STRUCTURE_TYPE_FRAMEBUFFER_ATTACHMENTS_CREATE_INFO) {
- uint32_t attachmentMask = 0;
- uint32_t colorCount = 0;
- const VkFramebufferAttachmentsCreateInfo *const pAttachmentsCreateInfo = pCurrent;
- for (uint32_t infoIndex = 0,
- count = min(sizeof(attachmentMask) * 8,
- pAttachmentsCreateInfo->attachmentImageInfoCount);
- infoIndex < count; ++infoIndex) {
- if (pAttachmentsCreateInfo->pAttachmentImageInfos[infoIndex].usage &
- VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT) {
- attachmentMask |= 1 << infoIndex;
- ++colorCount;
- if (colorCount == OBS_COLOR_ATTACHMENT_LIMIT)
- break;
- }
- }
- generate_framebuffer_variants(func, device, pCreateInfo, pAllocator, framebuffer_data,
- pAttachmentsCreateInfo, colorCount);
- framebuffer_data->color_attachment_mask = attachmentMask;
- break;
- }
- pCurrent = baseIn.pNext;
- }
- }
- return result;
- }
- static void VKAPI_CALL OBS_DestroyFramebuffer(VkDevice device, VkFramebuffer framebuffer,
- const VkAllocationCallbacks *pAllocator)
- {
- struct vk_data *const data = get_device_data(device);
- struct vk_device_funcs *const funcs = &data->funcs;
- if (data->valid && (framebuffer != VK_NULL_HANDLE)) {
- struct vk_framebuffer_data *framebuffer_data = get_framebuffer_data(data, framebuffer);
- if (framebuffer_data) {
- for (size_t i = 0; i < _countof(framebuffer_data->alternates); ++i) {
- if (framebuffer_data->alternates[i] != VK_NULL_HANDLE) {
- funcs->DestroyFramebuffer(device, framebuffer_data->alternates[i], pAllocator);
- framebuffer_data->alternates[i] = VK_NULL_HANDLE;
- }
- }
- remove_free_framebuffer_data(data, framebuffer, pAllocator);
- }
- }
- funcs->DestroyFramebuffer(device, framebuffer, pAllocator);
- }
- static const VkRenderPassBeginInfo *process_render_pass_begin_info(const VkRenderPassBeginInfo *pRenderPassBegin,
- VkRenderPassBeginInfo *pAlternateBegin,
- struct vk_data *data)
- {
- const void *pCurrent = pRenderPassBegin->pNext;
- while (pCurrent) {
- VkBaseInStructure baseIn;
- memcpy(&baseIn, pCurrent, sizeof(baseIn));
- if (baseIn.sType == VK_STRUCTURE_TYPE_RENDER_PASS_ATTACHMENT_BEGIN_INFO) {
- struct vk_framebuffer_data *const framebuffer_data =
- get_framebuffer_data(data, pRenderPassBegin->framebuffer);
- if (framebuffer_data) {
- const VkRenderPassAttachmentBeginInfo *const pAttachmentInfo = pCurrent;
- uint32_t swapMask = 0;
- uint32_t colorIndex = 0;
- for (uint32_t infoIndex = 0, count = pAttachmentInfo->attachmentCount;
- infoIndex < count; ++infoIndex) {
- if (framebuffer_data->color_attachment_mask & (1 << infoIndex)) {
- if (get_swap_view_data(data,
- pAttachmentInfo->pAttachments[infoIndex])) {
- swapMask |= 1 << colorIndex;
- }
- ++colorIndex;
- if (colorIndex == OBS_COLOR_ATTACHMENT_LIMIT)
- break;
- }
- }
- if (swapMask > 0) {
- VkFramebuffer alternate = framebuffer_data->alternates[swapMask];
- if (alternate != VK_NULL_HANDLE) {
- *pAlternateBegin = *pRenderPassBegin;
- pAlternateBegin->framebuffer = framebuffer_data->alternates[swapMask];
- pRenderPassBegin = pAlternateBegin;
- }
- }
- }
- break;
- }
- pCurrent = baseIn.pNext;
- }
- return pRenderPassBegin;
- }
- static void VKAPI_CALL OBS_CmdBeginRenderPass(VkCommandBuffer commandBuffer,
- const VkRenderPassBeginInfo *pRenderPassBegin, VkSubpassContents contents)
- {
- struct vk_data *const data = get_device_data_by_command_buffer(commandBuffer);
- VkRenderPassBeginInfo alternateBegin;
- if (data->valid) {
- pRenderPassBegin = process_render_pass_begin_info(pRenderPassBegin, &alternateBegin, data);
- }
- data->funcs.CmdBeginRenderPass(commandBuffer, pRenderPassBegin, contents);
- }
- static void VKAPI_CALL OBS_CmdBeginRenderPass2KHR(VkCommandBuffer commandBuffer,
- const VkRenderPassBeginInfo *pRenderPassBegin,
- const VkSubpassBeginInfo *pSubpassBeginInfo)
- {
- struct vk_data *const data = get_device_data_by_command_buffer(commandBuffer);
- VkRenderPassBeginInfo alternateBegin;
- if (data->valid) {
- pRenderPassBegin = process_render_pass_begin_info(pRenderPassBegin, &alternateBegin, data);
- }
- data->funcs.CmdBeginRenderPass2KHR(commandBuffer, pRenderPassBegin, pSubpassBeginInfo);
- }
- static void VKAPI_CALL OBS_CmdBeginRenderPass2(VkCommandBuffer commandBuffer,
- const VkRenderPassBeginInfo *pRenderPassBegin,
- const VkSubpassBeginInfo *pSubpassBeginInfo)
- {
- struct vk_data *const data = get_device_data_by_command_buffer(commandBuffer);
- VkRenderPassBeginInfo alternateBegin;
- if (data->valid) {
- pRenderPassBegin = process_render_pass_begin_info(pRenderPassBegin, &alternateBegin, data);
- }
- data->funcs.CmdBeginRenderPass2(commandBuffer, pRenderPassBegin, pSubpassBeginInfo);
- }
- static void VKAPI_CALL OBS_DestroySwapchainKHR(VkDevice device, VkSwapchainKHR sc, const VkAllocationCallbacks *ac)
- {
- struct vk_data *data = get_device_data(device);
- struct vk_device_funcs *funcs = &data->funcs;
- PFN_vkDestroySwapchainKHR destroy_swapchain = funcs->DestroySwapchainKHR;
- if ((sc != VK_NULL_HANDLE) && data->valid) {
- struct vk_swap_data *swap = get_swap_data(data, sc);
- if (swap) {
- if (data->cur_swap == swap) {
- vk_shtex_free(data);
- }
- vk_free(ac, swap->swap_images);
- remove_free_swap_data(data, sc, ac);
- }
- }
- destroy_swapchain(device, sc, ac);
- }
- static VkResult VKAPI_CALL OBS_CreateWin32SurfaceKHR(VkInstance inst, const VkWin32SurfaceCreateInfoKHR *info,
- const VkAllocationCallbacks *ac, VkSurfaceKHR *surf)
- {
- struct vk_inst_data *idata = get_inst_data(inst);
- struct vk_inst_funcs *ifuncs = &idata->funcs;
- VkResult res = ifuncs->CreateWin32SurfaceKHR(inst, info, ac, surf);
- if ((res == VK_SUCCESS) && idata->valid)
- add_surf_data(idata, *surf, info->hwnd, ac);
- return res;
- }
- static void VKAPI_CALL OBS_DestroySurfaceKHR(VkInstance inst, VkSurfaceKHR surf, const VkAllocationCallbacks *ac)
- {
- struct vk_inst_data *idata = get_inst_data(inst);
- struct vk_inst_funcs *ifuncs = &idata->funcs;
- PFN_vkDestroySurfaceKHR destroy_surface = ifuncs->DestroySurfaceKHR;
- if ((surf != VK_NULL_HANDLE) && idata->valid)
- remove_free_surf_data(idata, surf, ac);
- destroy_surface(inst, surf, ac);
- }
- /* clang-format off */
- #define GETPROCADDR(func) \
- if (!strcmp(pName, "vk" #func)) \
- return (PFN_vkVoidFunction)&OBS_##func;
- #define GETPROCADDR_IF_SUPPORTED(func) \
- if (!strcmp(pName, "vk" #func)) \
- return funcs->func ? (PFN_vkVoidFunction)&OBS_##func : NULL;
- /* clang-format on */
- static PFN_vkVoidFunction VKAPI_CALL OBS_GetDeviceProcAddr(VkDevice device, const char *pName)
- {
- struct vk_data *data = get_device_data(device);
- struct vk_device_funcs *funcs = &data->funcs;
- debug_procaddr("vkGetDeviceProcAddr(%p, \"%s\")", device, pName);
- GETPROCADDR(GetDeviceProcAddr);
- GETPROCADDR(DestroyDevice);
- GETPROCADDR_IF_SUPPORTED(CreateSwapchainKHR);
- GETPROCADDR_IF_SUPPORTED(DestroySwapchainKHR);
- GETPROCADDR_IF_SUPPORTED(QueuePresentKHR);
- GETPROCADDR(CreateImageView);
- GETPROCADDR(DestroyImageView);
- GETPROCADDR(CreateFramebuffer);
- GETPROCADDR(DestroyFramebuffer);
- GETPROCADDR(CmdBeginRenderPass);
- GETPROCADDR_IF_SUPPORTED(CmdBeginRenderPass2KHR);
- GETPROCADDR_IF_SUPPORTED(CmdBeginRenderPass2);
- if (funcs->GetDeviceProcAddr == NULL)
- return NULL;
- return funcs->GetDeviceProcAddr(device, pName);
- }
- /* bad layers require spec violation */
- #define RETURN_FP_FOR_NULL_INSTANCE 1
- static PFN_vkVoidFunction VKAPI_CALL OBS_GetInstanceProcAddr(VkInstance instance, const char *pName)
- {
- debug_procaddr("vkGetInstanceProcAddr(%p, \"%s\")", instance, pName);
- /* instance chain functions we intercept */
- GETPROCADDR(GetInstanceProcAddr);
- GETPROCADDR(CreateInstance);
- #if RETURN_FP_FOR_NULL_INSTANCE
- /* other instance chain functions we intercept */
- GETPROCADDR(DestroyInstance);
- GETPROCADDR(CreateWin32SurfaceKHR);
- GETPROCADDR(DestroySurfaceKHR);
- /* device chain functions we intercept */
- GETPROCADDR(GetDeviceProcAddr);
- GETPROCADDR(CreateDevice);
- GETPROCADDR(DestroyDevice);
- if (instance == NULL)
- return NULL;
- struct vk_inst_funcs *const funcs = get_inst_funcs(instance);
- #else
- if (instance == NULL)
- return NULL;
- struct vk_inst_funcs *const funcs = get_inst_funcs(instance);
- /* other instance chain functions we intercept */
- GETPROCADDR(DestroyInstance);
- GETPROCADDR_IF_SUPPORTED(CreateWin32SurfaceKHR);
- GETPROCADDR_IF_SUPPORTED(DestroySurfaceKHR);
- /* device chain functions we intercept */
- GETPROCADDR(GetDeviceProcAddr);
- GETPROCADDR(CreateDevice);
- GETPROCADDR(DestroyDevice);
- #endif
- const PFN_vkGetInstanceProcAddr gipa = funcs->GetInstanceProcAddr;
- return gipa ? gipa(instance, pName) : NULL;
- }
- #undef GETPROCADDR
- #ifndef _WIN64
- #pragma comment(linker, "/EXPORT:OBS_Negotiate=_OBS_Negotiate@4")
- #endif
- __declspec(dllexport) VkResult VKAPI_CALL OBS_Negotiate(VkNegotiateLayerInterface *nli)
- {
- if (nli->loaderLayerInterfaceVersion >= 2) {
- nli->sType = LAYER_NEGOTIATE_INTERFACE_STRUCT;
- nli->pNext = NULL;
- nli->pfnGetInstanceProcAddr = OBS_GetInstanceProcAddr;
- nli->pfnGetDeviceProcAddr = OBS_GetDeviceProcAddr;
- nli->pfnGetPhysicalDeviceProcAddr = NULL;
- }
- const uint32_t cur_ver = CURRENT_LOADER_LAYER_INTERFACE_VERSION;
- if (nli->loaderLayerInterfaceVersion > cur_ver) {
- nli->loaderLayerInterfaceVersion = cur_ver;
- }
- if (!vulkan_seen) {
- init_obj_list(&instances);
- init_obj_list(&devices);
- vulkan_seen = true;
- }
- return VK_SUCCESS;
- }
- bool hook_vulkan(void)
- {
- static bool hooked = false;
- if (!hooked && vulkan_seen) {
- hlog("Hooked Vulkan");
- hooked = true;
- }
- return hooked;
- }
|