libobs/graphics: Add Linux-only gs_query_dmabuf_* functions

When sharing DMA-BUFs it is required the announce the underlying
hardware capabilities via supported modifiers.

Add new device_query_dmabuf_capabilities vfunc to gs_exports and connect it
to the egl implementation stubs in the supported render platforms. Add a new
public method gs_query_dmabuf_capabilities() that calls the vfunc above.

Add new device_query_dmabuf_modifiers vfunc to gs_exports and connect it
to the egl implementation in the supported render platforms. Add a new
public method gs_query_dmabuf_modifiers() that calls the vfunc above.

docs/sphinx/reference-libobs-graphics-graphics.rst

@@ -1005,6 +1005,50 @@ Texture Functions
 
 ---------------------
 
+.. type:: enum gs_dmabuf_flags
+
+   DMA-BUF capabilities:
+
+   - GS_DMABUF_FLAG_NONE
+   - GS_DMABUF_FLAG_SUPPORTS_IMPLICIT_MODIFIERS  - Renderer supports implicit modifiers
+
+---------------------
+
+.. function:: bool *gs_query_dmabuf_capabilities(enum gs_dmabuf_flags *dmabuf_flags, uint32_t **drm_formats, size_t *n_formats)
+
+   **Linux only:** Queries the capabilities for DMA-BUFs.
+
+   Graphics cards can optimize frame buffers by storing them in custom layouts,
+   depending on their hardware features. These layouts can make these frame
+   buffers unsuitable for linear processing. This function allows querying whether
+   the graphics card in use supports implicit modifiers, and the supported texture
+   formats.
+
+   The caller must free the `drm_formats` array with `bfree()` after use.
+
+   :param dmabuf_flags: Pointer to receive a capability bitmap
+   :param drm_formats:  Pointer to receive an array of DRM formats
+   :param n_formats:    Pointer to receive the number of formats
+   :rtype:              bool
+
+---------------------
+
+.. function:: bool *gs_query_dmabuf_modifiers_for_format(uint32_t drm_format, uint64_t **modifiers, size_t *n_modifiers)
+
+   **Linux only:** Queries the supported DMA-BUF modifiers for a given format.
+
+   This function queries all supported explicit modifiers for a format,
+   stores them as an array and returns the number of supported modifiers.
+
+   The caller must free the `modifiers` array with `bfree()` after use.
+
+   :param drm_format:   DRM format of the DMA-BUF buffer
+   :param modifiers:    Pointer to receive an array of modifiers
+   :param n_modifiers:  Pointer to receive the number of modifiers
+   :rtype:              bool
+
+---------------------
+
 .. function:: gs_texture_t *gs_texture_create_from_iosurface(void *iosurf)
 
    **Mac only:** Creates a texture from an IOSurface.

libobs-opengl/gl-nix.c

@@ -134,3 +134,21 @@ extern struct gs_texture *device_texture_create_from_dmabuf(
 		device, width, height, drm_format, color_format, n_planes, fds,
 		strides, offsets, modifiers);
 }
+
+extern bool device_query_dmabuf_capabilities(gs_device_t *device,
+					     enum gs_dmabuf_flags *dmabuf_flags,
+					     uint32_t **drm_formats,
+					     size_t *n_formats)
+{
+	return gl_vtable->device_query_dmabuf_capabilities(
+		device, dmabuf_flags, drm_formats, n_formats);
+}
+
+extern bool device_query_dmabuf_modifiers_for_format(gs_device_t *device,
+						     uint32_t drm_format,
+						     uint64_t **modifiers,
+						     size_t *n_modifiers)
+{
+	return gl_vtable->device_query_dmabuf_modifiers_for_format(
+		device, drm_format, modifiers, n_modifiers);
+}

libobs-opengl/gl-nix.h

@@ -59,4 +59,13 @@ struct gl_winsys_vtable {
 		uint32_t drm_format, enum gs_color_format color_format,
 		uint32_t n_planes, const int *fds, const uint32_t *strides,
 		const uint32_t *offsets, const uint64_t *modifiers);
+
+	bool (*device_query_dmabuf_capabilities)(
+		gs_device_t *device, enum gs_dmabuf_flags *dmabuf_flags,
+		uint32_t **drm_formats, size_t *n_formats);
+
+	bool (*device_query_dmabuf_modifiers_for_format)(gs_device_t *device,
+							 uint32_t drm_format,
+							 uint64_t **modifiers,
+							 size_t *n_modifiers);
 };

libobs-opengl/gl-wayland-egl.c

@@ -363,6 +363,30 @@ static struct gs_texture *gl_wayland_egl_device_texture_create_from_dmabuf(
 					  fds, strides, offsets, modifiers);
 }
 
+static bool gl_wayland_egl_device_query_dmabuf_capabilities(
+	gs_device_t *device, enum gs_dmabuf_flags *dmabuf_flags,
+	uint32_t **drm_formats, size_t *n_formats)
+{
+	UNUSED_PARAMETER(device);
+	UNUSED_PARAMETER(dmabuf_flags);
+	UNUSED_PARAMETER(drm_formats);
+	UNUSED_PARAMETER(n_formats);
+
+	return false;
+}
+
+static bool gl_wayland_egl_device_query_dmabuf_modifiers_for_format(
+	gs_device_t *device, uint32_t drm_format, uint64_t **modifiers,
+	size_t *n_modifiers)
+{
+	UNUSED_PARAMETER(device);
+	UNUSED_PARAMETER(drm_format);
+	UNUSED_PARAMETER(modifiers);
+	UNUSED_PARAMETER(n_modifiers);
+
+	return false;
+}
+
 static const struct gl_winsys_vtable egl_wayland_winsys_vtable = {
 	.windowinfo_create = gl_wayland_egl_windowinfo_create,
 	.windowinfo_destroy = gl_wayland_egl_windowinfo_destroy,
@@ -380,6 +404,10 @@ static const struct gl_winsys_vtable egl_wayland_winsys_vtable = {
 	.device_present = gl_wayland_egl_device_present,
 	.device_texture_create_from_dmabuf =
 		gl_wayland_egl_device_texture_create_from_dmabuf,
+	.device_query_dmabuf_capabilities =
+		gl_wayland_egl_device_query_dmabuf_capabilities,
+	.device_query_dmabuf_modifiers_for_format =
+		gl_wayland_egl_device_query_dmabuf_modifiers_for_format,
 };
 
 const struct gl_winsys_vtable *gl_wayland_egl_get_winsys_vtable(void)

libobs-opengl/gl-x11-egl.c

@@ -649,6 +649,30 @@ static struct gs_texture *gl_x11_egl_device_texture_create_from_dmabuf(
 					  fds, strides, offsets, modifiers);
 }
 
+static bool gl_x11_egl_device_query_dmabuf_capabilities(
+	gs_device_t *device, enum gs_dmabuf_flags *dmabuf_flags,
+	uint32_t **drm_formats, size_t *n_formats)
+{
+	UNUSED_PARAMETER(device);
+	UNUSED_PARAMETER(dmabuf_flags);
+	UNUSED_PARAMETER(drm_formats);
+	UNUSED_PARAMETER(n_formats);
+
+	return false;
+}
+
+static bool gl_x11_egl_device_query_dmabuf_modifiers_for_format(
+	gs_device_t *device, uint32_t drm_format, uint64_t **modifiers,
+	size_t *n_modifiers)
+{
+	UNUSED_PARAMETER(device);
+	UNUSED_PARAMETER(drm_format);
+	UNUSED_PARAMETER(modifiers);
+	UNUSED_PARAMETER(n_modifiers);
+
+	return false;
+}
+
 static const struct gl_winsys_vtable egl_x11_winsys_vtable = {
 	.windowinfo_create = gl_x11_egl_windowinfo_create,
 	.windowinfo_destroy = gl_x11_egl_windowinfo_destroy,
@@ -666,6 +690,10 @@ static const struct gl_winsys_vtable egl_x11_winsys_vtable = {
 	.device_present = gl_x11_egl_device_present,
 	.device_texture_create_from_dmabuf =
 		gl_x11_egl_device_texture_create_from_dmabuf,
+	.device_query_dmabuf_capabilities =
+		gl_x11_egl_device_query_dmabuf_capabilities,
+	.device_query_dmabuf_modifiers_for_format =
+		gl_x11_egl_device_query_dmabuf_modifiers_for_format,
 };
 
 const struct gl_winsys_vtable *gl_x11_egl_get_winsys_vtable(void)

libobs-opengl/gl-x11-glx.c

@@ -599,6 +599,30 @@ static struct gs_texture *gl_x11_glx_device_texture_create_from_dmabuf(
 	return NULL;
 }
 
+static bool gl_x11_glx_device_query_dmabuf_capabilities(
+	gs_device_t *device, enum gs_dmabuf_flags *dmabuf_flags,
+	uint32_t **drm_formats, size_t *n_formats)
+{
+	UNUSED_PARAMETER(device);
+	UNUSED_PARAMETER(dmabuf_flags);
+	UNUSED_PARAMETER(drm_formats);
+	UNUSED_PARAMETER(n_formats);
+
+	return false;
+}
+
+static bool gl_x11_glx_device_query_dmabuf_modifiers_for_format(
+	gs_device_t *device, uint32_t drm_format, uint64_t **modifiers,
+	size_t *n_modifiers)
+{
+	UNUSED_PARAMETER(device);
+	UNUSED_PARAMETER(drm_format);
+	UNUSED_PARAMETER(modifiers);
+	UNUSED_PARAMETER(n_modifiers);
+
+	return false;
+}
+
 static const struct gl_winsys_vtable glx_winsys_vtable = {
 	.windowinfo_create = gl_x11_glx_windowinfo_create,
 	.windowinfo_destroy = gl_x11_glx_windowinfo_destroy,
@@ -616,6 +640,10 @@ static const struct gl_winsys_vtable glx_winsys_vtable = {
 	.device_present = gl_x11_glx_device_present,
 	.device_texture_create_from_dmabuf =
 		gl_x11_glx_device_texture_create_from_dmabuf,
+	.device_query_dmabuf_capabilities =
+		gl_x11_glx_device_query_dmabuf_capabilities,
+	.device_query_dmabuf_modifiers_for_format =
+		gl_x11_glx_device_query_dmabuf_modifiers_for_format,
 };
 
 const struct gl_winsys_vtable *gl_x11_glx_get_winsys_vtable(void)

libobs/graphics/device-exports.h

@@ -180,6 +180,16 @@ EXPORT gs_texture_t *device_texture_create_from_dmabuf(
 	uint32_t n_planes, const int *fds, const uint32_t *strides,
 	const uint32_t *offsets, const uint64_t *modifiers);
 
+EXPORT bool
+device_query_dmabuf_capabilities(gs_device_t *device,
+				 enum gs_dmabuf_flags *gs_dmabuf_flags,
+				 uint32_t **drm_formats, size_t *n_formats);
+
+EXPORT bool device_query_dmabuf_modifiers_for_format(gs_device_t *device,
+						     uint32_t drm_format,
+						     uint64_t **modifiers,
+						     size_t *n_modifiers);
+
 #endif
 
 #ifdef __cplusplus

libobs/graphics/graphics-imports.c

@@ -227,6 +227,8 @@ bool load_graphics_imports(struct gs_exports *exports, void *module,
 	GRAPHICS_IMPORT_OPTIONAL(device_unregister_loss_callbacks);
 #elif __linux__
 	GRAPHICS_IMPORT(device_texture_create_from_dmabuf);
+	GRAPHICS_IMPORT(device_query_dmabuf_capabilities);
+	GRAPHICS_IMPORT(device_query_dmabuf_modifiers_for_format);
 #endif
 
 	return success;

libobs/graphics/graphics-internal.h

@@ -337,6 +337,13 @@ struct gs_exports {
 		uint32_t drm_format, enum gs_color_format color_format,
 		uint32_t n_planes, const int *fds, const uint32_t *strides,
 		const uint32_t *offsets, const uint64_t *modifiers);
+	bool (*device_query_dmabuf_capabilities)(
+		gs_device_t *device, enum gs_dmabuf_flags *dmabuf_flags,
+		uint32_t **drm_formats, size_t *n_formats);
+	bool (*device_query_dmabuf_modifiers_for_format)(gs_device_t *device,
+							 uint32_t drm_format,
+							 uint64_t **modifiers,
+							 size_t *n_modifiers);
 #endif
 };
 

libobs/graphics/graphics.c

@@ -1392,6 +1392,25 @@ gs_texture_t *gs_texture_create_from_dmabuf(
 		n_planes, fds, strides, offsets, modifiers);
 }
 
+bool gs_query_dmabuf_capabilities(enum gs_dmabuf_flags *dmabuf_flags,
+				  uint32_t **drm_formats, size_t *n_formats)
+{
+	graphics_t *graphics = thread_graphics;
+
+	return graphics->exports.device_query_dmabuf_capabilities(
+		graphics->device, dmabuf_flags, drm_formats, n_formats);
+}
+
+bool gs_query_dmabuf_modifiers_for_format(uint32_t drm_format,
+					  uint64_t **modifiers,
+					  size_t *n_modifiers)
+{
+	graphics_t *graphics = thread_graphics;
+
+	return graphics->exports.device_query_dmabuf_modifiers_for_format(
+		graphics->device, drm_format, modifiers, n_modifiers);
+}
+
 #endif
 
 gs_texture_t *gs_cubetexture_create(uint32_t size,

libobs/graphics/graphics.h

@@ -943,6 +943,19 @@ EXPORT gs_texture_t *gs_texture_create_from_dmabuf(
 	const uint32_t *strides, const uint32_t *offsets,
 	const uint64_t *modifiers);
 
+enum gs_dmabuf_flags {
+	GS_DMABUF_FLAG_NONE = 0,
+	GS_DMABUF_FLAG_IMPLICIT_MODIFIERS_SUPPORTED = (1 << 0),
+};
+
+EXPORT bool gs_query_dmabuf_capabilities(enum gs_dmabuf_flags *dmabuf_flags,
+					 uint32_t **drm_formats,
+					 size_t *n_formats);
+
+EXPORT bool gs_query_dmabuf_modifiers_for_format(uint32_t drm_format,
+						 uint64_t **modifiers,
+						 size_t *n_modifiers);
+
 #endif
 
 /* inline functions used by modules */