libobs, libobs-opengl: add drm format param

This adds the drmbuf format as a parameter separate from the obs texture
format that will be used. drmbuf's may have a variety of formats that we
need to pass correctly to get a usable texture which may correspond to
multi-platform texture formats.

libobs-opengl/gl-egl-common.c

@@ -50,33 +50,6 @@ typedef void(APIENTRYP PFNGLEGLIMAGETARGETTEXTURE2DOESPROC)(
 	GLenum target, GLeglImageOES image);
 static PFNGLEGLIMAGETARGETTEXTURE2DOESPROC glEGLImageTargetTexture2DOES;
 
-/* copied from drm_fourcc.h */
-
-#define fourcc_code(a, b, c, d)                                \
-	((__u32)(a) | ((__u32)(b) << 8) | ((__u32)(c) << 16) | \
-	 ((__u32)(d) << 24))
-#define DRM_FORMAT_INVALID 0
-#define DRM_FORMAT_R8 fourcc_code('R', '8', ' ', ' ') /* [7:0] R */
-#define DRM_FORMAT_R16 \
-	fourcc_code('R', '1', '6', ' ') /* [15:0] R little endian */
-#define DRM_FORMAT_RG88 \
-	fourcc_code('R', 'G', '8', '8') /* [15:0] R:G 8:8 little endian */
-#define DRM_FORMAT_ABGR8888        \
-	fourcc_code('A', 'B', '2', \
-		    '4') /* [31:0] A:B:G:R 8:8:8:8 little endian */
-#define DRM_FORMAT_ABGR2101010     \
-	fourcc_code('A', 'B', '3', \
-		    '0') /* [31:0] A:B:G:R 2:10:10:10 little endian */
-#define DRM_FORMAT_ABGR16161616F   \
-	fourcc_code('A', 'B', '4', \
-		    'H') /* [63:0] A:B:G:R 16:16:16:16 little endian */
-#define DRM_FORMAT_ARGB8888        \
-	fourcc_code('A', 'R', '2', \
-		    '4') /* [31:0] A:R:G:B 8:8:8:8 little endian */
-#define DRM_FORMAT_XRGB8888        \
-	fourcc_code('X', 'R', '2', \
-		    '4') /* [31:0] x:R:G:B 8:8:8:8 little endian */
-
 static bool find_gl_extension(const char *extension)
 {
 	GLint n, i;
@@ -113,42 +86,6 @@ static bool init_egl_image_target_texture_2d_ext(void)
 	return true;
 }
 
-static inline enum gs_color_format gs_format_to_drm_format(uint32_t drm_format)
-{
-	switch (drm_format) {
-	case GS_R8:
-		return DRM_FORMAT_R8;
-	case GS_RGBA:
-		return DRM_FORMAT_ABGR8888;
-	case GS_BGRX:
-		return DRM_FORMAT_XRGB8888;
-	case GS_BGRA:
-		return DRM_FORMAT_ARGB8888;
-	case GS_R10G10B10A2:
-		return DRM_FORMAT_ABGR2101010;
-	case GS_R16:
-		return DRM_FORMAT_R16;
-	case GS_RGBA16F:
-		return DRM_FORMAT_ABGR16161616F;
-	case GS_R8G8:
-		return DRM_FORMAT_RG88;
-	case GS_A8:
-	case GS_R16F:
-	case GS_RGBA16:
-	case GS_RG16F:
-	case GS_R32F:
-	case GS_RG32F:
-	case GS_RGBA32F:
-	case GS_DXT1:
-	case GS_DXT3:
-	case GS_DXT5:
-	case GS_UNKNOWN:
-		return DRM_FORMAT_INVALID;
-	}
-
-	return DRM_FORMAT_INVALID;
-}
-
 static EGLImageKHR
 create_dmabuf_egl_image(EGLDisplay egl_display, unsigned int width,
 			unsigned int height, uint32_t drm_format,
@@ -242,24 +179,17 @@ create_dmabuf_egl_image(EGLDisplay egl_display, unsigned int width,
 
 struct gs_texture *
 gl_egl_create_dmabuf_image(EGLDisplay egl_display, unsigned int width,
-			   unsigned int height,
+			   unsigned int height, uint32_t drm_format,
 			   enum gs_color_format color_format, uint32_t n_planes,
 			   const int *fds, const uint32_t *strides,
 			   const uint32_t *offsets, const uint64_t *modifiers)
 {
 	struct gs_texture *texture = NULL;
 	EGLImage egl_image;
-	uint32_t drm_format;
 
 	if (!init_egl_image_target_texture_2d_ext())
 		return NULL;
 
-	drm_format = gs_format_to_drm_format(color_format);
-	if (drm_format == DRM_FORMAT_INVALID) {
-		blog(LOG_ERROR, "Invalid or unsupported image format");
-		return NULL;
-	}
-
 	egl_image = create_dmabuf_egl_image(egl_display, width, height,
 					    drm_format, n_planes, fds, strides,
 					    offsets, modifiers);

libobs-opengl/gl-egl-common.h

@@ -8,7 +8,7 @@ const char *gl_egl_error_to_string(EGLint error_number);
 
 struct gs_texture *
 gl_egl_create_dmabuf_image(EGLDisplay egl_display, unsigned int width,
-			   unsigned int height,
+			   unsigned int height, uint32_t drm_format,
 			   enum gs_color_format color_format, uint32_t n_planes,
 			   const int *fds, const uint32_t *strides,
 			   const uint32_t *offsets, const uint64_t *modifiers);

libobs-opengl/gl-nix.c

@@ -126,11 +126,11 @@ extern void device_present(gs_device_t *device)
 
 extern struct gs_texture *device_texture_create_from_dmabuf(
 	gs_device_t *device, unsigned int width, unsigned int height,
-	enum gs_color_format color_format, uint32_t n_planes, const int *fds,
-	const uint32_t *strides, const uint32_t *offsets,
-	const uint64_t *modifiers)
+	uint32_t drm_format, enum gs_color_format color_format,
+	uint32_t n_planes, const int *fds, const uint32_t *strides,
+	const uint32_t *offsets, const uint64_t *modifiers)
 {
 	return gl_vtable->device_texture_create_from_dmabuf(
-		device, width, height, color_format, n_planes, fds, strides,
-		offsets, modifiers);
+		device, width, height, drm_format, color_format, n_planes, fds,
+		strides, offsets, modifiers);
 }

libobs-opengl/gl-nix.h

@@ -56,7 +56,7 @@ struct gl_winsys_vtable {
 
 	struct gs_texture *(*device_texture_create_from_dmabuf)(
 		gs_device_t *device, unsigned int width, unsigned int height,
-		enum gs_color_format color_format, uint32_t n_planes,
-		const int *fds, const uint32_t *strides,
+		uint32_t drm_format, enum gs_color_format color_format,
+		uint32_t n_planes, const int *fds, const uint32_t *strides,
 		const uint32_t *offsets, const uint64_t *modifiers);
 };

libobs-opengl/gl-wayland-egl.c

@@ -323,15 +323,15 @@ static void gl_wayland_egl_device_present(gs_device_t *device)
 
 static struct gs_texture *gl_wayland_egl_device_texture_create_from_dmabuf(
 	gs_device_t *device, unsigned int width, unsigned int height,
-	enum gs_color_format color_format, uint32_t n_planes, const int *fds,
-	const uint32_t *strides, const uint32_t *offsets,
-	const uint64_t *modifiers)
+	uint32_t drm_format, enum gs_color_format color_format,
+	uint32_t n_planes, const int *fds, const uint32_t *strides,
+	const uint32_t *offsets, const uint64_t *modifiers)
 {
 	struct gl_platform *plat = device->plat;
 
 	return gl_egl_create_dmabuf_image(plat->display, width, height,
-					  color_format, n_planes, fds, strides,
-					  offsets, modifiers);
+					  drm_format, color_format, n_planes,
+					  fds, strides, offsets, modifiers);
 }
 
 static const struct gl_winsys_vtable egl_wayland_winsys_vtable = {

libobs-opengl/gl-x11-egl.c

@@ -637,15 +637,15 @@ static void gl_x11_egl_device_present(gs_device_t *device)
 
 static struct gs_texture *gl_x11_egl_device_texture_create_from_dmabuf(
 	gs_device_t *device, unsigned int width, unsigned int height,
-	enum gs_color_format color_format, uint32_t n_planes, const int *fds,
-	const uint32_t *strides, const uint32_t *offsets,
-	const uint64_t *modifiers)
+	uint32_t drm_format, enum gs_color_format color_format,
+	uint32_t n_planes, const int *fds, const uint32_t *strides,
+	const uint32_t *offsets, const uint64_t *modifiers)
 {
 	struct gl_platform *plat = device->plat;
 
 	return gl_egl_create_dmabuf_image(plat->edisplay, width, height,
-					  color_format, n_planes, fds, strides,
-					  offsets, modifiers);
+					  drm_format, color_format, n_planes,
+					  fds, strides, offsets, modifiers);
 }
 
 static const struct gl_winsys_vtable egl_x11_winsys_vtable = {

libobs-opengl/gl-x11-glx.c

@@ -581,13 +581,14 @@ static void gl_x11_glx_device_present(gs_device_t *device)
 
 static struct gs_texture *gl_x11_glx_device_texture_create_from_dmabuf(
 	gs_device_t *device, unsigned int width, unsigned int height,
-	enum gs_color_format color_format, uint32_t n_planes, const int *fds,
-	const uint32_t *strides, const uint32_t *offsets,
-	const uint64_t *modifiers)
+	uint32_t drm_format, enum gs_color_format color_format,
+	uint32_t n_planes, const int *fds, const uint32_t *strides,
+	const uint32_t *offsets, const uint64_t *modifiers)
 {
 	UNUSED_PARAMETER(device);
 	UNUSED_PARAMETER(width);
 	UNUSED_PARAMETER(height);
+	UNUSED_PARAMETER(drm_format);
 	UNUSED_PARAMETER(color_format);
 	UNUSED_PARAMETER(n_planes);
 	UNUSED_PARAMETER(fds);

libobs/graphics/device-exports.h

@@ -174,9 +174,9 @@ EXPORT void device_debug_marker_end(gs_device_t *device);
 
 EXPORT gs_texture_t *device_texture_create_from_dmabuf(
 	gs_device_t *device, unsigned int width, unsigned int height,
-	enum gs_color_format color_format, uint32_t n_planes, const int *fds,
-	const uint32_t *strides, const uint32_t *offsets,
-	const uint64_t *modifiers);
+	uint32_t drm_format, enum gs_color_format color_format,
+	uint32_t n_planes, const int *fds, const uint32_t *strides,
+	const uint32_t *offsets, const uint64_t *modifiers);
 
 #endif
 

libobs/graphics/graphics-internal.h

@@ -330,8 +330,8 @@ struct gs_exports {
 #elif __linux__
 	struct gs_texture *(*device_texture_create_from_dmabuf)(
 		gs_device_t *device, unsigned int width, unsigned int height,
-		enum gs_color_format color_format, uint32_t n_planes,
-		const int *fds, const uint32_t *strides,
+		uint32_t drm_format, enum gs_color_format color_format,
+		uint32_t n_planes, const int *fds, const uint32_t *strides,
 		const uint32_t *offsets, const uint64_t *modifiers);
 #endif
 };

libobs/graphics/graphics.c

@@ -1365,19 +1365,17 @@ gs_texture_t *gs_texture_create(uint32_t width, uint32_t height,
 
 #if __linux__
 
-gs_texture_t *gs_texture_create_from_dmabuf(unsigned int width,
-					    unsigned int height,
-					    enum gs_color_format color_format,
-					    uint32_t n_planes, const int *fds,
-					    const uint32_t *strides,
-					    const uint32_t *offsets,
-					    const uint64_t *modifiers)
+gs_texture_t *gs_texture_create_from_dmabuf(
+	unsigned int width, unsigned int height, uint32_t drm_format,
+	enum gs_color_format color_format, uint32_t n_planes, const int *fds,
+	const uint32_t *strides, const uint32_t *offsets,
+	const uint64_t *modifiers)
 {
 	graphics_t *graphics = thread_graphics;
 
 	return graphics->exports.device_texture_create_from_dmabuf(
-		graphics->device, width, height, color_format, n_planes, fds,
-		strides, offsets, modifiers);
+		graphics->device, width, height, drm_format, color_format,
+		n_planes, fds, strides, offsets, modifiers);
 }
 
 #endif

libobs/graphics/graphics.h

@@ -917,12 +917,11 @@ EXPORT void gs_unregister_loss_callbacks(void *data);
 
 #elif __linux__
 
-EXPORT gs_texture_t *
-gs_texture_create_from_dmabuf(unsigned int width, unsigned int height,
-			      enum gs_color_format color_format,
-			      uint32_t n_planes, const int *fds,
-			      const uint32_t *strides, const uint32_t *offsets,
-			      const uint64_t *modifiers);
+EXPORT gs_texture_t *gs_texture_create_from_dmabuf(
+	unsigned int width, unsigned int height, uint32_t drm_format,
+	enum gs_color_format color_format, uint32_t n_planes, const int *fds,
+	const uint32_t *strides, const uint32_t *offsets,
+	const uint64_t *modifiers);
 
 #endif