libobs-opengl: Clean up macOS code with better error handling

Adds more error handling to macOS code and also updates the code to
use more ObjC-appropriate code styling.

Variable names have been changed to be more expressive and platform-
specific constants are used instead of custom FourCharCodes.

libobs-opengl/gl-cocoa.m

@@ -46,19 +46,20 @@ static NSOpenGLContext *gl_context_create(NSOpenGLContext *share)
     NSOpenGLPixelFormatAttribute attributes[] = {NSOpenGLPFADoubleBuffer, NSOpenGLPFAOpenGLProfile,
                                                  NSOpenGLProfileVersion3_2Core, 0};
 
-    NSOpenGLPixelFormat *pf;
-    pf = [[NSOpenGLPixelFormat alloc] initWithAttributes:attributes];
-    if (!pf) {
+    NSOpenGLPixelFormat *pixelFormat;
+    pixelFormat = [[NSOpenGLPixelFormat alloc] initWithAttributes:attributes];
+    if (!pixelFormat) {
         blog(LOG_ERROR, "Failed to create pixel format");
-        return NULL;
+        return nil;
     }
 
     NSOpenGLContext *context;
-    context = [[NSOpenGLContext alloc] initWithFormat:pf shareContext:share];
-    [pf release];
+    context = [[NSOpenGLContext alloc] initWithFormat:pixelFormat shareContext:share];
+    [pixelFormat release];
+
     if (!context) {
         blog(LOG_ERROR, "Failed to create context");
-        return NULL;
+        return nil;
     }
 
     [context clearDrawable];
@@ -75,19 +76,19 @@ struct gl_platform *gl_platform_create(gs_device_t *device __unused, uint32_t ad
     }
 
     [context makeCurrentContext];
-    GLint interval = 0;
-    [context setValues:&interval forParameter:NSOpenGLContextParameterSwapInterval];
-    const bool success = gladLoadGL() != 0;
+    GLint swapInterval = 0;
+    [context setValues:&swapInterval forParameter:NSOpenGLContextParameterSwapInterval];
+    const bool isOpenGLLoaded = gladLoadGL() != 0;
 
-    if (!success) {
+    if (!isOpenGLLoaded) {
         blog(LOG_ERROR, "gladLoadGL failed");
         [context release];
         return NULL;
     }
 
-    struct gl_platform *plat = bzalloc(sizeof(struct gl_platform));
-    plat->context = context;
-    return plat;
+    struct gl_platform *platform = bzalloc(sizeof(struct gl_platform));
+    platform->context = context;
+    return platform;
 }
 
 void gl_platform_destroy(struct gl_platform *platform)
@@ -103,101 +104,145 @@ void gl_platform_destroy(struct gl_platform *platform)
 
 bool gl_platform_init_swapchain(struct gs_swap_chain *swap)
 {
-    NSOpenGLContext *parent = swap->device->plat->context;
-    NSOpenGLContext *context = gl_context_create(parent);
-    bool success = context != nil;
-    if (success) {
-        CGLContextObj parent_obj = [parent CGLContextObj];
-        CGLLockContext(parent_obj);
-
-        [parent makeCurrentContext];
-        struct gs_init_data *init_data = &swap->info;
-        swap->wi->texture = device_texture_create(swap->device, init_data->cx, init_data->cy, init_data->format, 1,
-                                                  NULL, GS_RENDER_TARGET);
+    NSOpenGLContext *platformContext = swap->device->plat->context;
+    NSOpenGLContext *swapContext = gl_context_create(platformContext);
+
+    BOOL hasFrameBuffer = NO;
+
+    if (swapContext) {
+        CGLContextObj platformContextObj = platformContext.CGLContextObj;
+        CGLContextObj swapContextObj = nil;
+
+        CGLLockContext(platformContextObj);
+
+        [platformContext makeCurrentContext];
+
+        struct gs_init_data *initData = &swap->info;
+        gs_texture_t *framebufferTexture = device_texture_create(swap->device, initData->cx, initData->cy,
+                                                                 initData->format, 1, NULL, GS_RENDER_TARGET);
+
+        if (!framebufferTexture) {
+            blog(LOG_ERROR, "gl_platform_init_swapchain: Unable to generate backing texture for frame buffer.");
+            goto init_swapchain_cleanup;
+        }
         glFlush();
+
         [NSOpenGLContext clearCurrentContext];
 
-        CGLContextObj context_obj = [context CGLContextObj];
-        CGLLockContext(context_obj);
+        swapContextObj = swapContext.CGLContextObj;
+        CGLLockContext(swapContextObj);
 
-        [context makeCurrentContext];
+        [swapContext makeCurrentContext];
 
 #pragma clang diagnostic push
 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
-        [context setView:swap->wi->view];
+        [swapContext setView:swap->wi->view];
 #pragma clang diagnostic pop
-        GLint interval = 0;
-        [context setValues:&interval forParameter:NSOpenGLContextParameterSwapInterval];
-        gl_gen_framebuffers(1, &swap->wi->fbo);
-        gl_bind_framebuffer(GL_FRAMEBUFFER, swap->wi->fbo);
-        glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, swap->wi->texture->texture, 0);
-        gl_success("glFrameBufferTexture2D");
-        glFlush();
-        [NSOpenGLContext clearCurrentContext];
 
-        CGLUnlockContext(context_obj);
+        GLint swapInterval = 0;
+        [swapContext setValues:&swapInterval forParameter:NSOpenGLContextParameterSwapInterval];
 
-        CGLUnlockContext(parent_obj);
+        GLuint framebufferObjectId = 0;
+        BOOL hasGeneratedFramebuffer = gl_gen_framebuffers(1, &framebufferObjectId);
+        BOOL hasBoundFrameBuffer = gl_bind_framebuffer(GL_FRAMEBUFFER, framebufferObjectId);
+
+        if (!(hasGeneratedFramebuffer && hasBoundFrameBuffer)) {
+            goto init_swapchain_cleanup;
+        }
 
-        swap->wi->context = context;
+        glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, framebufferTexture->texture, 0);
+        BOOL hasBoundFramebufferTexture = gl_success("glFrameBufferTexture2D");
+
+        if (!hasBoundFramebufferTexture) {
+            goto init_swapchain_cleanup;
+        }
+
+        glFlush();
+        gl_bind_framebuffer(GL_FRAMEBUFFER, 0);
+
+        hasFrameBuffer = YES;
+        swap->wi->context = swapContext;
+        swap->wi->fbo = framebufferObjectId;
+        swap->wi->texture = framebufferTexture;
+
+init_swapchain_cleanup:
+        [NSOpenGLContext clearCurrentContext];
+        if (swapContextObj) {
+            CGLUnlockContext(swapContextObj);
+        }
+        CGLUnlockContext(platformContextObj);
     }
 
-    return success;
+    return (swapContext && hasFrameBuffer);
 }
 
 void gl_platform_cleanup_swapchain(struct gs_swap_chain *swap)
 {
-    NSOpenGLContext *parent = swap->device->plat->context;
-    CGLContextObj parent_obj = [parent CGLContextObj];
-    CGLLockContext(parent_obj);
+    NSOpenGLContext *platformContext = swap->device->plat->context;
+    NSOpenGLContext *swapContext = swap->wi->context;
+    gs_texture_t *framebufferTexture = swap->wi->texture;
+    GLuint framebufferObjectId = swap->wi->fbo;
 
-    NSOpenGLContext *context = swap->wi->context;
-    CGLContextObj context_obj = [context CGLContextObj];
-    CGLLockContext(context_obj);
+    if (platformContext && swapContext) {
+        CGLContextObj platformContextObj = platformContext.CGLContextObj;
+        CGLLockContext(platformContextObj);
+        CGLContextObj swapContextObj = swapContext.CGLContextObj;
+        CGLLockContext(swapContextObj);
 
-    [context makeCurrentContext];
-    gl_delete_framebuffers(1, &swap->wi->fbo);
-    glFlush();
-    [NSOpenGLContext clearCurrentContext];
+        [swapContext makeCurrentContext];
+
+        gl_delete_framebuffers(1, &framebufferObjectId);
+        glFlush();
+        [NSOpenGLContext clearCurrentContext];
 
-    CGLUnlockContext(context_obj);
+        CGLUnlockContext(swapContextObj);
 
-    [parent makeCurrentContext];
-    gs_texture_destroy(swap->wi->texture);
-    glFlush();
-    [NSOpenGLContext clearCurrentContext];
-    swap->wi->context = nil;
+        [platformContext makeCurrentContext];
+        gs_texture_destroy(framebufferTexture);
+        glFlush();
+        [NSOpenGLContext clearCurrentContext];
+
+        CGLUnlockContext(platformContextObj);
 
-    CGLUnlockContext(parent_obj);
+        swap->wi->fbo = 0;
+        swap->wi->texture = NULL;
+        swap->wi->context = nil;
+    }
 }
 
 struct gl_windowinfo *gl_windowinfo_create(const struct gs_init_data *info)
 {
-    if (!info)
-        return NULL;
-
-    if (!info->window.view)
+    if (!(info && info->window.view)) {
         return NULL;
+    }
 
-    struct gl_windowinfo *wi = bzalloc(sizeof(struct gl_windowinfo));
+    NSView *projectorView = info->window.view;
 
-    wi->view = info->window.view;
-    wi->view.window.colorSpace = NSColorSpace.sRGBColorSpace;
 #pragma clang diagnostic push
 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
-    wi->view.wantsBestResolutionOpenGLSurface = YES;
+    projectorView.wantsBestResolutionOpenGLSurface = YES;
 #pragma clang diagnostic pop
 
-    return wi;
+    if (projectorView.window) {
+        projectorView.window.colorSpace = NSColorSpace.sRGBColorSpace;
+    }
+
+    struct gl_windowinfo *windowInfo = bzalloc(sizeof(struct gl_windowinfo));
+
+    windowInfo->view = projectorView;
+
+    return windowInfo;
 }
 
-void gl_windowinfo_destroy(struct gl_windowinfo *wi)
+void gl_windowinfo_destroy(struct gl_windowinfo *windowInfo)
 {
-    if (!wi)
+    if (!windowInfo)
         return;
 
-    wi->view = nil;
-    bfree(wi);
+    windowInfo->view = nil;
+    bfree(windowInfo);
+}
+
 void updateSwapchainFramebuffer(gs_device_t *device, OBSFrameBufferUpdateData updateData)
 {
     if (!(device && updateData.swapchain)) {
@@ -306,17 +351,16 @@ void gl_update(gs_device_t *device)
     }
 }
 
-void gl_clear_context(gs_device_t *device)
+void gl_clear_context(gs_device_t *device __unused)
 {
-    UNUSED_PARAMETER(device);
     [NSOpenGLContext clearCurrentContext];
 }
 
 void device_enter_context(gs_device_t *device)
 {
-    CGLLockContext([device->plat->context CGLContextObj]);
-
-    [device->plat->context makeCurrentContext];
+    NSOpenGLContext *platformContext = device->plat->context;
+    CGLLockContext(platformContext.CGLContextObj);
+    [platformContext makeCurrentContext];
 }
 
 void device_leave_context(gs_device_t *device)
@@ -330,7 +374,8 @@ void device_leave_context(gs_device_t *device)
     device->cur_swap = NULL;
     device->cur_fbo = NULL;
 
-    CGLUnlockContext([device->plat->context CGLContextObj]);
+    NSOpenGLContext *platformContext = device->plat->context;
+    CGLUnlockContext(platformContext.CGLContextObj);
 }
 
 void *device_get_device_obj(gs_device_t *device)
@@ -356,24 +401,53 @@ bool device_is_present_ready(gs_device_t *device __unused)
 
 void device_present(gs_device_t *device)
 {
+    if (!device) {
+        blog(LOG_ERROR, "device_present: Called without valid device reference");
+        return;
+    }
+
+    if (!(device->cur_swap && device->cur_swap->wi && device->cur_swap->wi->context)) {
+        blog(LOG_ERROR, "device_present: Called without valid window info or NSOpenGLContext");
+        return;
+    }
+
+    gs_swapchain_t *currentSwapchain = device->cur_swap;
+    struct gl_windowinfo *info = currentSwapchain->wi;
+
     glFlush();
+
+    NSOpenGLContext *currentContext = NSOpenGLContext.currentContext;
+    NSOpenGLContext *swapChainContext = info->context;
+    CGLContextObj swapChainContextObj = [swapChainContext CGLContextObj];
+
     [NSOpenGLContext clearCurrentContext];
+    CGLLockContext(swapChainContextObj);
+    [swapChainContext makeCurrentContext];
 
-    CGLLockContext([device->cur_swap->wi->context CGLContextObj]);
+    bool hasBoundReadFramebuffer = gl_bind_framebuffer(GL_READ_FRAMEBUFFER, info->fbo);
+    bool hasBoundDrawFramebuffer = gl_bind_framebuffer(GL_DRAW_FRAMEBUFFER, 0);
+
+    if (!(hasBoundReadFramebuffer && hasBoundDrawFramebuffer)) {
+        goto device_present_cleanup;
+    }
+
+    const UInt32 width = currentSwapchain->info.cx;
+    const UInt32 height = currentSwapchain->info.cy;
 
-    [device->cur_swap->wi->context makeCurrentContext];
-    gl_bind_framebuffer(GL_READ_FRAMEBUFFER, device->cur_swap->wi->fbo);
-    gl_bind_framebuffer(GL_DRAW_FRAMEBUFFER, 0);
-    const uint32_t width = device->cur_swap->info.cx;
-    const uint32_t height = device->cur_swap->info.cy;
     glBlitFramebuffer(0, 0, width, height, 0, height, width, 0, GL_COLOR_BUFFER_BIT, GL_NEAREST);
-    [device->cur_swap->wi->context flushBuffer];
-    glFlush();
-    [NSOpenGLContext clearCurrentContext];
+    bool didBlitFramebuffer = gl_success("glBlitFramebuffer");
+
+    if (!didBlitFramebuffer) {
+        goto device_present_cleanup;
+    }
 
-    CGLUnlockContext([device->cur_swap->wi->context CGLContextObj]);
+    [swapChainContext flushBuffer];
+    glFlush();
 
-    [device->plat->context makeCurrentContext];
+device_present_cleanup:
+    CGLUnlockContext(swapChainContextObj);
+    [NSOpenGLContext clearCurrentContext];
+    [currentContext makeCurrentContext];
 }
 
 bool device_is_monitor_hdr(gs_device_t *device __unused, void *monitor __unused)
@@ -394,29 +468,42 @@ gs_texture_t *device_texture_create_from_iosurface(gs_device_t *device, void *io
     IOSurfaceRef ref = (IOSurfaceRef) iosurf;
     struct gs_texture_2d *tex = bzalloc(sizeof(struct gs_texture_2d));
 
-    OSType pf = IOSurfaceGetPixelFormat(ref);
+    OSType pixelFormat = IOSurfaceGetPixelFormat(ref);
 
-    FourCharCode l10r_code = 0;
-    l10r_code = ('l' << 24) | ('1' << 16) | ('0' << 8) | 'r';
+    GLenum color_format = 0;
+    GLenum internal_format = 0;
+    GLenum texture_type = 0;
 
-    FourCharCode bgra_code = 0;
-    bgra_code = ('B' << 24) | ('G' << 16) | ('R' << 8) | 'A';
+    switch (pixelFormat) {
+        case kCVPixelFormatType_ARGB2101010LEPacked: {
+            color_format = GL_BGRA;
+            internal_format = convert_gs_internal_format(GS_R10G10B10A2);
+            texture_type = GL_UNSIGNED_INT_2_10_10_10_REV;
 
-    const bool l10r = pf == l10r_code;
-    if (pf == 0)
-        blog(LOG_ERROR, "Invalid IOSurface Buffer");
-    else if ((pf != bgra_code) && !l10r)
-        blog(LOG_ERROR, "Unexpected pixel format: %d (%c%c%c%c)", pf, pf >> 24, pf >> 16, pf >> 8, pf);
-
-    const enum gs_color_format color_format = l10r ? GS_R10G10B10A2 : GS_BGRA;
+            break;
+        }
+        case kCVPixelFormatType_32BGRA: {
+            color_format = convert_gs_format(GS_BGRA);
+            internal_format = convert_gs_internal_format(GS_BGRA);
+            texture_type = GL_UNSIGNED_INT_8_8_8_8_REV;
+            break;
+        }
+        case 0:
+            blog(LOG_ERROR, "Invalid IOSurface Buffer");
+            goto fail;
+        default:
+            blog(LOG_ERROR, "Unexpected pixel format: %d (%c%c%c%c)", pixelFormat, pixelFormat >> 24, pixelFormat >> 16,
+                 pixelFormat >> 8, pixelFormat);
+            goto fail;
+    }
 
     tex->base.device = device;
     tex->base.type = GS_TEXTURE_2D;
     tex->base.format = color_format;
     tex->base.levels = 1;
-    tex->base.gl_format = l10r ? GL_BGRA : convert_gs_format(color_format);
-    tex->base.gl_internal_format = convert_gs_internal_format(color_format);
-    tex->base.gl_type = l10r ? GL_UNSIGNED_INT_2_10_10_10_REV : GL_UNSIGNED_INT_8_8_8_8_REV;
+    tex->base.gl_format = color_format;
+    tex->base.gl_internal_format = internal_format;
+    tex->base.gl_type = texture_type;
     tex->base.gl_target = GL_TEXTURE_RECTANGLE_ARB;
     tex->base.is_dynamic = false;
     tex->base.is_render_target = false;
@@ -480,20 +567,22 @@ bool gs_texture_rebind_iosurface(gs_texture_t *texture, void *iosurf)
     if (!iosurf)
         return false;
 
-    FourCharCode l10r_code = 0;
-    l10r_code = ('l' << 24) | ('1' << 16) | ('0' << 8) | 'r';
-
-    FourCharCode bgra_code = 0;
-    bgra_code = ('B' << 24) | ('G' << 16) | ('R' << 8) | 'A';
-
     struct gs_texture_2d *tex = (struct gs_texture_2d *) texture;
     IOSurfaceRef ref = (IOSurfaceRef) iosurf;
 
-    OSType pf = IOSurfaceGetPixelFormat(ref);
-    if (pf == 0) {
-        blog(LOG_ERROR, "Invalid IOSurface buffer");
-    } else if ((pf != bgra_code) && (pf != l10r_code)) {
-        blog(LOG_ERROR, "Unexpected pixel format: %d (%c%c%c%c)", pf, pf >> 24, pf >> 16, pf >> 8, pf);
+    OSType pixelFormat = IOSurfaceGetPixelFormat(ref);
+
+    switch (pixelFormat) {
+        case kCVPixelFormatType_ARGB2101010LEPacked:
+        case kCVPixelFormatType_32BGRA:
+            break;
+        case 0:
+            blog(LOG_ERROR, "Invalid IOSurface Buffer");
+            return false;
+        default:
+            blog(LOG_ERROR, "Unexpected pixel format: %d (%c%c%c%c)", pixelFormat, pixelFormat >> 24, pixelFormat >> 16,
+                 pixelFormat >> 8, pixelFormat);
+            return false;
     }
 
     tex->width = (uint32_t) IOSurfaceGetWidth(ref);