Add support for modeless UI creation

I realized that I had intended modeless UI to be usable by plugins, but
it had been pointed out to me that modeless really needs to return a
pointer/handle to the user interface object that was created.

libobs/obs-internal.h

@@ -55,6 +55,11 @@ struct ui_callback {
 	bool (*callback)(void *data, void *ui_data);
 };
 
+struct ui_modeless {
+	struct obs_ui_info ui_info;
+	void *(*callback)(void *data, void *ui_data);
+};
+
 /* ------------------------------------------------------------------------- */
 
 struct obs_video {
@@ -107,6 +112,7 @@ struct obs_subsystem {
 	DARRAY(struct encoder_info) encoder_types;
 	DARRAY(struct service_info) service_types;
 	DARRAY(struct ui_callback)  ui_callbacks;
+	DARRAY(struct ui_modeless)  ui_modeless_callbacks;
 
 	signal_handler_t            signals;
 	proc_handler_t              procs;

libobs/obs-module.c

@@ -76,18 +76,24 @@ complete:
 	dstr_free(&enum_name);
 }
 
-static void module_load_ui_exports(struct obs_module *mod)
+struct generic_ui_callback {
+	struct obs_ui_info ui_info;
+	void *callback;
+};
+
+static void module_load_ui_exports(struct obs_module *mod,
+		const char *main_export, struct darray *array)
 {
 	bool (*enum_func)(size_t idx, struct obs_ui_info *info);
 	struct obs_ui_info ui_info;
 	size_t i = 0;
 
-	enum_func = os_dlsym(mod->module, "enum_ui");
+	enum_func = os_dlsym(mod->module, main_export);
 	if (!enum_func)
 		return;
 
 	while (enum_func(i++, &ui_info)) {
-		struct ui_callback callback;
+		struct generic_ui_callback callback;
 		struct dstr name;
 
 		dstr_init_copy(&name, ui_info.name);
@@ -104,7 +110,8 @@ static void module_load_ui_exports(struct obs_module *mod)
 			                  "'%s', but the function was not "
 			                  "found", mod->name, name.array);
 		} else {
-			da_push_back(obs->ui_callbacks, &callback);
+			darray_push_back(sizeof(struct generic_ui_callback),
+					array, &callback);
 		}
 
 		dstr_free(&name);
@@ -192,7 +199,9 @@ int obs_load_module(const char *path)
 	module_load_exports(&mod, &obs->encoder_types.da, "encoders",
 			sizeof(struct encoder_info), load_encoder_info);
 
-	module_load_ui_exports(&mod);
+	module_load_ui_exports(&mod, "enum_ui", &obs->ui_callbacks.da);
+	module_load_ui_exports(&mod, "enum_modeless_ui",
+			&obs->ui_modeless_callbacks.da);
 
 	da_push_back(obs->modules, &mod);
 	return MODULE_SUCCESS;

libobs/obs-ui.h

@@ -43,22 +43,24 @@ struct obs_ui_info {
  * separation of UI code from core code (which may often be in differing
  * languages)
  *
- *   A module with UI calls needs to export this function:
+ *   A module with UI calls needs to export one or both of these functions:
  *       + enum_ui
+ *       + enum_modeless_ui
  *
  *   The enum_ui function provides an obs_ui_info structure for each
- * input/output/etc.  For example, to export Qt-specific configuration
- * functions, the exports might be something like:
+ * input/output/etc.  Modeless UI should be exported enum_modeless_ui.  For
+ * example, to export Qt-specific configuration functions, the exports might
+ * be something like:
  *       + mysource_config_qt
  *       + myoutput_config_qt
- *       + myencoder_config_panel_qt
+ *       + myencoder_config_qt
  *
  *    ..And the values given to enum_ui would be something this:
  *
  *    struct obs_ui_info ui_list[] = {
- *            {"mysource",  "config",       "qt"},
- *            {"myoutput",  "config",       "qt"},
- *            {"myencoder", "config_panel", "qt"}
+ *            {"mysource",  "config", "qt"},
+ *            {"myoutput",  "config", "qt"},
+ *            {"myencoder", "config", "qt"}
  *    };
  *
  *   'qt' could be replaced with 'wx' or something similar if using wxWidgets,
@@ -73,6 +75,13 @@ struct obs_ui_info {
  *            ui_info: pointer to the ui data for this enumeration
  *       Return value: false when no more available.
  *
+ * ---------------------------------------------------------
+ *   bool enum_modeless_ui(size_t idx, struct obs_ui_info *ui_info);
+ *
+ *                idx: index of the enumeration
+ *            ui_info: pointer to the ui data for this enumeration
+ *       Return value: false when no more available.
+ *
  * ===========================================
  *   Export Format
  * ===========================================
@@ -82,8 +91,7 @@ struct obs_ui_info {
  *       bool [name]_[task]_[target](void *data, void *ui_data);
  *
  *     [name]: specifies the name of the input/output/encoder/etc.
- *     [task]: specifies the task of the user interface, most often 'config',
- *             or 'config_panel'
+ *     [task]: specifies the task of the user interface, most often 'config'
  *   [target]: specifies the target or toolkit of the user interface, such as
  *             but not limited to 'qt', 'wx', 'win32', 'cocoa', etc.  If
  *             a custom solution is desired, it can use a program-specific
@@ -107,19 +115,32 @@ struct obs_ui_info {
  *
  *   In this example, the ui_data variable should ideally be a pointer to the
  * QWidget parent, if any.
+ *
+ * ===========================================
+ *   Modeless UI
+ * ===========================================
+ *   Modeless user interface calls are supported, but they must be exported
+ * through enum_modeless_ui, and the format is slightly different:
+ *
+ *       void *[name]_[task]_[target](void *data, void *ui_data);
+ *
+ *   Modeless UI calls return immediately, and typically are supposed to return
+ * a pointer or handle to the specific UI object that was created.  For
+ * example, a win32 modeless would return an HWND, a Qt object would return
+ * a pointer to a QWidget.
  */
 
 /**
  * ===========================================
  *   obs_call_ui
  * ===========================================
- * Requests UI to be displayed
+ * Requests modal UI to be displayed
  *
- *   This is typically used for things like creating dialogs/panels/etc for
+ *   This is typically used for things like creating modal dialogs/etc for
  * specific toolkits.
  *
  *           name: Name of the input/output/etc type that UI was requested for
- *           task: Task of the user interface (i.e. "config", "config_panel")
+ *           task: Task of the user interface (usually "config")
  *         target: Desired target (i.e. "qt", "wx", "gtk3", "win32", etc)
  *           data: Pointer to the obs input/output/etc
  *        ui_data: UI-specific data, usually a parent pointer/handle (if any)
@@ -135,6 +156,24 @@ struct obs_ui_info {
 EXPORT int obs_call_ui(const char *name, const char *task, const char *target,
 		void *data, void *ui_data);
 
+/**
+ * ===========================================
+ *   obs_create_ui
+ * ===========================================
+ * Requests modeless UI to be created
+ *
+ *           name: Name of the input/output/etc type that UI was requested for
+ *           task: Task of the user interface
+ *         target: Desired target (i.e. "qt", "wx", "gtk3", "win32", etc)
+ *           data: Pointer to the obs input/output/etc
+ *        ui_data: UI-specific data, usually a parent pointer/handle (if any)
+ *
+ *   Return value: Pointer to the target-specific modeless object, or NULL if
+ *                 not found or failed.
+ */
+EXPORT void *obs_create_ui(const char *name, const char *task,
+		const char *target, void *data, void *ui_data);
+
 #ifdef __cplusplus
 }
 #endif

libobs/obs.c

@@ -327,6 +327,7 @@ void obs_shutdown(void)
 	da_free(obs->output_types);
 	da_free(obs->service_types);
 	da_free(obs->ui_callbacks);
+	da_free(obs->ui_modeless_callbacks);
 
 	obs_free_data();
 	obs_free_video();
@@ -468,6 +469,22 @@ static inline struct ui_callback *get_ui_callback(const char *name,
 	return NULL;
 }
 
+static inline struct ui_modeless *get_modeless_ui_callback(const char *name,
+		const char *task, const char *target)
+{
+	for (size_t i = 0; i < obs->ui_modeless_callbacks.num; i++) {
+		struct ui_modeless *callback;
+		callback = obs->ui_modeless_callbacks.array+i;
+
+		if (strcmp(callback->ui_info.name,   name)   == 0 &&
+		    strcmp(callback->ui_info.task,   task)   == 0 &&
+		    strcmp(callback->ui_info.target, target) == 0)
+			return callback;
+	}
+
+	return NULL;
+}
+
 int obs_call_ui(const char *name, const char *task, const char *target,
 		void *data, void *ui_data)
 {
@@ -483,6 +500,16 @@ int obs_call_ui(const char *name, const char *task, const char *target,
 	return errorcode;
 }
 
+void *obs_create_ui(const char *name, const char *task, const char *target,
+		void *data, void *ui_data)
+{
+	struct ui_modeless *callback;
+	int errorcode = OBS_UI_NOTFOUND;
+
+	callback = get_modeless_ui_callback(name, task, target);
+	return callback ? callback->callback(data, ui_data) : NULL;
+}
+
 bool obs_add_source(obs_source_t source)
 {
 	struct calldata params = {0};