deps/file-updater: Add file updater util. lib

This allows plugins to update and cache data files from a remote source.

Here are the steps that occur when the API initiates an update check:

1.) It checks to see if the local files are greater than the cached
    files.  If the local version is newer (for whatever reason), it
    replaces the cached version(s) with the local version.

2.) A packages.json file is downloaded from the specified URL.  That
    packages.json file contains a version number and a list of files to
    be updated.

3.) If the downloaded package version is greater than the cached
    version, executes step 4-5 on each file.

4.) Checks the version for the file to update in packages.json, and if
    the version is greater than the cached version, proceeds to step 5,
    otherwise repeat step 4-5 for other files.

5.) Calls the callback given to the update function (if any) with the
    file information (file name, buffer, etc), and if the callback
    returns true, allows the cached file to be updated and replaced,
    otherwise goes back to step 4-6 for the rest of the files.

NOTE: Files are never modified directly.  All file saving/modification
is performed in a temporary directory, and then files are moved to their
destination.  This should eliminate any possibility of file corruption
(or at least dramatically reduce the possibility).

deps/CMakeLists.txt

@@ -6,6 +6,7 @@ endif()
 add_subdirectory(glad)
 add_subdirectory(ipc-util)
 add_subdirectory(libff)
+add_subdirectory(file-updater)
 
 find_package(Jansson 2.5 QUIET)
 

deps/file-updater/CMakeLists.txt

@@ -0,0 +1,32 @@
+project(file-updater)
+
+find_package(Libcurl REQUIRED)
+
+include_directories(${LIBCURL_INCLUDE_DIRS})
+
+if(WIN32 AND NOT MINGW)
+	include_directories(../w32-pthreads)
+	set(file-updater_PLATFORM_DEPS
+		w32-pthreads)
+endif()
+
+set(file-updater_HEADERS
+	file-updater/file-updater.h)
+set(file-updater_SOURCES
+	file-updater/file-updater.c)
+
+add_library(file-updater STATIC
+	${file-updater_SOURCES}
+	${file-updater_HEADERS})
+
+target_include_directories(file-updater
+	PUBLIC .)
+
+if(NOT MSVC AND NOT MINGW)
+	target_compile_options(file-updater PRIVATE -fPIC)
+endif()
+
+target_link_libraries(file-updater
+	${LIBCURL_LIBRARIES}
+	${file-updater_PLATFORM_DEPS}
+	libobs)

deps/file-updater/file-updater/file-updater.c

@@ -0,0 +1,423 @@
+#include <util/threading.h>
+#include <util/platform.h>
+#include <util/darray.h>
+#include <util/dstr.h>
+#include <obs-data.h>
+#include <curl/curl.h>
+#include "file-updater.h"
+
+#define warn(msg, ...) \
+	blog(LOG_WARNING, "%s"msg, info->log_prefix, ##__VA_ARGS__)
+#define info(msg, ...) \
+	blog(LOG_WARNING, "%s"msg, info->log_prefix, ##__VA_ARGS__)
+
+struct update_info {
+	char error[CURL_ERROR_SIZE];
+	struct curl_slist *header;
+	DARRAY(uint8_t) file_data;
+	char *user_agent;
+	CURL *curl;
+	char *url;
+
+	/* directories */
+	char *local;
+	char *cache;
+	char *temp;
+
+	const char *remote_url;
+	obs_data_t *local_package;
+	obs_data_t *cache_package;
+	obs_data_t *remote_package;
+
+	confirm_file_callback_t callback;
+	void *param;
+
+	pthread_t thread;
+	bool thread_created;
+	char *log_prefix;
+};
+
+void update_info_destroy(struct update_info *info)
+{
+	if (!info)
+		return;
+
+	if (info->thread_created)
+		pthread_join(info->thread, NULL);
+
+	da_free(info->file_data);
+	bfree(info->log_prefix);
+	bfree(info->user_agent);
+	bfree(info->temp);
+	bfree(info->cache);
+	bfree(info->local);
+	bfree(info->url);
+
+	if (info->header)
+		curl_slist_free_all(info->header);
+	if (info->curl)
+		curl_easy_cleanup(info->curl);
+	if (info->local_package)
+		obs_data_release(info->local_package);
+	if (info->cache_package)
+		obs_data_release(info->cache_package);
+	if (info->remote_package)
+		obs_data_release(info->remote_package);
+	bfree(info);
+}
+
+static size_t http_write(uint8_t *ptr, size_t size, size_t nmemb,
+		struct update_info *info)
+{
+	size_t total = size * nmemb;
+	if (total)
+		da_push_back_array(info->file_data, ptr, total);
+
+	return total;
+}
+
+static bool do_http_request(struct update_info *info, const char *url)
+{
+	CURLcode code;
+	uint8_t null_terminator = 0;
+
+	da_resize(info->file_data, 0);
+	curl_easy_setopt(info->curl, CURLOPT_URL, url);
+	curl_easy_setopt(info->curl, CURLOPT_HTTPHEADER, info->header);
+	curl_easy_setopt(info->curl, CURLOPT_ERRORBUFFER, info->error);
+	curl_easy_setopt(info->curl, CURLOPT_WRITEFUNCTION, http_write);
+	curl_easy_setopt(info->curl, CURLOPT_WRITEDATA, info);
+	curl_easy_setopt(info->curl, CURLOPT_FAILONERROR, true);
+
+	code = curl_easy_perform(info->curl);
+	if (code != CURLE_OK) {
+		warn("Remote update of URL \"%s\" failed: %s", url,
+				info->error);
+		return false;
+	}
+
+	da_push_back(info->file_data, &null_terminator);
+
+	return true;
+}
+
+static char *get_path(const char *dir, const char *file)
+{
+	struct dstr str = {0};
+
+	dstr_copy(&str, dir);
+
+	if (str.array && dstr_end(&str) != '/' && dstr_end(&str) != '\\')
+		dstr_cat_ch(&str, '/');
+
+	dstr_cat(&str, file);
+	return str.array;
+}
+
+static inline obs_data_t *get_package(const char *base_path, const char *file)
+{
+	char *full_path = get_path(base_path, file);
+	obs_data_t *package = obs_data_create_from_json_file(full_path);
+	bfree(full_path);
+	return package;
+}
+
+static bool init_update(struct update_info *info)
+{
+	struct dstr user_agent = {0};
+
+	info->curl = curl_easy_init();
+	if (!info->curl) {
+		warn("Could not initialize Curl");
+		return false;
+	}
+
+	info->local_package = get_package(info->local, "package.json");
+	info->cache_package = get_package(info->cache, "package.json");
+
+	dstr_copy(&user_agent, "User-Agent: ");
+	dstr_cat(&user_agent, info->user_agent);
+
+	info->header = curl_slist_append(info->header, user_agent.array);
+
+	dstr_free(&user_agent);
+	return true;
+}
+
+static void copy_local_to_cache(struct update_info *info, const char *file)
+{
+	char *local_file_path = get_path(info->local, file);
+	char *cache_file_path = get_path(info->cache, file);
+	char *temp_file_path  = get_path(info->temp,  file);
+
+	os_copyfile(local_file_path, temp_file_path);
+	os_unlink(cache_file_path);
+	os_rename(temp_file_path, cache_file_path);
+
+	bfree(local_file_path);
+	bfree(cache_file_path);
+	bfree(temp_file_path);
+}
+
+static void enum_files(obs_data_t *package,
+		bool (*enum_func)(void *param, obs_data_t *file),
+		void *param)
+{
+	obs_data_array_t *array = obs_data_get_array(package, "files");
+	size_t num;
+
+	if (!array)
+		return;
+
+	num = obs_data_array_count(array);
+
+	for (size_t i = 0; i < num; i++) {
+		obs_data_t *file = obs_data_array_item(array, i);
+		bool continue_enum = enum_func(param, file);
+		obs_data_release(file);
+
+		if (!continue_enum)
+			break;
+	}
+
+	obs_data_array_release(array);
+}
+
+struct file_update_data {
+	const char *name;
+	int version;
+	bool newer;
+	bool found;
+};
+
+static bool newer_than_cache(void *param, obs_data_t *cache_file)
+{
+	struct file_update_data *input = param;
+	const char *name = obs_data_get_string(cache_file, "name");
+	int version = (int)obs_data_get_int(cache_file, "version");
+
+	if (strcmp(input->name, name) == 0) {
+		input->found = true;
+		input->newer = input->version > version;
+		return false;
+	}
+
+	return true;
+}
+
+static bool update_files_to_local(void *param, obs_data_t *local_file)
+{
+	struct update_info *info = param;
+	struct file_update_data data = {
+		.name = obs_data_get_string(local_file, "name"),
+		.version = (int)obs_data_get_int(local_file, "version")
+	};
+
+	enum_files(info->cache_package, newer_than_cache, &data);
+	if (data.newer || !data.found)
+		copy_local_to_cache(info, data.name);
+
+	return true;
+}
+
+static int update_local_version(struct update_info *info)
+{
+	int local_version;
+	int cache_version = 0;
+
+	local_version = (int)obs_data_get_int(info->local_package, "version");
+	cache_version = (int)obs_data_get_int(info->cache_package, "version");
+
+	/* if local cached version is out of date, copy new version */
+	if (cache_version < local_version) {
+		enum_files(info->local_package, update_files_to_local, info);
+		copy_local_to_cache(info, "package.json");
+
+		obs_data_release(info->cache_package);
+		obs_data_addref(info->local_package);
+		info->cache_package = info->local_package;
+
+		return local_version;
+	}
+
+	return cache_version;
+}
+
+static inline bool do_relative_http_request(struct update_info *info,
+		const char *url, const char *file)
+{
+	char *full_url = get_path(url, file);
+	bool success = do_http_request(info, full_url);
+	bfree(full_url);
+	return success;
+}
+
+static inline void write_file_data(struct update_info *info,
+		const char *base_path, const char *file)
+{
+	char *full_path = get_path(base_path, file);
+	os_quick_write_utf8_file(full_path,
+			(char*)info->file_data.array,
+			info->file_data.num - 1, false);
+	bfree(full_path);
+}
+
+static inline void replace_file(const char *src_base_path,
+		const char *dst_base_path, const char *file)
+{
+	char *src_path = get_path(src_base_path, file);
+	char *dst_path = get_path(dst_base_path, file);
+
+	if (src_path && dst_path) {
+		os_unlink(dst_path);
+		os_rename(src_path, dst_path);
+	}
+
+	bfree(dst_path);
+	bfree(src_path);
+}
+
+static bool update_remote_files(void *param, obs_data_t *remote_file)
+{
+	struct update_info *info = param;
+
+	struct file_update_data data = {
+		.name = obs_data_get_string(remote_file, "name"),
+		.version = (int)obs_data_get_int(remote_file, "version")
+	};
+
+	enum_files(info->cache_package, newer_than_cache, &data);
+	if (!data.newer && data.found)
+		return true;
+
+	if (!do_relative_http_request(info, info->remote_url, data.name))
+		return true;
+
+	if (info->callback) {
+		struct file_download_data download_data;
+		bool confirm;
+
+		download_data.name = data.name;
+		download_data.version = data.version;
+		download_data.buffer.da = info->file_data.da;
+
+		confirm = info->callback(info->param, &download_data);
+
+		info->file_data.da = download_data.buffer.da;
+
+		if (!confirm) {
+			info("Update file '%s' (version %d) rejected",
+					data.name, data.version);
+			return true;
+		}
+	}
+
+	write_file_data(info, info->temp, data.name);
+	replace_file(info->temp, info->cache, data.name);
+
+	info("Successfully updated file '%s' (version %d)",
+			data.name, data.version);
+	return true;
+}
+
+static void update_remote_version(struct update_info *info, int cur_version)
+{
+	int remote_version;
+
+	if (!do_http_request(info, info->url))
+		return;
+
+	if (!info->file_data.array || info->file_data.array[0] != '{') {
+		warn("Remote package does not exist or is not valid json");
+		return;
+	}
+
+	info->remote_package = obs_data_create_from_json(info->file_data.array);
+	if (!info->remote_package) {
+		warn("Failed to initialize remote package json");
+		return;
+	}
+
+	remote_version = (int)obs_data_get_int(info->remote_package, "version");
+	if (remote_version <= cur_version)
+		return;
+
+	write_file_data(info, info->temp, "package.json");
+
+	info->remote_url = obs_data_get_string(info->remote_package, "url");
+	if (!info->remote_url) {
+		warn("No remote url in package file");
+		return;
+	}
+
+	/* download new files */
+	enum_files(info->remote_package, update_remote_files, info);
+
+	replace_file(info->temp, info->cache, "package.json");
+
+	info("Successfully updated package (version %d)", remote_version);
+	return;
+}
+
+static void *update_thread(void *data)
+{
+	struct update_info *info = data;
+	int cur_version;
+
+	if (!init_update(info))
+		return NULL;
+
+	cur_version = update_local_version(info);
+	update_remote_version(info, cur_version);
+	os_rmdir(info->temp);
+	return NULL;
+}
+
+update_info_t *update_info_create(
+		const char *log_prefix,
+		const char *user_agent,
+		const char *update_url,
+		const char *local_dir,
+		const char *cache_dir,
+		confirm_file_callback_t confirm_callback,
+		void *param)
+{
+	struct update_info *info;
+	struct dstr dir = {0};
+
+	if (!log_prefix)
+		log_prefix = "";
+
+	if (os_mkdir(cache_dir) < 0) {
+		blog(LOG_WARNING, "%sCould not create cache directory %s",
+				log_prefix, cache_dir);
+		return NULL;
+	}
+
+	dstr_copy(&dir, cache_dir);
+	if (dstr_end(&dir) != '/' && dstr_end(&dir) != '\\')
+		dstr_cat_ch(&dir, '/');
+	dstr_cat(&dir, ".temp");
+
+	if (os_mkdir(dir.array) < 0) {
+		blog(LOG_WARNING, "%sCould not create temp directory %s",
+				log_prefix, cache_dir);
+		dstr_free(&dir);
+		return NULL;
+	}
+
+	info = bzalloc(sizeof(*info));
+	info->log_prefix = bstrdup(log_prefix);
+	info->user_agent = bstrdup(user_agent);
+	info->temp = dir.array;
+	info->local = bstrdup(local_dir);
+	info->cache = bstrdup(cache_dir);
+	info->url = get_path(update_url, "package.json");
+	info->callback = confirm_callback;
+	info->param = param;
+
+	if (pthread_create(&info->thread, NULL, update_thread, info) == 0)
+		info->thread_created = true;
+
+	return info;
+}

deps/file-updater/file-updater/file-updater.h

@@ -0,0 +1,24 @@
+#pragma once
+
+struct update_info;
+typedef struct update_info update_info_t;
+
+struct file_download_data {
+	const char *name;
+	int version;
+
+	DARRAY(uint8_t) buffer;
+};
+
+typedef bool (*confirm_file_callback_t)(void *param,
+		struct file_download_data *file);
+
+update_info_t *update_info_create(
+		const char *log_prefix,
+		const char *user_agent,
+		const char *update_url,
+		const char *local_dir,
+		const char *cache_dir,
+		confirm_file_callback_t confirm_callback,
+		void *param);
+void update_info_destroy(update_info_t *info);