libobs-d3d11: Add checksum to shader cache

A few reports came in of cache files with the correct size but full of
null bytes, presumably from a system crash and an SSD lying about buffer
flushes. This commit adds a checksum at the end of the compiled bytecode
so we don't try to use invalid data.

Co-Authored-By: derrod <[email protected]>
(cherry picked from commit 1641812580572d2914e4e06abf5b3f2f2cd21bc3)

libobs-d3d11/d3d11-shader.cpp

@@ -205,13 +205,13 @@ void gs_shader::BuildConstantBuffer()
 		gs_shader_set_default(&params[i]);
 }
 
-static uint64_t fnv1a_hash(const char *str)
+static uint64_t fnv1a_hash(const char *str, size_t len)
 {
 	const uint64_t FNV_OFFSET = 14695981039346656037ULL;
 	const uint64_t FNV_PRIME = 1099511628211ULL;
 	uint64_t hash = FNV_OFFSET;
-	while (*str) {
-		hash ^= (uint64_t)*str++;
+	for (size_t i = 0; i < len; i++) {
+		hash ^= (uint64_t)str[i];
 		hash *= FNV_PRIME;
 	}
 	return hash;
@@ -223,31 +223,64 @@ void gs_shader::Compile(const char *shaderString, const char *file,
 	ComPtr<ID3D10Blob> errorsBlob;
 	HRESULT hr;
 
+	bool is_cached = false;
 	char hashstr[20];
 
 	if (!shaderString)
 		throw "No shader string specified";
 
-	uint64_t hash = fnv1a_hash(shaderString);
+	size_t shaderStrLen = strlen(shaderString);
+	uint64_t hash = fnv1a_hash(shaderString, shaderStrLen);
 	snprintf(hashstr, sizeof(hashstr), "%02llx", hash);
 
 	BPtr program_data =
 		os_get_program_data_path_ptr("obs-studio/shader-cache");
 	auto cachePath = filesystem::u8path(program_data.Get()) / hashstr;
+	// Increment if on-disk format changes
+	cachePath += ".v2";
 
 	std::fstream cacheFile;
+	cacheFile.exceptions(fstream::badbit | fstream::eofbit);
+
 	if (filesystem::exists(cachePath) && !filesystem::is_empty(cachePath))
 		cacheFile.open(cachePath, ios::in | ios::binary | ios::ate);
 
 	if (cacheFile.is_open()) {
-		streampos len = cacheFile.tellg();
-		cacheFile.seekg(0, ios::beg);
-
-		device->d3dCreateBlob(len, shader);
-		cacheFile.read((char *)(*shader)->GetBufferPointer(), len);
-	} else {
-		hr = device->d3dCompile(shaderString, strlen(shaderString),
-					file, NULL, NULL, "main", target,
+		uint64_t checksum;
+
+		try {
+			streampos len = cacheFile.tellg();
+			// Not enough data for checksum + shader
+			if (len <= sizeof(checksum))
+				throw length_error("File truncated");
+
+			cacheFile.seekg(0, ios::beg);
+
+			len -= sizeof(checksum);
+			device->d3dCreateBlob(len, shader);
+			cacheFile.read((char *)(*shader)->GetBufferPointer(),
+				       len);
+			uint64_t calculated_checksum = fnv1a_hash(
+				(char *)(*shader)->GetBufferPointer(), len);
+
+			cacheFile.read((char *)&checksum, sizeof(checksum));
+			if (calculated_checksum != checksum)
+				throw exception("Checksum mismatch");
+
+			is_cached = true;
+		} catch (const exception &e) {
+			// Something went wrong reading the cache file, delete it
+			blog(LOG_WARNING,
+			     "Loading shader cache file failed with \"%s\": %s",
+			     e.what(), file);
+			cacheFile.close();
+			filesystem::remove(cachePath);
+		}
+	}
+
+	if (!is_cached) {
+		hr = device->d3dCompile(shaderString, shaderStrLen, file, NULL,
+					NULL, "main", target,
 					D3D10_SHADER_OPTIMIZATION_LEVEL3, 0,
 					shader, errorsBlob.Assign());
 		if (FAILED(hr)) {
@@ -259,8 +292,23 @@ void gs_shader::Compile(const char *shaderString, const char *file,
 
 		cacheFile.open(cachePath, ios::out | ios::binary);
 		if (cacheFile.is_open()) {
-			cacheFile.write((char *)(*shader)->GetBufferPointer(),
+			try {
+				uint64_t calculated_checksum = fnv1a_hash(
+					(char *)(*shader)->GetBufferPointer(),
+					(*shader)->GetBufferSize());
+
+				cacheFile.write(
+					(char *)(*shader)->GetBufferPointer(),
 					(*shader)->GetBufferSize());
+				cacheFile.write((char *)&calculated_checksum,
+						sizeof(calculated_checksum));
+			} catch (const exception &e) {
+				blog(LOG_WARNING,
+				     "Writing shader cache file failed with \"%s\": %s",
+				     e.what(), file);
+				cacheFile.close();
+				filesystem::remove(cachePath);
+			}
 		}
 	}