libobs: Precompute more accurate matrices

These are generated with .NET BigInteger fractions, and late divide.

libobs/data/color.effect

@@ -20,17 +20,17 @@ float3 srgb_nonlinear_to_linear(float3 v)
 
 float3 rec709_to_rec2020(float3 v)
 {
-	float r = dot(v, float3(0.62740389593469914, 0.32928303837788397, 0.043313065687417190));
-	float g = dot(v, float3(0.069097289358232047, 0.91954039507545904, 0.011362315566309173));
-	float b = dot(v, float3(0.016391438875150235, 0.088013307877225860, 0.89559525324762468));
+	float r = dot(v, float3(0.62740389593469903, 0.32928303837788370, 0.043313065687417225));
+	float g = dot(v, float3(0.069097289358232075, 0.91954039507545871, 0.011362315566309178));
+	float b = dot(v, float3(0.016391438875150280, 0.088013307877225749, 0.89559525324762401));
 	return float3(r, g, b);
 }
 
 float3 rec2020_to_rec709(float3 v)
 {
-	float r = dot(v, float3(1.6604910021084343, -0.58764113878854973, -0.072849863319884745));
-	float g = dot(v, float3(-0.12455047452159063, 1.1328998971259603, -0.0083494226043695080));
-	float b = dot(v, float3(-0.018150763354905199, -0.10057889800800746, 1.1187296613629123));
+	float r = dot(v, float3(1.6604910021084345, -0.58764113878854951, -0.072849863319884883));
+	float g = dot(v, float3(-0.12455047452159074, 1.1328998971259603, -0.0083494226043694768));
+	float b = dot(v, float3(-0.018150763354905303, -0.10057889800800739, 1.1187296613629127));
 	return float3(r, g, b);
 }
 

plugins/obs-filters/data/color.effect

@@ -10,17 +10,17 @@ float3 srgb_linear_to_nonlinear(float3 v)
 
 float3 rec709_to_rec2020(float3 v)
 {
-	float r = dot(v, float3(0.62740389593469914, 0.32928303837788397, 0.043313065687417190));
-	float g = dot(v, float3(0.069097289358232047, 0.91954039507545904, 0.011362315566309173));
-	float b = dot(v, float3(0.016391438875150235, 0.088013307877225860, 0.89559525324762468));
+	float r = dot(v, float3(0.62740389593469903, 0.32928303837788370, 0.043313065687417225));
+	float g = dot(v, float3(0.069097289358232075, 0.91954039507545871, 0.011362315566309178));
+	float b = dot(v, float3(0.016391438875150280, 0.088013307877225749, 0.89559525324762401));
 	return float3(r, g, b);
 }
 
 float3 rec2020_to_rec709(float3 v)
 {
-	float r = dot(v, float3(1.6604910021084343, -0.58764113878854973, -0.072849863319884745));
-	float g = dot(v, float3(-0.12455047452159063, 1.1328998971259603, -0.0083494226043695080));
-	float b = dot(v, float3(-0.018150763354905199, -0.10057889800800746, 1.1187296613629123));
+	float r = dot(v, float3(1.6604910021084345, -0.58764113878854951, -0.072849863319884883));
+	float g = dot(v, float3(-0.12455047452159074, 1.1328998971259603, -0.0083494226043694768));
+	float b = dot(v, float3(-0.018150763354905303, -0.10057889800800739, 1.1187296613629127));
 	return float3(r, g, b);
 }