[media] v4l2-tpg-core: drop SYCC, use higher precision 601 conversion matrix

The SYCC Y'CbCr encoding is identical to the 601 encoding. Since the
SYCC define is about to be removed for use in the kernel we need to
drop it in the TPG code as well.

This patch also adds a 4th decimal to the 601 conversion matrix.
That was specified by the sYCC spec and it makes sense to use this
across the board.

[mchehab@s-opensource.com: fix conflicts with LaTeX math expression patch]
Signed-off-by: Hans Verkuil <hans.verkuil@cisco.com>

Signed-off-by: Mauro Carvalho Chehab <mchehab@s-opensource.com>

Documentation/media/uapi/v4l/pixfmt-007.rst

@@ -97,11 +97,11 @@ the following ``V4L2_YCBCR_ENC_601`` encoding:
 
 .. math::
 
-    Y' = 0.299R' + 0.587G' + 0.114B'
+    Y' = 0.2990R' + 0.5870G' + 0.1140B'
 
-    Cb = -0.169R' - 0.331G' + 0.5B'
+    Cb = -0.1687R' - 0.3313G' + 0.5B'
 
-    Cr = 0.5R' - 0.419G' - 0.081B'
+    Cr = 0.5R' - 0.4187G' - 0.0813B'
 
 Y' is clamped to the range [0…1] and Cb and Cr are clamped to the range
 [-0.5…0.5]. This conversion to Y'CbCr is identical to the one defined in
@@ -241,11 +241,11 @@ scaled and offset:
 
 .. math::
 
-    Y' = \frac{219}{256} * (0.299R' + 0.587G' + 0.114B') + \frac{16}{256}
+    Y' = \frac{219}{256} * (0.2990R' + 0.5870G' + 0.1140B') + \frac{16}{256}
 
-    Cb = \frac{224}{256} * (-0.169R' - 0.331G' + 0.5B')
+    Cb = \frac{224}{256} * (-0.1687R' - 0.3313G' + 0.5B')
 
-    Cr = \frac{224}{256} * (0.5R' - 0.419G' - 0.081B')
+    Cr = \frac{224}{256} * (0.5R' - 0.4187G' - 0.0813B')
 
 Y' is clamped to the range [0…1] and Cb and Cr are clamped to the range
 [-0.5…0.5]. The non-standard xvYCC 709 or xvYCC 601 encodings can be
@@ -440,11 +440,11 @@ the following ``V4L2_YCBCR_ENC_601`` encoding:
 
 .. math::
 
-    Y' = 0.299R' + 0.587G' + 0.114B'
+    Y' = 0.2990R' + 0.5870G' + 0.1140B'
 
-    Cb = -0.169R' - 0.331G' + 0.5B'
+    Cb = -0.1687R' - 0.3313G' + 0.5B'
 
-    Cr = 0.5R' - 0.419G' - 0.081B'
+    Cr = 0.5R' - 0.4187G' - 0.0813B'
 
 Y' is clamped to the range [0…1] and Cb and Cr are clamped to the range
 [-0.5…0.5]. This transform is identical to one defined in SMPTE
@@ -842,11 +842,11 @@ the following ``V4L2_YCBCR_ENC_601`` encoding:
 
 .. math::
 
-    Y' = 0.299R' + 0.587G' + 0.114B'
+    Y' = 0.2990R' + 0.5870G' + 0.1140B'
 
-    Cb = -0.169R' - 0.331G' + 0.5B'
+    Cb = -0.1687R' - 0.3313G' + 0.5B'
 
-    Cr = 0.5R' - 0.419G' - 0.081B'
+    Cr = 0.5R' - 0.4187G' - 0.0813B'
 
 Y' is clamped to the range [0…1] and Cb and Cr are clamped to the range
 [-0.5…0.5]. The Y'CbCr quantization is limited range. This transform is
@@ -940,11 +940,11 @@ the following ``V4L2_YCBCR_ENC_601`` encoding:
 
 .. math::
 
-    Y' = 0.299R' + 0.587G' + 0.114B'
+    Y' = 0.2990R' + 0.5870G' + 0.1140B'
 
-    Cb = -0.169R' - 0.331G' + 0.5B'
+    Cb = -0.1687R' - 0.3313G' + 0.5B'
 
-    Cr = 0.5R' - 0.419G' - 0.081B'
+    Cr = 0.5R' - 0.4187G' - 0.0813B'
 
 Y' is clamped to the range [0…1] and Cb and Cr are clamped to the range
 [-0.5…0.5]. The Y'CbCr quantization is limited range. This transform is

drivers/media/common/v4l2-tpg/v4l2-tpg-core.c

@@ -504,14 +504,14 @@ static void color_to_ycbcr(struct tpg_data *tpg, int r, int g, int b,
 #define COEFF(v, r) ((int)(0.5 + (v) * (r) * 256.0))
 
 	static const int bt601[3][3] = {
-		{ COEFF(0.299, 219),  COEFF(0.587, 219),  COEFF(0.114, 219)  },
-		{ COEFF(-0.169, 224), COEFF(-0.331, 224), COEFF(0.5, 224)    },
-		{ COEFF(0.5, 224),    COEFF(-0.419, 224), COEFF(-0.081, 224) },
+		{ COEFF(0.299, 219),   COEFF(0.587, 219),   COEFF(0.114, 219)   },
+		{ COEFF(-0.1687, 224), COEFF(-0.3313, 224), COEFF(0.5, 224)     },
+		{ COEFF(0.5, 224),     COEFF(-0.4187, 224), COEFF(-0.0813, 224) },
 	};
 	static const int bt601_full[3][3] = {
-		{ COEFF(0.299, 255),  COEFF(0.587, 255),  COEFF(0.114, 255)  },
-		{ COEFF(-0.169, 255), COEFF(-0.331, 255), COEFF(0.5, 255)    },
-		{ COEFF(0.5, 255),    COEFF(-0.419, 255), COEFF(-0.081, 255) },
+		{ COEFF(0.299, 255),   COEFF(0.587, 255),   COEFF(0.114, 255)   },
+		{ COEFF(-0.1687, 255), COEFF(-0.3313, 255), COEFF(0.5, 255)     },
+		{ COEFF(0.5, 255),     COEFF(-0.4187, 255), COEFF(-0.0813, 255) },
 	};
 	static const int rec709[3][3] = {
 		{ COEFF(0.2126, 219),  COEFF(0.7152, 219),  COEFF(0.0722, 219)  },
@@ -558,7 +558,6 @@ static void color_to_ycbcr(struct tpg_data *tpg, int r, int g, int b,
 
 	switch (tpg->real_ycbcr_enc) {
 	case V4L2_YCBCR_ENC_601:
-	case V4L2_YCBCR_ENC_SYCC:
 		rgb2ycbcr(full ? bt601_full : bt601, r, g, b, y_offset, y, cb, cr);
 		break;
 	case V4L2_YCBCR_ENC_XV601:
@@ -674,7 +673,6 @@ static void ycbcr_to_color(struct tpg_data *tpg, int y, int cb, int cr,
 
 	switch (tpg->real_ycbcr_enc) {
 	case V4L2_YCBCR_ENC_601:
-	case V4L2_YCBCR_ENC_SYCC:
 		ycbcr2rgb(full ? bt601_full : bt601, y, cb, cr, y_offset, r, g, b);
 		break;
 	case V4L2_YCBCR_ENC_XV601: