drm/amd/display: fix recout_skip calculation when rotating 180 or 270

Fixed fliped landscape and fliped portrait hard hang.

Signed-off-by: Yongqiang Sun <yongqiang.sun@amd.com>
Reviewed-by: Tony Cheng <Tony.Cheng@amd.com>
Acked-by: Harry Wentland <harry.wentland@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>

drivers/gpu/drm/amd/display/dc/core/dc_resource.c

@@ -498,26 +498,15 @@ static void calculate_viewport(struct pipe_ctx *pipe_ctx)
 	data->viewport_c.height = (data->viewport.height + vpc_div - 1) / vpc_div;
 
 	/* Handle hsplit */
-	if (pri_split || sec_split) {
-		/* HMirror XOR Secondary_pipe XOR Rotation_180 */
-		bool right_view = (sec_split != plane_state->horizontal_mirror) !=
-					(plane_state->rotation == ROTATION_ANGLE_180);
-
-		if (plane_state->rotation == ROTATION_ANGLE_90
-				|| plane_state->rotation == ROTATION_ANGLE_270)
-			/* Secondary_pipe XOR Rotation_270 */
-			right_view = (plane_state->rotation == ROTATION_ANGLE_270) != sec_split;
-
-		if (right_view) {
-			data->viewport.x +=  data->viewport.width / 2;
-			data->viewport_c.x +=  data->viewport_c.width / 2;
-			/* Ceil offset pipe */
-			data->viewport.width = (data->viewport.width + 1) / 2;
-			data->viewport_c.width = (data->viewport_c.width + 1) / 2;
-		} else {
-			data->viewport.width /= 2;
-			data->viewport_c.width /= 2;
-		}
+	if (sec_split) {
+		data->viewport.x +=  data->viewport.width / 2;
+		data->viewport_c.x +=  data->viewport_c.width / 2;
+		/* Ceil offset pipe */
+		data->viewport.width = (data->viewport.width + 1) / 2;
+		data->viewport_c.width = (data->viewport_c.width + 1) / 2;
+	} else if (pri_split) {
+		data->viewport.width /= 2;
+		data->viewport_c.width /= 2;
 	}
 
 	if (plane_state->rotation == ROTATION_ANGLE_90 ||
@@ -534,6 +523,16 @@ static void calculate_recout(struct pipe_ctx *pipe_ctx, struct view *recout_skip
 	struct rect surf_src = plane_state->src_rect;
 	struct rect surf_clip = plane_state->clip_rect;
 	int recout_full_x, recout_full_y;
+	bool pri_split = pipe_ctx->bottom_pipe &&
+			pipe_ctx->bottom_pipe->plane_state == pipe_ctx->plane_state;
+	bool sec_split = pipe_ctx->top_pipe &&
+			pipe_ctx->top_pipe->plane_state == pipe_ctx->plane_state;
+
+	if (stream->view_format == VIEW_3D_FORMAT_SIDE_BY_SIDE ||
+		stream->view_format == VIEW_3D_FORMAT_TOP_AND_BOTTOM) {
+		pri_split = false;
+		sec_split = false;
+	}
 
 	if (pipe_ctx->plane_state->rotation == ROTATION_ANGLE_90 ||
 			pipe_ctx->plane_state->rotation == ROTATION_ANGLE_270)
@@ -569,23 +568,35 @@ static void calculate_recout(struct pipe_ctx *pipe_ctx, struct view *recout_skip
 
 	/* Handle h & vsplit */
 	if (pipe_ctx->top_pipe && pipe_ctx->top_pipe->plane_state ==
-		pipe_ctx->plane_state) {
-		if (stream->view_format == VIEW_3D_FORMAT_TOP_AND_BOTTOM) {
-			pipe_ctx->plane_res.scl_data.recout.y += pipe_ctx->plane_res.scl_data.recout.height / 2;
-			/* Floor primary pipe, ceil 2ndary pipe */
-			pipe_ctx->plane_res.scl_data.recout.height = (pipe_ctx->plane_res.scl_data.recout.height + 1) / 2;
-		} else {
-			pipe_ctx->plane_res.scl_data.recout.x += pipe_ctx->plane_res.scl_data.recout.width / 2;
-			pipe_ctx->plane_res.scl_data.recout.width = (pipe_ctx->plane_res.scl_data.recout.width + 1) / 2;
-		}
+			pipe_ctx->plane_state && stream->view_format == VIEW_3D_FORMAT_TOP_AND_BOTTOM) {
+		pipe_ctx->plane_res.scl_data.recout.y += pipe_ctx->plane_res.scl_data.recout.height / 2;
+		/* Floor primary pipe, ceil 2ndary pipe */
+		pipe_ctx->plane_res.scl_data.recout.height = (pipe_ctx->plane_res.scl_data.recout.height + 1) / 2;
 	} else if (pipe_ctx->bottom_pipe &&
-			pipe_ctx->bottom_pipe->plane_state == pipe_ctx->plane_state) {
-		if (stream->view_format == VIEW_3D_FORMAT_TOP_AND_BOTTOM)
-			pipe_ctx->plane_res.scl_data.recout.height /= 2;
-		else
+			pipe_ctx->bottom_pipe->plane_state == pipe_ctx->plane_state
+			&& stream->view_format == VIEW_3D_FORMAT_TOP_AND_BOTTOM)
+		pipe_ctx->plane_res.scl_data.recout.height /= 2;
+
+	if (pri_split || sec_split) {
+		/* HMirror XOR Secondary_pipe XOR Rotation_180 */
+		bool right_view = (sec_split != plane_state->horizontal_mirror) !=
+					(plane_state->rotation == ROTATION_ANGLE_180);
+
+		if (plane_state->rotation == ROTATION_ANGLE_90
+				|| plane_state->rotation == ROTATION_ANGLE_270)
+			/* Secondary_pipe XOR Rotation_270 */
+			right_view = (plane_state->rotation == ROTATION_ANGLE_270) != sec_split;
+
+		if (right_view) {
+			pipe_ctx->plane_res.scl_data.recout.x +=
+					pipe_ctx->plane_res.scl_data.recout.width / 2;
+			/* Ceil offset pipe */
+			pipe_ctx->plane_res.scl_data.recout.width =
+					(pipe_ctx->plane_res.scl_data.recout.width + 1) / 2;
+		} else {
 			pipe_ctx->plane_res.scl_data.recout.width /= 2;
+		}
 	}
-
 	/* Unclipped recout offset = stream dst offset + ((surf dst offset - stream surf_src offset)
 	 * 				* 1/ stream scaling ratio) - (surf surf_src offset * 1/ full scl
 	 * 				ratio)
@@ -601,6 +612,21 @@ static void calculate_recout(struct pipe_ctx *pipe_ctx, struct view *recout_skip
 
 	recout_skip->width = pipe_ctx->plane_res.scl_data.recout.x - recout_full_x;
 	recout_skip->height = pipe_ctx->plane_res.scl_data.recout.y - recout_full_y;
+
+	/*Adjust recout_skip for rotation */
+	if ((pri_split || sec_split) && (plane_state->rotation == ROTATION_ANGLE_270 || plane_state->rotation == ROTATION_ANGLE_180)) {
+		bool right_view = (sec_split != plane_state->horizontal_mirror) !=
+					(plane_state->rotation == ROTATION_ANGLE_180);
+
+		if (plane_state->rotation == ROTATION_ANGLE_90
+				|| plane_state->rotation == ROTATION_ANGLE_270)
+			/* Secondary_pipe XOR Rotation_270 */
+			right_view = (plane_state->rotation == ROTATION_ANGLE_270) != sec_split;
+		if (!right_view)
+			recout_skip->width = pipe_ctx->plane_res.scl_data.recout.x + pipe_ctx->plane_res.scl_data.recout.width / 2 - recout_full_x;
+		else
+			recout_skip->width = pipe_ctx->plane_res.scl_data.recout.x - pipe_ctx->plane_res.scl_data.recout.width / 2 - recout_full_x;
+	}
 }
 
 static void calculate_scaling_ratios(struct pipe_ctx *pipe_ctx)