From 5ee8886bff635bbf253927b3ad2ff7281737ba09 Mon Sep 17 00:00:00 2001 From: triple-groove Date: Sun, 22 Feb 2026 16:08:10 -0800 Subject: [PATCH 4/5] fix(dsc): use bits_per_component for flatnessDetThresh in DP DSC path EvoSetDpDscParamsC9() (nvkms-evo4.c, Ada/Blackwell) and EvoSetDpDscParams() (nvkms-evo3.c, Turing/Ampere) compute the DSC encoder's flatness_det_thresh register using bitsPerPixelX16 instead of bits_per_component. For 8bpc content at 8.0 bpp this produces 2 << (129 - 8) = 2 << 121, which is undefined behavior for a 32-bit integer. On x86-64, the shift wraps to 2 << 25 = 0x4000000, which truncates to 0 in the 10-bit register field. The VESA DSC spec defines: flatness_det_thresh = 2 << (bpc - 8) where bpc is 8, 10, or 12, yielding values of 2, 8, or 32. The HDMI DSC path in the same files already uses the correct formula. The existing code had an XXX comment acknowledging the bug: // XXX: I'm pretty sure that this is wrong. Fix: extract bits_per_component from PPS byte 3 bits [7:4], which is already computed correctly by the PPS generator. Signed-off-by: triple-groove --- src/nvidia-modeset/src/nvkms-evo3.c | 11 +++++++---- src/nvidia-modeset/src/nvkms-evo4.c | 11 +++++++---- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/nvidia-modeset/src/nvkms-evo3.c b/src/nvidia-modeset/src/nvkms-evo3.c index 455836e80c96..4f329eca9909 100644 --- a/src/nvidia-modeset/src/nvkms-evo3.c +++ b/src/nvidia-modeset/src/nvkms-evo3.c @@ -7211,10 +7211,13 @@ static void EvoSetDpDscParams(const NVDispEvoRec *pDispEvo, nvAssert(pDscInfo->type == NV_DSC_INFO_EVO_TYPE_DP); - // XXX: I'm pretty sure that this is wrong. - // BitsPerPixelx16 is something like (24 * 16) = 384, and 2 << (384 - 8) is - // an insanely large number. - flatnessDetThresh = (2 << (pDscInfo->dp.bitsPerPixelX16 - 8)); /* ??? */ + // Fix: use bits_per_component from PPS byte 3 [7:4], not bitsPerPixelX16. + // DSC spec: flatness_det_thresh = 2 << (bpc - 8). + // PPS DW[0] packs bytes [3][2][1][0] as [31:24][23:16][15:8][7:0]. + { + NvU32 bpc = (pDscInfo->dp.pps[0] >> 28) & 0xF; // PPS byte 3 bits[7:4] + flatnessDetThresh = (bpc >= 8) ? (2 << (bpc - 8)) : 2; + } nvAssert((pDscInfo->dp.dscMode == NV_DSC_EVO_MODE_DUAL) || (pDscInfo->dp.dscMode == NV_DSC_EVO_MODE_SINGLE)); diff --git a/src/nvidia-modeset/src/nvkms-evo4.c b/src/nvidia-modeset/src/nvkms-evo4.c index a2b94b16e180..e44a0d96bf0c 100644 --- a/src/nvidia-modeset/src/nvkms-evo4.c +++ b/src/nvidia-modeset/src/nvkms-evo4.c @@ -1413,10 +1413,13 @@ static void EvoSetDpDscParamsC9(const NVDispEvoRec *pDispEvo, nvAssert(pDscInfo->type == NV_DSC_INFO_EVO_TYPE_DP); - // XXX: I'm pretty sure that this is wrong. - // BitsPerPixelx16 is something like (24 * 16) = 384, and 2 << (384 - 8) is - // an insanely large number. - flatnessDetThresh = (2 << (pDscInfo->dp.bitsPerPixelX16 - 8)); /* ??? */ + // Fix: use bits_per_component from PPS byte 3 [7:4], not bitsPerPixelX16. + // DSC spec: flatness_det_thresh = 2 << (bpc - 8). + // PPS DW[0] packs bytes [3][2][1][0] as [31:24][23:16][15:8][7:0]. + { + NvU32 bpc = (pDscInfo->dp.pps[0] >> 28) & 0xF; + flatnessDetThresh = (bpc >= 8) ? (2 << (bpc - 8)) : 2; + } nvAssert((pDscInfo->dp.dscMode == NV_DSC_EVO_MODE_DUAL) || (pDscInfo->dp.dscMode == NV_DSC_EVO_MODE_SINGLE)); -- 2.53.0