libavcdec: Fix integer overflow issue in ui_max_frame_num

In avc MaxFrameNum can be 65536 which is of 17 bits due to which interger overflow was happening for i2_max_frm_num and ui_max_frame_num. This has been fixed. Bug: 369676522 Test: poc in bug description Change-Id: I858eea6bf8eea1e2cee6d4a7c28a84705eb51792
harishdm · Oct 24, 2024 · 7d6f857 · 7d6f857
1 parent 1a896d4
commit 7d6f857
Show file tree

Hide file tree

Showing 3 changed files with 6 additions and 6 deletions.
diff --git a/decoder/ih264d_dpb_mgr.c b/decoder/ih264d_dpb_mgr.c
@@ -730,7 +730,7 @@ WORD32 ih264d_ref_idx_reordering(dec_struct_t *ps_dec, UWORD8 uc_lx)
     dpb_manager_t *ps_dpb_mgr = ps_dec->ps_dpb_mgr;
     UWORD16 u4_cur_pic_num = ps_dec->ps_cur_slice->u2_frame_num;
     /*< Maximum Picture Number Minus 1 */
-    UWORD16 ui_max_frame_num =
+    UWORD32 ui_max_frame_num =
                     ps_dec->ps_cur_sps->u2_u4_max_pic_num_minus1 + 1;
 
     WORD32 i, count = 0;
@@ -776,7 +776,7 @@ WORD32 ih264d_ref_idx_reordering(dec_struct_t *ps_dec, UWORD8 uc_lx)
             {
                 // diffPicNum is +ve
                 i_temp = (WORD32)u2_pred_frame_num + (WORD32)ui_nextUev;
-                if(i_temp >= ui_max_frame_num)
+                if(i_temp >= (WORD32)ui_max_frame_num)
                     i_temp -= ui_max_frame_num;
             }
             /* Find the dpb with the matching picNum (picNum==frameNum for framePic) */

diff --git a/decoder/ih264d_parse_headers.c b/decoder/ih264d_parse_headers.c
@@ -584,7 +584,7 @@ WORD32 ih264d_parse_sps(dec_struct_t *ps_dec, dec_bit_stream_t *ps_bitstrm)
     UWORD8 i;
     dec_seq_params_t *ps_seq = NULL;
     UWORD8 u1_profile_idc, u1_level_idc, u1_seq_parameter_set_id, u1_mb_aff_flag = 0;
-    UWORD16 i2_max_frm_num;
+    UWORD32 u4_max_frm_num;
     UWORD32 *pu4_bitstrm_buf = ps_bitstrm->pu4_buffer;
     UWORD32 *pu4_bitstrm_ofst = &ps_bitstrm->u4_ofst;
     UWORD8 u1_frm, uc_constraint_set0_flag, uc_constraint_set1_flag;
@@ -794,8 +794,8 @@ WORD32 ih264d_parse_sps(dec_struct_t *ps_dec, dec_bit_stream_t *ps_bitstrm)
     COPYTHECONTEXT("SPS: log2_max_frame_num_minus4",
                     (ps_seq->u1_bits_in_frm_num - 4));
 
-    i2_max_frm_num = (1 << (ps_seq->u1_bits_in_frm_num));
-    ps_seq->u2_u4_max_pic_num_minus1 = i2_max_frm_num - 1;
+    u4_max_frm_num = (1 << (ps_seq->u1_bits_in_frm_num));
+    ps_seq->u2_u4_max_pic_num_minus1 = u4_max_frm_num - 1;
     /*--------------------------------------------------------------------*/
     /* Decode picture order count and related values                      */
     /*--------------------------------------------------------------------*/

diff --git a/decoder/ih264d_structs.h b/decoder/ih264d_structs.h
@@ -343,7 +343,7 @@ typedef struct
     UWORD8 u1_arbitrary_slice_order_allowed_flag;
     UWORD8 u1_redundant_slices_allowed_flag;
     UWORD8 u1_bits_in_frm_num; /** Number of bits in frame num */
-    UWORD16 u2_u4_max_pic_num_minus1; /** Maximum frame num minus 1 */
+    UWORD32 u2_u4_max_pic_num_minus1; /** Maximum frame num minus 1 */
     UWORD8 u1_pic_order_cnt_type; /** 0 - 2 indicates the method to code picture order count */
     UWORD8 u1_log2_max_pic_order_cnt_lsb_minus;
     WORD32 i4_max_pic_order_cntLsb;