summaryrefslogtreecommitdiff
path: root/libavcodec/h264_cabac.c
diff options
context:
space:
mode:
Diffstat (limited to 'libavcodec/h264_cabac.c')
-rw-r--r--libavcodec/h264_cabac.c68
1 files changed, 43 insertions, 25 deletions
diff --git a/libavcodec/h264_cabac.c b/libavcodec/h264_cabac.c
index 5e7f3e7768..68d7282717 100644
--- a/libavcodec/h264_cabac.c
+++ b/libavcodec/h264_cabac.c
@@ -2,20 +2,20 @@
* H.26L/H.264/AVC/JVT/14496-10/... cabac decoding
* Copyright (c) 2003 Michael Niedermayer <michaelni@gmx.at>
*
- * This file is part of Libav.
+ * This file is part of FFmpeg.
*
- * Libav is free software; you can redistribute it and/or
+ * FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
- * Libav is distributed in the hope that it will be useful,
+ * FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
- * License along with Libav; if not, write to the Free Software
+ * License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
@@ -26,9 +26,11 @@
*/
#define CABAC(h) 1
+#define UNCHECKED_BITSTREAM_READER 1
#define INT_BIT (CHAR_BIT * sizeof(int))
#include "libavutil/attributes.h"
+#include "libavutil/avassert.h"
#include "libavutil/timer.h"
#include "config.h"
#include "cabac.h"
@@ -45,8 +47,6 @@
#include "x86/h264_i386.h"
#endif
-#include <assert.h>
-
/* Cabac pre state table */
static const int8_t cabac_context_init_I[1024][2] =
@@ -1284,7 +1284,7 @@ void ff_h264_init_cabac_states(const H264Context *h, H264SliceContext *sl)
static int decode_cabac_field_decoding_flag(const H264Context *h, H264SliceContext *sl)
{
- const long mbb_xy = sl->mb_xy - 2L*h->mb_stride;
+ const int mbb_xy = sl->mb_xy - 2*h->mb_stride;
unsigned long ctx = 0;
@@ -1501,7 +1501,7 @@ static int decode_cabac_mb_mvd(H264SliceContext *sl, int ctxbase, int amvd, int
int mvd;
if(!get_cabac(&sl->cabac, &sl->cabac_state[ctxbase+((amvd-3)>>(INT_BIT-1))+((amvd-33)>>(INT_BIT-1))+2])){
-// if(!get_cabac(&h->cabac, &h->cabac_state[ctxbase+(amvd>2)+(amvd>32)])){
+// if(!get_cabac(&sl->cabac, &sl->cabac_state[ctxbase+(amvd>2)+(amvd>32)])){
*mvda= 0;
return 0;
}
@@ -1540,8 +1540,12 @@ static int decode_cabac_mb_mvd(H264SliceContext *sl, int ctxbase, int amvd, int
int amvd1 = sl->mvd_cache[list][scan8[n] - 1][1] +\
sl->mvd_cache[list][scan8[n] - 8][1];\
\
- mx += decode_cabac_mb_mvd(sl, 40, amvd0, &mpx);\
- my += decode_cabac_mb_mvd(sl, 47, amvd1, &mpy);\
+ int mxd = decode_cabac_mb_mvd(sl, 40, amvd0, &mpx);\
+ int myd = decode_cabac_mb_mvd(sl, 47, amvd1, &mpy);\
+ if (mxd == INT_MIN || myd == INT_MIN) \
+ return AVERROR_INVALIDDATA; \
+ mx += mxd;\
+ my += myd;\
}
static av_always_inline int get_cabac_cbf_ctx(H264SliceContext *sl,
@@ -1640,7 +1644,9 @@ decode_cabac_residual_internal(const H264Context *h, H264SliceContext *sl,
cc.range = sl->cabac.range;
cc.low = sl->cabac.low;
cc.bytestream= sl->cabac.bytestream;
+#if !UNCHECKED_BITSTREAM_READER || ARCH_AARCH64
cc.bytestream_end = sl->cabac.bytestream_end;
+#endif
#else
#define CC &sl->cabac
#endif
@@ -1689,7 +1695,7 @@ decode_cabac_residual_internal(const H264Context *h, H264SliceContext *sl,
}
#endif
}
- assert(coeff_count > 0);
+ av_assert2(coeff_count > 0);
if( is_dc ) {
if( cat == 3 )
@@ -1701,7 +1707,7 @@ decode_cabac_residual_internal(const H264Context *h, H264SliceContext *sl,
if( max_coeff == 64 )
fill_rectangle(&sl->non_zero_count_cache[scan8[n]], 2, 2, 8, coeff_count, 1);
else {
- assert( cat == 1 || cat == 2 || cat == 4 || cat == 7 || cat == 8 || cat == 11 || cat == 12 );
+ av_assert2( cat == 1 || cat == 2 || cat == 4 || cat == 7 || cat == 8 || cat == 11 || cat == 12 );
sl->non_zero_count_cache[scan8[n]] = coeff_count;
}
}
@@ -1917,6 +1923,7 @@ int ff_h264_decode_mb_cabac(const H264Context *h, H264SliceContext *sl)
mb_xy = sl->mb_xy = sl->mb_x + sl->mb_y*h->mb_stride;
+ ff_tlog(h->avctx, "pic:%d mb:%d/%d\n", h->poc.frame_num, sl->mb_x, sl->mb_y);
if (sl->slice_type_nos != AV_PICTURE_TYPE_I) {
int skip;
/* a skipped mb needs the aff flag from the following mb */
@@ -1955,7 +1962,7 @@ int ff_h264_decode_mb_cabac(const H264Context *h, H264SliceContext *sl)
if (sl->slice_type_nos == AV_PICTURE_TYPE_B) {
int ctx = 0;
- assert(sl->slice_type_nos == AV_PICTURE_TYPE_B);
+ av_assert2(sl->slice_type_nos == AV_PICTURE_TYPE_B);
if (!IS_DIRECT(sl->left_type[LTOP] - 1))
ctx++;
@@ -2008,7 +2015,7 @@ int ff_h264_decode_mb_cabac(const H264Context *h, H264SliceContext *sl)
mb_type = decode_cabac_intra_mb_type(sl, 3, 1);
if (sl->slice_type == AV_PICTURE_TYPE_SI && mb_type)
mb_type--;
- assert(sl->slice_type_nos == AV_PICTURE_TYPE_I);
+ av_assert2(sl->slice_type_nos == AV_PICTURE_TYPE_I);
decode_intra_mb:
partition_count = 0;
cbp = ff_h264_i_mb_type_info[mb_type].cbp;
@@ -2024,6 +2031,7 @@ decode_intra_mb:
const int mb_size = ff_h264_mb_sizes[sps->chroma_format_idc] *
sps->bit_depth_luma >> 3;
const uint8_t *ptr;
+ int ret;
// We assume these blocks are very rare so we do not optimize it.
// FIXME The two following lines get the bitstream position in the cabac
@@ -2040,7 +2048,9 @@ decode_intra_mb:
sl->intra_pcm_ptr = ptr;
ptr += mb_size;
- ff_init_cabac_decoder(&sl->cabac, ptr, sl->cabac.bytestream_end - ptr);
+ ret = ff_init_cabac_decoder(&sl->cabac, ptr, sl->cabac.bytestream_end - ptr);
+ if (ret < 0)
+ return ret;
// All blocks are present
h->cbp_table[mb_xy] = 0xf7ef;
@@ -2071,7 +2081,7 @@ decode_intra_mb:
int pred = pred_intra_mode(h, sl, i);
sl->intra4x4_pred_mode_cache[scan8[i]] = decode_cabac_mb_intra4x4_pred_mode(sl, pred);
- ff_dlog(h->avctx, "i4x4 pred=%d mode=%d\n", pred,
+ ff_tlog(h->avctx, "i4x4 pred=%d mode=%d\n", pred,
sl->intra4x4_pred_mode_cache[scan8[i]]);
}
}
@@ -2126,10 +2136,10 @@ decode_intra_mb:
for( i = 0; i < 4; i++ ) {
if(IS_DIRECT(sl->sub_mb_type[i])) continue;
if(IS_DIR(sl->sub_mb_type[i], 0, list)){
- int rc = sl->ref_count[list] << MB_MBAFF(sl);
+ unsigned rc = sl->ref_count[list] << MB_MBAFF(sl);
if (rc > 1) {
ref[list][i] = decode_cabac_mb_ref(sl, list, 4 * i);
- if (ref[list][i] >= (unsigned) rc) {
+ if (ref[list][i] >= rc) {
av_log(h->avctx, AV_LOG_ERROR, "Reference %d >= %d\n", ref[list][i], rc);
return -1;
}
@@ -2212,10 +2222,11 @@ decode_intra_mb:
if(IS_16X16(mb_type)){
for (list = 0; list < sl->list_count; list++) {
if(IS_DIR(mb_type, 0, list)){
- int ref, rc = sl->ref_count[list] << MB_MBAFF(sl);
+ int ref;
+ unsigned rc = sl->ref_count[list] << MB_MBAFF(sl);
if (rc > 1) {
ref= decode_cabac_mb_ref(sl, list, 0);
- if (ref >= (unsigned) rc) {
+ if (ref >= rc) {
av_log(h->avctx, AV_LOG_ERROR, "Reference %d >= %d\n", ref, rc);
return -1;
}
@@ -2240,10 +2251,11 @@ decode_intra_mb:
for (list = 0; list < sl->list_count; list++) {
for(i=0; i<2; i++){
if(IS_DIR(mb_type, i, list)){
- int ref, rc = sl->ref_count[list] << MB_MBAFF(sl);
+ int ref;
+ unsigned rc = sl->ref_count[list] << MB_MBAFF(sl);
if (rc > 1) {
ref= decode_cabac_mb_ref(sl, list, 8 * i);
- if (ref >= (unsigned) rc) {
+ if (ref >= rc) {
av_log(h->avctx, AV_LOG_ERROR, "Reference %d >= %d\n", ref, rc);
return -1;
}
@@ -2271,14 +2283,15 @@ decode_intra_mb:
}
}
}else{
- assert(IS_8X16(mb_type));
+ av_assert2(IS_8X16(mb_type));
for (list = 0; list < sl->list_count; list++) {
for(i=0; i<2; i++){
if(IS_DIR(mb_type, i, list)){ //FIXME optimize
- int ref, rc = sl->ref_count[list] << MB_MBAFF(sl);
+ int ref;
+ unsigned rc = sl->ref_count[list] << MB_MBAFF(sl);
if (rc > 1) {
ref = decode_cabac_mb_ref(sl, list, 4 * i);
- if (ref >= (unsigned) rc) {
+ if (ref >= rc) {
av_log(h->avctx, AV_LOG_ERROR, "Reference %d >= %d\n", ref, rc);
return -1;
}
@@ -2317,6 +2330,11 @@ decode_intra_mb:
cbp = decode_cabac_mb_cbp_luma(sl);
if(decode_chroma)
cbp |= decode_cabac_mb_cbp_chroma(sl) << 4;
+ } else {
+ if (!decode_chroma && cbp>15) {
+ av_log(h->avctx, AV_LOG_ERROR, "gray chroma\n");
+ return AVERROR_INVALIDDATA;
+ }
}
h->cbp_table[mb_xy] = sl->cbp = cbp;
View Lorry Controller Status