mpeg12dec: Always close reader on error

A reader left open may lead to hangs.

Signed-off-by: Vittorio Giovara <vittorio.giovara@gmail.com>
Signed-off-by: Diego Biurrun <diego@biurrun.de>

1 files changed, 48 insertions, 15 deletions

diff --git a/libavcodec/mpeg12dec.c b/libavcodec/mpeg12dec.c
index 711b310c34..882fdf3d82 100644
--- a/libavcodec/mpeg12dec.c
+++ b/libavcodec/mpeg12dec.c
@@ -128,9 +128,10 @@ static int mpeg_decode_motion(MpegEncContext *s, int fcode, int pred)
     return sign_extend(val, 5 + shift);
 }
 
+#define MAX_INDEX (64 - 1)
 #define check_scantable_index(ctx, x)                                         \
     do {                                                                      \
-        if ((x) > 63) {                                                       \
+        if ((x) > MAX_INDEX) {                                                \
             av_log(ctx->avctx, AV_LOG_ERROR, "ac-tex damaged at %d %d\n",     \
                    ctx->mb_x, ctx->mb_y);                                     \
             return AVERROR_INVALIDDATA;                                       \
@@ -170,7 +171,8 @@ static inline int mpeg1_decode_block_intra(MpegEncContext *s,
                 break;
             } else if (level != 0) {
                 i += run;
-                check_scantable_index(s, i);
+                if (i > MAX_INDEX)
+                    break;
                 j = scantable[i];
                 level = (level * qscale * quant_matrix[j]) >> 4;
                 level = (level - 1) | 1;
@@ -192,7 +194,8 @@ static inline int mpeg1_decode_block_intra(MpegEncContext *s,
                     LAST_SKIP_BITS(re, &s->gb, 8);
                 }
                 i += run;
-                check_scantable_index(s, i);
+                if (i > MAX_INDEX)
+                    break;
                 j = scantable[i];
                 if (level < 0) {
                     level = -level;
@@ -209,6 +212,9 @@ static inline int mpeg1_decode_block_intra(MpegEncContext *s,
         }
         CLOSE_READER(re, &s->gb);
     }
+
+    check_scantable_index(s, i);
+
     s->block_last_index[n] = i;
     return 0;
 }
@@ -250,7 +256,8 @@ static inline int mpeg1_decode_block_inter(MpegEncContext *s,
 
             if (level != 0) {
                 i += run;
-                check_scantable_index(s, i);
+                if (i > MAX_INDEX)
+                    break;
                 j = scantable[i];
                 level = ((level * 2 + 1) * qscale * quant_matrix[j]) >> 5;
                 level = (level - 1) | 1;
@@ -272,7 +279,8 @@ static inline int mpeg1_decode_block_inter(MpegEncContext *s,
                     SKIP_BITS(re, &s->gb, 8);
                 }
                 i += run;
-                check_scantable_index(s, i);
+                if (i > MAX_INDEX)
+                    break;
                 j = scantable[i];
                 if (level < 0) {
                     level = -level;
@@ -294,6 +302,9 @@ end:
         LAST_SKIP_BITS(re, &s->gb, 2);
         CLOSE_READER(re, &s->gb);
     }
+
+    check_scantable_index(s, i);
+
     s->block_last_index[n] = i;
     return 0;
 }
@@ -330,7 +341,8 @@ static inline int mpeg1_fast_decode_block_inter(MpegEncContext *s,
 
             if (level != 0) {
                 i += run;
-                check_scantable_index(s, i);
+                if (i > MAX_INDEX)
+                    break;
                 j = scantable[i];
                 level = ((level * 2 + 1) * qscale) >> 1;
                 level = (level - 1) | 1;
@@ -352,7 +364,8 @@ static inline int mpeg1_fast_decode_block_inter(MpegEncContext *s,
                     SKIP_BITS(re, &s->gb, 8);
                 }
                 i += run;
-                check_scantable_index(s, i);
+                if (i > MAX_INDEX)
+                    break;
                 j = scantable[i];
                 if (level < 0) {
                     level = -level;
@@ -374,6 +387,9 @@ end:
         LAST_SKIP_BITS(re, &s->gb, 2);
         CLOSE_READER(re, &s->gb);
     }
+
+    check_scantable_index(s, i);
+
     s->block_last_index[n] = i;
     return 0;
 }
@@ -419,7 +435,8 @@ static inline int mpeg2_decode_block_non_intra(MpegEncContext *s,
 
             if (level != 0) {
                 i += run;
-                check_scantable_index(s, i);
+                if (i > MAX_INDEX)
+                    break;
                 j = scantable[i];
                 level = ((level * 2 + 1) * qscale * quant_matrix[j]) >> 5;
                 level = (level ^ SHOW_SBITS(re, &s->gb, 1)) -
@@ -434,7 +451,8 @@ static inline int mpeg2_decode_block_non_intra(MpegEncContext *s,
                 SKIP_BITS(re, &s->gb, 12);
 
                 i += run;
-                check_scantable_index(s, i);
+                if (i > MAX_INDEX)
+                    break;
                 j = scantable[i];
                 if (level < 0) {
                     level = ((-level * 2 + 1) * qscale * quant_matrix[j]) >> 5;
@@ -456,6 +474,8 @@ end:
     }
     block[63] ^= (mismatch & 1);
 
+    check_scantable_index(s, i);
+
     s->block_last_index[n] = i;
     return 0;
 }
@@ -489,7 +509,8 @@ static inline int mpeg2_fast_decode_block_non_intra(MpegEncContext *s,
 
         if (level != 0) {
             i += run;
-            check_scantable_index(s, i);
+            if (i > MAX_INDEX)
+                break;
             j = scantable[i];
             level = ((level * 2 + 1) * qscale) >> 1;
             level = (level ^ SHOW_SBITS(re, &s->gb, 1)) -
@@ -504,7 +525,8 @@ static inline int mpeg2_fast_decode_block_non_intra(MpegEncContext *s,
             SKIP_BITS(re, &s->gb, 12);
 
             i += run;
-            check_scantable_index(s, i);
+            if (i > MAX_INDEX)
+                break;
             j = scantable[i];
             if (level < 0) {
                 level = ((-level * 2 + 1) * qscale) >> 1;
@@ -522,6 +544,9 @@ static inline int mpeg2_fast_decode_block_non_intra(MpegEncContext *s,
 end:
     LAST_SKIP_BITS(re, &s->gb, 2);
     CLOSE_READER(re, &s->gb);
+
+    check_scantable_index(s, i);
+
     s->block_last_index[n] = i;
     return 0;
 }
@@ -572,7 +597,8 @@ static inline int mpeg2_decode_block_intra(MpegEncContext *s,
                 break;
             } else if (level != 0) {
                 i += run;
-                check_scantable_index(s, i);
+                if (i > MAX_INDEX)
+                    break;
                 j = scantable[i];
                 level = (level * qscale * quant_matrix[j]) >> 4;
                 level = (level ^ SHOW_SBITS(re, &s->gb, 1)) -
@@ -586,7 +612,8 @@ static inline int mpeg2_decode_block_intra(MpegEncContext *s,
                 level = SHOW_SBITS(re, &s->gb, 12);
                 SKIP_BITS(re, &s->gb, 12);
                 i += run;
-                check_scantable_index(s, i);
+                if (i > MAX_INDEX)
+                    break;
                 j = scantable[i];
                 if (level < 0) {
                     level = (-level * qscale * quant_matrix[j]) >> 4;
@@ -603,6 +630,8 @@ static inline int mpeg2_decode_block_intra(MpegEncContext *s,
     }
     block[63] ^= mismatch & 1;
 
+    check_scantable_index(s, i);
+
     s->block_last_index[n] = i;
     return 0;
 }
@@ -650,7 +679,8 @@ static inline int mpeg2_fast_decode_block_intra(MpegEncContext *s,
                 break;
             } else if (level != 0) {
                 i += run;
-                check_scantable_index(s, i);
+                if (i > MAX_INDEX)
+                    break;
                 j = scantable[i];
                 level = (level * qscale * quant_matrix[j]) >> 4;
                 level = (level ^ SHOW_SBITS(re, &s->gb, 1)) -
@@ -664,7 +694,8 @@ static inline int mpeg2_fast_decode_block_intra(MpegEncContext *s,
                 level = SHOW_SBITS(re, &s->gb, 12);
                 SKIP_BITS(re, &s->gb, 12);
                 i += run;
-                check_scantable_index(s, i);
+                if (i > MAX_INDEX)
+                    break;
                 j = scantable[i];
                 if (level < 0) {
                     level = (-level * qscale * quant_matrix[j]) >> 4;
@@ -679,6 +710,8 @@ static inline int mpeg2_fast_decode_block_intra(MpegEncContext *s,
         CLOSE_READER(re, &s->gb);
     }
 
+    check_scantable_index(s, i);
+
     s->block_last_index[n] = i;
     return 0;
 }