[Backport] Security bug 933743v5.12.3

To M73: MSE: Prevent OOB in AVC conversion to AnnexB

Overflowing size_t buffer indexer could allow OOB unless overflow is
caught.

BUG=933743
Reviewed-on: https://chromium-review.googlesource.com/c/1490832

Change-Id: I9955fe1deb807171d73bdb7b48629fc747f99df6
Reviewed-by: Allan Sandfeld Jensen <allan.jensen@qt.io>

3 files changed, 8 insertions, 8 deletions

diff --git a/chromium/media/filters/ffmpeg_h265_to_annex_b_bitstream_converter.cc b/chromium/media/filters/ffmpeg_h265_to_annex_b_bitstream_converter.cc
index 21b6b2ebad0..200794e63c9 100644
--- a/chromium/media/filters/ffmpeg_h265_to_annex_b_bitstream_converter.cc
+++ b/chromium/media/filters/ffmpeg_h265_to_annex_b_bitstream_converter.cc
@@ -52,7 +52,7 @@ bool FFmpegH265ToAnnexBBitstreamConverter::ConvertPacket(AVPacket* packet) {
   // allow that (see crbug.com/455379).
   input_frame.insert(input_frame.end(),
                      packet->data, packet->data + packet->size);
-  int nalu_size_len = hevc_config_->lengthSizeMinusOne + 1;
+  size_t nalu_size_len = hevc_config_->lengthSizeMinusOne + 1;
   if (!mp4::AVC::ConvertFrameToAnnexB(nalu_size_len, &input_frame,
                                       &subsamples)) {
     DVLOG(1) << "AnnexB conversion failed";
diff --git a/chromium/media/formats/mp4/avc.cc b/chromium/media/formats/mp4/avc.cc
index 509335b07e9..6512ad65394 100644
--- a/chromium/media/formats/mp4/avc.cc
+++ b/chromium/media/formats/mp4/avc.cc
@@ -21,9 +21,9 @@ static const uint8_t kAnnexBStartCode[] = {0, 0, 0, 1};
 static const int kAnnexBStartCodeSize = 4;
 
 static bool ConvertAVCToAnnexBInPlaceForLengthSize4(std::vector<uint8_t>* buf) {
-  const int kLengthSize = 4;
+  const size_t kLengthSize = 4;
   size_t pos = 0;
-  while (pos + kLengthSize < buf->size()) {
+  while (buf->size() > kLengthSize && buf->size() - kLengthSize > pos) {
     uint32_t nal_length = (*buf)[pos];
     nal_length = (nal_length << 8) + (*buf)[pos+1];
     nal_length = (nal_length << 8) + (*buf)[pos+2];
@@ -61,7 +61,7 @@ int AVC::FindSubsampleIndex(const std::vector<uint8_t>& buffer,
 }
 
 // static
-bool AVC::ConvertFrameToAnnexB(int length_size,
+bool AVC::ConvertFrameToAnnexB(size_t length_size,
                                std::vector<uint8_t>* buffer,
                                std::vector<SubsampleEntry>* subsamples) {
   RCHECK(length_size == 1 || length_size == 2 || length_size == 4);
@@ -77,8 +77,8 @@ bool AVC::ConvertFrameToAnnexB(int length_size,
   buffer->reserve(temp.size() + 32);
 
   size_t pos = 0;
-  while (pos + length_size < temp.size()) {
-    int nal_length = temp[pos];
+  while (temp.size() > length_size && temp.size() - length_size > pos) {
+    size_t nal_length = temp[pos];
     if (length_size == 2) nal_length = (nal_length << 8) + temp[pos+1];
     pos += length_size;
 
@@ -87,7 +87,7 @@ bool AVC::ConvertFrameToAnnexB(int length_size,
       return false;
     }
 
-    RCHECK(pos + nal_length <= temp.size());
+    RCHECK(temp.size() >= nal_length && temp.size() - nal_length >= pos);
     buffer->insert(buffer->end(), kAnnexBStartCode,
                    kAnnexBStartCode + kAnnexBStartCodeSize);
     if (subsamples && !subsamples->empty()) {
diff --git a/chromium/media/formats/mp4/avc.h b/chromium/media/formats/mp4/avc.h
index 655aa2f8653..3c32eb7fa88 100644
--- a/chromium/media/formats/mp4/avc.h
+++ b/chromium/media/formats/mp4/avc.h
@@ -26,7 +26,7 @@ struct AVCDecoderConfigurationRecord;
 
 class MEDIA_EXPORT AVC {
  public:
-  static bool ConvertFrameToAnnexB(int length_size,
+  static bool ConvertFrameToAnnexB(size_t length_size,
                                    std::vector<uint8_t>* buffer,
                                    std::vector<SubsampleEntry>* subsamples);