vp8 encoder: fix some integer overflowsmaster

cap the bitrate to 1000Mbps to avoid many instances of bitrate * 3 / 2
overflowing.

this adds coverage for 2048x2048 in the default test for VP8 with TODOs
for issues at that resolution for VP9 and at max resolution for both.

Bug: b/189602769
Bug: chromium:1264506
Bug: webm:1748
Bug: webm:1749
Bug: webm:1750
Bug: webm:1751
Change-Id: Iedee4dd8d3609c2504271f94d22433dfcd828429

2 files changed, 41 insertions, 6 deletions

diff --git a/test/realtime_test.cc b/test/realtime_test.cc
index 63a5347d9..b32a35513 100644
--- a/test/realtime_test.cc
+++ b/test/realtime_test.cc
@@ -7,6 +7,8 @@
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
+#include <limits.h>
+
 #include "test/codec_factory.h"
 #include "test/encode_test_driver.h"
 #include "test/util.h"
@@ -52,6 +54,22 @@ class RealtimeTest
     frame_packets_++;
   }
 
+  bool IsVP9() const {
+#if CONFIG_VP9_ENCODER
+    return codec_ == &libvpx_test::kVP9;
+#else
+    return false;
+#endif
+  }
+
+  void TestIntegerOverflow(unsigned int width, unsigned int height) {
+    ::libvpx_test::RandomVideoSource video;
+    video.SetSize(width, height);
+    video.set_limit(20);
+    cfg_.rc_target_bitrate = UINT_MAX;
+    ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
+  }
+
   int frame_packets_;
 };
 
@@ -64,11 +82,26 @@ TEST_P(RealtimeTest, RealtimeFirstPassProducesFrames) {
 }
 
 TEST_P(RealtimeTest, IntegerOverflow) {
-  ::libvpx_test::RandomVideoSource video;
-  video.SetSize(800, 480);
-  video.set_limit(20);
-  cfg_.rc_target_bitrate = 140000000;
-  ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
+  if (IsVP9()) {
+    // TODO(https://crbug.com/webm/1749): This should match VP8.
+    TestIntegerOverflow(800, 480);
+  } else {
+    TestIntegerOverflow(2048, 2048);
+  }
+}
+
+TEST_P(RealtimeTest, IntegerOverflowLarge) {
+  if (IsVP9()) {
+    GTEST_SKIP() << "TODO(https://crbug.com/webm/1750): Enable this test after "
+                    "undefined sanitizer warnings are fixed.";
+    // TestIntegerOverflow(16384, 16384);
+  } else {
+    GTEST_SKIP()
+        << "TODO(https://crbug.com/webm/1748,https://crbug.com/webm/1751):"
+        << " Enable this test after bitstream errors & undefined sanitizer "
+           "warnings are fixed.";
+    // TestIntegerOverflow(16383, 16383);
+  }
 }
 
 VP8_INSTANTIATE_TEST_SUITE(RealtimeTest,
diff --git a/vp8/vp8_cx_iface.c b/vp8/vp8_cx_iface.c
index 893b7a513..ab954c46f 100644
--- a/vp8/vp8_cx_iface.c
+++ b/vp8/vp8_cx_iface.c
@@ -339,7 +339,9 @@ static vpx_codec_err_t set_vp8e_config(VP8_CONFIG *oxcf,
     oxcf->end_usage = USAGE_CONSTANT_QUALITY;
   }
 
-  oxcf->target_bandwidth = cfg.rc_target_bitrate;
+  // Cap the target rate to 1000 Mbps to avoid some integer overflows in
+  // target bandwidth calculations.
+  oxcf->target_bandwidth = VPXMIN(cfg.rc_target_bitrate, 1000000);
   oxcf->rc_max_intra_bitrate_pct = vp8_cfg.rc_max_intra_bitrate_pct;
   oxcf->gf_cbr_boost_pct = vp8_cfg.gf_cbr_boost_pct;