GetBackwardReferences: fail on alloc error1.2.1

previously failures in the call to
VP8LBackwardReferencesTraceBackwards() would be ignored which, though it
wouldn't result in a crash, would produce non-deterministic output

Change-Id: Id9890a60883c3270ec75e968506d46eea32b76d4
(cherry picked from commit e3cfafaf719c2e163d3548d7a415da96fdff714f)
(cherry picked from commit 20ef03ee351d4ff03fc5ff3ec4804a879d1b9d5c)

1 files changed, 10 insertions, 9 deletions

diff --git a/src/enc/backward_references_enc.c b/src/enc/backward_references_enc.c
index 519b36a0..d5e931ef 100644
--- a/src/enc/backward_references_enc.c
+++ b/src/enc/backward_references_enc.c
@@ -976,15 +976,16 @@ static int GetBackwardReferences(int width, int height,
       const VP8LHashChain* const hash_chain_tmp =
           (lz77_types_best[i] == kLZ77Standard) ? hash_chain : &hash_chain_box;
       const int cache_bits = (i == 1) ? 0 : *cache_bits_best;
-      if (VP8LBackwardReferencesTraceBackwards(width, height, argb, cache_bits,
-                                               hash_chain_tmp, &refs[i],
-                                               refs_tmp)) {
-        double bit_cost_trace;
-        VP8LHistogramCreate(histo, refs_tmp, cache_bits);
-        bit_cost_trace = VP8LHistogramEstimateBits(histo);
-        if (bit_cost_trace < bit_costs_best[i]) {
-          BackwardRefsSwap(refs_tmp, &refs[i]);
-        }
+      double bit_cost_trace;
+      if (!VP8LBackwardReferencesTraceBackwards(width, height, argb, cache_bits,
+                                                hash_chain_tmp, &refs[i],
+                                                refs_tmp)) {
+        goto Error;
+      }
+      VP8LHistogramCreate(histo, refs_tmp, cache_bits);
+      bit_cost_trace = VP8LHistogramEstimateBits(histo);
+      if (bit_cost_trace < bit_costs_best[i]) {
+        BackwardRefsSwap(refs_tmp, &refs[i]);
       }
     }