VP10: some changes to palette mode

Account for rounding in distortion calculation in k-means;
carry out rounding before duplicates removal of base colors;
replace numbers with macros;
use prefix increment.

Slight coding gain (<0.1%) on screen_content testset.

Change-Id: Ie8bd241266da6b82c7b2874befc3a0c72b4fcd8c

6 files changed, 26 insertions, 20 deletions

diff --git a/vp10/common/entropymode.c b/vp10/common/entropymode.c
index 370e1c287..e930ff3b8 100644
--- a/vp10/common/entropymode.c
+++ b/vp10/common/entropymode.c
@@ -285,7 +285,8 @@ const vpx_tree_index vp10_palette_size_tree[TREE_SIZE(PALETTE_SIZES)] = {
 };
 
 // TODO(huisu): tune these probs
-const vpx_prob vp10_default_palette_y_size_prob[10][PALETTE_SIZES - 1] = {
+const vpx_prob
+vp10_default_palette_y_size_prob[PALETTE_BLOCK_SIZES][PALETTE_SIZES - 1] = {
     {  96,  89, 100,  64,  77, 130},
     {  22,  15,  44,  16,  34,  82},
     {  30,  19,  57,  18,  38,  86},
@@ -298,7 +299,8 @@ const vpx_prob vp10_default_palette_y_size_prob[10][PALETTE_SIZES - 1] = {
     {  98, 105, 142,  63,  64, 152},
 };
 
-const vpx_prob vp10_default_palette_uv_size_prob[10][PALETTE_SIZES - 1] = {
+const vpx_prob
+vp10_default_palette_uv_size_prob[PALETTE_BLOCK_SIZES][PALETTE_SIZES - 1] = {
     { 160, 196, 228, 213, 175, 230},
     {  87, 148, 208, 141, 166, 163},
     {  72, 151, 204, 139, 155, 161},
@@ -311,7 +313,9 @@ const vpx_prob vp10_default_palette_uv_size_prob[10][PALETTE_SIZES - 1] = {
     {  72,  55,  66,  68,  79, 107},
 };
 
-const vpx_prob vp10_default_palette_y_mode_prob[10][3] = {
+const vpx_prob
+vp10_default_palette_y_mode_prob[PALETTE_BLOCK_SIZES][PALETTE_Y_MODE_CONTEXTS]
+                                                      = {
     { 240,  180,  100, },
     { 240,  180,  100, },
     { 240,  180,  100, },
diff --git a/vp10/encoder/bitstream.c b/vp10/encoder/bitstream.c
index ca9b17b80..8ecdb955c 100644
--- a/vp10/encoder/bitstream.c
+++ b/vp10/encoder/bitstream.c
@@ -48,7 +48,7 @@ static const struct vp10_token palette_size_encodings[] = {
     {0, 1}, {2, 2}, {6, 3}, {14, 4}, {30, 5}, {62, 6}, {63, 6},
 };
 static const struct vp10_token
-palette_color_encodings[PALETTE_MAX_SIZE - 1][8] = {
+palette_color_encodings[PALETTE_MAX_SIZE - 1][PALETTE_MAX_SIZE] = {
     {{0, 1}, {1, 1}},  // 2 colors
     {{0, 1}, {2, 2}, {3, 2}},  // 3 colors
     {{0, 1}, {2, 2}, {6, 3}, {7, 3}},  // 4 colors
diff --git a/vp10/encoder/encoder.h b/vp10/encoder/encoder.h
index a957b6382..b59741624 100644
--- a/vp10/encoder/encoder.h
+++ b/vp10/encoder/encoder.h
@@ -458,8 +458,8 @@ typedef struct VP10_COMP {
   int y_mode_costs[INTRA_MODES][INTRA_MODES][INTRA_MODES];
   int switchable_interp_costs[SWITCHABLE_FILTER_CONTEXTS][SWITCHABLE_FILTERS];
   int partition_cost[PARTITION_CONTEXTS][PARTITION_TYPES];
-  int palette_y_size_cost[10][PALETTE_SIZES];
-  int palette_uv_size_cost[10][PALETTE_SIZES];
+  int palette_y_size_cost[PALETTE_BLOCK_SIZES][PALETTE_SIZES];
+  int palette_uv_size_cost[PALETTE_BLOCK_SIZES][PALETTE_SIZES];
   int palette_y_color_cost[PALETTE_MAX_SIZE - 1][PALETTE_COLOR_CONTEXTS]
                                                  [PALETTE_COLORS];
   int palette_uv_color_cost[PALETTE_MAX_SIZE - 1][PALETTE_COLOR_CONTEXTS]
diff --git a/vp10/encoder/palette.c b/vp10/encoder/palette.c
index 04e911884..b6fd6533e 100644
--- a/vp10/encoder/palette.c
+++ b/vp10/encoder/palette.c
@@ -8,14 +8,15 @@
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
+#include <math.h>
 #include "vp10/encoder/palette.h"
 
 static double calc_dist(const double *p1, const double *p2, int dim) {
   double dist = 0;
   int i = 0;
 
-  for (i = 0; i < dim; i++) {
-    dist = dist + (p1[i] - p2[i]) * (p1[i] - p2[i]);
+  for (i = 0; i < dim; ++i) {
+    dist = dist + (p1[i] - round(p2[i])) * (p1[i] - round(p2[i]));
   }
   return dist;
 }
@@ -25,10 +26,10 @@ void vp10_calc_indices(const double *data, const double *centroids,
   int i, j;
   double min_dist, this_dist;
 
-  for (i = 0; i < n; i++) {
+  for (i = 0; i < n; ++i) {
     min_dist = calc_dist(data + i * dim, centroids, dim);
     indices[i] = 0;
-    for (j = 1; j < k; j++) {
+    for (j = 1; j < k; ++j) {
       this_dist = calc_dist(data + i * dim, centroids + j * dim, dim);
       if (this_dist < min_dist) {
         min_dist = this_dist;
@@ -47,23 +48,23 @@ static void calc_centroids(const double *data, double *centroids,
   memset(count, 0, sizeof(count[0]) * k);
   memset(centroids, 0, sizeof(centroids[0]) * k * dim);
 
-  for (i = 0; i < n; i++) {
+  for (i = 0; i < n; ++i) {
     index = indices[i];
     assert(index < k);
-    count[index]++;
-    for (j = 0; j < dim; j++) {
+    ++count[index];
+    for (j = 0; j < dim; ++j) {
       centroids[index * dim + j] += data[i * dim + j];
     }
   }
 
-  for (i = 0; i < k; i++) {
+  for (i = 0; i < k; ++i) {
     if (count[i] == 0) {
       // TODO(huisu): replace rand() with something else.
       memcpy(centroids + i * dim, data + (rand() % n) * dim,
                  sizeof(centroids[0]) * dim);
     } else {
       const double norm = 1.0 / count[i];
-      for (j = 0; j < dim; j++)
+      for (j = 0; j < dim; ++j)
         centroids[i * dim + j] *= norm;
     }
   }
@@ -75,7 +76,7 @@ static double calc_total_dist(const double *data, const double *centroids,
   int i;
   (void) k;
 
-  for (i = 0; i < n; i++)
+  for (i = 0; i < n; ++i)
     dist += calc_dist(data + i * dim, centroids + indices[i] * dim, dim);
 
   return dist;
@@ -107,7 +108,7 @@ int vp10_k_means(const double *data, double *centroids, uint8_t *indices,
     memcpy(pre_centroids, centroids, sizeof(pre_centroids[0]) * k * dim);
     memcpy(pre_indices, indices, sizeof(pre_indices[0]) * n);
     pre_dist = this_dist;
-    i++;
+    ++i;
   }
 
   return i;
@@ -169,7 +170,7 @@ int vp10_count_colors_highbd(const uint8_t *src8, int stride, int rows,
   for (r = 0; r < rows; ++r) {
     for (c = 0; c < cols; ++c) {
       val = src[r * stride + c];
-      val_count[val]++;
+      ++val_count[val];
     }
   }
 
diff --git a/vp10/encoder/rd.c b/vp10/encoder/rd.c
index 806b8542b..ee02bfc80 100644
--- a/vp10/encoder/rd.c
+++ b/vp10/encoder/rd.c
@@ -85,7 +85,7 @@ static void fill_mode_costs(VP10_COMP *cpi) {
     vp10_cost_tokens(cpi->switchable_interp_costs[i],
                     fc->switchable_interp_prob[i], vp10_switchable_interp_tree);
 
-  for (i = 0; i < 10; ++i) {
+  for (i = 0; i < PALETTE_BLOCK_SIZES; ++i) {
     vp10_cost_tokens(cpi->palette_y_size_cost[i],
                      vp10_default_palette_y_size_prob[i],
                      vp10_palette_size_tree);
diff --git a/vp10/encoder/rdopt.c b/vp10/encoder/rdopt.c
index c9ad94557..76e41ef16 100644
--- a/vp10/encoder/rdopt.c
+++ b/vp10/encoder/rdopt.c
@@ -821,7 +821,8 @@ void rd_pick_palette_intra_sby(VP10_COMP *cpi, MACROBLOCK *x, BLOCK_SIZE bsize,
       vp10_k_means(data, centroids, indices, pre_indices, rows * cols,
                    n, 1, max_itr);
       vp10_insertion_sort(centroids, n);
-
+      for (i = 0; i < n; ++i)
+        centroids[i] = round(centroids[i]);
       // remove duplicates
       i = 1;
       k = n;