From 5ed5e90f2ae299cbec66996860d794771a85fee8 Mon Sep 17 00:00:00 2001 From: Nicolas George Date: Tue, 1 Jan 2013 17:06:04 +0100 Subject: lavc/dvdsubenc: improve color distance function. Consider the color space as an hypercone with apex alpha=0 and base alpha=1 instead of an hypercube. Make the encoder consider very transparent colors more similar even if the hue is very different. This corresponds roughly to using the alpha as a weight for the color difference. Only 4 bits of alpha are used, because this is what dvdsub uses, and it avoids overflows. Fix trac ticket #2005. --- libavcodec/dvdsubenc.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'libavcodec/dvdsubenc.c') diff --git a/libavcodec/dvdsubenc.c b/libavcodec/dvdsubenc.c index bf54aa2863..cd041c3f24 100644 --- a/libavcodec/dvdsubenc.c +++ b/libavcodec/dvdsubenc.c @@ -94,10 +94,14 @@ static void dvd_encode_rle(uint8_t **pq, static int color_distance(uint32_t a, uint32_t b) { int r = 0, d, i; + int alpha_a = 8, alpha_b = 8; - for (i = 0; i < 32; i += 8) { - d = ((a >> i) & 0xFF) - ((b >> i) & 0xFF); + for (i = 24; i >= 0; i -= 8) { + d = alpha_a * (int)((a >> i) & 0xFF) - + alpha_b * (int)((b >> i) & 0xFF); r += d * d; + alpha_a = a >> 28; + alpha_b = b >> 28; } return r; } @@ -130,7 +134,8 @@ static void count_colors(AVCodecContext *avctx, unsigned hits[33], if (match) { best_d = INT_MAX; for (j = 0; j < 16; j++) { - d = color_distance(color & 0xFFFFFF, dvdc->global_palette[j]); + d = color_distance(0xFF000000 | color, + 0xFF000000 | dvdc->global_palette[j]); if (d < best_d) { best_d = d; best_j = j; -- cgit v1.2.1