* libtiff/tiffiop.h: add TIFF_NOSANITIZE_UNSIGNED_INT_OVERFLOW macro to

disable CLang warnings raised by -fsanitize=undefined,unsigned-integer-overflow
* libtiff/tif_predict.c: decorate legitimate functions where unsigned int
overflow occur with TIFF_NOSANITIZE_UNSIGNED_INT_OVERFLOW
* libtiff/tif_dirread.c: avoid unsigned int overflow in EstimateStripByteCounts()
and BYTECOUNTLOOKSBAD when file is too short.
* libtiff/tif_jpeg.c: avoid (harmless) unsigned int overflow on tiled images.
* libtiff/tif_fax3.c: avoid unsigned int overflow in Fax3Encode2DRow(). Could
potentially be a bug with huge rows.
* libtiff/tif_getimage.c: avoid many (harmless) unsigned int overflows.

7 files changed, 87 insertions, 51 deletions

diff --git a/ChangeLog b/ChangeLog
index 977bef28..39fb10ec 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,16 @@
+2017-06-18  Even Rouault <even.rouault at spatialys.com>
+
+	* libtiff/tiffiop.h: add TIFF_NOSANITIZE_UNSIGNED_INT_OVERFLOW macro to
+	disable CLang warnings raised by -fsanitize=undefined,unsigned-integer-overflow
+	* libtiff/tif_predict.c: decorate legitimate functions where unsigned int
+	overflow occur with TIFF_NOSANITIZE_UNSIGNED_INT_OVERFLOW
+	* libtiff/tif_dirread.c: avoid unsigned int overflow in EstimateStripByteCounts()
+	and BYTECOUNTLOOKSBAD when file is too short.
+	* libtiff/tif_jpeg.c: avoid (harmless) unsigned int overflow on tiled images.
+	* libtiff/tif_fax3.c: avoid unsigned int overflow in Fax3Encode2DRow(). Could
+	potentially be a bug with huge rows.
+	* libtiff/tif_getimage.c: avoid many (harmless) unsigned int overflows.
+
 2017-06-12  Even Rouault <even.rouault at spatialys.com>
 
 	* libtiff/tif_dirread.c: TIFFFetchStripThing(): limit the number of items
diff --git a/libtiff/tif_dirread.c b/libtiff/tif_dirread.c
index bf8cef22..2e2cdccc 100644
--- a/libtiff/tif_dirread.c
+++ b/libtiff/tif_dirread.c
@@ -1,4 +1,4 @@
-/* $Id: tif_dirread.c,v 1.211 2017-06-12 19:13:49 erouault Exp $ */
+/* $Id: tif_dirread.c,v 1.212 2017-06-18 10:31:50 erouault Exp $ */
 
 /*
  * Copyright (c) 1988-1997 Sam Leffler
@@ -4002,7 +4002,8 @@ TIFFReadDirectory(TIFF* tif)
 		#define	BYTECOUNTLOOKSBAD \
 		    ( (tif->tif_dir.td_stripbytecount[0] == 0 && tif->tif_dir.td_stripoffset[0] != 0) || \
 		      (tif->tif_dir.td_compression == COMPRESSION_NONE && \
-		       tif->tif_dir.td_stripbytecount[0] > TIFFGetFileSize(tif) - tif->tif_dir.td_stripoffset[0]) || \
+		       (tif->tif_dir.td_stripoffset[0] <= TIFFGetFileSize(tif) && \
+		        tif->tif_dir.td_stripbytecount[0] > TIFFGetFileSize(tif) - tif->tif_dir.td_stripoffset[0])) || \
 		      (tif->tif_mode == O_RDONLY && \
 		       tif->tif_dir.td_compression == COMPRESSION_NONE && \
 		       tif->tif_dir.td_stripbytecount[0] < TIFFScanlineSize64(tif) * tif->tif_dir.td_imagelength) )
@@ -4386,7 +4387,11 @@ EstimateStripByteCounts(TIFF* tif, TIFFDirEntry* dir, uint16 dircount)
 			}
 			space+=datasize;
 		}
-		space = filesize - space;
+		if( filesize < space )
+                    /* we should perhaps return in error ? */
+                    space = filesize;
+                else
+                    space = filesize - space;
 		if (td->td_planarconfig == PLANARCONFIG_SEPARATE)
 			space /= td->td_samplesperpixel;
 		for (strip = 0; strip < td->td_nstrips; strip++)
diff --git a/libtiff/tif_fax3.c b/libtiff/tif_fax3.c
index 087ceddd..5fd51411 100644
--- a/libtiff/tif_fax3.c
+++ b/libtiff/tif_fax3.c
@@ -1,4 +1,4 @@
-/* $Id: tif_fax3.c,v 1.80 2017-04-27 19:50:01 erouault Exp $ */
+/* $Id: tif_fax3.c,v 1.81 2017-06-18 10:31:50 erouault Exp $ */
 
 /*
  * Copyright (c) 1990-1997 Sam Leffler
@@ -1043,7 +1043,11 @@ Fax3Encode2DRow(TIFF* tif, unsigned char* bp, unsigned char* rp, uint32 bits)
 	for (;;) {
 		b2 = finddiff2(rp, b1, bits, PIXEL(rp,b1));
 		if (b2 >= a1) {
-			int32 d = b1 - a1;
+			/* Naive computation triggers -fsanitize=undefined,unsigned-integer-overflow */
+			/* although it is correct unless the difference between both is < 31 bit */
+			/* int32 d = b1 - a1; */
+			int32 d = (b1 >= a1 && b1 - a1 <= 3U) ? (int32)(b1 - a1):
+			          (b1 < a1 && a1 - b1 <= 3U) ? -(int32)(a1 - b1) : 0x7FFFFFFF;
 			if (!(-3 <= d && d <= 3)) {	/* horizontal mode */
 				a2 = finddiff2(bp, a1, bits, PIXEL(bp,a1));
 				putcode(tif, &horizcode);
diff --git a/libtiff/tif_getimage.c b/libtiff/tif_getimage.c
index d44a93c1..a592daa7 100644
--- a/libtiff/tif_getimage.c
+++ b/libtiff/tif_getimage.c
@@ -1,4 +1,4 @@
-/* $Id: tif_getimage.c,v 1.107 2017-05-29 11:29:06 erouault Exp $ */
+/* $Id: tif_getimage.c,v 1.108 2017-06-18 10:31:50 erouault Exp $ */
 
 /*
  * Copyright (c) 1991-1997 Sam Leffler
@@ -1212,8 +1212,8 @@ DECLAREContigPutFunc(put8bitcmaptile)
     int samplesperpixel = img->samplesperpixel;
 
     (void) y;
-    while (h-- > 0) {
-	for (x = w; x-- > 0;)
+    for( ; h > 0; --h) {
+	for (x = w; x > 0; --x)
         {
 	    *cp++ = PALmap[*pp][0];
             pp += samplesperpixel;
@@ -1232,7 +1232,7 @@ DECLAREContigPutFunc(put4bitcmaptile)
 
     (void) x; (void) y;
     fromskew /= 2;
-    while (h-- > 0) {
+    for( ; h > 0; --h) {
 	uint32* bw;
 	UNROLL2(w, bw = PALmap[*pp++], *cp++ = *bw++);
 	cp += toskew;
@@ -1249,7 +1249,7 @@ DECLAREContigPutFunc(put2bitcmaptile)
 
     (void) x; (void) y;
     fromskew /= 4;
-    while (h-- > 0) {
+    for( ; h > 0; --h) {
 	uint32* bw;
 	UNROLL4(w, bw = PALmap[*pp++], *cp++ = *bw++);
 	cp += toskew;
@@ -1266,7 +1266,7 @@ DECLAREContigPutFunc(put1bitcmaptile)
 
     (void) x; (void) y;
     fromskew /= 8;
-    while (h-- > 0) {
+    for( ; h > 0; --h) {
 	uint32* bw;
 	UNROLL8(w, bw = PALmap[*pp++], *cp++ = *bw++);
 	cp += toskew;
@@ -1283,8 +1283,8 @@ DECLAREContigPutFunc(putgreytile)
     uint32** BWmap = img->BWmap;
 
     (void) y;
-    while (h-- > 0) {
-	for (x = w; x-- > 0;)
+    for( ; h > 0; --h) {
+	for (x = w; x > 0; --x)
         {
 	    *cp++ = BWmap[*pp][0];
             pp += samplesperpixel;
@@ -1303,8 +1303,8 @@ DECLAREContigPutFunc(putagreytile)
     uint32** BWmap = img->BWmap;
 
     (void) y;
-    while (h-- > 0) {
-	for (x = w; x-- > 0;)
+    for( ; h > 0; --h) {
+	for (x = w; x > 0; --x)
         {
             *cp++ = BWmap[*pp][0] & ((uint32)*(pp+1) << 24 | ~A1);
             pp += samplesperpixel;
@@ -1323,10 +1323,10 @@ DECLAREContigPutFunc(put16bitbwtile)
     uint32** BWmap = img->BWmap;
 
     (void) y;
-    while (h-- > 0) {
+    for( ; h > 0; --h) {
         uint16 *wp = (uint16 *) pp;
 
-	for (x = w; x-- > 0;)
+	for (x = w; x > 0; --x)
         {
             /* use high order byte of 16bit value */
 
@@ -1348,7 +1348,7 @@ DECLAREContigPutFunc(put1bitbwtile)
 
     (void) x; (void) y;
     fromskew /= 8;
-    while (h-- > 0) {
+    for( ; h > 0; --h) {
 	uint32* bw;
 	UNROLL8(w, bw = BWmap[*pp++], *cp++ = *bw++);
 	cp += toskew;
@@ -1365,7 +1365,7 @@ DECLAREContigPutFunc(put2bitbwtile)
 
     (void) x; (void) y;
     fromskew /= 4;
-    while (h-- > 0) {
+    for( ; h > 0; --h) {
 	uint32* bw;
 	UNROLL4(w, bw = BWmap[*pp++], *cp++ = *bw++);
 	cp += toskew;
@@ -1382,7 +1382,7 @@ DECLAREContigPutFunc(put4bitbwtile)
 
     (void) x; (void) y;
     fromskew /= 2;
-    while (h-- > 0) {
+    for( ; h > 0; --h) {
 	uint32* bw;
 	UNROLL2(w, bw = BWmap[*pp++], *cp++ = *bw++);
 	cp += toskew;
@@ -1399,7 +1399,7 @@ DECLAREContigPutFunc(putRGBcontig8bittile)
 
     (void) x; (void) y;
     fromskew *= samplesperpixel;
-    while (h-- > 0) {
+    for( ; h > 0; --h) {
 	UNROLL8(w, NOP,
 	    *cp++ = PACK(pp[0], pp[1], pp[2]);
 	    pp += samplesperpixel);
@@ -1418,7 +1418,7 @@ DECLAREContigPutFunc(putRGBAAcontig8bittile)
 
     (void) x; (void) y;
     fromskew *= samplesperpixel;
-    while (h-- > 0) {
+    for( ; h > 0; --h) {
 	UNROLL8(w, NOP,
 	    *cp++ = PACK4(pp[0], pp[1], pp[2], pp[3]);
 	    pp += samplesperpixel);
@@ -1436,10 +1436,10 @@ DECLAREContigPutFunc(putRGBUAcontig8bittile)
 	int samplesperpixel = img->samplesperpixel;
 	(void) y;
 	fromskew *= samplesperpixel;
-	while (h-- > 0) {
+	for( ; h > 0; --h) {
 		uint32 r, g, b, a;
 		uint8* m;
-		for (x = w; x-- > 0;) {
+		for (x = w; x > 0; --x) {
 			a = pp[3];
 			m = img->UaToAa+((size_t) a<<8);
 			r = m[pp[0]];
@@ -1462,8 +1462,8 @@ DECLAREContigPutFunc(putRGBcontig16bittile)
 	uint16 *wp = (uint16 *)pp;
 	(void) y;
 	fromskew *= samplesperpixel;
-	while (h-- > 0) {
-		for (x = w; x-- > 0;) {
+	for( ; h > 0; --h) {
+		for (x = w; x > 0; --x) {
 			*cp++ = PACK(img->Bitdepth16To8[wp[0]],
 			    img->Bitdepth16To8[wp[1]],
 			    img->Bitdepth16To8[wp[2]]);
@@ -1484,8 +1484,8 @@ DECLAREContigPutFunc(putRGBAAcontig16bittile)
 	uint16 *wp = (uint16 *)pp;
 	(void) y;
 	fromskew *= samplesperpixel;
-	while (h-- > 0) {
-		for (x = w; x-- > 0;) {
+	for( ; h > 0; --h) {
+		for (x = w; x > 0; --x) {
 			*cp++ = PACK4(img->Bitdepth16To8[wp[0]],
 			    img->Bitdepth16To8[wp[1]],
 			    img->Bitdepth16To8[wp[2]],
@@ -1507,10 +1507,10 @@ DECLAREContigPutFunc(putRGBUAcontig16bittile)
 	uint16 *wp = (uint16 *)pp;
 	(void) y;
 	fromskew *= samplesperpixel;
-	while (h-- > 0) {
+	for( ; h > 0; --h) {
 		uint32 r,g,b,a;
 		uint8* m;
-		for (x = w; x-- > 0;) {
+		for (x = w; x > 0; --x) {
 			a = img->Bitdepth16To8[wp[3]];
 			m = img->UaToAa+((size_t) a<<8);
 			r = m[img->Bitdepth16To8[wp[0]]];
@@ -1536,7 +1536,7 @@ DECLAREContigPutFunc(putRGBcontig8bitCMYKtile)
 
     (void) x; (void) y;
     fromskew *= samplesperpixel;
-    while (h-- > 0) {
+    for( ; h > 0; --h) {
 	UNROLL8(w, NOP,
 	    k = 255 - pp[3];
 	    r = (k*(255-pp[0]))/255;
@@ -1562,8 +1562,8 @@ DECLAREContigPutFunc(putRGBcontig8bitCMYKMaptile)
 
     (void) y;
     fromskew *= samplesperpixel;
-    while (h-- > 0) {
-	for (x = w; x-- > 0;) {
+    for( ; h > 0; --h) {
+	for (x = w; x > 0; --x) {
 	    k = 255 - pp[3];
 	    r = (k*(255-pp[0]))/255;
 	    g = (k*(255-pp[1]))/255;
@@ -1592,7 +1592,7 @@ static void name(\
 DECLARESepPutFunc(putRGBseparate8bittile)
 {
     (void) img; (void) x; (void) y; (void) a;
-    while (h-- > 0) {
+    for( ; h > 0; --h) {
 	UNROLL8(w, NOP, *cp++ = PACK(*r++, *g++, *b++));
 	SKEW(r, g, b, fromskew);
 	cp += toskew;
@@ -1605,7 +1605,7 @@ DECLARESepPutFunc(putRGBseparate8bittile)
 DECLARESepPutFunc(putRGBAAseparate8bittile)
 {
 	(void) img; (void) x; (void) y; 
-	while (h-- > 0) {
+	for( ; h > 0; --h) {
 		UNROLL8(w, NOP, *cp++ = PACK4(*r++, *g++, *b++, *a++));
 		SKEW4(r, g, b, a, fromskew);
 		cp += toskew;
@@ -1618,9 +1618,9 @@ DECLARESepPutFunc(putRGBAAseparate8bittile)
 DECLARESepPutFunc(putCMYKseparate8bittile)
 {
 	(void) img; (void) y;
-	while (h-- > 0) {
+	for( ; h > 0; --h) {
 		uint32 rv, gv, bv, kv;
-		for (x = w; x-- > 0;) {
+		for (x = w; x > 0; --x) {
 			kv = 255 - *a++;
 			rv = (kv*(255-*r++))/255;
 			gv = (kv*(255-*g++))/255;
@@ -1638,10 +1638,10 @@ DECLARESepPutFunc(putCMYKseparate8bittile)
 DECLARESepPutFunc(putRGBUAseparate8bittile)
 {
 	(void) img; (void) y;
-	while (h-- > 0) {
+	for( ; h > 0; --h) {
 		uint32 rv, gv, bv, av;
 		uint8* m;
-		for (x = w; x-- > 0;) {
+		for (x = w; x > 0; --x) {
 			av = *a++;
 			m = img->UaToAa+((size_t) av<<8);
 			rv = m[*r++];
@@ -1663,7 +1663,7 @@ DECLARESepPutFunc(putRGBseparate16bittile)
 	uint16 *wg = (uint16*) g;
 	uint16 *wb = (uint16*) b;
 	(void) img; (void) y; (void) a;
-	while (h-- > 0) {
+	for( ; h > 0; --h) {
 		for (x = 0; x < w; x++)
 			*cp++ = PACK(img->Bitdepth16To8[*wr++],
 			    img->Bitdepth16To8[*wg++],
@@ -1683,7 +1683,7 @@ DECLARESepPutFunc(putRGBAAseparate16bittile)
 	uint16 *wb = (uint16*) b;
 	uint16 *wa = (uint16*) a;
 	(void) img; (void) y;
-	while (h-- > 0) {
+	for( ; h > 0; --h) {
 		for (x = 0; x < w; x++)
 			*cp++ = PACK4(img->Bitdepth16To8[*wr++],
 			    img->Bitdepth16To8[*wg++],
@@ -1704,10 +1704,10 @@ DECLARESepPutFunc(putRGBUAseparate16bittile)
 	uint16 *wb = (uint16*) b;
 	uint16 *wa = (uint16*) a;
 	(void) img; (void) y;
-	while (h-- > 0) {
+	for( ; h > 0; --h) {
 		uint32 r2,g2,b2,a2;
 		uint8* m;
-		for (x = w; x-- > 0;) {
+		for (x = w; x > 0; --x) {
 			a2 = img->Bitdepth16To8[*wa++];
 			m = img->UaToAa+((size_t) a2<<8);
 			r2 = m[img->Bitdepth16To8[*wr++]];
@@ -1729,8 +1729,8 @@ DECLAREContigPutFunc(putcontig8bitCIELab)
 	uint32 r, g, b;
 	(void) y;
 	fromskew *= 3;
-	while (h-- > 0) {
-		for (x = w; x-- > 0;) {
+	for( ; h > 0; --h) {
+		for (x = w; x > 0; --x) {
 			TIFFCIELabToXYZ(img->cielab,
 					(unsigned char)pp[0],
 					(signed char)pp[1],
@@ -2226,7 +2226,7 @@ DECLARESepPutFunc(putseparate8bitYCbCr11tile)
 	(void) y;
 	(void) a;
 	/* TODO: naming of input vars is still off, change obfuscating declaration inside define, or resolve obfuscation */
-	while (h-- > 0) {
+	for( ; h > 0; --h) {
 		x = w;
 		do {
 			uint32 dr, dg, db;
diff --git a/libtiff/tif_jpeg.c b/libtiff/tif_jpeg.c
index 1553e86b..aaecf1ee 100644
--- a/libtiff/tif_jpeg.c
+++ b/libtiff/tif_jpeg.c
@@ -1,4 +1,4 @@
-/* $Id: tif_jpeg.c,v 1.127 2017-01-31 13:02:27 erouault Exp $ */
+/* $Id: tif_jpeg.c,v 1.128 2017-06-18 10:31:50 erouault Exp $ */
 
 /*
  * Copyright (c) 1994-1997 Sam Leffler
@@ -1043,13 +1043,13 @@ JPEGPreDecode(TIFF* tif, uint16 s)
 	/*
 	 * Check image parameters and set decompression parameters.
 	 */
-	segment_width = td->td_imagewidth;
-	segment_height = td->td_imagelength - tif->tif_row;
 	if (isTiled(tif)) {
                 segment_width = td->td_tilewidth;
                 segment_height = td->td_tilelength;
 		sp->bytesperline = TIFFTileRowSize(tif);
 	} else {
+		segment_width = td->td_imagewidth;
+		segment_height = td->td_imagelength - tif->tif_row;
 		if (segment_height > td->td_rowsperstrip)
 			segment_height = td->td_rowsperstrip;
 		sp->bytesperline = TIFFScanlineSize(tif);
diff --git a/libtiff/tif_predict.c b/libtiff/tif_predict.c
index 7a60a39e..9ae1f57a 100644
--- a/libtiff/tif_predict.c
+++ b/libtiff/tif_predict.c
@@ -1,4 +1,4 @@
-/* $Id: tif_predict.c,v 1.43 2017-05-10 15:21:16 erouault Exp $ */
+/* $Id: tif_predict.c,v 1.44 2017-06-18 10:31:50 erouault Exp $ */
 
 /*
  * Copyright (c) 1988-1997 Sam Leffler
@@ -277,6 +277,7 @@ PredictorSetupEncode(TIFF* tif)
 /* - when storing into the byte stream, we explicitly mask with 0xff so */
 /*   as to make icc -check=conversions happy (not necessary by the standard) */
 
+TIFF_NOSANITIZE_UNSIGNED_INT_OVERFLOW
 static int
 horAcc8(TIFF* tif, uint8* cp0, tmsize_t cc)
 {
@@ -344,6 +345,7 @@ swabHorAcc16(TIFF* tif, uint8* cp0, tmsize_t cc)
         return horAcc16(tif, cp0, cc);
 }
 
+TIFF_NOSANITIZE_UNSIGNED_INT_OVERFLOW
 static int
 horAcc16(TIFF* tif, uint8* cp0, tmsize_t cc)
 {
@@ -378,6 +380,7 @@ swabHorAcc32(TIFF* tif, uint8* cp0, tmsize_t cc)
 	return horAcc32(tif, cp0, cc);
 }
 
+TIFF_NOSANITIZE_UNSIGNED_INT_OVERFLOW
 static int
 horAcc32(TIFF* tif, uint8* cp0, tmsize_t cc)
 {
@@ -503,6 +506,7 @@ PredictorDecodeTile(TIFF* tif, uint8* op0, tmsize_t occ0, uint16 s)
 		return 0;
 }
 
+TIFF_NOSANITIZE_UNSIGNED_INT_OVERFLOW
 static int
 horDiff8(TIFF* tif, uint8* cp0, tmsize_t cc)
 {
@@ -556,6 +560,7 @@ horDiff8(TIFF* tif, uint8* cp0, tmsize_t cc)
 	return 1;
 }
 
+TIFF_NOSANITIZE_UNSIGNED_INT_OVERFLOW
 static int
 horDiff16(TIFF* tif, uint8* cp0, tmsize_t cc)
 {
@@ -595,6 +600,7 @@ swabHorDiff16(TIFF* tif, uint8* cp0, tmsize_t cc)
     return 1;
 }
 
+TIFF_NOSANITIZE_UNSIGNED_INT_OVERFLOW
 static int
 horDiff32(TIFF* tif, uint8* cp0, tmsize_t cc)
 {
@@ -637,6 +643,7 @@ swabHorDiff32(TIFF* tif, uint8* cp0, tmsize_t cc)
 /*
  * Floating point predictor differencing routine.
  */
+TIFF_NOSANITIZE_UNSIGNED_INT_OVERFLOW
 static int
 fpDiff(TIFF* tif, uint8* cp0, tmsize_t cc)
 {
diff --git a/libtiff/tiffiop.h b/libtiff/tiffiop.h
index 5294ee78..73591340 100644
--- a/libtiff/tiffiop.h
+++ b/libtiff/tiffiop.h
@@ -1,4 +1,4 @@
-/* $Id: tiffiop.h,v 1.90 2016-12-02 21:56:56 erouault Exp $ */
+/* $Id: tiffiop.h,v 1.91 2017-06-18 10:31:50 erouault Exp $ */
 
 /*
  * Copyright (c) 1988-1997 Sam Leffler
@@ -315,6 +315,13 @@ typedef size_t TIFFIOSize_t;
 #define _TIFF_off_t off_t
 #endif
 
+#if __clang_major__ >= 4 || (__clang_major__ == 3 && __clang_minor__ >= 8)
+#define TIFF_NOSANITIZE_UNSIGNED_INT_OVERFLOW __attribute__((no_sanitize("unsigned-integer-overflow")))
+#else
+#define TIFF_NOSANITIZE_UNSIGNED_INT_OVERFLOW
+#endif
+
+
 #if defined(__cplusplus)
 extern "C" {
 #endif