From b5cba5cff937e5336ff23380785da80cab09146c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20B=C3=ADlka?= Date: Sat, 11 Jul 2015 17:44:10 +0200 Subject: Handle overflow in __hcreate_r Hi, As in bugzilla entry there is overflow in hsearch when looking for prime number as SIZE_MAX - 1 is divisible by 5. We fix that by rejecting large inputs before looking for prime. * misc/hsearch_r.c (__hcreate_r): Handle overflow. (cherry picked from commit 2f5c1750558fe64bac361f52d6827ab1bcfe52bc) --- misc/hsearch_r.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'misc/hsearch_r.c') diff --git a/misc/hsearch_r.c b/misc/hsearch_r.c index 81c27d800c..4414a25cba 100644 --- a/misc/hsearch_r.c +++ b/misc/hsearch_r.c @@ -19,7 +19,7 @@ #include #include #include - +#include #include /* [Aho,Sethi,Ullman] Compilers: Principles, Techniques and Tools, 1986 @@ -73,6 +73,13 @@ hcreate_r (nel, htab) return 0; } + if (nel >= SIZE_MAX / sizeof (_ENTRY)) + { + __set_errno (ENOMEM); + return 0; + } + + /* There is still another table active. Return with error. */ if (htab->table != NULL) return 0; -- cgit v1.2.1 From 3c9e8d9477aba0f514171bb4706670052544479b Mon Sep 17 00:00:00 2001 From: Florian Weimer Date: Thu, 28 Jan 2016 13:59:11 +0100 Subject: Improve check against integer wraparound in hcreate_r [BZ #18240] (cherry picked from commit bae7c7c764413b23e61cb099ce33be4c4ee259bb) --- misc/hsearch_r.c | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) (limited to 'misc/hsearch_r.c') diff --git a/misc/hsearch_r.c b/misc/hsearch_r.c index 4414a25cba..af5521234c 100644 --- a/misc/hsearch_r.c +++ b/misc/hsearch_r.c @@ -46,15 +46,12 @@ static int isprime (unsigned int number) { /* no even number will be passed */ - unsigned int div = 3; - - while (div * div < number && number % div != 0) - div += 2; - - return number % div != 0; + for (unsigned int div = 3; div <= number / div; div += 2) + if (number % div == 0) + return 0; + return 1; } - /* Before using the hash table we must allocate memory for it. Test for an existing table are done. We allocate one element more as the found prime number says. This is done for more effective @@ -73,13 +70,6 @@ hcreate_r (nel, htab) return 0; } - if (nel >= SIZE_MAX / sizeof (_ENTRY)) - { - __set_errno (ENOMEM); - return 0; - } - - /* There is still another table active. Return with error. */ if (htab->table != NULL) return 0; @@ -88,10 +78,19 @@ hcreate_r (nel, htab) use will not work. */ if (nel < 3) nel = 3; - /* Change nel to the first prime number not smaller as nel. */ - nel |= 1; /* make odd */ - while (!isprime (nel)) - nel += 2; + + /* Change nel to the first prime number in the range [nel, UINT_MAX - 2], + The '- 2' means 'nel += 2' cannot overflow. */ + for (nel |= 1; ; nel += 2) + { + if (UINT_MAX - 2 < nel) + { + __set_errno (ENOMEM); + return 0; + } + if (isprime (nel)) + break; + } htab->size = nel; htab->filled = 0; -- cgit v1.2.1