[3.7] gh-98517: Fix buffer overflows in _sha3 module (GH-98519) (GH-98528)

This is a port of the applicable part of XKCP's fix [1] for
CVE-2022-37454 and avoids the segmentation fault and the infinite
loop in the test cases published in [2].

[1]: https://github.com/XKCP/XKCP/commit/fdc6fef075f4e81d6b1bc38364248975e08e340a
[2]: https://mouha.be/sha-3-buffer-overflow/

Regression test added by: Gregory P. Smith [Google LLC] <greg@krypto.org>
(cherry picked from commit 0e4e058602d93b88256ff90bbef501ba20be9dd3)

Co-authored-by: Theo Buehler <botovq@users.noreply.github.com>

1 files changed, 8 insertions, 7 deletions

diff --git a/Modules/_sha3/kcp/KeccakSponge.inc b/Modules/_sha3/kcp/KeccakSponge.inc
index e10739deaf..cf92e4db4d 100644
--- a/Modules/_sha3/kcp/KeccakSponge.inc
+++ b/Modules/_sha3/kcp/KeccakSponge.inc
@@ -171,7 +171,7 @@ int SpongeAbsorb(SpongeInstance *instance, const unsigned char *data, size_t dat
     i = 0;
     curData = data;
     while(i < dataByteLen) {
-        if ((instance->byteIOIndex == 0) && (dataByteLen >= (i + rateInBytes))) {
+        if ((instance->byteIOIndex == 0) && (dataByteLen-i >= rateInBytes)) {
 #ifdef SnP_FastLoop_Absorb
             /* processing full blocks first */
 
@@ -199,10 +199,10 @@ int SpongeAbsorb(SpongeInstance *instance, const unsigned char *data, size_t dat
         }
         else {
             /* normal lane: using the message queue */
-
-            partialBlock = (unsigned int)(dataByteLen - i);
-            if (partialBlock+instance->byteIOIndex > rateInBytes)
+            if (dataByteLen-i > rateInBytes-instance->byteIOIndex)
                 partialBlock = rateInBytes-instance->byteIOIndex;
+            else
+                partialBlock = (unsigned int)(dataByteLen - i);
             #ifdef KeccakReference
             displayBytes(1, "Block to be absorbed (part)", curData, partialBlock);
             #endif
@@ -281,7 +281,7 @@ int SpongeSqueeze(SpongeInstance *instance, unsigned char *data, size_t dataByte
     i = 0;
     curData = data;
     while(i < dataByteLen) {
-        if ((instance->byteIOIndex == rateInBytes) && (dataByteLen >= (i + rateInBytes))) {
+        if ((instance->byteIOIndex == rateInBytes) && (dataByteLen-i >= rateInBytes)) {
             for(j=dataByteLen-i; j>=rateInBytes; j-=rateInBytes) {
                 SnP_Permute(instance->state);
                 SnP_ExtractBytes(instance->state, curData, 0, rateInBytes);
@@ -299,9 +299,10 @@ int SpongeSqueeze(SpongeInstance *instance, unsigned char *data, size_t dataByte
                 SnP_Permute(instance->state);
                 instance->byteIOIndex = 0;
             }
-            partialBlock = (unsigned int)(dataByteLen - i);
-            if (partialBlock+instance->byteIOIndex > rateInBytes)
+            if (dataByteLen-i > rateInBytes-instance->byteIOIndex)
                 partialBlock = rateInBytes-instance->byteIOIndex;
+            else
+                partialBlock = (unsigned int)(dataByteLen - i);
             i += partialBlock;
 
             SnP_ExtractBytes(instance->state, curData, instance->byteIOIndex, partialBlock);