cmd/compile: fix range analysis of small signed integers

For int8, int16, and int32, comparing their unsigned value to MaxInt64
to determine non-negativity doesn't make sense, because they have
negative values whose unsigned representation is smaller than that.
Fix is simply to compare with the appropriate upper bound based on the
value type's size.

Fixes #32560.

Change-Id: Ie7afad7a56af92bd890ba5ff33c86d1df06cfd9a
Reviewed-on: https://go-review.googlesource.com/c/go/+/181797
Run-TryBot: Matthew Dempsky <mdempsky@google.com>
Reviewed-by: Josh Bleecher Snyder <josharian@gmail.com>
Reviewed-by: David Chase <drchase@google.com>
Reviewed-by: Keith Randall <khr@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>

1 files changed, 16 insertions, 2 deletions

diff --git a/src/cmd/compile/internal/ssa/prove.go b/src/cmd/compile/internal/ssa/prove.go
index a73cd613f2..7c69327990 100644
--- a/src/cmd/compile/internal/ssa/prove.go
+++ b/src/cmd/compile/internal/ssa/prove.go
@@ -553,15 +553,29 @@ func (ft *factsTable) isNonNegative(v *Value) bool {
 		return true
 	}
 
+	var max int64
+	switch v.Type.Size() {
+	case 1:
+		max = math.MaxInt8
+	case 2:
+		max = math.MaxInt16
+	case 4:
+		max = math.MaxInt32
+	case 8:
+		max = math.MaxInt64
+	default:
+		panic("unexpected integer size")
+	}
+
 	// Check if the recorded limits can prove that the value is positive
-	if l, has := ft.limits[v.ID]; has && (l.min >= 0 || l.umax <= math.MaxInt64) {
+	if l, has := ft.limits[v.ID]; has && (l.min >= 0 || l.umax <= uint64(max)) {
 		return true
 	}
 
 	// Check if v = x+delta, and we can use x's limits to prove that it's positive
 	if x, delta := isConstDelta(v); x != nil {
 		if l, has := ft.limits[x.ID]; has {
-			if delta > 0 && l.min >= -delta && l.max <= math.MaxInt64-delta {
+			if delta > 0 && l.min >= -delta && l.max <= max-delta {
 				return true
 			}
 			if delta < 0 && l.min >= -delta {