summaryrefslogtreecommitdiff
path: root/libgo/go/unicode
diff options
context:
space:
mode:
Diffstat (limited to 'libgo/go/unicode')
-rw-r--r--libgo/go/unicode/utf16/utf16.go101
-rw-r--r--libgo/go/unicode/utf16/utf16_test.go91
-rw-r--r--libgo/go/unicode/utf8/string.go199
-rw-r--r--libgo/go/unicode/utf8/string_test.go114
-rw-r--r--libgo/go/unicode/utf8/utf8.go391
-rw-r--r--libgo/go/unicode/utf8/utf8_test.go344
6 files changed, 1240 insertions, 0 deletions
diff --git a/libgo/go/unicode/utf16/utf16.go b/libgo/go/unicode/utf16/utf16.go
new file mode 100644
index 00000000000..2b2eb28f2df
--- /dev/null
+++ b/libgo/go/unicode/utf16/utf16.go
@@ -0,0 +1,101 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package utf16 implements encoding and decoding of UTF-16 sequences.
+package utf16
+
+import "unicode"
+
+const (
+ // 0xd800-0xdc00 encodes the high 10 bits of a pair.
+ // 0xdc00-0xe000 encodes the low 10 bits of a pair.
+ // the value is those 20 bits plus 0x10000.
+ surr1 = 0xd800
+ surr2 = 0xdc00
+ surr3 = 0xe000
+
+ surrSelf = 0x10000
+)
+
+// IsSurrogate returns true if the specified Unicode code point
+// can appear in a surrogate pair.
+func IsSurrogate(r rune) bool {
+ return surr1 <= r && r < surr3
+}
+
+// DecodeRune returns the UTF-16 decoding of a surrogate pair.
+// If the pair is not a valid UTF-16 surrogate pair, DecodeRune returns
+// the Unicode replacement code point U+FFFD.
+func DecodeRune(r1, r2 rune) rune {
+ if surr1 <= r1 && r1 < surr2 && surr2 <= r2 && r2 < surr3 {
+ return (rune(r1)-surr1)<<10 | (rune(r2) - surr2) + 0x10000
+ }
+ return unicode.ReplacementChar
+}
+
+// EncodeRune returns the UTF-16 surrogate pair r1, r2 for the given rune.
+// If the rune is not a valid Unicode code point or does not need encoding,
+// EncodeRune returns U+FFFD, U+FFFD.
+func EncodeRune(r rune) (r1, r2 rune) {
+ if r < surrSelf || r > unicode.MaxRune || IsSurrogate(r) {
+ return unicode.ReplacementChar, unicode.ReplacementChar
+ }
+ r -= surrSelf
+ return surr1 + (r>>10)&0x3ff, surr2 + r&0x3ff
+}
+
+// Encode returns the UTF-16 encoding of the Unicode code point sequence s.
+func Encode(s []rune) []uint16 {
+ n := len(s)
+ for _, v := range s {
+ if v >= surrSelf {
+ n++
+ }
+ }
+
+ a := make([]uint16, n)
+ n = 0
+ for _, v := range s {
+ switch {
+ case v < 0, surr1 <= v && v < surr3, v > unicode.MaxRune:
+ v = unicode.ReplacementChar
+ fallthrough
+ case v < surrSelf:
+ a[n] = uint16(v)
+ n++
+ default:
+ r1, r2 := EncodeRune(v)
+ a[n] = uint16(r1)
+ a[n+1] = uint16(r2)
+ n += 2
+ }
+ }
+ return a[0:n]
+}
+
+// Decode returns the Unicode code point sequence represented
+// by the UTF-16 encoding s.
+func Decode(s []uint16) []rune {
+ a := make([]rune, len(s))
+ n := 0
+ for i := 0; i < len(s); i++ {
+ switch r := s[i]; {
+ case surr1 <= r && r < surr2 && i+1 < len(s) &&
+ surr2 <= s[i+1] && s[i+1] < surr3:
+ // valid surrogate sequence
+ a[n] = DecodeRune(rune(r), rune(s[i+1]))
+ i++
+ n++
+ case surr1 <= r && r < surr3:
+ // invalid surrogate sequence
+ a[n] = unicode.ReplacementChar
+ n++
+ default:
+ // normal rune
+ a[n] = rune(r)
+ n++
+ }
+ }
+ return a[0:n]
+}
diff --git a/libgo/go/unicode/utf16/utf16_test.go b/libgo/go/unicode/utf16/utf16_test.go
new file mode 100644
index 00000000000..d453b2f984d
--- /dev/null
+++ b/libgo/go/unicode/utf16/utf16_test.go
@@ -0,0 +1,91 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package utf16_test
+
+import (
+ "reflect"
+ "testing"
+ "unicode"
+ . "unicode/utf16"
+)
+
+type encodeTest struct {
+ in []rune
+ out []uint16
+}
+
+var encodeTests = []encodeTest{
+ {[]rune{1, 2, 3, 4}, []uint16{1, 2, 3, 4}},
+ {[]rune{0xffff, 0x10000, 0x10001, 0x12345, 0x10ffff},
+ []uint16{0xffff, 0xd800, 0xdc00, 0xd800, 0xdc01, 0xd808, 0xdf45, 0xdbff, 0xdfff}},
+ {[]rune{'a', 'b', 0xd7ff, 0xd800, 0xdfff, 0xe000, 0x110000, -1},
+ []uint16{'a', 'b', 0xd7ff, 0xfffd, 0xfffd, 0xe000, 0xfffd, 0xfffd}},
+}
+
+func TestEncode(t *testing.T) {
+ for _, tt := range encodeTests {
+ out := Encode(tt.in)
+ if !reflect.DeepEqual(out, tt.out) {
+ t.Errorf("Encode(%x) = %x; want %x", tt.in, out, tt.out)
+ }
+ }
+}
+
+func TestEncodeRune(t *testing.T) {
+ for i, tt := range encodeTests {
+ j := 0
+ for _, r := range tt.in {
+ r1, r2 := EncodeRune(r)
+ if r < 0x10000 || r > unicode.MaxRune {
+ if j >= len(tt.out) {
+ t.Errorf("#%d: ran out of tt.out", i)
+ break
+ }
+ if r1 != unicode.ReplacementChar || r2 != unicode.ReplacementChar {
+ t.Errorf("EncodeRune(%#x) = %#x, %#x; want 0xfffd, 0xfffd", r, r1, r2)
+ }
+ j++
+ } else {
+ if j+1 >= len(tt.out) {
+ t.Errorf("#%d: ran out of tt.out", i)
+ break
+ }
+ if r1 != rune(tt.out[j]) || r2 != rune(tt.out[j+1]) {
+ t.Errorf("EncodeRune(%#x) = %#x, %#x; want %#x, %#x", r, r1, r2, tt.out[j], tt.out[j+1])
+ }
+ j += 2
+ dec := DecodeRune(r1, r2)
+ if dec != r {
+ t.Errorf("DecodeRune(%#x, %#x) = %#x; want %#x", r1, r2, dec, r)
+ }
+ }
+ }
+ if j != len(tt.out) {
+ t.Errorf("#%d: EncodeRune didn't generate enough output", i)
+ }
+ }
+}
+
+type decodeTest struct {
+ in []uint16
+ out []rune
+}
+
+var decodeTests = []decodeTest{
+ {[]uint16{1, 2, 3, 4}, []rune{1, 2, 3, 4}},
+ {[]uint16{0xffff, 0xd800, 0xdc00, 0xd800, 0xdc01, 0xd808, 0xdf45, 0xdbff, 0xdfff},
+ []rune{0xffff, 0x10000, 0x10001, 0x12345, 0x10ffff}},
+ {[]uint16{0xd800, 'a'}, []rune{0xfffd, 'a'}},
+ {[]uint16{0xdfff}, []rune{0xfffd}},
+}
+
+func TestDecode(t *testing.T) {
+ for _, tt := range decodeTests {
+ out := Decode(tt.in)
+ if !reflect.DeepEqual(out, tt.out) {
+ t.Errorf("Decode(%x) = %x; want %x", tt.in, out, tt.out)
+ }
+ }
+}
diff --git a/libgo/go/unicode/utf8/string.go b/libgo/go/unicode/utf8/string.go
new file mode 100644
index 00000000000..443decf056c
--- /dev/null
+++ b/libgo/go/unicode/utf8/string.go
@@ -0,0 +1,199 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package utf8
+
+import "errors"
+
+// String wraps a regular string with a small structure that provides more
+// efficient indexing by code point index, as opposed to byte index.
+// Scanning incrementally forwards or backwards is O(1) per index operation
+// (although not as fast a range clause going forwards). Random access is
+// O(N) in the length of the string, but the overhead is less than always
+// scanning from the beginning.
+// If the string is ASCII, random access is O(1).
+// Unlike the built-in string type, String has internal mutable state and
+// is not thread-safe.
+type String struct {
+ str string
+ numRunes int
+ // If width > 0, the rune at runePos starts at bytePos and has the specified width.
+ width int
+ bytePos int
+ runePos int
+ nonASCII int // byte index of the first non-ASCII rune.
+}
+
+// NewString returns a new UTF-8 string with the provided contents.
+func NewString(contents string) *String {
+ return new(String).Init(contents)
+}
+
+// Init initializes an existing String to hold the provided contents.
+// It returns a pointer to the initialized String.
+func (s *String) Init(contents string) *String {
+ s.str = contents
+ s.bytePos = 0
+ s.runePos = 0
+ for i := 0; i < len(contents); i++ {
+ if contents[i] >= RuneSelf {
+ // Not ASCII.
+ s.numRunes = RuneCountInString(contents)
+ _, s.width = DecodeRuneInString(contents)
+ s.nonASCII = i
+ return s
+ }
+ }
+ // ASCII is simple. Also, the empty string is ASCII.
+ s.numRunes = len(contents)
+ s.width = 0
+ s.nonASCII = len(contents)
+ return s
+}
+
+// String returns the contents of the String. This method also means the
+// String is directly printable by fmt.Print.
+func (s *String) String() string {
+ return s.str
+}
+
+// RuneCount returns the number of runes (Unicode code points) in the String.
+func (s *String) RuneCount() int {
+ return s.numRunes
+}
+
+// IsASCII returns a boolean indicating whether the String contains only ASCII bytes.
+func (s *String) IsASCII() bool {
+ return s.width == 0
+}
+
+// Slice returns the string sliced at rune positions [i:j].
+func (s *String) Slice(i, j int) string {
+ // ASCII is easy. Let the compiler catch the indexing error if there is one.
+ if j < s.nonASCII {
+ return s.str[i:j]
+ }
+ if i < 0 || j > s.numRunes || i > j {
+ panic(sliceOutOfRange)
+ }
+ if i == j {
+ return ""
+ }
+ // For non-ASCII, after At(i), bytePos is always the position of the indexed character.
+ var low, high int
+ switch {
+ case i < s.nonASCII:
+ low = i
+ case i == s.numRunes:
+ low = len(s.str)
+ default:
+ s.At(i)
+ low = s.bytePos
+ }
+ switch {
+ case j == s.numRunes:
+ high = len(s.str)
+ default:
+ s.At(j)
+ high = s.bytePos
+ }
+ return s.str[low:high]
+}
+
+// At returns the rune with index i in the String. The sequence of runes is the same
+// as iterating over the contents with a "for range" clause.
+func (s *String) At(i int) rune {
+ // ASCII is easy. Let the compiler catch the indexing error if there is one.
+ if i < s.nonASCII {
+ return rune(s.str[i])
+ }
+
+ // Now we do need to know the index is valid.
+ if i < 0 || i >= s.numRunes {
+ panic(outOfRange)
+ }
+
+ var r rune
+
+ // Five easy common cases: within 1 spot of bytePos/runePos, or the beginning, or the end.
+ // With these cases, all scans from beginning or end work in O(1) time per rune.
+ switch {
+
+ case i == s.runePos-1: // backing up one rune
+ r, s.width = DecodeLastRuneInString(s.str[0:s.bytePos])
+ s.runePos = i
+ s.bytePos -= s.width
+ return r
+ case i == s.runePos+1: // moving ahead one rune
+ s.runePos = i
+ s.bytePos += s.width
+ fallthrough
+ case i == s.runePos:
+ r, s.width = DecodeRuneInString(s.str[s.bytePos:])
+ return r
+ case i == 0: // start of string
+ r, s.width = DecodeRuneInString(s.str)
+ s.runePos = 0
+ s.bytePos = 0
+ return r
+
+ case i == s.numRunes-1: // last rune in string
+ r, s.width = DecodeLastRuneInString(s.str)
+ s.runePos = i
+ s.bytePos = len(s.str) - s.width
+ return r
+ }
+
+ // We need to do a linear scan. There are three places to start from:
+ // 1) The beginning
+ // 2) bytePos/runePos.
+ // 3) The end
+ // Choose the closest in rune count, scanning backwards if necessary.
+ forward := true
+ if i < s.runePos {
+ // Between beginning and pos. Which is closer?
+ // Since both i and runePos are guaranteed >= nonASCII, that's the
+ // lowest location we need to start from.
+ if i < (s.runePos-s.nonASCII)/2 {
+ // Scan forward from beginning
+ s.bytePos, s.runePos = s.nonASCII, s.nonASCII
+ } else {
+ // Scan backwards from where we are
+ forward = false
+ }
+ } else {
+ // Between pos and end. Which is closer?
+ if i-s.runePos < (s.numRunes-s.runePos)/2 {
+ // Scan forward from pos
+ } else {
+ // Scan backwards from end
+ s.bytePos, s.runePos = len(s.str), s.numRunes
+ forward = false
+ }
+ }
+ if forward {
+ // TODO: Is it much faster to use a range loop for this scan?
+ for {
+ r, s.width = DecodeRuneInString(s.str[s.bytePos:])
+ if s.runePos == i {
+ break
+ }
+ s.runePos++
+ s.bytePos += s.width
+ }
+ } else {
+ for {
+ r, s.width = DecodeLastRuneInString(s.str[0:s.bytePos])
+ s.runePos--
+ s.bytePos -= s.width
+ if s.runePos == i {
+ break
+ }
+ }
+ }
+ return r
+}
+
+var outOfRange = errors.New("utf8.String: index out of range")
+var sliceOutOfRange = errors.New("utf8.String: slice index out of range")
diff --git a/libgo/go/unicode/utf8/string_test.go b/libgo/go/unicode/utf8/string_test.go
new file mode 100644
index 00000000000..2c139bea98c
--- /dev/null
+++ b/libgo/go/unicode/utf8/string_test.go
@@ -0,0 +1,114 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package utf8_test
+
+import (
+ "math/rand"
+ "testing"
+ . "unicode/utf8"
+)
+
+func TestScanForwards(t *testing.T) {
+ for _, s := range testStrings {
+ runes := []rune(s)
+ str := NewString(s)
+ if str.RuneCount() != len(runes) {
+ t.Errorf("%s: expected %d runes; got %d", s, len(runes), str.RuneCount())
+ break
+ }
+ for i, expect := range runes {
+ got := str.At(i)
+ if got != expect {
+ t.Errorf("%s[%d]: expected %c (%U); got %c (%U)", s, i, expect, expect, got, got)
+ }
+ }
+ }
+}
+
+func TestScanBackwards(t *testing.T) {
+ for _, s := range testStrings {
+ runes := []rune(s)
+ str := NewString(s)
+ if str.RuneCount() != len(runes) {
+ t.Errorf("%s: expected %d runes; got %d", s, len(runes), str.RuneCount())
+ break
+ }
+ for i := len(runes) - 1; i >= 0; i-- {
+ expect := runes[i]
+ got := str.At(i)
+ if got != expect {
+ t.Errorf("%s[%d]: expected %c (%U); got %c (%U)", s, i, expect, expect, got, got)
+ }
+ }
+ }
+}
+
+func randCount() int {
+ if testing.Short() {
+ return 100
+ }
+ return 100000
+}
+
+func TestRandomAccess(t *testing.T) {
+ for _, s := range testStrings {
+ if len(s) == 0 {
+ continue
+ }
+ runes := []rune(s)
+ str := NewString(s)
+ if str.RuneCount() != len(runes) {
+ t.Errorf("%s: expected %d runes; got %d", s, len(runes), str.RuneCount())
+ break
+ }
+ for j := 0; j < randCount(); j++ {
+ i := rand.Intn(len(runes))
+ expect := runes[i]
+ got := str.At(i)
+ if got != expect {
+ t.Errorf("%s[%d]: expected %c (%U); got %c (%U)", s, i, expect, expect, got, got)
+ }
+ }
+ }
+}
+
+func TestRandomSliceAccess(t *testing.T) {
+ for _, s := range testStrings {
+ if len(s) == 0 || s[0] == '\x80' { // the bad-UTF-8 string fools this simple test
+ continue
+ }
+ runes := []rune(s)
+ str := NewString(s)
+ if str.RuneCount() != len(runes) {
+ t.Errorf("%s: expected %d runes; got %d", s, len(runes), str.RuneCount())
+ break
+ }
+ for k := 0; k < randCount(); k++ {
+ i := rand.Intn(len(runes))
+ j := rand.Intn(len(runes) + 1)
+ if i > j { // include empty strings
+ continue
+ }
+ expect := string(runes[i:j])
+ got := str.Slice(i, j)
+ if got != expect {
+ t.Errorf("%s[%d:%d]: expected %q got %q", s, i, j, expect, got)
+ }
+ }
+ }
+}
+
+func TestLimitSliceAccess(t *testing.T) {
+ for _, s := range testStrings {
+ str := NewString(s)
+ if str.Slice(0, 0) != "" {
+ t.Error("failure with empty slice at beginning")
+ }
+ nr := RuneCountInString(s)
+ if str.Slice(nr, nr) != "" {
+ t.Error("failure with empty slice at end")
+ }
+ }
+}
diff --git a/libgo/go/unicode/utf8/utf8.go b/libgo/go/unicode/utf8/utf8.go
new file mode 100644
index 00000000000..a5f9983b332
--- /dev/null
+++ b/libgo/go/unicode/utf8/utf8.go
@@ -0,0 +1,391 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package utf8 implements functions and constants to support text encoded in
+// UTF-8. This package calls a Unicode character a rune for brevity.
+package utf8
+
+import "unicode" // only needed for a couple of constants
+
+// Numbers fundamental to the encoding.
+const (
+ RuneError = unicode.ReplacementChar // the "error" Rune or "replacement character".
+ RuneSelf = 0x80 // characters below Runeself are represented as themselves in a single byte.
+ UTFMax = 4 // maximum number of bytes of a UTF-8 encoded Unicode character.
+)
+
+const (
+ t1 = 0x00 // 0000 0000
+ tx = 0x80 // 1000 0000
+ t2 = 0xC0 // 1100 0000
+ t3 = 0xE0 // 1110 0000
+ t4 = 0xF0 // 1111 0000
+ t5 = 0xF8 // 1111 1000
+
+ maskx = 0x3F // 0011 1111
+ mask2 = 0x1F // 0001 1111
+ mask3 = 0x0F // 0000 1111
+ mask4 = 0x07 // 0000 0111
+
+ rune1Max = 1<<7 - 1
+ rune2Max = 1<<11 - 1
+ rune3Max = 1<<16 - 1
+ rune4Max = 1<<21 - 1
+)
+
+func decodeRuneInternal(p []byte) (r rune, size int, short bool) {
+ n := len(p)
+ if n < 1 {
+ return RuneError, 0, true
+ }
+ c0 := p[0]
+
+ // 1-byte, 7-bit sequence?
+ if c0 < tx {
+ return rune(c0), 1, false
+ }
+
+ // unexpected continuation byte?
+ if c0 < t2 {
+ return RuneError, 1, false
+ }
+
+ // need first continuation byte
+ if n < 2 {
+ return RuneError, 1, true
+ }
+ c1 := p[1]
+ if c1 < tx || t2 <= c1 {
+ return RuneError, 1, false
+ }
+
+ // 2-byte, 11-bit sequence?
+ if c0 < t3 {
+ r = rune(c0&mask2)<<6 | rune(c1&maskx)
+ if r <= rune1Max {
+ return RuneError, 1, false
+ }
+ return r, 2, false
+ }
+
+ // need second continuation byte
+ if n < 3 {
+ return RuneError, 1, true
+ }
+ c2 := p[2]
+ if c2 < tx || t2 <= c2 {
+ return RuneError, 1, false
+ }
+
+ // 3-byte, 16-bit sequence?
+ if c0 < t4 {
+ r = rune(c0&mask3)<<12 | rune(c1&maskx)<<6 | rune(c2&maskx)
+ if r <= rune2Max {
+ return RuneError, 1, false
+ }
+ return r, 3, false
+ }
+
+ // need third continuation byte
+ if n < 4 {
+ return RuneError, 1, true
+ }
+ c3 := p[3]
+ if c3 < tx || t2 <= c3 {
+ return RuneError, 1, false
+ }
+
+ // 4-byte, 21-bit sequence?
+ if c0 < t5 {
+ r = rune(c0&mask4)<<18 | rune(c1&maskx)<<12 | rune(c2&maskx)<<6 | rune(c3&maskx)
+ if r <= rune3Max {
+ return RuneError, 1, false
+ }
+ return r, 4, false
+ }
+
+ // error
+ return RuneError, 1, false
+}
+
+func decodeRuneInStringInternal(s string) (r rune, size int, short bool) {
+ n := len(s)
+ if n < 1 {
+ return RuneError, 0, true
+ }
+ c0 := s[0]
+
+ // 1-byte, 7-bit sequence?
+ if c0 < tx {
+ return rune(c0), 1, false
+ }
+
+ // unexpected continuation byte?
+ if c0 < t2 {
+ return RuneError, 1, false
+ }
+
+ // need first continuation byte
+ if n < 2 {
+ return RuneError, 1, true
+ }
+ c1 := s[1]
+ if c1 < tx || t2 <= c1 {
+ return RuneError, 1, false
+ }
+
+ // 2-byte, 11-bit sequence?
+ if c0 < t3 {
+ r = rune(c0&mask2)<<6 | rune(c1&maskx)
+ if r <= rune1Max {
+ return RuneError, 1, false
+ }
+ return r, 2, false
+ }
+
+ // need second continuation byte
+ if n < 3 {
+ return RuneError, 1, true
+ }
+ c2 := s[2]
+ if c2 < tx || t2 <= c2 {
+ return RuneError, 1, false
+ }
+
+ // 3-byte, 16-bit sequence?
+ if c0 < t4 {
+ r = rune(c0&mask3)<<12 | rune(c1&maskx)<<6 | rune(c2&maskx)
+ if r <= rune2Max {
+ return RuneError, 1, false
+ }
+ return r, 3, false
+ }
+
+ // need third continuation byte
+ if n < 4 {
+ return RuneError, 1, true
+ }
+ c3 := s[3]
+ if c3 < tx || t2 <= c3 {
+ return RuneError, 1, false
+ }
+
+ // 4-byte, 21-bit sequence?
+ if c0 < t5 {
+ r = rune(c0&mask4)<<18 | rune(c1&maskx)<<12 | rune(c2&maskx)<<6 | rune(c3&maskx)
+ if r <= rune3Max {
+ return RuneError, 1, false
+ }
+ return r, 4, false
+ }
+
+ // error
+ return RuneError, 1, false
+}
+
+// FullRune reports whether the bytes in p begin with a full UTF-8 encoding of a rune.
+// An invalid encoding is considered a full Rune since it will convert as a width-1 error rune.
+func FullRune(p []byte) bool {
+ _, _, short := decodeRuneInternal(p)
+ return !short
+}
+
+// FullRuneInString is like FullRune but its input is a string.
+func FullRuneInString(s string) bool {
+ _, _, short := decodeRuneInStringInternal(s)
+ return !short
+}
+
+// DecodeRune unpacks the first UTF-8 encoding in p and returns the rune and its width in bytes.
+func DecodeRune(p []byte) (r rune, size int) {
+ r, size, _ = decodeRuneInternal(p)
+ return
+}
+
+// DecodeRuneInString is like DecodeRune but its input is a string.
+func DecodeRuneInString(s string) (r rune, size int) {
+ r, size, _ = decodeRuneInStringInternal(s)
+ return
+}
+
+// DecodeLastRune unpacks the last UTF-8 encoding in p
+// and returns the rune and its width in bytes.
+func DecodeLastRune(p []byte) (r rune, size int) {
+ end := len(p)
+ if end == 0 {
+ return RuneError, 0
+ }
+ start := end - 1
+ r = rune(p[start])
+ if r < RuneSelf {
+ return r, 1
+ }
+ // guard against O(n^2) behavior when traversing
+ // backwards through strings with long sequences of
+ // invalid UTF-8.
+ lim := end - UTFMax
+ if lim < 0 {
+ lim = 0
+ }
+ for start--; start >= lim; start-- {
+ if RuneStart(p[start]) {
+ break
+ }
+ }
+ if start < 0 {
+ start = 0
+ }
+ r, size = DecodeRune(p[start:end])
+ if start+size != end {
+ return RuneError, 1
+ }
+ return r, size
+}
+
+// DecodeLastRuneInString is like DecodeLastRune but its input is a string.
+func DecodeLastRuneInString(s string) (r rune, size int) {
+ end := len(s)
+ if end == 0 {
+ return RuneError, 0
+ }
+ start := end - 1
+ r = rune(s[start])
+ if r < RuneSelf {
+ return r, 1
+ }
+ // guard against O(n^2) behavior when traversing
+ // backwards through strings with long sequences of
+ // invalid UTF-8.
+ lim := end - UTFMax
+ if lim < 0 {
+ lim = 0
+ }
+ for start--; start >= lim; start-- {
+ if RuneStart(s[start]) {
+ break
+ }
+ }
+ if start < 0 {
+ start = 0
+ }
+ r, size = DecodeRuneInString(s[start:end])
+ if start+size != end {
+ return RuneError, 1
+ }
+ return r, size
+}
+
+// RuneLen returns the number of bytes required to encode the rune.
+func RuneLen(r rune) int {
+ switch {
+ case r <= rune1Max:
+ return 1
+ case r <= rune2Max:
+ return 2
+ case r <= rune3Max:
+ return 3
+ case r <= rune4Max:
+ return 4
+ }
+ return -1
+}
+
+// EncodeRune writes into p (which must be large enough) the UTF-8 encoding of the rune.
+// It returns the number of bytes written.
+func EncodeRune(p []byte, r rune) int {
+ // Negative values are erroneous. Making it unsigned addresses the problem.
+ if uint32(r) <= rune1Max {
+ p[0] = byte(r)
+ return 1
+ }
+
+ if uint32(r) <= rune2Max {
+ p[0] = t2 | byte(r>>6)
+ p[1] = tx | byte(r)&maskx
+ return 2
+ }
+
+ if uint32(r) > unicode.MaxRune {
+ r = RuneError
+ }
+
+ if uint32(r) <= rune3Max {
+ p[0] = t3 | byte(r>>12)
+ p[1] = tx | byte(r>>6)&maskx
+ p[2] = tx | byte(r)&maskx
+ return 3
+ }
+
+ p[0] = t4 | byte(r>>18)
+ p[1] = tx | byte(r>>12)&maskx
+ p[2] = tx | byte(r>>6)&maskx
+ p[3] = tx | byte(r)&maskx
+ return 4
+}
+
+// RuneCount returns the number of runes in p. Erroneous and short
+// encodings are treated as single runes of width 1 byte.
+func RuneCount(p []byte) int {
+ i := 0
+ var n int
+ for n = 0; i < len(p); n++ {
+ if p[i] < RuneSelf {
+ i++
+ } else {
+ _, size := DecodeRune(p[i:])
+ i += size
+ }
+ }
+ return n
+}
+
+// RuneCountInString is like RuneCount but its input is a string.
+func RuneCountInString(s string) (n int) {
+ for _ = range s {
+ n++
+ }
+ return
+}
+
+// RuneStart reports whether the byte could be the first byte of
+// an encoded rune. Second and subsequent bytes always have the top
+// two bits set to 10.
+func RuneStart(b byte) bool { return b&0xC0 != 0x80 }
+
+// Valid reports whether p consists entirely of valid UTF-8-encoded runes.
+func Valid(p []byte) bool {
+ i := 0
+ for i < len(p) {
+ if p[i] < RuneSelf {
+ i++
+ } else {
+ _, size := DecodeRune(p[i:])
+ if size == 1 {
+ // All valid runes of size of 1 (those
+ // below RuneSelf) were handled above.
+ // This must be a RuneError.
+ return false
+ }
+ i += size
+ }
+ }
+ return true
+}
+
+// ValidString reports whether s consists entirely of valid UTF-8-encoded runes.
+func ValidString(s string) bool {
+ for i, r := range s {
+ if r == RuneError {
+ // The RuneError value can be an error
+ // sentinel value (if it's size 1) or the same
+ // value encoded properly. Decode it to see if
+ // it's the 1 byte sentinel value.
+ _, size := DecodeRuneInString(s[i:])
+ if size == 1 {
+ return false
+ }
+ }
+ }
+ return true
+}
diff --git a/libgo/go/unicode/utf8/utf8_test.go b/libgo/go/unicode/utf8/utf8_test.go
new file mode 100644
index 00000000000..63514265bb9
--- /dev/null
+++ b/libgo/go/unicode/utf8/utf8_test.go
@@ -0,0 +1,344 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package utf8_test
+
+import (
+ "bytes"
+ "testing"
+ . "unicode/utf8"
+)
+
+type Utf8Map struct {
+ r rune
+ str string
+}
+
+var utf8map = []Utf8Map{
+ {0x0000, "\x00"},
+ {0x0001, "\x01"},
+ {0x007e, "\x7e"},
+ {0x007f, "\x7f"},
+ {0x0080, "\xc2\x80"},
+ {0x0081, "\xc2\x81"},
+ {0x00bf, "\xc2\xbf"},
+ {0x00c0, "\xc3\x80"},
+ {0x00c1, "\xc3\x81"},
+ {0x00c8, "\xc3\x88"},
+ {0x00d0, "\xc3\x90"},
+ {0x00e0, "\xc3\xa0"},
+ {0x00f0, "\xc3\xb0"},
+ {0x00f8, "\xc3\xb8"},
+ {0x00ff, "\xc3\xbf"},
+ {0x0100, "\xc4\x80"},
+ {0x07ff, "\xdf\xbf"},
+ {0x0800, "\xe0\xa0\x80"},
+ {0x0801, "\xe0\xa0\x81"},
+ {0xfffe, "\xef\xbf\xbe"},
+ {0xffff, "\xef\xbf\xbf"},
+ {0x10000, "\xf0\x90\x80\x80"},
+ {0x10001, "\xf0\x90\x80\x81"},
+ {0x10fffe, "\xf4\x8f\xbf\xbe"},
+ {0x10ffff, "\xf4\x8f\xbf\xbf"},
+ {0xFFFD, "\xef\xbf\xbd"},
+}
+
+var testStrings = []string{
+ "",
+ "abcd",
+ "☺☻☹",
+ "日a本b語ç日ð本Ê語þ日¥本¼語i日©",
+ "日a本b語ç日ð本Ê語þ日¥本¼語i日©日a本b語ç日ð本Ê語þ日¥本¼語i日©日a本b語ç日ð本Ê語þ日¥本¼語i日©",
+ "\x80\x80\x80\x80",
+}
+
+func TestFullRune(t *testing.T) {
+ for i := 0; i < len(utf8map); i++ {
+ m := utf8map[i]
+ b := []byte(m.str)
+ if !FullRune(b) {
+ t.Errorf("FullRune(%q) (%U) = false, want true", b, m.r)
+ }
+ s := m.str
+ if !FullRuneInString(s) {
+ t.Errorf("FullRuneInString(%q) (%U) = false, want true", s, m.r)
+ }
+ b1 := b[0 : len(b)-1]
+ if FullRune(b1) {
+ t.Errorf("FullRune(%q) = true, want false", b1)
+ }
+ s1 := string(b1)
+ if FullRuneInString(s1) {
+ t.Errorf("FullRune(%q) = true, want false", s1)
+ }
+ }
+}
+
+func TestEncodeRune(t *testing.T) {
+ for i := 0; i < len(utf8map); i++ {
+ m := utf8map[i]
+ b := []byte(m.str)
+ var buf [10]byte
+ n := EncodeRune(buf[0:], m.r)
+ b1 := buf[0:n]
+ if !bytes.Equal(b, b1) {
+ t.Errorf("EncodeRune(%#04x) = %q want %q", m.r, b1, b)
+ }
+ }
+}
+
+func TestDecodeRune(t *testing.T) {
+ for i := 0; i < len(utf8map); i++ {
+ m := utf8map[i]
+ b := []byte(m.str)
+ r, size := DecodeRune(b)
+ if r != m.r || size != len(b) {
+ t.Errorf("DecodeRune(%q) = %#04x, %d want %#04x, %d", b, r, size, m.r, len(b))
+ }
+ s := m.str
+ r, size = DecodeRuneInString(s)
+ if r != m.r || size != len(b) {
+ t.Errorf("DecodeRune(%q) = %#04x, %d want %#04x, %d", s, r, size, m.r, len(b))
+ }
+
+ // there's an extra byte that bytes left behind - make sure trailing byte works
+ r, size = DecodeRune(b[0:cap(b)])
+ if r != m.r || size != len(b) {
+ t.Errorf("DecodeRune(%q) = %#04x, %d want %#04x, %d", b, r, size, m.r, len(b))
+ }
+ s = m.str + "\x00"
+ r, size = DecodeRuneInString(s)
+ if r != m.r || size != len(b) {
+ t.Errorf("DecodeRuneInString(%q) = %#04x, %d want %#04x, %d", s, r, size, m.r, len(b))
+ }
+
+ // make sure missing bytes fail
+ wantsize := 1
+ if wantsize >= len(b) {
+ wantsize = 0
+ }
+ r, size = DecodeRune(b[0 : len(b)-1])
+ if r != RuneError || size != wantsize {
+ t.Errorf("DecodeRune(%q) = %#04x, %d want %#04x, %d", b[0:len(b)-1], r, size, RuneError, wantsize)
+ }
+ s = m.str[0 : len(m.str)-1]
+ r, size = DecodeRuneInString(s)
+ if r != RuneError || size != wantsize {
+ t.Errorf("DecodeRuneInString(%q) = %#04x, %d want %#04x, %d", s, r, size, RuneError, wantsize)
+ }
+
+ // make sure bad sequences fail
+ if len(b) == 1 {
+ b[0] = 0x80
+ } else {
+ b[len(b)-1] = 0x7F
+ }
+ r, size = DecodeRune(b)
+ if r != RuneError || size != 1 {
+ t.Errorf("DecodeRune(%q) = %#04x, %d want %#04x, %d", b, r, size, RuneError, 1)
+ }
+ s = string(b)
+ r, size = DecodeRune(b)
+ if r != RuneError || size != 1 {
+ t.Errorf("DecodeRuneInString(%q) = %#04x, %d want %#04x, %d", s, r, size, RuneError, 1)
+ }
+
+ }
+}
+
+// Check that DecodeRune and DecodeLastRune correspond to
+// the equivalent range loop.
+func TestSequencing(t *testing.T) {
+ for _, ts := range testStrings {
+ for _, m := range utf8map {
+ for _, s := range []string{ts + m.str, m.str + ts, ts + m.str + ts} {
+ testSequence(t, s)
+ }
+ }
+ }
+}
+
+// Check that a range loop and a []int conversion visit the same runes.
+// Not really a test of this package, but the assumption is used here and
+// it's good to verify
+func TestIntConversion(t *testing.T) {
+ for _, ts := range testStrings {
+ runes := []rune(ts)
+ if RuneCountInString(ts) != len(runes) {
+ t.Errorf("%q: expected %d runes; got %d", ts, len(runes), RuneCountInString(ts))
+ break
+ }
+ i := 0
+ for _, r := range ts {
+ if r != runes[i] {
+ t.Errorf("%q[%d]: expected %c (%U); got %c (%U)", ts, i, runes[i], runes[i], r, r)
+ }
+ i++
+ }
+ }
+}
+
+func testSequence(t *testing.T, s string) {
+ type info struct {
+ index int
+ r rune
+ }
+ index := make([]info, len(s))
+ b := []byte(s)
+ si := 0
+ j := 0
+ for i, r := range s {
+ if si != i {
+ t.Errorf("Sequence(%q) mismatched index %d, want %d", s, si, i)
+ return
+ }
+ index[j] = info{i, r}
+ j++
+ r1, size1 := DecodeRune(b[i:])
+ if r != r1 {
+ t.Errorf("DecodeRune(%q) = %#04x, want %#04x", s[i:], r1, r)
+ return
+ }
+ r2, size2 := DecodeRuneInString(s[i:])
+ if r != r2 {
+ t.Errorf("DecodeRuneInString(%q) = %#04x, want %#04x", s[i:], r2, r)
+ return
+ }
+ if size1 != size2 {
+ t.Errorf("DecodeRune/DecodeRuneInString(%q) size mismatch %d/%d", s[i:], size1, size2)
+ return
+ }
+ si += size1
+ }
+ j--
+ for si = len(s); si > 0; {
+ r1, size1 := DecodeLastRune(b[0:si])
+ r2, size2 := DecodeLastRuneInString(s[0:si])
+ if size1 != size2 {
+ t.Errorf("DecodeLastRune/DecodeLastRuneInString(%q, %d) size mismatch %d/%d", s, si, size1, size2)
+ return
+ }
+ if r1 != index[j].r {
+ t.Errorf("DecodeLastRune(%q, %d) = %#04x, want %#04x", s, si, r1, index[j].r)
+ return
+ }
+ if r2 != index[j].r {
+ t.Errorf("DecodeLastRuneInString(%q, %d) = %#04x, want %#04x", s, si, r2, index[j].r)
+ return
+ }
+ si -= size1
+ if si != index[j].index {
+ t.Errorf("DecodeLastRune(%q) index mismatch at %d, want %d", s, si, index[j].index)
+ return
+ }
+ j--
+ }
+ if si != 0 {
+ t.Errorf("DecodeLastRune(%q) finished at %d, not 0", s, si)
+ }
+}
+
+// Check that negative runes encode as U+FFFD.
+func TestNegativeRune(t *testing.T) {
+ errorbuf := make([]byte, UTFMax)
+ errorbuf = errorbuf[0:EncodeRune(errorbuf, RuneError)]
+ buf := make([]byte, UTFMax)
+ buf = buf[0:EncodeRune(buf, -1)]
+ if !bytes.Equal(buf, errorbuf) {
+ t.Errorf("incorrect encoding [% x] for -1; expected [% x]", buf, errorbuf)
+ }
+}
+
+type RuneCountTest struct {
+ in string
+ out int
+}
+
+var runecounttests = []RuneCountTest{
+ {"abcd", 4},
+ {"☺☻☹", 3},
+ {"1,2,3,4", 7},
+ {"\xe2\x00", 2},
+}
+
+func TestRuneCount(t *testing.T) {
+ for i := 0; i < len(runecounttests); i++ {
+ tt := runecounttests[i]
+ if out := RuneCountInString(tt.in); out != tt.out {
+ t.Errorf("RuneCountInString(%q) = %d, want %d", tt.in, out, tt.out)
+ }
+ if out := RuneCount([]byte(tt.in)); out != tt.out {
+ t.Errorf("RuneCount(%q) = %d, want %d", tt.in, out, tt.out)
+ }
+ }
+}
+
+type ValidTest struct {
+ in string
+ out bool
+}
+
+var validTests = []ValidTest{
+ {"", true},
+ {"a", true},
+ {"abc", true},
+ {"Ж", true},
+ {"ЖЖ", true},
+ {"брэд-ЛГТМ", true},
+ {"☺☻☹", true},
+ {string([]byte{66, 250}), false},
+ {string([]byte{66, 250, 67}), false},
+ {"a\uFFFDb", true},
+}
+
+func TestValid(t *testing.T) {
+ for i, tt := range validTests {
+ if Valid([]byte(tt.in)) != tt.out {
+ t.Errorf("%d. Valid(%q) = %v; want %v", i, tt.in, !tt.out, tt.out)
+ }
+ if ValidString(tt.in) != tt.out {
+ t.Errorf("%d. ValidString(%q) = %v; want %v", i, tt.in, !tt.out, tt.out)
+ }
+ }
+}
+
+func BenchmarkRuneCountTenASCIIChars(b *testing.B) {
+ for i := 0; i < b.N; i++ {
+ RuneCountInString("0123456789")
+ }
+}
+
+func BenchmarkRuneCountTenJapaneseChars(b *testing.B) {
+ for i := 0; i < b.N; i++ {
+ RuneCountInString("日本語日本語日本語日")
+ }
+}
+
+func BenchmarkEncodeASCIIRune(b *testing.B) {
+ buf := make([]byte, UTFMax)
+ for i := 0; i < b.N; i++ {
+ EncodeRune(buf, 'a')
+ }
+}
+
+func BenchmarkEncodeJapaneseRune(b *testing.B) {
+ buf := make([]byte, UTFMax)
+ for i := 0; i < b.N; i++ {
+ EncodeRune(buf, '本')
+ }
+}
+
+func BenchmarkDecodeASCIIRune(b *testing.B) {
+ a := []byte{'a'}
+ for i := 0; i < b.N; i++ {
+ DecodeRune(a)
+ }
+}
+
+func BenchmarkDecodeJapaneseRune(b *testing.B) {
+ nihon := []byte("本")
+ for i := 0; i < b.N; i++ {
+ DecodeRune(nihon)
+ }
+}
View Lorry Controller Status