diff options
| author | ian <ian@138bc75d-0d04-0410-961f-82ee72b054a4> | 2011-03-16 23:05:44 +0000 |
|---|---|---|
| committer | ian <ian@138bc75d-0d04-0410-961f-82ee72b054a4> | 2011-03-16 23:05:44 +0000 |
| commit | 31c6ec422702226aabab7d082da16663e6c3e72c (patch) | |
| tree | 44176975832a3faf1626836e70c97d5edd674122 /libgo | |
| parent | b3145af52cfb7c84d62a8e4ceeb165a7369718da (diff) | |
| download | gcc-31c6ec422702226aabab7d082da16663e6c3e72c.tar.gz | |
Update to current version of Go library (revision 94d654be2064).
git-svn-id: svn+ssh://gcc.gnu.org/svn/gcc/trunk@171076 138bc75d-0d04-0410-961f-82ee72b054a4
Diffstat (limited to 'libgo')
292 files changed, 16258 insertions, 4878 deletions
diff --git a/libgo/MERGE b/libgo/MERGE index 97e6655a625..e572b232450 100644 --- a/libgo/MERGE +++ b/libgo/MERGE @@ -1,4 +1,4 @@ -559f12e8fcd5 +94d654be2064 The first line of this file holds the Mercurial revision number of the last merge done from the master library sources. diff --git a/libgo/Makefile.am b/libgo/Makefile.am index e19d229a9e5..0715a9911bf 100644 --- a/libgo/Makefile.am +++ b/libgo/Makefile.am @@ -105,6 +105,7 @@ toolexeclibgo_DATA = \ bufio.gox \ bytes.gox \ cmath.gox \ + crypto.gox \ ebnf.gox \ exec.gox \ expvar.gox \ @@ -158,8 +159,10 @@ toolexeclibgoarchive_DATA = \ toolexeclibgocompressdir = $(toolexeclibgodir)/compress toolexeclibgocompress_DATA = \ + compress/bzip2.gox \ compress/flate.gox \ compress/gzip.gox \ + compress/lzw.gox \ compress/zlib.gox toolexeclibgocontainerdir = $(toolexeclibgodir)/container @@ -178,11 +181,13 @@ toolexeclibgocrypto_DATA = \ crypto/blowfish.gox \ crypto/cast5.gox \ crypto/cipher.gox \ + crypto/dsa.gox \ crypto/elliptic.gox \ crypto/hmac.gox \ crypto/md4.gox \ crypto/md5.gox \ crypto/ocsp.gox \ + crypto/openpgp.gox \ crypto/rand.gox \ crypto/rc4.gox \ crypto/ripemd160.gox \ @@ -201,6 +206,7 @@ toolexeclibgocryptoopenpgpdir = $(toolexeclibgocryptodir)/openpgp toolexeclibgocryptoopenpgp_DATA = \ crypto/openpgp/armor.gox \ crypto/openpgp/error.gox \ + crypto/openpgp/packet.gox \ crypto/openpgp/s2k.gox toolexeclibgodebugdir = $(toolexeclibgodir)/debug @@ -306,6 +312,11 @@ toolexeclibgoruntime_DATA = \ runtime/debug.gox \ runtime/pprof.gox +toolexeclibgosyncdir = $(toolexeclibgodir)/sync + +toolexeclibgosync_DATA = \ + sync/atomic.gox + toolexeclibgotestingdir = $(toolexeclibgodir)/testing toolexeclibgotesting_DATA = \ @@ -411,8 +422,6 @@ runtime_files = \ runtime/mfixalloc.c \ runtime/mgc0.c \ runtime/mheap.c \ - runtime/mheapmap32.c \ - runtime/mheapmap64.c \ runtime/msize.c \ runtime/proc.c \ runtime/thread.c \ @@ -489,6 +498,9 @@ go_cmath_files = \ go/cmath/sqrt.go \ go/cmath/tan.go +go_crypto_files = \ + go/crypto/crypto.go + go_ebnf_files = \ go/ebnf/ebnf.go \ go/ebnf/parser.go @@ -533,6 +545,7 @@ go_http_files = \ go/http/client.go \ go/http/dump.go \ go/http/fs.go \ + go/http/header.go \ go/http/lex.go \ go/http/persist.go \ go/http/request.go \ @@ -540,6 +553,7 @@ go_http_files = \ go/http/server.go \ go/http/status.go \ go/http/transfer.go \ + go/http/transport.go \ go/http/url.go go_image_files = \ @@ -693,6 +707,7 @@ go_os_files = \ go/os/env_unix.go \ go/os/error.go \ go/os/exec.go \ + go/os/exec_unix.go \ go/os/file.go \ go/os/file_unix.go \ go/os/getwd.go \ @@ -738,8 +753,6 @@ go_runtime_files = \ go/runtime/debug.go \ go/runtime/error.go \ go/runtime/extern.go \ - go/runtime/malloc_defs.go \ - go/runtime/runtime_defs.go \ go/runtime/sig.go \ go/runtime/softfloat64.go \ go/runtime/type.go \ @@ -781,11 +794,11 @@ go_strings_files = \ go/strings/strings.go go_sync_files = \ + go/sync/cond.go \ go/sync/mutex.go \ go/sync/once.go \ - go/sync/rwmutex.go -go_sync_c_files = \ - go/sync/cas.c + go/sync/rwmutex.go \ + go/sync/waitgroup.go if LIBGO_IS_SOLARIS go_syslog_file = go/syslog/syslog_solaris.go @@ -851,6 +864,12 @@ go_archive_zip_files = \ go/archive/zip/reader.go \ go/archive/zip/struct.go +go_compress_bzip2_files = \ + go/compress/bzip2/bit_reader.go \ + go/compress/bzip2/bzip2.go \ + go/compress/bzip2/huffman.go \ + go/compress/bzip2/move_to_front.go + go_compress_flate_files = \ go/compress/flate/deflate.go \ go/compress/flate/huffman_bit_writer.go \ @@ -864,6 +883,10 @@ go_compress_gzip_files = \ go/compress/gzip/gzip.go \ go/compress/gzip/gunzip.go +go_compress_lzw_files = \ + go/compress/lzw/reader.go \ + go/compress/lzw/writer.go + go_compress_zlib_files = \ go/compress/zlib/reader.go \ go/compress/zlib/writer.go @@ -911,6 +934,8 @@ go_crypto_cipher_files = \ go/crypto/cipher/io.go \ go/crypto/cipher/ocfb.go \ go/crypto/cipher/ofb.go +go_crypto_dsa_files = \ + go/crypto/dsa/dsa.go go_crypto_elliptic_files = \ go/crypto/elliptic/elliptic.go go_crypto_hmac_files = \ @@ -923,6 +948,11 @@ go_crypto_md5_files = \ go/crypto/md5/md5block.go go_crypto_ocsp_files = \ go/crypto/ocsp/ocsp.go +go_crypto_openpgp_files = \ + go/crypto/openpgp/canonical_text.go \ + go/crypto/openpgp/keys.go \ + go/crypto/openpgp/read.go \ + go/crypto/openpgp/write.go go_crypto_rand_files = \ go/crypto/rand/rand.go \ go/crypto/rand/rand_unix.go @@ -970,6 +1000,19 @@ go_crypto_openpgp_armor_files = \ go/crypto/openpgp/armor/encode.go go_crypto_openpgp_error_files = \ go/crypto/openpgp/error/error.go +go_crypto_openpgp_packet_files = \ + go/crypto/openpgp/packet/compressed.go \ + go/crypto/openpgp/packet/encrypted_key.go \ + go/crypto/openpgp/packet/literal.go \ + go/crypto/openpgp/packet/one_pass_signature.go \ + go/crypto/openpgp/packet/packet.go \ + go/crypto/openpgp/packet/private_key.go \ + go/crypto/openpgp/packet/public_key.go \ + go/crypto/openpgp/packet/reader.go \ + go/crypto/openpgp/packet/signature.go \ + go/crypto/openpgp/packet/symmetric_key_encrypted.go \ + go/crypto/openpgp/packet/symmetrically_encrypted.go \ + go/crypto/openpgp/packet/userid.go go_crypto_openpgp_s2k_files = \ go/crypto/openpgp/s2k/s2k.go @@ -1095,6 +1138,7 @@ go_net_dict_files = \ go/net/dict/dict.go go_net_textproto_files = \ + go/net/textproto/header.go \ go/net/textproto/pipeline.go \ go/net/textproto/reader.go \ go/net/textproto/textproto.go \ @@ -1116,6 +1160,11 @@ go_runtime_debug_files = \ go_runtime_pprof_files = \ go/runtime/pprof/pprof.go +go_sync_atomic_files = \ + go/sync/atomic/doc.go +go_sync_atomic_c_files = \ + go/sync/atomic/atomic.c + go_testing_iotest_files = \ go/testing/iotest/logger.go \ go/testing/iotest/reader.go \ @@ -1268,6 +1317,7 @@ libgo_go_objs = \ bytes/bytes.lo \ bytes/index.lo \ cmath/cmath.lo \ + crypto/crypto.lo \ ebnf/ebnf.lo \ exec/exec.lo \ expvar/expvar.lo \ @@ -1298,8 +1348,7 @@ libgo_go_objs = \ sort/sort.lo \ strconv/strconv.lo \ strings/strings.lo \ - sync/mutex.lo \ - sync/cas.lo \ + sync/sync.lo \ syslog/syslog.lo \ syslog/syslog_c.lo \ tabwriter/tabwriter.lo \ @@ -1313,8 +1362,10 @@ libgo_go_objs = \ xml/xml.lo \ archive/tar.lo \ archive/zip.lo \ + compress/bzip2.lo \ compress/flate.lo \ compress/gzip.lo \ + compress/lzw.lo \ compress/zlib.lo \ container/heap.lo \ container/list.lo \ @@ -1325,11 +1376,13 @@ libgo_go_objs = \ crypto/blowfish.lo \ crypto/cast5.lo \ crypto/cipher.lo \ + crypto/dsa.lo \ crypto/elliptic.lo \ crypto/hmac.lo \ crypto/md4.lo \ crypto/md5.lo \ crypto/ocsp.lo \ + crypto/openpgp.lo \ crypto/rand.lo \ crypto/rc4.lo \ crypto/ripemd160.lo \ @@ -1344,6 +1397,7 @@ libgo_go_objs = \ crypto/xtea.lo \ crypto/openpgp/armor.lo \ crypto/openpgp/error.lo \ + crypto/openpgp/packet.lo \ crypto/openpgp/s2k.lo \ debug/dwarf.lo \ debug/elf.lo \ @@ -1385,6 +1439,8 @@ libgo_go_objs = \ rpc/jsonrpc.lo \ runtime/debug.lo \ runtime/pprof.lo \ + sync/atomic.lo \ + sync/atomic_c.lo \ syscalls/syscall.lo \ syscalls/errno.lo \ testing/testing.lo \ @@ -1500,6 +1556,12 @@ cmath/check: $(CHECK_DEPS) $(CHECK) .PHONY: cmath/check +crypto/crypto.lo: $(go_crypto_files) hash.gox + $(BUILDPACKAGE) +crypto/check: $(CHECK_DEPS) + $(CHECK) +.PHONY: crypto/check + ebnf/ebnf.lo: $(go_ebnf_files) container/vector.gox go/scanner.gox \ go/token.gox os.gox strconv.gox unicode.gox utf8.gox $(BUILDPACKAGE) @@ -1507,7 +1569,7 @@ ebnf/check: $(CHECK_DEPS) $(CHECK) .PHONY: ebnf/check -exec/exec.lo: $(go_exec_files) os.gox strings.gox +exec/exec.lo: $(go_exec_files) os.gox strconv.gox strings.gox $(BUILDPACKAGE) exec/check: $(CHECK_DEPS) $(CHECK) @@ -1526,8 +1588,8 @@ flag/check: $(CHECK_DEPS) $(CHECK) .PHONY: flag/check -fmt/fmt.lo: $(go_fmt_files) bytes.gox io.gox os.gox reflect.gox strconv.gox \ - strings.gox unicode.gox utf8.gox +fmt/fmt.lo: $(go_fmt_files) bytes.gox io.gox math.gox os.gox reflect.gox \ + strconv.gox strings.gox unicode.gox utf8.gox $(BUILDPACKAGE) fmt/check: $(CHECK_DEPS) $(CHECK) @@ -1554,10 +1616,10 @@ html/check: $(CHECK_DEPS) $(CHECK) .PHONY: html/check -http/http.lo: $(go_http_files) bufio.gox bytes.gox container/list.gox \ - container/vector.gox crypto/rand.gox crypto/tls.gox \ - encoding/base64.gox fmt.gox io.gox io/ioutil.gox log.gox \ - mime.gox mime/multipart.gox net.gox os.gox path.gox sort.gox \ +http/http.lo: $(go_http_files) bufio.gox bytes.gox container/vector.gox \ + crypto/rand.gox crypto/tls.gox encoding/base64.gox fmt.gox \ + io.gox io/ioutil.gox log.gox mime.gox mime/multipart.gox \ + net.gox net/textproto.gox os.gox path.gox sort.gox \ strconv.gox strings.gox sync.gox time.gox utf8.gox $(BUILDPACKAGE) http/check: $(CHECK_DEPS) @@ -1576,9 +1638,10 @@ io/check: $(CHECK_DEPS) $(CHECK) .PHONY: io/check -json/json.lo: $(go_json_files) bytes.gox container/vector.gox fmt.gox io.gox \ - math.gox os.gox reflect.gox runtime.gox strconv.gox \ - strings.gox unicode.gox utf16.gox utf8.gox +json/json.lo: $(go_json_files) bytes.gox container/vector.gox \ + encoding/base64.gox fmt.gox io.gox math.gox os.gox \ + reflect.gox runtime.gox strconv.gox strings.gox unicode.gox \ + utf16.gox utf8.gox $(BUILDPACKAGE) json/check: $(CHECK_DEPS) $(CHECK) @@ -1611,14 +1674,14 @@ net/check: $(CHECK_DEPS) $(CHECK) .PHONY: net/check -netchan/netchan.lo: $(go_netchan_files) gob.gox log.gox net.gox os.gox \ +netchan/netchan.lo: $(go_netchan_files) gob.gox io.gox log.gox net.gox os.gox \ reflect.gox strconv.gox sync.gox time.gox $(BUILDPACKAGE) netchan/check: $(CHECK_DEPS) $(CHECK) .PHONY: netchan/check -os/os.lo: $(go_os_files) sync.gox syscall.gox +os/os.lo: $(go_os_files) runtime.gox sync.gox syscall.gox $(BUILDPACKAGE) os/check: $(CHECK_DEPS) $(CHECK) @@ -1706,10 +1769,8 @@ strings/check: $(CHECK_DEPS) $(CHECK) .PHONY: strings/check -sync/mutex.lo: $(go_sync_files) runtime.gox +sync/sync.lo: $(go_sync_files) runtime.gox sync/atomic.gox $(BUILDPACKAGE) -sync/cas.lo: $(go_sync_c_files) sync/mutex.lo - $(LTCOMPILE) -c -o sync/cas.lo $(srcdir)/go/sync/cas.c sync/check: $(CHECK_DEPS) $(CHECK) .PHONY: sync/check @@ -1806,6 +1867,13 @@ archive/zip/check: $(CHECK_DEPS) $(CHECK) .PHONY: archive/zip/check +compress/bzip2.lo: $(go_compress_bzip2_files) bufio.gox io.gox os.gox sort.gox + $(BUILDPACKAGE) +compress/bzip2/check: $(CHECK_DEPS) + @$(MKDIR_P) compress/bzip2 + $(CHECK) +.PHONY: compress/bzip2/check + compress/flate.lo: $(go_compress_flate_files) bufio.gox io.gox math.gox \ os.gox sort.gox strconv.gox $(BUILDPACKAGE) @@ -1822,6 +1890,13 @@ compress/gzip/check: $(CHECK_DEPS) $(CHECK) .PHONY: compress/gzip/check +compress/lzw.lo: $(go_compress_lzw_files) bufio.gox fmt.gox io.gox os.gox + $(BUILDPACKAGE) +compress/lzw/check: $(CHECK_DEPS) + @$(MKDIR_P) compress/lzw + $(CHECK) +.PHONY: compress/lzw/check + compress/zlib.lo: $(go_compress_zlib_files) bufio.gox compress/flate.gox \ hash.gox hash/adler32.gox io.gox os.gox $(BUILDPACKAGE) @@ -1893,6 +1968,13 @@ crypto/cipher/check: $(CHECK_DEPS) $(CHECK) .PHONY: crypto/cipher/check +crypto/dsa.lo: $(go_crypto_dsa_files) big.gox io.gox os.gox + $(BUILDPACKAGE) +crypto/dsa/check: $(CHECK_DEPS) + @$(MKDIR_P) crypto/dsa + $(CHECK) +.PHONY: crypto/dsa/check + crypto/elliptic.lo: $(go_crypto_elliptic_files) big.gox io.gox os.gox sync.gox $(BUILDPACKAGE) crypto/elliptic/check: $(CHECK_DEPS) @@ -1908,21 +1990,21 @@ crypto/hmac/check: $(CHECK_DEPS) $(CHECK) .PHONY: crypto/hmac/check -crypto/md4.lo: $(go_crypto_md4_files) hash.gox os.gox +crypto/md4.lo: $(go_crypto_md4_files) crypto.gox hash.gox os.gox $(BUILDPACKAGE) crypto/md4/check: $(CHECK_DEPS) @$(MKDIR_P) crypto/md4 $(CHECK) .PHONY: crypto/md4/check -crypto/md5.lo: $(go_crypto_md5_files) hash.gox os.gox +crypto/md5.lo: $(go_crypto_md5_files) crypto.gox hash.gox os.gox $(BUILDPACKAGE) crypto/md5/check: $(CHECK_DEPS) @$(MKDIR_P) crypto/md5 $(CHECK) .PHONY: crypto/md5/check -crypto/ocsp.lo: $(go_crypto_ocsp_files) asn1.gox crypto/rsa.gox \ +crypto/ocsp.lo: $(go_crypto_ocsp_files) asn1.gox crypto.gox crypto/rsa.gox \ crypto/sha1.gox crypto/x509.gox os.gox time.gox $(BUILDPACKAGE) crypto/ocsp/check: $(CHECK_DEPS) @@ -1930,8 +2012,18 @@ crypto/ocsp/check: $(CHECK_DEPS) $(CHECK) .PHONY: crypto/ocsp/check -crypto/rand.lo: $(go_crypto_rand_files) crypto/aes.gox io.gox os.gox sync.gox \ - time.gox +crypto/openpgp.lo: $(go_crypto_openpgp_files) crypto.gox \ + crypto/openpgp/armor.gox crypto/openpgp/error.gox \ + crypto/openpgp/packet.gox crypto/rsa.gox crypto/sha256.gox \ + hash.gox io.gox os.gox strconv.gox time.gox + $(BUILDPACKAGE) +crypto/openpgp/check: $(CHECK_DEPS) + @$(MKDIR_P) crypto/openpgp + $(CHECK) +.PHONY: crypto/openpgp/check + +crypto/rand.lo: $(go_crypto_rand_files) bufio.gox crypto/aes.gox io.gox \ + os.gox sync.gox time.gox $(BUILDPACKAGE) crypto/rand/check: $(CHECK_DEPS) @$(MKDIR_P) crypto/rand @@ -1945,14 +2037,14 @@ crypto/rc4/check: $(CHECK_DEPS) $(CHECK) .PHONY: crypto/rc4/check -crypto/ripemd160.lo: $(go_crypto_ripemd160_files) hash.gox os.gox +crypto/ripemd160.lo: $(go_crypto_ripemd160_files) crypto.gox hash.gox os.gox $(BUILDPACKAGE) crypto/ripemd160/check: $(CHECK_DEPS) @$(MKDIR_P) crypto/ripemd160 $(CHECK) .PHONY: crypto/ripemd160/check -crypto/rsa.lo: $(go_crypto_rsa_files) big.gox crypto/sha1.gox \ +crypto/rsa.lo: $(go_crypto_rsa_files) big.gox crypto.gox crypto/sha1.gox \ crypto/subtle.gox encoding/hex.gox hash.gox io.gox os.gox $(BUILDPACKAGE) crypto/rsa/check: $(CHECK_DEPS) @@ -1960,21 +2052,21 @@ crypto/rsa/check: $(CHECK_DEPS) $(CHECK) .PHONY: crypto/rsa/check -crypto/sha1.lo: $(go_crypto_sha1_files) hash.gox os.gox +crypto/sha1.lo: $(go_crypto_sha1_files) crypto.gox hash.gox os.gox $(BUILDPACKAGE) crypto/sha1/check: $(CHECK_DEPS) @$(MKDIR_P) crypto/sha1 $(CHECK) .PHONY: crypto/sha1/check -crypto/sha256.lo: $(go_crypto_sha256_files) hash.gox os.gox +crypto/sha256.lo: $(go_crypto_sha256_files) crypto.gox hash.gox os.gox $(BUILDPACKAGE) crypto/sha256/check: $(CHECK_DEPS) @$(MKDIR_P) crypto/sha256 $(CHECK) .PHONY: crypto/sha256/check -crypto/sha512.lo: $(go_crypto_sha512_files) hash.gox os.gox +crypto/sha512.lo: $(go_crypto_sha512_files) crypto.gox hash.gox os.gox $(BUILDPACKAGE) crypto/sha512/check: $(CHECK_DEPS) @$(MKDIR_P) crypto/sha512 @@ -1989,7 +2081,7 @@ crypto/subtle/check: $(CHECK_DEPS) .PHONY: crypto/subtle/check crypto/tls.lo: $(go_crypto_tls_files) big.gox bufio.gox bytes.gox \ - container/list.gox crypto/aes.gox crypto/cipher.gox \ + container/list.gox crypto.gox crypto/aes.gox crypto/cipher.gox \ crypto/elliptic.gox crypto/hmac.gox crypto/md5.gox \ crypto/rc4.gox crypto/rand.gox crypto/rsa.gox crypto/sha1.gox \ crypto/subtle.gox crypto/rsa.gox crypto/sha1.gox \ @@ -2009,8 +2101,8 @@ crypto/twofish/check: $(CHECK_DEPS) .PHONY: crypto/twofish/check crypto/x509.lo: $(go_crypto_x509_files) asn1.gox big.gox container/vector.gox \ - crypto/rsa.gox crypto/sha1.gox hash.gox os.gox strings.gox \ - time.gox + crypto.gox crypto/rsa.gox crypto/sha1.gox hash.gox os.gox \ + strings.gox time.gox $(BUILDPACKAGE) crypto/x509/check: $(CHECK_DEPS) @$(MKDIR_P) crypto/x509 @@ -2033,16 +2125,30 @@ crypto/openpgp/armor/check: $(CHECK_DEPS) $(CHECK) .PHONY: crypto/openpgp/armor/check -crypto/openpgp/error.lo: $(go_crypto_openpgp_error_files) +crypto/openpgp/error.lo: $(go_crypto_openpgp_error_files) strconv.gox $(BUILDPACKAGE) crypto/openpgp/error/check: $(CHECK_DEPS) @$(MKDIR_P) crypto/openpgp/error $(CHECK) .PHONY: crypto/openpgp/error/check -crypto/openpgp/s2k.lo: $(go_crypto_openpgp_s2k_files) crypto/md5.gox \ - crypto/openpgp/error.gox crypto/ripemd160.gox crypto/sha1.gox \ - crypto/sha256.gox crypto/sha512.gox hash.gox io.gox os.gox +crypto/openpgp/packet.lo: $(go_crypto_openpgp_packet_files) big.gox bytes.gox \ + compress/flate.gox compress/zlib.gox crypto.gox \ + crypto/aes.gox crypto/cast5.gox crypto/cipher.gox \ + crypto/openpgp/error.gox crypto/openpgp/s2k.gox \ + crypto/rand.gox crypto/rsa.gox crypto/sha1.gox \ + crypto/subtle.gox encoding/binary.gox hash.gox io.gox \ + io/ioutil.gox os.gox strconv.gox strings.gox + $(BUILDPACKAGE) +crypto/openpgp/packet/check: $(CHECK_DEPS) + @$(MKDIR_P) crypto/openpgp/packet + $(CHECK) +.PHONY: crypto/openpgp/packet/check + +crypto/openpgp/s2k.lo: $(go_crypto_openpgp_s2k_files) crypto.gox \ + crypto/md5.gox crypto/openpgp/error.gox crypto/ripemd160.gox \ + crypto/sha1.gox crypto/sha256.gox crypto/sha512.gox hash.gox \ + io.gox os.gox $(BUILDPACKAGE) crypto/openpgp/s2k/check: $(CHECK_DEPS) @$(MKDIR_P) crypto/openpgp/s2k @@ -2361,6 +2467,15 @@ runtime/pprof/check: $(CHECK_DEPS) $(CHECK) .PHONY: runtime/pprof/check +sync/atomic.lo: $(go_sync_atomic_files) + $(BUILDPACKAGE) +sync/atomic_c.lo: $(go_sync_atomic_c_files) sync/atomic.lo + $(LTCOMPILE) -c -o $@ $(srcdir)/go/sync/atomic/atomic.c +sync/atomic/check: $(CHECK_DEPS) + @$(MKDIR_P) sync/atomic + $(CHECK) +.PHONY: sync/atomic/check + testing/iotest.lo: $(go_testing_iotest_files) io.gox log.gox os.gox $(BUILDPACKAGE) testing/iotest/check: $(CHECK_DEPS) @@ -2410,6 +2525,8 @@ bytes.gox: bytes/bytes.lo $(BUILDGOX) cmath.gox: cmath/cmath.lo $(BUILDGOX) +crypto.gox: crypto/crypto.lo + $(BUILDGOX) ebnf.gox: ebnf/ebnf.lo $(BUILDGOX) exec.gox: exec/exec.lo @@ -2470,7 +2587,7 @@ strconv.gox: strconv/strconv.lo $(BUILDGOX) strings.gox: strings/strings.lo $(BUILDGOX) -sync.gox: sync/mutex.lo +sync.gox: sync/sync.lo $(BUILDGOX) syslog.gox: syslog/syslog.lo $(BUILDGOX) @@ -2502,10 +2619,14 @@ archive/tar.gox: archive/tar.lo archive/zip.gox: archive/zip.lo $(BUILDGOX) +compress/bzip2.gox: compress/bzip2.lo + $(BUILDGOX) compress/flate.gox: compress/flate.lo $(BUILDGOX) compress/gzip.gox: compress/gzip.lo $(BUILDGOX) +compress/lzw.gox: compress/lzw.lo + $(BUILDGOX) compress/zlib.gox: compress/zlib.lo $(BUILDGOX) @@ -2528,6 +2649,8 @@ crypto/cast5.gox: crypto/cast5.lo $(BUILDGOX) crypto/cipher.gox: crypto/cipher.lo $(BUILDGOX) +crypto/dsa.gox: crypto/dsa.lo + $(BUILDGOX) crypto/elliptic.gox: crypto/elliptic.lo $(BUILDGOX) crypto/hmac.gox: crypto/hmac.lo @@ -2538,6 +2661,8 @@ crypto/md5.gox: crypto/md5.lo $(BUILDGOX) crypto/ocsp.gox: crypto/ocsp.lo $(BUILDGOX) +crypto/openpgp.gox: crypto/openpgp.lo + $(BUILDGOX) crypto/rand.gox: crypto/rand.lo $(BUILDGOX) crypto/rc4.gox: crypto/rc4.lo @@ -2567,6 +2692,8 @@ crypto/openpgp/armor.gox: crypto/openpgp/armor.lo $(BUILDGOX) crypto/openpgp/error.gox: crypto/openpgp/error.lo $(BUILDGOX) +crypto/openpgp/packet.gox: crypto/openpgp/packet.lo + $(BUILDGOX) crypto/openpgp/s2k.gox: crypto/openpgp/s2k.lo $(BUILDGOX) @@ -2664,6 +2791,9 @@ runtime/debug.gox: runtime/debug.lo runtime/pprof.gox: runtime/pprof.lo $(BUILDGOX) +sync/atomic.gox: sync/atomic.lo + $(BUILDGOX) + testing/iotest.gox: testing/iotest.lo $(BUILDGOX) testing/quick.gox: testing/quick.lo @@ -2725,8 +2855,10 @@ TEST_PACKAGES = \ xml/check \ archive/tar/check \ archive/zip/check \ + compress/bzip2/check \ compress/flate/check \ compress/gzip/check \ + compress/lzw/check \ compress/zlib/check \ container/heap/check \ container/list/check \ @@ -2737,11 +2869,13 @@ TEST_PACKAGES = \ crypto/blowfish/check \ crypto/cast5/check \ crypto/cipher/check \ + crypto/dsa/check \ crypto/elliptic/check \ crypto/hmac/check \ crypto/md4/check \ crypto/md5/check \ crypto/ocsp/check \ + crypto/openpgp/check \ crypto/rand/check \ crypto/rc4/check \ crypto/ripemd160/check \ @@ -2755,6 +2889,7 @@ TEST_PACKAGES = \ crypto/x509/check \ crypto/xtea/check \ crypto/openpgp/armor/check \ + crypto/openpgp/packet/check \ crypto/openpgp/s2k/check \ debug/dwarf/check \ debug/elf/check \ @@ -2787,6 +2922,7 @@ TEST_PACKAGES = \ $(os_inotify_check) \ os/signal/check \ rpc/jsonrpc/check \ + sync/atomic/check \ testing/quick/check \ testing/script/check diff --git a/libgo/Makefile.in b/libgo/Makefile.in index 452c60857ff..c9f01879189 100644 --- a/libgo/Makefile.in +++ b/libgo/Makefile.in @@ -112,6 +112,7 @@ am__installdirs = "$(DESTDIR)$(toolexeclibdir)" \ "$(DESTDIR)$(toolexeclibgoosdir)" \ "$(DESTDIR)$(toolexeclibgorpcdir)" \ "$(DESTDIR)$(toolexeclibgoruntimedir)" \ + "$(DESTDIR)$(toolexeclibgosyncdir)" \ "$(DESTDIR)$(toolexeclibgotestingdir)" LIBRARIES = $(toolexeclib_LIBRARIES) ARFLAGS = cru @@ -122,43 +123,45 @@ libgobegin_a_OBJECTS = $(am_libgobegin_a_OBJECTS) LTLIBRARIES = $(toolexeclib_LTLIBRARIES) am__DEPENDENCIES_1 = am__DEPENDENCIES_2 = asn1/asn1.lo big/big.lo bufio/bufio.lo \ - bytes/bytes.lo bytes/index.lo cmath/cmath.lo ebnf/ebnf.lo \ - exec/exec.lo expvar/expvar.lo flag/flag.lo fmt/fmt.lo \ - gob/gob.lo hash/hash.lo html/html.lo http/http.lo \ + bytes/bytes.lo bytes/index.lo cmath/cmath.lo crypto/crypto.lo \ + ebnf/ebnf.lo exec/exec.lo expvar/expvar.lo flag/flag.lo \ + fmt/fmt.lo gob/gob.lo hash/hash.lo html/html.lo http/http.lo \ image/image.lo io/io.lo json/json.lo log/log.lo math/math.lo \ mime/mime.lo net/net.lo netchan/netchan.lo os/os.lo \ patch/patch.lo path/path.lo rand/rand.lo reflect/reflect.lo \ regexp/regexp.lo rpc/rpc.lo runtime/runtime.lo \ scanner/scanner.lo smtp/smtp.lo sort/sort.lo \ - strconv/strconv.lo strings/strings.lo sync/mutex.lo \ - sync/cas.lo syslog/syslog.lo syslog/syslog_c.lo \ - tabwriter/tabwriter.lo template/template.lo time/time.lo \ - try/try.lo unicode/unicode.lo utf16/utf16.lo utf8/utf8.lo \ + strconv/strconv.lo strings/strings.lo sync/sync.lo \ + syslog/syslog.lo syslog/syslog_c.lo tabwriter/tabwriter.lo \ + template/template.lo time/time.lo try/try.lo \ + unicode/unicode.lo utf16/utf16.lo utf8/utf8.lo \ websocket/websocket.lo xml/xml.lo archive/tar.lo \ - archive/zip.lo compress/flate.lo compress/gzip.lo \ - compress/zlib.lo container/heap.lo container/list.lo \ - container/ring.lo container/vector.lo crypto/aes.lo \ - crypto/block.lo crypto/blowfish.lo crypto/cast5.lo \ - crypto/cipher.lo crypto/elliptic.lo crypto/hmac.lo \ - crypto/md4.lo crypto/md5.lo crypto/ocsp.lo crypto/rand.lo \ + archive/zip.lo compress/bzip2.lo compress/flate.lo \ + compress/gzip.lo compress/lzw.lo compress/zlib.lo \ + container/heap.lo container/list.lo container/ring.lo \ + container/vector.lo crypto/aes.lo crypto/block.lo \ + crypto/blowfish.lo crypto/cast5.lo crypto/cipher.lo \ + crypto/dsa.lo crypto/elliptic.lo crypto/hmac.lo crypto/md4.lo \ + crypto/md5.lo crypto/ocsp.lo crypto/openpgp.lo crypto/rand.lo \ crypto/rc4.lo crypto/ripemd160.lo crypto/rsa.lo crypto/sha1.lo \ crypto/sha256.lo crypto/sha512.lo crypto/subtle.lo \ crypto/tls.lo crypto/twofish.lo crypto/x509.lo crypto/xtea.lo \ crypto/openpgp/armor.lo crypto/openpgp/error.lo \ - crypto/openpgp/s2k.lo debug/dwarf.lo debug/elf.lo \ - debug/gosym.lo debug/macho.lo debug/pe.lo debug/proc.lo \ - encoding/ascii85.lo encoding/base32.lo encoding/base64.lo \ - encoding/binary.lo encoding/git85.lo encoding/hex.lo \ - encoding/line.lo encoding/pem.lo exp/datafmt.lo exp/draw.lo \ - exp/eval.lo go/ast.lo go/doc.lo go/parser.lo go/printer.lo \ - go/scanner.lo go/token.lo go/typechecker.lo hash/adler32.lo \ - hash/crc32.lo hash/crc64.lo http/pprof.lo image/jpeg.lo \ - image/png.lo index/suffixarray.lo io/ioutil.lo \ - mime/multipart.lo net/dict.lo net/textproto.lo \ + crypto/openpgp/packet.lo crypto/openpgp/s2k.lo debug/dwarf.lo \ + debug/elf.lo debug/gosym.lo debug/macho.lo debug/pe.lo \ + debug/proc.lo encoding/ascii85.lo encoding/base32.lo \ + encoding/base64.lo encoding/binary.lo encoding/git85.lo \ + encoding/hex.lo encoding/line.lo encoding/pem.lo \ + exp/datafmt.lo exp/draw.lo exp/eval.lo go/ast.lo go/doc.lo \ + go/parser.lo go/printer.lo go/scanner.lo go/token.lo \ + go/typechecker.lo hash/adler32.lo hash/crc32.lo hash/crc64.lo \ + http/pprof.lo image/jpeg.lo image/png.lo index/suffixarray.lo \ + io/ioutil.lo mime/multipart.lo net/dict.lo net/textproto.lo \ $(am__DEPENDENCIES_1) os/signal.lo rpc/jsonrpc.lo \ - runtime/debug.lo runtime/pprof.lo syscalls/syscall.lo \ - syscalls/errno.lo testing/testing.lo testing/iotest.lo \ - testing/quick.lo testing/script.lo + runtime/debug.lo runtime/pprof.lo sync/atomic.lo \ + sync/atomic_c.lo syscalls/syscall.lo syscalls/errno.lo \ + testing/testing.lo testing/iotest.lo testing/quick.lo \ + testing/script.lo libgo_la_DEPENDENCIES = $(am__DEPENDENCIES_2) $(am__DEPENDENCIES_1) \ $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) \ $(am__DEPENDENCIES_1) @@ -204,8 +207,7 @@ am__libgo_la_SOURCES_DIST = runtime/go-append.c runtime/go-assert.c \ runtime/mcache.c runtime/mcentral.c \ runtime/mem_posix_memalign.c runtime/mem.c runtime/mfinal.c \ runtime/mfixalloc.c runtime/mgc0.c runtime/mheap.c \ - runtime/mheapmap32.c runtime/mheapmap64.c runtime/msize.c \ - runtime/proc.c runtime/thread.c \ + runtime/msize.c runtime/proc.c runtime/thread.c \ runtime/rtems-task-variable-add.c chan.c iface.c malloc.c \ map.c mprof.c reflect.c sigqueue.c string.c @HAVE_SYS_MMAN_H_FALSE@am__objects_1 = mem_posix_memalign.lo @@ -237,9 +239,8 @@ am__objects_3 = go-append.lo go-assert.lo go-assert-interface.lo \ go-unreflect.lo go-unsafe-new.lo go-unsafe-newarray.lo \ go-unsafe-pointer.lo go-unwind.lo mcache.lo mcentral.lo \ $(am__objects_1) mfinal.lo mfixalloc.lo mgc0.lo mheap.lo \ - mheapmap32.lo mheapmap64.lo msize.lo proc.lo thread.lo \ - $(am__objects_2) chan.lo iface.lo malloc.lo map.lo mprof.lo \ - reflect.lo sigqueue.lo string.lo + msize.lo proc.lo thread.lo $(am__objects_2) chan.lo iface.lo \ + malloc.lo map.lo mprof.lo reflect.lo sigqueue.lo string.lo am_libgo_la_OBJECTS = $(am__objects_3) libgo_la_OBJECTS = $(am_libgo_la_OBJECTS) DEFAULT_INCLUDES = -I.@am__isrc@ @@ -280,7 +281,7 @@ DATA = $(toolexeclibgo_DATA) $(toolexeclibgoarchive_DATA) \ $(toolexeclibgoio_DATA) $(toolexeclibgomime_DATA) \ $(toolexeclibgonet_DATA) $(toolexeclibgoos_DATA) \ $(toolexeclibgorpc_DATA) $(toolexeclibgoruntime_DATA) \ - $(toolexeclibgotesting_DATA) + $(toolexeclibgosync_DATA) $(toolexeclibgotesting_DATA) RECURSIVE_CLEAN_TARGETS = mostlyclean-recursive clean-recursive \ distclean-recursive maintainer-clean-recursive AM_RECURSIVE_TARGETS = $(RECURSIVE_TARGETS:-recursive=) \ @@ -546,6 +547,7 @@ toolexeclibgo_DATA = \ bufio.gox \ bytes.gox \ cmath.gox \ + crypto.gox \ ebnf.gox \ exec.gox \ expvar.gox \ @@ -597,8 +599,10 @@ toolexeclibgoarchive_DATA = \ toolexeclibgocompressdir = $(toolexeclibgodir)/compress toolexeclibgocompress_DATA = \ + compress/bzip2.gox \ compress/flate.gox \ compress/gzip.gox \ + compress/lzw.gox \ compress/zlib.gox toolexeclibgocontainerdir = $(toolexeclibgodir)/container @@ -615,11 +619,13 @@ toolexeclibgocrypto_DATA = \ crypto/blowfish.gox \ crypto/cast5.gox \ crypto/cipher.gox \ + crypto/dsa.gox \ crypto/elliptic.gox \ crypto/hmac.gox \ crypto/md4.gox \ crypto/md5.gox \ crypto/ocsp.gox \ + crypto/openpgp.gox \ crypto/rand.gox \ crypto/rc4.gox \ crypto/ripemd160.gox \ @@ -637,6 +643,7 @@ toolexeclibgocryptoopenpgpdir = $(toolexeclibgocryptodir)/openpgp toolexeclibgocryptoopenpgp_DATA = \ crypto/openpgp/armor.gox \ crypto/openpgp/error.gox \ + crypto/openpgp/packet.gox \ crypto/openpgp/s2k.gox toolexeclibgodebugdir = $(toolexeclibgodir)/debug @@ -725,6 +732,10 @@ toolexeclibgoruntime_DATA = \ runtime/debug.gox \ runtime/pprof.gox +toolexeclibgosyncdir = $(toolexeclibgodir)/sync +toolexeclibgosync_DATA = \ + sync/atomic.gox + toolexeclibgotestingdir = $(toolexeclibgodir)/testing toolexeclibgotesting_DATA = \ testing/iotest.gox \ @@ -821,8 +832,6 @@ runtime_files = \ runtime/mfixalloc.c \ runtime/mgc0.c \ runtime/mheap.c \ - runtime/mheapmap32.c \ - runtime/mheapmap64.c \ runtime/msize.c \ runtime/proc.c \ runtime/thread.c \ @@ -874,6 +883,9 @@ go_cmath_files = \ go/cmath/sqrt.go \ go/cmath/tan.go +go_crypto_files = \ + go/crypto/crypto.go + go_ebnf_files = \ go/ebnf/ebnf.go \ go/ebnf/parser.go @@ -918,6 +930,7 @@ go_http_files = \ go/http/client.go \ go/http/dump.go \ go/http/fs.go \ + go/http/header.go \ go/http/lex.go \ go/http/persist.go \ go/http/request.go \ @@ -925,6 +938,7 @@ go_http_files = \ go/http/server.go \ go/http/status.go \ go/http/transfer.go \ + go/http/transport.go \ go/http/url.go go_image_files = \ @@ -1051,6 +1065,7 @@ go_os_files = \ go/os/env_unix.go \ go/os/error.go \ go/os/exec.go \ + go/os/exec_unix.go \ go/os/file.go \ go/os/file_unix.go \ go/os/getwd.go \ @@ -1096,8 +1111,6 @@ go_runtime_files = \ go/runtime/debug.go \ go/runtime/error.go \ go/runtime/extern.go \ - go/runtime/malloc_defs.go \ - go/runtime/runtime_defs.go \ go/runtime/sig.go \ go/runtime/softfloat64.go \ go/runtime/type.go \ @@ -1128,12 +1141,11 @@ go_strings_files = \ go/strings/strings.go go_sync_files = \ + go/sync/cond.go \ go/sync/mutex.go \ go/sync/once.go \ - go/sync/rwmutex.go - -go_sync_c_files = \ - go/sync/cas.c + go/sync/rwmutex.go \ + go/sync/waitgroup.go @LIBGO_IS_SOLARIS_FALSE@go_syslog_file = go/syslog/syslog_unix.go @LIBGO_IS_SOLARIS_TRUE@go_syslog_file = go/syslog/syslog_solaris.go @@ -1196,6 +1208,12 @@ go_archive_zip_files = \ go/archive/zip/reader.go \ go/archive/zip/struct.go +go_compress_bzip2_files = \ + go/compress/bzip2/bit_reader.go \ + go/compress/bzip2/bzip2.go \ + go/compress/bzip2/huffman.go \ + go/compress/bzip2/move_to_front.go + go_compress_flate_files = \ go/compress/flate/deflate.go \ go/compress/flate/huffman_bit_writer.go \ @@ -1209,6 +1227,10 @@ go_compress_gzip_files = \ go/compress/gzip/gzip.go \ go/compress/gzip/gunzip.go +go_compress_lzw_files = \ + go/compress/lzw/reader.go \ + go/compress/lzw/writer.go + go_compress_zlib_files = \ go/compress/zlib/reader.go \ go/compress/zlib/writer.go @@ -1261,6 +1283,9 @@ go_crypto_cipher_files = \ go/crypto/cipher/ocfb.go \ go/crypto/cipher/ofb.go +go_crypto_dsa_files = \ + go/crypto/dsa/dsa.go + go_crypto_elliptic_files = \ go/crypto/elliptic/elliptic.go @@ -1278,6 +1303,12 @@ go_crypto_md5_files = \ go_crypto_ocsp_files = \ go/crypto/ocsp/ocsp.go +go_crypto_openpgp_files = \ + go/crypto/openpgp/canonical_text.go \ + go/crypto/openpgp/keys.go \ + go/crypto/openpgp/read.go \ + go/crypto/openpgp/write.go + go_crypto_rand_files = \ go/crypto/rand/rand.go \ go/crypto/rand/rand_unix.go @@ -1338,6 +1369,20 @@ go_crypto_openpgp_armor_files = \ go_crypto_openpgp_error_files = \ go/crypto/openpgp/error/error.go +go_crypto_openpgp_packet_files = \ + go/crypto/openpgp/packet/compressed.go \ + go/crypto/openpgp/packet/encrypted_key.go \ + go/crypto/openpgp/packet/literal.go \ + go/crypto/openpgp/packet/one_pass_signature.go \ + go/crypto/openpgp/packet/packet.go \ + go/crypto/openpgp/packet/private_key.go \ + go/crypto/openpgp/packet/public_key.go \ + go/crypto/openpgp/packet/reader.go \ + go/crypto/openpgp/packet/signature.go \ + go/crypto/openpgp/packet/symmetric_key_encrypted.go \ + go/crypto/openpgp/packet/symmetrically_encrypted.go \ + go/crypto/openpgp/packet/userid.go + go_crypto_openpgp_s2k_files = \ go/crypto/openpgp/s2k/s2k.go @@ -1484,6 +1529,7 @@ go_net_dict_files = \ go/net/dict/dict.go go_net_textproto_files = \ + go/net/textproto/header.go \ go/net/textproto/pipeline.go \ go/net/textproto/reader.go \ go/net/textproto/textproto.go \ @@ -1506,6 +1552,12 @@ go_runtime_debug_files = \ go_runtime_pprof_files = \ go/runtime/pprof/pprof.go +go_sync_atomic_files = \ + go/sync/atomic/doc.go + +go_sync_atomic_c_files = \ + go/sync/atomic/atomic.c + go_testing_iotest_files = \ go/testing/iotest/logger.go \ go/testing/iotest/reader.go \ @@ -1606,6 +1658,7 @@ libgo_go_objs = \ bytes/bytes.lo \ bytes/index.lo \ cmath/cmath.lo \ + crypto/crypto.lo \ ebnf/ebnf.lo \ exec/exec.lo \ expvar/expvar.lo \ @@ -1636,8 +1689,7 @@ libgo_go_objs = \ sort/sort.lo \ strconv/strconv.lo \ strings/strings.lo \ - sync/mutex.lo \ - sync/cas.lo \ + sync/sync.lo \ syslog/syslog.lo \ syslog/syslog_c.lo \ tabwriter/tabwriter.lo \ @@ -1651,8 +1703,10 @@ libgo_go_objs = \ xml/xml.lo \ archive/tar.lo \ archive/zip.lo \ + compress/bzip2.lo \ compress/flate.lo \ compress/gzip.lo \ + compress/lzw.lo \ compress/zlib.lo \ container/heap.lo \ container/list.lo \ @@ -1663,11 +1717,13 @@ libgo_go_objs = \ crypto/blowfish.lo \ crypto/cast5.lo \ crypto/cipher.lo \ + crypto/dsa.lo \ crypto/elliptic.lo \ crypto/hmac.lo \ crypto/md4.lo \ crypto/md5.lo \ crypto/ocsp.lo \ + crypto/openpgp.lo \ crypto/rand.lo \ crypto/rc4.lo \ crypto/ripemd160.lo \ @@ -1682,6 +1738,7 @@ libgo_go_objs = \ crypto/xtea.lo \ crypto/openpgp/armor.lo \ crypto/openpgp/error.lo \ + crypto/openpgp/packet.lo \ crypto/openpgp/s2k.lo \ debug/dwarf.lo \ debug/elf.lo \ @@ -1723,6 +1780,8 @@ libgo_go_objs = \ rpc/jsonrpc.lo \ runtime/debug.lo \ runtime/pprof.lo \ + sync/atomic.lo \ + sync/atomic_c.lo \ syscalls/syscall.lo \ syscalls/errno.lo \ testing/testing.lo \ @@ -1857,8 +1916,10 @@ TEST_PACKAGES = \ xml/check \ archive/tar/check \ archive/zip/check \ + compress/bzip2/check \ compress/flate/check \ compress/gzip/check \ + compress/lzw/check \ compress/zlib/check \ container/heap/check \ container/list/check \ @@ -1869,11 +1930,13 @@ TEST_PACKAGES = \ crypto/blowfish/check \ crypto/cast5/check \ crypto/cipher/check \ + crypto/dsa/check \ crypto/elliptic/check \ crypto/hmac/check \ crypto/md4/check \ crypto/md5/check \ crypto/ocsp/check \ + crypto/openpgp/check \ crypto/rand/check \ crypto/rc4/check \ crypto/ripemd160/check \ @@ -1887,6 +1950,7 @@ TEST_PACKAGES = \ crypto/x509/check \ crypto/xtea/check \ crypto/openpgp/armor/check \ + crypto/openpgp/packet/check \ crypto/openpgp/s2k/check \ debug/dwarf/check \ debug/elf/check \ @@ -1919,6 +1983,7 @@ TEST_PACKAGES = \ $(os_inotify_check) \ os/signal/check \ rpc/jsonrpc/check \ + sync/atomic/check \ testing/quick/check \ testing/script/check @@ -2146,8 +2211,6 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mfixalloc.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mgc0.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mheap.Plo@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mheapmap32.Plo@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mheapmap64.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mprof.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/msize.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/proc.Plo@am__quote@ @@ -2794,20 +2857,6 @@ mheap.lo: runtime/mheap.c @AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ @am__fastdepCC_FALSE@ $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o mheap.lo `test -f 'runtime/mheap.c' || echo '$(srcdir)/'`runtime/mheap.c -mheapmap32.lo: runtime/mheapmap32.c -@am__fastdepCC_TRUE@ $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT mheapmap32.lo -MD -MP -MF $(DEPDIR)/mheapmap32.Tpo -c -o mheapmap32.lo `test -f 'runtime/mheapmap32.c' || echo '$(srcdir)/'`runtime/mheapmap32.c -@am__fastdepCC_TRUE@ $(am__mv) $(DEPDIR)/mheapmap32.Tpo $(DEPDIR)/mheapmap32.Plo -@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='runtime/mheapmap32.c' object='mheapmap32.lo' libtool=yes @AMDEPBACKSLASH@ -@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ -@am__fastdepCC_FALSE@ $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o mheapmap32.lo `test -f 'runtime/mheapmap32.c' || echo '$(srcdir)/'`runtime/mheapmap32.c - -mheapmap64.lo: runtime/mheapmap64.c -@am__fastdepCC_TRUE@ $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT mheapmap64.lo -MD -MP -MF $(DEPDIR)/mheapmap64.Tpo -c -o mheapmap64.lo `test -f 'runtime/mheapmap64.c' || echo '$(srcdir)/'`runtime/mheapmap64.c -@am__fastdepCC_TRUE@ $(am__mv) $(DEPDIR)/mheapmap64.Tpo $(DEPDIR)/mheapmap64.Plo -@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='runtime/mheapmap64.c' object='mheapmap64.lo' libtool=yes @AMDEPBACKSLASH@ -@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ -@am__fastdepCC_FALSE@ $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o mheapmap64.lo `test -f 'runtime/mheapmap64.c' || echo '$(srcdir)/'`runtime/mheapmap64.c - msize.lo: runtime/msize.c @am__fastdepCC_TRUE@ $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT msize.lo -MD -MP -MF $(DEPDIR)/msize.Tpo -c -o msize.lo `test -f 'runtime/msize.c' || echo '$(srcdir)/'`runtime/msize.c @am__fastdepCC_TRUE@ $(am__mv) $(DEPDIR)/msize.Tpo $(DEPDIR)/msize.Plo @@ -3261,6 +3310,26 @@ uninstall-toolexeclibgoruntimeDATA: test -n "$$files" || exit 0; \ echo " ( cd '$(DESTDIR)$(toolexeclibgoruntimedir)' && rm -f" $$files ")"; \ cd "$(DESTDIR)$(toolexeclibgoruntimedir)" && rm -f $$files +install-toolexeclibgosyncDATA: $(toolexeclibgosync_DATA) + @$(NORMAL_INSTALL) + test -z "$(toolexeclibgosyncdir)" || $(MKDIR_P) "$(DESTDIR)$(toolexeclibgosyncdir)" + @list='$(toolexeclibgosync_DATA)'; test -n "$(toolexeclibgosyncdir)" || list=; \ + for p in $$list; do \ + if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ + echo "$$d$$p"; \ + done | $(am__base_list) | \ + while read files; do \ + echo " $(INSTALL_DATA) $$files '$(DESTDIR)$(toolexeclibgosyncdir)'"; \ + $(INSTALL_DATA) $$files "$(DESTDIR)$(toolexeclibgosyncdir)" || exit $$?; \ + done + +uninstall-toolexeclibgosyncDATA: + @$(NORMAL_UNINSTALL) + @list='$(toolexeclibgosync_DATA)'; test -n "$(toolexeclibgosyncdir)" || list=; \ + files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \ + test -n "$$files" || exit 0; \ + echo " ( cd '$(DESTDIR)$(toolexeclibgosyncdir)' && rm -f" $$files ")"; \ + cd "$(DESTDIR)$(toolexeclibgosyncdir)" && rm -f $$files install-toolexeclibgotestingDATA: $(toolexeclibgotesting_DATA) @$(NORMAL_INSTALL) test -z "$(toolexeclibgotestingdir)" || $(MKDIR_P) "$(DESTDIR)$(toolexeclibgotestingdir)" @@ -3598,7 +3667,7 @@ all-am: Makefile $(LIBRARIES) $(LTLIBRARIES) all-multi $(DATA) \ config.h installdirs: installdirs-recursive installdirs-am: - for dir in "$(DESTDIR)$(toolexeclibdir)" "$(DESTDIR)$(toolexeclibdir)" "$(DESTDIR)$(toolexeclibgodir)" "$(DESTDIR)$(toolexeclibgoarchivedir)" "$(DESTDIR)$(toolexeclibgocompressdir)" "$(DESTDIR)$(toolexeclibgocontainerdir)" "$(DESTDIR)$(toolexeclibgocryptodir)" "$(DESTDIR)$(toolexeclibgocryptoopenpgpdir)" "$(DESTDIR)$(toolexeclibgodebugdir)" "$(DESTDIR)$(toolexeclibgoencodingdir)" "$(DESTDIR)$(toolexeclibgoexpdir)" "$(DESTDIR)$(toolexeclibgogodir)" "$(DESTDIR)$(toolexeclibgohashdir)" "$(DESTDIR)$(toolexeclibgohttpdir)" "$(DESTDIR)$(toolexeclibgoimagedir)" "$(DESTDIR)$(toolexeclibgoindexdir)" "$(DESTDIR)$(toolexeclibgoiodir)" "$(DESTDIR)$(toolexeclibgomimedir)" "$(DESTDIR)$(toolexeclibgonetdir)" "$(DESTDIR)$(toolexeclibgoosdir)" "$(DESTDIR)$(toolexeclibgorpcdir)" "$(DESTDIR)$(toolexeclibgoruntimedir)" "$(DESTDIR)$(toolexeclibgotestingdir)"; do \ + for dir in "$(DESTDIR)$(toolexeclibdir)" "$(DESTDIR)$(toolexeclibdir)" "$(DESTDIR)$(toolexeclibgodir)" "$(DESTDIR)$(toolexeclibgoarchivedir)" "$(DESTDIR)$(toolexeclibgocompressdir)" "$(DESTDIR)$(toolexeclibgocontainerdir)" "$(DESTDIR)$(toolexeclibgocryptodir)" "$(DESTDIR)$(toolexeclibgocryptoopenpgpdir)" "$(DESTDIR)$(toolexeclibgodebugdir)" "$(DESTDIR)$(toolexeclibgoencodingdir)" "$(DESTDIR)$(toolexeclibgoexpdir)" "$(DESTDIR)$(toolexeclibgogodir)" "$(DESTDIR)$(toolexeclibgohashdir)" "$(DESTDIR)$(toolexeclibgohttpdir)" "$(DESTDIR)$(toolexeclibgoimagedir)" "$(DESTDIR)$(toolexeclibgoindexdir)" "$(DESTDIR)$(toolexeclibgoiodir)" "$(DESTDIR)$(toolexeclibgomimedir)" "$(DESTDIR)$(toolexeclibgonetdir)" "$(DESTDIR)$(toolexeclibgoosdir)" "$(DESTDIR)$(toolexeclibgorpcdir)" "$(DESTDIR)$(toolexeclibgoruntimedir)" "$(DESTDIR)$(toolexeclibgosyncdir)" "$(DESTDIR)$(toolexeclibgotestingdir)"; do \ test -z "$$dir" || $(MKDIR_P) "$$dir"; \ done install: install-recursive @@ -3672,7 +3741,7 @@ install-exec-am: install-multi install-toolexeclibLIBRARIES \ install-toolexeclibgoindexDATA install-toolexeclibgoioDATA \ install-toolexeclibgomimeDATA install-toolexeclibgonetDATA \ install-toolexeclibgoosDATA install-toolexeclibgorpcDATA \ - install-toolexeclibgoruntimeDATA \ + install-toolexeclibgoruntimeDATA install-toolexeclibgosyncDATA \ install-toolexeclibgotestingDATA install-html: install-html-recursive @@ -3732,6 +3801,7 @@ uninstall-am: uninstall-toolexeclibLIBRARIES \ uninstall-toolexeclibgomimeDATA uninstall-toolexeclibgonetDATA \ uninstall-toolexeclibgoosDATA uninstall-toolexeclibgorpcDATA \ uninstall-toolexeclibgoruntimeDATA \ + uninstall-toolexeclibgosyncDATA \ uninstall-toolexeclibgotestingDATA .MAKE: $(RECURSIVE_CLEAN_TARGETS) $(RECURSIVE_TARGETS) all all-multi \ @@ -3766,7 +3836,7 @@ uninstall-am: uninstall-toolexeclibLIBRARIES \ install-toolexeclibgoindexDATA install-toolexeclibgoioDATA \ install-toolexeclibgomimeDATA install-toolexeclibgonetDATA \ install-toolexeclibgoosDATA install-toolexeclibgorpcDATA \ - install-toolexeclibgoruntimeDATA \ + install-toolexeclibgoruntimeDATA install-toolexeclibgosyncDATA \ install-toolexeclibgotestingDATA installcheck installcheck-am \ installdirs installdirs-am maintainer-clean \ maintainer-clean-generic maintainer-clean-multi mostlyclean \ @@ -3790,6 +3860,7 @@ uninstall-am: uninstall-toolexeclibLIBRARIES \ uninstall-toolexeclibgomimeDATA uninstall-toolexeclibgonetDATA \ uninstall-toolexeclibgoosDATA uninstall-toolexeclibgorpcDATA \ uninstall-toolexeclibgoruntimeDATA \ + uninstall-toolexeclibgosyncDATA \ uninstall-toolexeclibgotestingDATA @@ -3872,6 +3943,12 @@ cmath/check: $(CHECK_DEPS) $(CHECK) .PHONY: cmath/check +crypto/crypto.lo: $(go_crypto_files) hash.gox + $(BUILDPACKAGE) +crypto/check: $(CHECK_DEPS) + $(CHECK) +.PHONY: crypto/check + ebnf/ebnf.lo: $(go_ebnf_files) container/vector.gox go/scanner.gox \ go/token.gox os.gox strconv.gox unicode.gox utf8.gox $(BUILDPACKAGE) @@ -3879,7 +3956,7 @@ ebnf/check: $(CHECK_DEPS) $(CHECK) .PHONY: ebnf/check -exec/exec.lo: $(go_exec_files) os.gox strings.gox +exec/exec.lo: $(go_exec_files) os.gox strconv.gox strings.gox $(BUILDPACKAGE) exec/check: $(CHECK_DEPS) $(CHECK) @@ -3898,8 +3975,8 @@ flag/check: $(CHECK_DEPS) $(CHECK) .PHONY: flag/check -fmt/fmt.lo: $(go_fmt_files) bytes.gox io.gox os.gox reflect.gox strconv.gox \ - strings.gox unicode.gox utf8.gox +fmt/fmt.lo: $(go_fmt_files) bytes.gox io.gox math.gox os.gox reflect.gox \ + strconv.gox strings.gox unicode.gox utf8.gox $(BUILDPACKAGE) fmt/check: $(CHECK_DEPS) $(CHECK) @@ -3926,10 +4003,10 @@ html/check: $(CHECK_DEPS) $(CHECK) .PHONY: html/check -http/http.lo: $(go_http_files) bufio.gox bytes.gox container/list.gox \ - container/vector.gox crypto/rand.gox crypto/tls.gox \ - encoding/base64.gox fmt.gox io.gox io/ioutil.gox log.gox \ - mime.gox mime/multipart.gox net.gox os.gox path.gox sort.gox \ +http/http.lo: $(go_http_files) bufio.gox bytes.gox container/vector.gox \ + crypto/rand.gox crypto/tls.gox encoding/base64.gox fmt.gox \ + io.gox io/ioutil.gox log.gox mime.gox mime/multipart.gox \ + net.gox net/textproto.gox os.gox path.gox sort.gox \ strconv.gox strings.gox sync.gox time.gox utf8.gox $(BUILDPACKAGE) http/check: $(CHECK_DEPS) @@ -3948,9 +4025,10 @@ io/check: $(CHECK_DEPS) $(CHECK) .PHONY: io/check -json/json.lo: $(go_json_files) bytes.gox container/vector.gox fmt.gox io.gox \ - math.gox os.gox reflect.gox runtime.gox strconv.gox \ - strings.gox unicode.gox utf16.gox utf8.gox +json/json.lo: $(go_json_files) bytes.gox container/vector.gox \ + encoding/base64.gox fmt.gox io.gox math.gox os.gox \ + reflect.gox runtime.gox strconv.gox strings.gox unicode.gox \ + utf16.gox utf8.gox $(BUILDPACKAGE) json/check: $(CHECK_DEPS) $(CHECK) @@ -3983,14 +4061,14 @@ net/check: $(CHECK_DEPS) $(CHECK) .PHONY: net/check -netchan/netchan.lo: $(go_netchan_files) gob.gox log.gox net.gox os.gox \ +netchan/netchan.lo: $(go_netchan_files) gob.gox io.gox log.gox net.gox os.gox \ reflect.gox strconv.gox sync.gox time.gox $(BUILDPACKAGE) netchan/check: $(CHECK_DEPS) $(CHECK) .PHONY: netchan/check -os/os.lo: $(go_os_files) sync.gox syscall.gox +os/os.lo: $(go_os_files) runtime.gox sync.gox syscall.gox $(BUILDPACKAGE) os/check: $(CHECK_DEPS) $(CHECK) @@ -4078,10 +4156,8 @@ strings/check: $(CHECK_DEPS) $(CHECK) .PHONY: strings/check -sync/mutex.lo: $(go_sync_files) runtime.gox +sync/sync.lo: $(go_sync_files) runtime.gox sync/atomic.gox $(BUILDPACKAGE) -sync/cas.lo: $(go_sync_c_files) sync/mutex.lo - $(LTCOMPILE) -c -o sync/cas.lo $(srcdir)/go/sync/cas.c sync/check: $(CHECK_DEPS) $(CHECK) .PHONY: sync/check @@ -4178,6 +4254,13 @@ archive/zip/check: $(CHECK_DEPS) $(CHECK) .PHONY: archive/zip/check +compress/bzip2.lo: $(go_compress_bzip2_files) bufio.gox io.gox os.gox sort.gox + $(BUILDPACKAGE) +compress/bzip2/check: $(CHECK_DEPS) + @$(MKDIR_P) compress/bzip2 + $(CHECK) +.PHONY: compress/bzip2/check + compress/flate.lo: $(go_compress_flate_files) bufio.gox io.gox math.gox \ os.gox sort.gox strconv.gox $(BUILDPACKAGE) @@ -4194,6 +4277,13 @@ compress/gzip/check: $(CHECK_DEPS) $(CHECK) .PHONY: compress/gzip/check +compress/lzw.lo: $(go_compress_lzw_files) bufio.gox fmt.gox io.gox os.gox + $(BUILDPACKAGE) +compress/lzw/check: $(CHECK_DEPS) + @$(MKDIR_P) compress/lzw + $(CHECK) +.PHONY: compress/lzw/check + compress/zlib.lo: $(go_compress_zlib_files) bufio.gox compress/flate.gox \ hash.gox hash/adler32.gox io.gox os.gox $(BUILDPACKAGE) @@ -4265,6 +4355,13 @@ crypto/cipher/check: $(CHECK_DEPS) $(CHECK) .PHONY: crypto/cipher/check +crypto/dsa.lo: $(go_crypto_dsa_files) big.gox io.gox os.gox + $(BUILDPACKAGE) +crypto/dsa/check: $(CHECK_DEPS) + @$(MKDIR_P) crypto/dsa + $(CHECK) +.PHONY: crypto/dsa/check + crypto/elliptic.lo: $(go_crypto_elliptic_files) big.gox io.gox os.gox sync.gox $(BUILDPACKAGE) crypto/elliptic/check: $(CHECK_DEPS) @@ -4280,21 +4377,21 @@ crypto/hmac/check: $(CHECK_DEPS) $(CHECK) .PHONY: crypto/hmac/check -crypto/md4.lo: $(go_crypto_md4_files) hash.gox os.gox +crypto/md4.lo: $(go_crypto_md4_files) crypto.gox hash.gox os.gox $(BUILDPACKAGE) crypto/md4/check: $(CHECK_DEPS) @$(MKDIR_P) crypto/md4 $(CHECK) .PHONY: crypto/md4/check -crypto/md5.lo: $(go_crypto_md5_files) hash.gox os.gox +crypto/md5.lo: $(go_crypto_md5_files) crypto.gox hash.gox os.gox $(BUILDPACKAGE) crypto/md5/check: $(CHECK_DEPS) @$(MKDIR_P) crypto/md5 $(CHECK) .PHONY: crypto/md5/check -crypto/ocsp.lo: $(go_crypto_ocsp_files) asn1.gox crypto/rsa.gox \ +crypto/ocsp.lo: $(go_crypto_ocsp_files) asn1.gox crypto.gox crypto/rsa.gox \ crypto/sha1.gox crypto/x509.gox os.gox time.gox $(BUILDPACKAGE) crypto/ocsp/check: $(CHECK_DEPS) @@ -4302,8 +4399,18 @@ crypto/ocsp/check: $(CHECK_DEPS) $(CHECK) .PHONY: crypto/ocsp/check -crypto/rand.lo: $(go_crypto_rand_files) crypto/aes.gox io.gox os.gox sync.gox \ - time.gox +crypto/openpgp.lo: $(go_crypto_openpgp_files) crypto.gox \ + crypto/openpgp/armor.gox crypto/openpgp/error.gox \ + crypto/openpgp/packet.gox crypto/rsa.gox crypto/sha256.gox \ + hash.gox io.gox os.gox strconv.gox time.gox + $(BUILDPACKAGE) +crypto/openpgp/check: $(CHECK_DEPS) + @$(MKDIR_P) crypto/openpgp + $(CHECK) +.PHONY: crypto/openpgp/check + +crypto/rand.lo: $(go_crypto_rand_files) bufio.gox crypto/aes.gox io.gox \ + os.gox sync.gox time.gox $(BUILDPACKAGE) crypto/rand/check: $(CHECK_DEPS) @$(MKDIR_P) crypto/rand @@ -4317,14 +4424,14 @@ crypto/rc4/check: $(CHECK_DEPS) $(CHECK) .PHONY: crypto/rc4/check -crypto/ripemd160.lo: $(go_crypto_ripemd160_files) hash.gox os.gox +crypto/ripemd160.lo: $(go_crypto_ripemd160_files) crypto.gox hash.gox os.gox $(BUILDPACKAGE) crypto/ripemd160/check: $(CHECK_DEPS) @$(MKDIR_P) crypto/ripemd160 $(CHECK) .PHONY: crypto/ripemd160/check -crypto/rsa.lo: $(go_crypto_rsa_files) big.gox crypto/sha1.gox \ +crypto/rsa.lo: $(go_crypto_rsa_files) big.gox crypto.gox crypto/sha1.gox \ crypto/subtle.gox encoding/hex.gox hash.gox io.gox os.gox $(BUILDPACKAGE) crypto/rsa/check: $(CHECK_DEPS) @@ -4332,21 +4439,21 @@ crypto/rsa/check: $(CHECK_DEPS) $(CHECK) .PHONY: crypto/rsa/check -crypto/sha1.lo: $(go_crypto_sha1_files) hash.gox os.gox +crypto/sha1.lo: $(go_crypto_sha1_files) crypto.gox hash.gox os.gox $(BUILDPACKAGE) crypto/sha1/check: $(CHECK_DEPS) @$(MKDIR_P) crypto/sha1 $(CHECK) .PHONY: crypto/sha1/check -crypto/sha256.lo: $(go_crypto_sha256_files) hash.gox os.gox +crypto/sha256.lo: $(go_crypto_sha256_files) crypto.gox hash.gox os.gox $(BUILDPACKAGE) crypto/sha256/check: $(CHECK_DEPS) @$(MKDIR_P) crypto/sha256 $(CHECK) .PHONY: crypto/sha256/check -crypto/sha512.lo: $(go_crypto_sha512_files) hash.gox os.gox +crypto/sha512.lo: $(go_crypto_sha512_files) crypto.gox hash.gox os.gox $(BUILDPACKAGE) crypto/sha512/check: $(CHECK_DEPS) @$(MKDIR_P) crypto/sha512 @@ -4361,7 +4468,7 @@ crypto/subtle/check: $(CHECK_DEPS) .PHONY: crypto/subtle/check crypto/tls.lo: $(go_crypto_tls_files) big.gox bufio.gox bytes.gox \ - container/list.gox crypto/aes.gox crypto/cipher.gox \ + container/list.gox crypto.gox crypto/aes.gox crypto/cipher.gox \ crypto/elliptic.gox crypto/hmac.gox crypto/md5.gox \ crypto/rc4.gox crypto/rand.gox crypto/rsa.gox crypto/sha1.gox \ crypto/subtle.gox crypto/rsa.gox crypto/sha1.gox \ @@ -4381,8 +4488,8 @@ crypto/twofish/check: $(CHECK_DEPS) .PHONY: crypto/twofish/check crypto/x509.lo: $(go_crypto_x509_files) asn1.gox big.gox container/vector.gox \ - crypto/rsa.gox crypto/sha1.gox hash.gox os.gox strings.gox \ - time.gox + crypto.gox crypto/rsa.gox crypto/sha1.gox hash.gox os.gox \ + strings.gox time.gox $(BUILDPACKAGE) crypto/x509/check: $(CHECK_DEPS) @$(MKDIR_P) crypto/x509 @@ -4405,16 +4512,30 @@ crypto/openpgp/armor/check: $(CHECK_DEPS) $(CHECK) .PHONY: crypto/openpgp/armor/check -crypto/openpgp/error.lo: $(go_crypto_openpgp_error_files) +crypto/openpgp/error.lo: $(go_crypto_openpgp_error_files) strconv.gox $(BUILDPACKAGE) crypto/openpgp/error/check: $(CHECK_DEPS) @$(MKDIR_P) crypto/openpgp/error $(CHECK) .PHONY: crypto/openpgp/error/check -crypto/openpgp/s2k.lo: $(go_crypto_openpgp_s2k_files) crypto/md5.gox \ - crypto/openpgp/error.gox crypto/ripemd160.gox crypto/sha1.gox \ - crypto/sha256.gox crypto/sha512.gox hash.gox io.gox os.gox +crypto/openpgp/packet.lo: $(go_crypto_openpgp_packet_files) big.gox bytes.gox \ + compress/flate.gox compress/zlib.gox crypto.gox \ + crypto/aes.gox crypto/cast5.gox crypto/cipher.gox \ + crypto/openpgp/error.gox crypto/openpgp/s2k.gox \ + crypto/rand.gox crypto/rsa.gox crypto/sha1.gox \ + crypto/subtle.gox encoding/binary.gox hash.gox io.gox \ + io/ioutil.gox os.gox strconv.gox strings.gox + $(BUILDPACKAGE) +crypto/openpgp/packet/check: $(CHECK_DEPS) + @$(MKDIR_P) crypto/openpgp/packet + $(CHECK) +.PHONY: crypto/openpgp/packet/check + +crypto/openpgp/s2k.lo: $(go_crypto_openpgp_s2k_files) crypto.gox \ + crypto/md5.gox crypto/openpgp/error.gox crypto/ripemd160.gox \ + crypto/sha1.gox crypto/sha256.gox crypto/sha512.gox hash.gox \ + io.gox os.gox $(BUILDPACKAGE) crypto/openpgp/s2k/check: $(CHECK_DEPS) @$(MKDIR_P) crypto/openpgp/s2k @@ -4733,6 +4854,15 @@ runtime/pprof/check: $(CHECK_DEPS) $(CHECK) .PHONY: runtime/pprof/check +sync/atomic.lo: $(go_sync_atomic_files) + $(BUILDPACKAGE) +sync/atomic_c.lo: $(go_sync_atomic_c_files) sync/atomic.lo + $(LTCOMPILE) -c -o $@ $(srcdir)/go/sync/atomic/atomic.c +sync/atomic/check: $(CHECK_DEPS) + @$(MKDIR_P) sync/atomic + $(CHECK) +.PHONY: sync/atomic/check + testing/iotest.lo: $(go_testing_iotest_files) io.gox log.gox os.gox $(BUILDPACKAGE) testing/iotest/check: $(CHECK_DEPS) @@ -4777,6 +4907,8 @@ bytes.gox: bytes/bytes.lo $(BUILDGOX) cmath.gox: cmath/cmath.lo $(BUILDGOX) +crypto.gox: crypto/crypto.lo + $(BUILDGOX) ebnf.gox: ebnf/ebnf.lo $(BUILDGOX) exec.gox: exec/exec.lo @@ -4837,7 +4969,7 @@ strconv.gox: strconv/strconv.lo $(BUILDGOX) strings.gox: strings/strings.lo $(BUILDGOX) -sync.gox: sync/mutex.lo +sync.gox: sync/sync.lo $(BUILDGOX) syslog.gox: syslog/syslog.lo $(BUILDGOX) @@ -4869,10 +5001,14 @@ archive/tar.gox: archive/tar.lo archive/zip.gox: archive/zip.lo $(BUILDGOX) +compress/bzip2.gox: compress/bzip2.lo + $(BUILDGOX) compress/flate.gox: compress/flate.lo $(BUILDGOX) compress/gzip.gox: compress/gzip.lo $(BUILDGOX) +compress/lzw.gox: compress/lzw.lo + $(BUILDGOX) compress/zlib.gox: compress/zlib.lo $(BUILDGOX) @@ -4895,6 +5031,8 @@ crypto/cast5.gox: crypto/cast5.lo $(BUILDGOX) crypto/cipher.gox: crypto/cipher.lo $(BUILDGOX) +crypto/dsa.gox: crypto/dsa.lo + $(BUILDGOX) crypto/elliptic.gox: crypto/elliptic.lo $(BUILDGOX) crypto/hmac.gox: crypto/hmac.lo @@ -4905,6 +5043,8 @@ crypto/md5.gox: crypto/md5.lo $(BUILDGOX) crypto/ocsp.gox: crypto/ocsp.lo $(BUILDGOX) +crypto/openpgp.gox: crypto/openpgp.lo + $(BUILDGOX) crypto/rand.gox: crypto/rand.lo $(BUILDGOX) crypto/rc4.gox: crypto/rc4.lo @@ -4934,6 +5074,8 @@ crypto/openpgp/armor.gox: crypto/openpgp/armor.lo $(BUILDGOX) crypto/openpgp/error.gox: crypto/openpgp/error.lo $(BUILDGOX) +crypto/openpgp/packet.gox: crypto/openpgp/packet.lo + $(BUILDGOX) crypto/openpgp/s2k.gox: crypto/openpgp/s2k.lo $(BUILDGOX) @@ -5031,6 +5173,9 @@ runtime/debug.gox: runtime/debug.lo runtime/pprof.gox: runtime/pprof.lo $(BUILDGOX) +sync/atomic.gox: sync/atomic.lo + $(BUILDGOX) + testing/iotest.gox: testing/iotest.lo $(BUILDGOX) testing/quick.gox: testing/quick.lo diff --git a/libgo/go/archive/zip/reader.go b/libgo/go/archive/zip/reader.go index 579ba160294..d8d9bba60bc 100644 --- a/libgo/go/archive/zip/reader.go +++ b/libgo/go/archive/zip/reader.go @@ -42,6 +42,10 @@ type File struct { bodyOffset int64 } +func (f *File) hasDataDescriptor() bool { + return f.Flags&0x8 != 0 +} + // OpenReader will open the Zip file specified by name and return a Reader. func OpenReader(name string) (*Reader, os.Error) { f, err := os.Open(name, os.O_RDONLY, 0644) @@ -93,7 +97,16 @@ func (f *File) Open() (rc io.ReadCloser, err os.Error) { return } } - r := io.NewSectionReader(f.zipr, off+f.bodyOffset, int64(f.CompressedSize)) + size := int64(f.CompressedSize) + if f.hasDataDescriptor() { + if size == 0 { + // permit SectionReader to see the rest of the file + size = f.zipsize - (off + f.bodyOffset) + } else { + size += dataDescriptorLen + } + } + r := io.NewSectionReader(f.zipr, off+f.bodyOffset, size) switch f.Method { case 0: // store (no compression) rc = nopCloser{r} @@ -103,7 +116,7 @@ func (f *File) Open() (rc io.ReadCloser, err os.Error) { err = UnsupportedMethod } if rc != nil { - rc = &checksumReader{rc, crc32.NewIEEE(), f.CRC32} + rc = &checksumReader{rc, crc32.NewIEEE(), f, r} } return } @@ -111,7 +124,8 @@ func (f *File) Open() (rc io.ReadCloser, err os.Error) { type checksumReader struct { rc io.ReadCloser hash hash.Hash32 - sum uint32 + f *File + zipr io.Reader // for reading the data descriptor } func (r *checksumReader) Read(b []byte) (n int, err os.Error) { @@ -120,7 +134,12 @@ func (r *checksumReader) Read(b []byte) (n int, err os.Error) { if err != os.EOF { return } - if r.hash.Sum32() != r.sum { + if r.f.hasDataDescriptor() { + if err = readDataDescriptor(r.zipr, r.f); err != nil { + return + } + } + if r.hash.Sum32() != r.f.CRC32 { err = ChecksumError } return @@ -205,6 +224,18 @@ func readDirectoryHeader(f *File, r io.Reader) (err os.Error) { return } +func readDataDescriptor(r io.Reader, f *File) (err os.Error) { + defer func() { + if rerr, ok := recover().(os.Error); ok { + err = rerr + } + }() + read(r, &f.CRC32) + read(r, &f.CompressedSize) + read(r, &f.UncompressedSize) + return +} + func readDirectoryEnd(r io.ReaderAt, size int64) (d *directoryEnd, err os.Error) { // look for directoryEndSignature in the last 1k, then in the last 65k var b []byte diff --git a/libgo/go/archive/zip/reader_test.go b/libgo/go/archive/zip/reader_test.go index 3c24f1467cf..72e8cccfd47 100644 --- a/libgo/go/archive/zip/reader_test.go +++ b/libgo/go/archive/zip/reader_test.go @@ -52,6 +52,15 @@ var tests = []ZipTest{ }, {Name: "readme.zip"}, {Name: "readme.notzip", Error: FormatError}, + { + Name: "dd.zip", + File: []ZipTestFile{ + { + Name: "filename", + Content: []byte("This is a test textfile.\n"), + }, + }, + }, } func TestReader(t *testing.T) { @@ -102,16 +111,18 @@ func readTestZip(t *testing.T, zt ZipTest) { } // test invalid checksum - z.File[0].CRC32++ // invalidate - r, err := z.File[0].Open() - if err != nil { - t.Error(err) - return - } - var b bytes.Buffer - _, err = io.Copy(&b, r) - if err != ChecksumError { - t.Errorf("%s: copy error=%v, want %v", z.File[0].Name, err, ChecksumError) + if !z.File[0].hasDataDescriptor() { // skip test when crc32 in dd + z.File[0].CRC32++ // invalidate + r, err := z.File[0].Open() + if err != nil { + t.Error(err) + return + } + var b bytes.Buffer + _, err = io.Copy(&b, r) + if err != ChecksumError { + t.Errorf("%s: copy error=%v, want %v", z.File[0].Name, err, ChecksumError) + } } } diff --git a/libgo/go/archive/zip/struct.go b/libgo/go/archive/zip/struct.go index 8a8c727d474..bfe0aae2e9a 100644 --- a/libgo/go/archive/zip/struct.go +++ b/libgo/go/archive/zip/struct.go @@ -4,6 +4,7 @@ const ( fileHeaderSignature = 0x04034b50 directoryHeaderSignature = 0x02014b50 directoryEndSignature = 0x06054b50 + dataDescriptorLen = 12 ) type FileHeader struct { diff --git a/libgo/go/asn1/marshal.go b/libgo/go/asn1/marshal.go index 24548714b21..57b8f20ba7f 100644 --- a/libgo/go/asn1/marshal.go +++ b/libgo/go/asn1/marshal.go @@ -317,7 +317,7 @@ func marshalBody(out *forkableWriter, value reflect.Value, params fieldParameter switch v := value.(type) { case *reflect.BoolValue: if v.Get() { - return out.WriteByte(1) + return out.WriteByte(255) } else { return out.WriteByte(0) } diff --git a/libgo/go/bufio/bufio.go b/libgo/go/bufio/bufio.go index c13456a6326..eae5c5ce975 100644 --- a/libgo/go/bufio/bufio.go +++ b/libgo/go/bufio/bufio.go @@ -286,7 +286,8 @@ func (b *Reader) ReadSlice(delim byte) (line []byte, err os.Error) { // returning a slice containing the data up to and including the delimiter. // If ReadBytes encounters an error before finding a delimiter, // it returns the data read before the error and the error itself (often os.EOF). -// ReadBytes returns err != nil if and only if line does not end in delim. +// ReadBytes returns err != nil if and only if the returned data does not end in +// delim. func (b *Reader) ReadBytes(delim byte) (line []byte, err os.Error) { // Use ReadSlice to look for array, // accumulating full buffers. @@ -332,7 +333,8 @@ func (b *Reader) ReadBytes(delim byte) (line []byte, err os.Error) { // returning a string containing the data up to and including the delimiter. // If ReadString encounters an error before finding a delimiter, // it returns the data read before the error and the error itself (often os.EOF). -// ReadString returns err != nil if and only if line does not end in delim. +// ReadString returns err != nil if and only if the returned data does not end in +// delim. func (b *Reader) ReadString(delim byte) (line string, err os.Error) { bytes, e := b.ReadBytes(delim) return string(bytes), e @@ -383,6 +385,9 @@ func (b *Writer) Flush() os.Error { if b.err != nil { return b.err } + if b.n == 0 { + return nil + } n, e := b.wr.Write(b.buf[0:b.n]) if n < b.n && e == nil { e = io.ErrShortWrite diff --git a/libgo/go/bytes/buffer.go b/libgo/go/bytes/buffer.go index 62cf82810e7..1acd4e05cae 100644 --- a/libgo/go/bytes/buffer.go +++ b/libgo/go/bytes/buffer.go @@ -154,17 +154,20 @@ func (b *Buffer) ReadFrom(r io.Reader) (n int64, err os.Error) { } // WriteTo writes data to w until the buffer is drained or an error -// occurs. The return value n is the number of bytes written. +// occurs. The return value n is the number of bytes written; it always +// fits into an int, but it is int64 to match the io.WriterTo interface. // Any error encountered during the write is also returned. func (b *Buffer) WriteTo(w io.Writer) (n int64, err os.Error) { b.lastRead = opInvalid - for b.off < len(b.buf) { + if b.off < len(b.buf) { m, e := w.Write(b.buf[b.off:]) - n += int64(m) b.off += m + n = int64(m) if e != nil { return n, e } + // otherwise all bytes were written, by definition of + // Write method in io.Writer } // Buffer is now empty; reset. b.Truncate(0) @@ -301,6 +304,36 @@ func (b *Buffer) UnreadByte() os.Error { return nil } +// ReadBytes reads until the first occurrence of delim in the input, +// returning a slice containing the data up to and including the delimiter. +// If ReadBytes encounters an error before finding a delimiter, +// it returns the data read before the error and the error itself (often os.EOF). +// ReadBytes returns err != nil if and only if the returned data does not end in +// delim. +func (b *Buffer) ReadBytes(delim byte) (line []byte, err os.Error) { + i := IndexByte(b.buf[b.off:], delim) + size := i + 1 + if i < 0 { + size = len(b.buf) - b.off + err = os.EOF + } + line = make([]byte, size) + copy(line, b.buf[b.off:]) + b.off += size + return +} + +// ReadString reads until the first occurrence of delim in the input, +// returning a string containing the data up to and including the delimiter. +// If ReadString encounters an error before finding a delimiter, +// it returns the data read before the error and the error itself (often os.EOF). +// ReadString returns err != nil if and only if the returned data does not end +// in delim. +func (b *Buffer) ReadString(delim byte) (line string, err os.Error) { + bytes, err := b.ReadBytes(delim) + return string(bytes), err +} + // NewBuffer creates and initializes a new Buffer using buf as its initial // contents. It is intended to prepare a Buffer to read existing data. It // can also be used to size the internal buffer for writing. To do that, diff --git a/libgo/go/bytes/buffer_test.go b/libgo/go/bytes/buffer_test.go index 509793d24a8..56a2d927539 100644 --- a/libgo/go/bytes/buffer_test.go +++ b/libgo/go/bytes/buffer_test.go @@ -6,6 +6,7 @@ package bytes_test import ( . "bytes" + "os" "rand" "testing" "utf8" @@ -238,7 +239,7 @@ func TestMixedReadsAndWrites(t *testing.T) { func TestNil(t *testing.T) { var b *Buffer if b.String() != "<nil>" { - t.Errorf("expcted <nil>; got %q", b.String()) + t.Errorf("expected <nil>; got %q", b.String()) } } @@ -347,3 +348,38 @@ func TestNext(t *testing.T) { } } } + +var readBytesTests = []struct { + buffer string + delim byte + expected []string + err os.Error +}{ + {"", 0, []string{""}, os.EOF}, + {"a\x00", 0, []string{"a\x00"}, nil}, + {"abbbaaaba", 'b', []string{"ab", "b", "b", "aaab"}, nil}, + {"hello\x01world", 1, []string{"hello\x01"}, nil}, + {"foo\nbar", 0, []string{"foo\nbar"}, os.EOF}, + {"alpha\nbeta\ngamma\n", '\n', []string{"alpha\n", "beta\n", "gamma\n"}, nil}, + {"alpha\nbeta\ngamma", '\n', []string{"alpha\n", "beta\n", "gamma"}, os.EOF}, +} + +func TestReadBytes(t *testing.T) { + for _, test := range readBytesTests { + buf := NewBufferString(test.buffer) + var err os.Error + for _, expected := range test.expected { + var bytes []byte + bytes, err = buf.ReadBytes(test.delim) + if string(bytes) != expected { + t.Errorf("expected %q, got %q", expected, bytes) + } + if err != nil { + break + } + } + if err != test.err { + t.Errorf("expected error %v, got %v", test.err, err) + } + } +} diff --git a/libgo/go/compress/bzip2/bit_reader.go b/libgo/go/compress/bzip2/bit_reader.go new file mode 100644 index 00000000000..50f0ec836b4 --- /dev/null +++ b/libgo/go/compress/bzip2/bit_reader.go @@ -0,0 +1,88 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bzip2 + +import ( + "bufio" + "io" + "os" +) + +// bitReader wraps an io.Reader and provides the ability to read values, +// bit-by-bit, from it. Its Read* methods don't return the usual os.Error +// because the error handling was verbose. Instead, any error is kept and can +// be checked afterwards. +type bitReader struct { + r byteReader + n uint64 + bits uint + err os.Error +} + +// bitReader needs to read bytes from an io.Reader. We attempt to cast the +// given io.Reader to this interface and, if it doesn't already fit, we wrap in +// a bufio.Reader. +type byteReader interface { + ReadByte() (byte, os.Error) +} + +func newBitReader(r io.Reader) bitReader { + byter, ok := r.(byteReader) + if !ok { + byter = bufio.NewReader(r) + } + return bitReader{r: byter} +} + +// ReadBits64 reads the given number of bits and returns them in the +// least-significant part of a uint64. In the event of an error, it returns 0 +// and the error can be obtained by calling Error(). +func (br *bitReader) ReadBits64(bits uint) (n uint64) { + for bits > br.bits { + b, err := br.r.ReadByte() + if err == os.EOF { + err = io.ErrUnexpectedEOF + } + if err != nil { + br.err = err + return 0 + } + br.n <<= 8 + br.n |= uint64(b) + br.bits += 8 + } + + // br.n looks like this (assuming that br.bits = 14 and bits = 6): + // Bit: 111111 + // 5432109876543210 + // + // (6 bits, the desired output) + // |-----| + // V V + // 0101101101001110 + // ^ ^ + // |------------| + // br.bits (num valid bits) + // + // This the next line right shifts the desired bits into the + // least-significant places and masks off anything above. + n = (br.n >> (br.bits - bits)) & ((1 << bits) - 1) + br.bits -= bits + return +} + +func (br *bitReader) ReadBits(bits uint) (n int) { + n64 := br.ReadBits64(bits) + return int(n64) +} + +func (br *bitReader) ReadBit() bool { + n := br.ReadBits(1) + return n != 0 +} + +func (br *bitReader) Error() os.Error { + return br.err +} diff --git a/libgo/go/compress/bzip2/bzip2.go b/libgo/go/compress/bzip2/bzip2.go new file mode 100644 index 00000000000..9e97edec175 --- /dev/null +++ b/libgo/go/compress/bzip2/bzip2.go @@ -0,0 +1,390 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package bzip2 implements bzip2 decompression. +package bzip2 + +import ( + "io" + "os" +) + +// There's no RFC for bzip2. I used the Wikipedia page for reference and a lot +// of guessing: http://en.wikipedia.org/wiki/Bzip2 +// The source code to pyflate was useful for debugging: +// http://www.paul.sladen.org/projects/pyflate + +// A StructuralError is returned when the bzip2 data is found to be +// syntactically invalid. +type StructuralError string + +func (s StructuralError) String() string { + return "bzip2 data invalid: " + string(s) +} + +// A reader decompresses bzip2 compressed data. +type reader struct { + br bitReader + setupDone bool // true if we have parsed the bzip2 header. + blockSize int // blockSize in bytes, i.e. 900 * 1024. + eof bool + buf []byte // stores Burrows-Wheeler transformed data. + c [256]uint // the `C' array for the inverse BWT. + tt []uint32 // mirrors the `tt' array in the bzip2 source and contains the P array in the upper 24 bits. + tPos uint32 // Index of the next output byte in tt. + + preRLE []uint32 // contains the RLE data still to be processed. + preRLEUsed int // number of entries of preRLE used. + lastByte int // the last byte value seen. + byteRepeats uint // the number of repeats of lastByte seen. + repeats uint // the number of copies of lastByte to output. +} + +// NewReader returns an io.Reader which decompresses bzip2 data from r. +func NewReader(r io.Reader) io.Reader { + bz2 := new(reader) + bz2.br = newBitReader(r) + return bz2 +} + +const bzip2FileMagic = 0x425a // "BZ" +const bzip2BlockMagic = 0x314159265359 +const bzip2FinalMagic = 0x177245385090 + +// setup parses the bzip2 header. +func (bz2 *reader) setup() os.Error { + br := &bz2.br + + magic := br.ReadBits(16) + if magic != bzip2FileMagic { + return StructuralError("bad magic value") + } + + t := br.ReadBits(8) + if t != 'h' { + return StructuralError("non-Huffman entropy encoding") + } + + level := br.ReadBits(8) + if level < '1' || level > '9' { + return StructuralError("invalid compression level") + } + + bz2.blockSize = 100 * 1024 * (int(level) - '0') + bz2.tt = make([]uint32, bz2.blockSize) + return nil +} + +func (bz2 *reader) Read(buf []byte) (n int, err os.Error) { + if bz2.eof { + return 0, os.EOF + } + + if !bz2.setupDone { + err = bz2.setup() + brErr := bz2.br.Error() + if brErr != nil { + err = brErr + } + if err != nil { + return 0, err + } + bz2.setupDone = true + } + + n, err = bz2.read(buf) + brErr := bz2.br.Error() + if brErr != nil { + err = brErr + } + return +} + +func (bz2 *reader) read(buf []byte) (n int, err os.Error) { + // bzip2 is a block based compressor, except that it has a run-length + // preprocessing step. The block based nature means that we can + // preallocate fixed-size buffers and reuse them. However, the RLE + // preprocessing would require allocating huge buffers to store the + // maximum expansion. Thus we process blocks all at once, except for + // the RLE which we decompress as required. + + for (bz2.repeats > 0 || bz2.preRLEUsed < len(bz2.preRLE)) && n < len(buf) { + // We have RLE data pending. + + // The run-length encoding works like this: + // Any sequence of four equal bytes is followed by a length + // byte which contains the number of repeats of that byte to + // include. (The number of repeats can be zero.) Because we are + // decompressing on-demand our state is kept in the reader + // object. + + if bz2.repeats > 0 { + buf[n] = byte(bz2.lastByte) + n++ + bz2.repeats-- + if bz2.repeats == 0 { + bz2.lastByte = -1 + } + continue + } + + bz2.tPos = bz2.preRLE[bz2.tPos] + b := byte(bz2.tPos) + bz2.tPos >>= 8 + bz2.preRLEUsed++ + + if bz2.byteRepeats == 3 { + bz2.repeats = uint(b) + bz2.byteRepeats = 0 + continue + } + + if bz2.lastByte == int(b) { + bz2.byteRepeats++ + } else { + bz2.byteRepeats = 0 + } + bz2.lastByte = int(b) + + buf[n] = b + n++ + } + + if n > 0 { + return + } + + // No RLE data is pending so we need to read a block. + + br := &bz2.br + magic := br.ReadBits64(48) + if magic == bzip2FinalMagic { + br.ReadBits64(32) // ignored CRC + bz2.eof = true + return 0, os.EOF + } else if magic != bzip2BlockMagic { + return 0, StructuralError("bad magic value found") + } + + err = bz2.readBlock() + if err != nil { + return 0, err + } + + return bz2.read(buf) +} + +// readBlock reads a bzip2 block. The magic number should already have been consumed. +func (bz2 *reader) readBlock() (err os.Error) { + br := &bz2.br + br.ReadBits64(32) // skip checksum. TODO: check it if we can figure out what it is. + randomized := br.ReadBits(1) + if randomized != 0 { + return StructuralError("deprecated randomized files") + } + origPtr := uint(br.ReadBits(24)) + + // If not every byte value is used in the block (i.e., it's text) then + // the symbol set is reduced. The symbols used are stored as a + // two-level, 16x16 bitmap. + symbolRangeUsedBitmap := br.ReadBits(16) + symbolPresent := make([]bool, 256) + numSymbols := 0 + for symRange := uint(0); symRange < 16; symRange++ { + if symbolRangeUsedBitmap&(1<<(15-symRange)) != 0 { + bits := br.ReadBits(16) + for symbol := uint(0); symbol < 16; symbol++ { + if bits&(1<<(15-symbol)) != 0 { + symbolPresent[16*symRange+symbol] = true + numSymbols++ + } + } + } + } + + // A block uses between two and six different Huffman trees. + numHuffmanTrees := br.ReadBits(3) + if numHuffmanTrees < 2 || numHuffmanTrees > 6 { + return StructuralError("invalid number of Huffman trees") + } + + // The Huffman tree can switch every 50 symbols so there's a list of + // tree indexes telling us which tree to use for each 50 symbol block. + numSelectors := br.ReadBits(15) + treeIndexes := make([]uint8, numSelectors) + + // The tree indexes are move-to-front transformed and stored as unary + // numbers. + mtfTreeDecoder := newMTFDecoderWithRange(numHuffmanTrees) + for i := range treeIndexes { + c := 0 + for { + inc := br.ReadBits(1) + if inc == 0 { + break + } + c++ + } + if c >= numHuffmanTrees { + return StructuralError("tree index too large") + } + treeIndexes[i] = uint8(mtfTreeDecoder.Decode(c)) + } + + // The list of symbols for the move-to-front transform is taken from + // the previously decoded symbol bitmap. + symbols := make([]byte, numSymbols) + nextSymbol := 0 + for i := 0; i < 256; i++ { + if symbolPresent[i] { + symbols[nextSymbol] = byte(i) + nextSymbol++ + } + } + mtf := newMTFDecoder(symbols) + + numSymbols += 2 // to account for RUNA and RUNB symbols + huffmanTrees := make([]huffmanTree, numHuffmanTrees) + + // Now we decode the arrays of code-lengths for each tree. + lengths := make([]uint8, numSymbols) + for i := 0; i < numHuffmanTrees; i++ { + // The code lengths are delta encoded from a 5-bit base value. + length := br.ReadBits(5) + for j := 0; j < numSymbols; j++ { + for { + if !br.ReadBit() { + break + } + if br.ReadBit() { + length-- + } else { + length++ + } + } + if length < 0 || length > 20 { + return StructuralError("Huffman length out of range") + } + lengths[j] = uint8(length) + } + huffmanTrees[i], err = newHuffmanTree(lengths) + if err != nil { + return err + } + } + + selectorIndex := 1 // the next tree index to use + currentHuffmanTree := huffmanTrees[treeIndexes[0]] + bufIndex := 0 // indexes bz2.buf, the output buffer. + // The output of the move-to-front transform is run-length encoded and + // we merge the decoding into the Huffman parsing loop. These two + // variables accumulate the repeat count. See the Wikipedia page for + // details. + repeat := 0 + repeat_power := 0 + + // The `C' array (used by the inverse BWT) needs to be zero initialised. + for i := range bz2.c { + bz2.c[i] = 0 + } + + decoded := 0 // counts the number of symbols decoded by the current tree. + for { + if decoded == 50 { + currentHuffmanTree = huffmanTrees[treeIndexes[selectorIndex]] + selectorIndex++ + decoded = 0 + } + + v := currentHuffmanTree.Decode(br) + decoded++ + + if v < 2 { + // This is either the RUNA or RUNB symbol. + if repeat == 0 { + repeat_power = 1 + } + repeat += repeat_power << v + repeat_power <<= 1 + + // This limit of 2 million comes from the bzip2 source + // code. It prevents repeat from overflowing. + if repeat > 2*1024*1024 { + return StructuralError("repeat count too large") + } + continue + } + + if repeat > 0 { + // We have decoded a complete run-length so we need to + // replicate the last output symbol. + for i := 0; i < repeat; i++ { + b := byte(mtf.First()) + bz2.tt[bufIndex] = uint32(b) + bz2.c[b]++ + bufIndex++ + } + repeat = 0 + } + + if int(v) == numSymbols-1 { + // This is the EOF symbol. Because it's always at the + // end of the move-to-front list, and nevers gets moved + // to the front, it has this unique value. + break + } + + // Since two metasymbols (RUNA and RUNB) have values 0 and 1, + // one would expect |v-2| to be passed to the MTF decoder. + // However, the front of the MTF list is never referenced as 0, + // it's always referenced with a run-length of 1. Thus 0 + // doesn't need to be encoded and we have |v-1| in the next + // line. + b := byte(mtf.Decode(int(v - 1))) + bz2.tt[bufIndex] = uint32(b) + bz2.c[b]++ + bufIndex++ + } + + if origPtr >= uint(bufIndex) { + return StructuralError("origPtr out of bounds") + } + + // We have completed the entropy decoding. Now we can perform the + // inverse BWT and setup the RLE buffer. + bz2.preRLE = bz2.tt[:bufIndex] + bz2.preRLEUsed = 0 + bz2.tPos = inverseBWT(bz2.preRLE, origPtr, bz2.c[:]) + bz2.lastByte = -1 + bz2.byteRepeats = 0 + bz2.repeats = 0 + + return nil +} + +// inverseBWT implements the inverse Burrows-Wheeler transform as described in +// http://www.hpl.hp.com/techreports/Compaq-DEC/SRC-RR-124.pdf, section 4.2. +// In that document, origPtr is called `I' and c is the `C' array after the +// first pass over the data. It's an argument here because we merge the first +// pass with the Huffman decoding. +// +// This also implements the `single array' method from the bzip2 source code +// which leaves the output, still shuffled, in the bottom 8 bits of tt with the +// index of the next byte in the top 24-bits. The index of the first byte is +// returned. +func inverseBWT(tt []uint32, origPtr uint, c []uint) uint32 { + sum := uint(0) + for i := 0; i < 256; i++ { + sum += c[i] + c[i] = sum - c[i] + } + + for i := range tt { + b := tt[i] & 0xff + tt[c[b]] |= uint32(i) << 8 + c[b]++ + } + + return tt[origPtr] >> 8 +} diff --git a/libgo/go/compress/bzip2/bzip2_test.go b/libgo/go/compress/bzip2/bzip2_test.go new file mode 100644 index 00000000000..156eea83ff2 --- /dev/null +++ b/libgo/go/compress/bzip2/bzip2_test.go @@ -0,0 +1,158 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bzip2 + +import ( + "bytes" + "encoding/hex" + "io" + "io/ioutil" + "os" + "testing" +) + +func TestBitReader(t *testing.T) { + buf := bytes.NewBuffer([]byte{0xaa}) + br := newBitReader(buf) + if n := br.ReadBits(1); n != 1 { + t.Errorf("read 1 wrong") + } + if n := br.ReadBits(1); n != 0 { + t.Errorf("read 2 wrong") + } + if n := br.ReadBits(1); n != 1 { + t.Errorf("read 3 wrong") + } + if n := br.ReadBits(1); n != 0 { + t.Errorf("read 4 wrong") + } +} + +func TestBitReaderLarge(t *testing.T) { + buf := bytes.NewBuffer([]byte{0x12, 0x34, 0x56, 0x78}) + br := newBitReader(buf) + if n := br.ReadBits(32); n != 0x12345678 { + t.Errorf("got: %x want: %x", n, 0x12345678) + } +} + +func readerFromHex(s string) io.Reader { + data, err := hex.DecodeString(s) + if err != nil { + panic("readerFromHex: bad input") + } + return bytes.NewBuffer(data) +} + +func decompressHex(s string) (out []byte, err os.Error) { + r := NewReader(readerFromHex(s)) + return ioutil.ReadAll(r) +} + +func TestHelloWorldBZ2(t *testing.T) { + out, err := decompressHex(helloWorldBZ2Hex) + if err != nil { + t.Errorf("error from Read: %s", err) + return + } + + if !bytes.Equal(helloWorld, out) { + t.Errorf("got %x, want %x", out, helloWorld) + } +} + +func testZeros(t *testing.T, inHex string, n int) { + out, err := decompressHex(inHex) + if err != nil { + t.Errorf("error from Read: %s", err) + return + } + + expected := make([]byte, n) + + if !bytes.Equal(expected, out) { + allZeros := true + for _, b := range out { + if b != 0 { + allZeros = false + break + } + } + t.Errorf("incorrect result, got %d bytes (allZeros: %t)", len(out), allZeros) + } +} + +func Test32Zeros(t *testing.T) { + testZeros(t, thirtyTwoZerosBZ2Hex, 32) +} + +func Test1MBZeros(t *testing.T) { + testZeros(t, oneMBZerosBZ2Hex, 1024*1024) +} + +func testRandomData(t *testing.T, compressedHex, uncompressedHex string) { + out, err := decompressHex(compressedHex) + if err != nil { + t.Errorf("error from Read: %s", err) + return + } + + expected, _ := hex.DecodeString(uncompressedHex) + + if !bytes.Equal(out, expected) { + t.Errorf("incorrect result\ngot: %x\nwant: %x", out, expected) + } +} + +func TestRandomData1(t *testing.T) { + testRandomData(t, randBZ2Hex, randHex) +} + +func TestRandomData2(t *testing.T) { + // This test involves several repeated bytes in the output, but they + // should trigger RLE decoding. + testRandomData(t, rand2BZ2Hex, rand2Hex) +} + +func TestRandomData3(t *testing.T) { + // This test uses the full range of symbols. + testRandomData(t, rand3BZ2Hex, rand3Hex) +} + +func Test1MBSawtooth(t *testing.T) { + out, err := decompressHex(oneMBSawtoothBZ2Hex) + if err != nil { + t.Errorf("error from Read: %s", err) + return + } + + expected := make([]byte, 1024*1024) + + for i := range expected { + expected[i] = byte(i) + } + + if !bytes.Equal(out, expected) { + t.Error("incorrect result") + } +} + +const helloWorldBZ2Hex = "425a68393141592653594eece83600000251800010400006449080200031064c4101a7a9a580bb9431f8bb9229c28482776741b0" + +var helloWorld = []byte("hello world\n") + +const thirtyTwoZerosBZ2Hex = "425a6839314159265359b5aa5098000000600040000004200021008283177245385090b5aa5098" +const oneMBZerosBZ2Hex = "425a683931415926535938571ce50008084000c0040008200030cc0529a60806c4201e2ee48a70a12070ae39ca" + +const randBZ2Hex = "425a6839314159265359905d990d0001957fffffffffffafffffffffffffffffbfff6fffdfffffffffffffffffffffffffffffc002b6dd75676ed5b77720098320d11a64626981323d4da47a83131a13d09e8040f534cd4f4d27a464d193008cd09804601347a980026350c9886234d36864193d1351b44c136919e90340d26127a4cd264c32023009898981310c0344c340027a8303427a99a04c00003534c230d034f5006468d268cf54d36a3009a69a62626261311b40026013d34201a6934c9a604c98ca6c8460989fa9346234d30d3469a2604fd4131a7aa6d0046043d4c62098479269e89e835190d018d4c046001a11e801a0264792321932308c43a130688c260d46686804cd01a9e80981193684c6a68c00000004c4c20c04627a4c0000260003400d04c0681a01334026009a6f48041466132581ec5212b081d96b0effc16543e2228b052fcd30f2567ee8d970e0f10aabca68dd8270591c376cfc1baae0dba00aaff2d6caf6b211322c997cc18eaee5927f75185336bf907021324c71626c1dd20e22b9b0977f05d0f901eaa51db9fbaf7c603b4c87bc82890e6dd7e61d0079e27ec050dd788fd958152061cd01e222f9547cb9efc465d775b6fc98bac7d387bffd151ae09dadf19494f7a638e2eae58e550faba5fe6820ea520eb986096de4e527d80def3ba625e71fbefdcf7e7844e0a25d29b52dcd1344fca083737d42692aab38d230485f3c8ed54c2ed31f15cf0270c8143765b10b92157233fa1dfe0d7ce8ffe70b8b8f7250071701dfe9f1c94de362c9031455951c93eb098a6b50ee45c6131fefc3b6f9643e21f4adc59497138e246f5c57d834aa67c4f10d8bd8b3908d8130dd7388409c299a268eab3664fa4907c5c31574874bd8d388a4ab22b339660804e53e1b8d05867d40e3082560608d35d5d2c6054e8bab23da28f61f83efd41d25529ad6ea15fb50505cacfabb0902166427354ca3830a2c8415f21b19e592690fbe447020d685a4bcd16ecc4ff1a1c0e572627d0ef6265c008a43fc243240541061ed7840606be466d1c0dac2c53250ed567507d926c844154560d631960c65e15157829b2c7f16859f111a3a8cb72bf24ffa57a680c3be67b1be67c8dd8aea73ac2437a78df5b686d427080ebc01bd30b71a49f6ea31dc0f08e4849e38face96717690239538bc08b6cc5aa8d467cb9c36aa83d40ac7e58bddbfa185b22065e89a86c0145569d9e23726651aec49e31588d70f40fe9a4449dcf4f89eac220171e9c938e803dc195679651004b79ad33cc0c13aeeba5941b33ffeeb8fbe16e76c7811445c67b4269c90479433ddf9e8ed1d00c166b6c17217fb22c3ef1b0c1c7e28e185446a111c37f1ea6c07a59fbcc6546ecc6968d36ba58bc5489a5640647e426b0c39350cb6f07d5dc7a717648c4ec7f841467597ae1f65f408fd2d9940a4b1b860b3c9ae351dcae0b4425f7e8538710f2e40b7f70d13b51ac05ccc6ecda8264a88cad2d721d18132a9b9110a9e759c2483c77dcefc7e464ec88588174cb0c9abff93230ea0bed8decdd8ed8bfe2b5df0a253803678df04fab44c03b9ab7cc97d6e6d6fd0c4c840ce0efc498436f453bbb181603459471f2b588724592b222ec990614db530e10cadd84705621cfdd9261fa44a5f5806a2d74b575056b3c915255c65678f9c16e6dc00a99180fef1a840aff0e842ac02731080cc92782538360a60a727991013984da4fad95f79d5030677b7528d076b2483685fca4429edf804682fdc110dfc2f7c30e23e20a72e039108a0ad6fdee2f76985a4b4be4f5afc6101bf9d5042b657a05dc914e1424241766434" +const randHex = "c95138082bdf2b9bfa5b1072b23f729735d42c785eeb94320fb14c265b9c2ca421d01a3db986df1ac2acde5a0e6bf955d6f95e61261540905928e195f1a66644cc7f37281744fff4dc6df35566a494c41a8167151950eb74f5fc45f85ad0e5ed28b49adfe218aa7ec1707e8e1d55825f61f72beda3b4c006b8c9188d7336a5d875329b1b58c27cc4e89ecbae02c7712400c39dd131d2c6de82e2863da51d472bdfb21ecce62cc9cf769ed28aedc7583d755da45a0d90874bda269dd53283a9bdfd05f95fc8e9a304bb338ea1a2111894678c18134f17d31a15d9bfc1237894650f3e715e2548639ecbddb845cfe4a46a7b3a3c540f48629488e8c869f1e9f3f4c552243a8105b20eb8e264994214349dae83b165fd6c2a5b8e83fce09fc0a80d3281c8d53a9a08095bd19cbc1388df23975646ed259e003d39261ee68cbece8bcf32971f7fe7e588e8ba8f5e8597909abaea693836a79a1964050ed910a45a0f13a58cd2d3ae18992c5b23082407fd920d0bf01e33118a017bb5e39f44931346845af52128f7965206759433a346034ea481671f501280067567619f5ecef6cded077f92ed7f3b3ce8e308c80f34ba06939e9303f91b4318c8c1dd4cc223c1f057ac0c91211c629cd30e46ee9ec1d9fd493086b7bc2bc83e33f08749a5d430b0ed4f79d70f481940c9b0930b16321886a0df4fa5a1465d5208c7d3494a7987d9a5e42aa256f0c9523947f8318d0ef0af3d59a45cfc2418d0785c9a548b32b81e7de18be7d55a69a4c156bbb3d7579c0ac8e9c72b24646e54b0d0e8725f8f49fb44ae3c6b9d0287be118586255a90a4a83483ed0328518037e52aa959c5748ed83e13023e532306be98b8288da306bbb040bcf5d92176f84a9306dc6b274b040370b61d71fde58dd6d20e6fee348eae0c54bd0a5a487b2d005f329794f2a902c296af0a4c1f638f63292a1fa18e006c1b1838636f4de71c73635b25660d32e88a0917e1a5677f6a02ca65585b82cbd99fb4badbfa97a585da1e6cadf6737b4ec6ca33f245d66ee6a9fae6785d69b003c17b9fc6ec34fe5824ab8caae5e8e14dc6f9e116e7bf4a60c04388783c8ae929e1b46b3ef3bbe81b38f2fa6da771bf39dfba2374d3d2ed356b8e2c42081d885a91a3afb2f31986d2f9873354c48cf5448492c32e62385af423aa4f83db6d1b2669650379a1134b0a04cbca0862d6f9743c791cbb527d36cd5d1f0fc7f503831c8bd1b7a0ef8ae1a5ed1155dfdd9e32b6bb33138112d3d476b802179cb85a2a6c354ccfed2f31604fbd8d6ec4baf9f1c8454f72c6588c06a7df3178c43a6970bfa02dd6f74cb5ec3b63f9eddaa17db5cbf27fac6de8e57c384afd0954179f7b5690c3bee42abc4fa79b4b12101a9cf5f0b9aecdda945def0bd04163237247d3539850e123fe18139f316fa0256d5bd2faa8" + +const oneMBSawtoothBZ2Hex = "425a683931415926535971931ea00006ddffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe007de00000000000000024c00130001300000000000000000000000000000000000000000000000000000000126000980009800000000000000000000000000000000000000000000000000000000930004c0004c000000000000000000000000000000000000000000000000000000004980026000260000000000000000000000000000000000000000000000000000000009aaaaa0000000000000000000000000000000000000000000000000000000000000000498002600026000000000000000000000000000000000000000000000000000000007fc42271980d044c0a822607411304a08982d044c1a82260f411308a08984d044c2a82261741130ca08986d044c3a82261f411310a08988d044c4a822627411314a0898ad044c5a82262f411318a0898cd044c6a82263741131ca0898ed044c7a82263f411320a08990d044c8a822647411324a08992d044c9a82264f411328a08994d044caa82265741132ca08996d044cba82265f411330a08998d044cca822667411334a0899ad044cda82266f411338a0899cd044cea82267741133ca0899ed044cfa82267f411340a089a0d044d0a822687411344a089a2d044d1a82268f411348a089a4d044d2a82269741134ca089a6d044d3a82269f411350a089a8d044d4a8226a7411354a089aad044d5a8226af411358a089acd044d6a8226b741135ca089aed044d7a8226bf411360a089b0d044d8a8226c7411364a089b2d044d9a8226cf411368a089b4d044daa8226d741136ca089b6d044dba8226df411370a089b8d044dca8226e7411374a089bad044dda8226ef411378a089bcd044dea8226f741137ca089bed044dfa8226ff411380a089c0d044e0a822707411384a089c2d044e1a82270f411388a089c4d044e2a82271741138ca089c59089c69089c71089c79089c81089c89089c91089c99089ca1089ca9089cb1089cb9089cc1089cc9089cd1089cd9089ce1089ce9089cf1089cf9089d01089d09089d11089d19089d21089d29089d31089d39089d41089d49089d51089d59089d61089d69089d71089d79089d81089d89089d91089d99089da1089da9089db1089db9089dc1089dc9089dd1089dd9089de1089de9089df1089df9089e01089e09089e11089e19089e21089e29089e31089e39089e41089e49089e51089e59089e61089e69089e71089e79089e81089e89089e91089e99089ea1089ea9089eb1089eb9089ec1089ec9089ed1089ed9089ee1089ee9089ef1089ef9089f01089f09089f11089f19089f21089f29089f31089f39089f41089f49089f51089f59089f61089f69089f71089f79089f81089f89089f91089f99089fa1089fa9089fb1089fb9089fc1089fc9089fd1089fd9089fe1089fe9089ff1089ff98a0ac9329acf23ba884804fdd3ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0034f800000000000024c00130001300000000000000000000000000000000000000000000000000000000126000980009800000000000000000000000000000000000000000000000000000000930004c0004c000000000000000000000000000000000000000000000000000000004980026000260000000000000000000000000000000000000000000000000000000024c0013000130000000000000000000000000000000000000000000000000000000002955540000000000000000000000000000000000000000000000000000000000000001ff108c00846024230221181908c108460a4230621183908c20846124230a21185908c308461a4230e21187908c40846224231221189908c508462a423162118b908c60846324231a2118d908c708463a4231e2118f908c80846424232221191908c908464a4232621193908ca0846524232a21195908cb08465a4232e21197908cc0846624233221199908cd08466a423362119b908ce0846724233a2119d908cf08467a4233e2119f908d008468242342211a1908d108468a42346211a3908d20846924234a211a5908d308469a4234e211a7908d40846a242352211a9908d50846aa42356211ab908d60846b24235a211ad908d70846ba4235e211af908d80846c242362211b1908d90846ca42366211b3908da0846d24236a211b5908db0846da4236e211b7908dc0846e242372211b9908dd0846ea42376211bb908de0846f24237a211bd908df0846fa4237e211bf908e008470242382211c1908e108470a42386211c3908e20847124238a211c5908e2f8c211c6c8471d211c7c84721211c8c84725211c9c84729211cac8472d211cbc84731211ccc84735211cdc84739211cec8473d211cfc84741211d0c84745211d1c84749211d2c8474d211d3c84751211d4c84755211d5c84759211d6c8475d211d7c84761211d8c84765211d9c84769211dac8476d211dbc84771211dcc84775211ddc84779211dec8477d211dfc84781211e0c84785211e1c84789211e2c8478d211e3c84791211e4c84795211e5c84799211e6c8479d211e7c847a1211e8c847a5211e9c847a9211eac847ad211ebc847b1211ecc847b5211edc847b9211eec847bd211efc847c1211f0c847c5211f1c847c9211f2c847cd211f3c847d1211f4c847d5211f5c847d9211f6c847dd211f7c847e1211f8c847e5211f9c847e9211fac847ed211fbc847f1211fcc847f5211fdc847f9211fec847fd211ff8bb9229c284803a8b6248" + +const rand2BZ2Hex = "425a6839314159265359d992d0f60000137dfe84020310091c1e280e100e042801099210094806c0110002e70806402000546034000034000000f2830000032000d3403264049270eb7a9280d308ca06ad28f6981bee1bf8160727c7364510d73a1e123083421b63f031f63993a0f40051fbf177245385090d992d0f60" +const rand2Hex = "92d5652616ac444a4a04af1a8a3964aca0450d43d6cf233bd03233f4ba92f8719e6c2a2bd4f5f88db07ecd0da3a33b263483db9b2c158786ad6363be35d17335ba" + +const rand3BZ2Hex = "425a68393141592653593be669d00000327ffffffffffffffffffffffffffffffffffff7ffffffffffffffffffffffffffffffc002b3b2b1b6e2bae400004c00132300004c0d268c004c08c0130026001a008683234c0684c34008c230261a04c0260064d07a8d00034000d27a1268c9931a8d327a3427a41faa69ea0da264c1a34219326869b51b49a6469a3268c689fa53269a62794687a9a68f5189994c9e487a8f534fd49a3d34043629e8c93d04da4f4648d30d4f44d3234c4d3023d0840680984d309934c234d3131a000640984f536a6132601300130130c8d00d04d1841ea7a8d31a02609b40023460010c01a34d4c1a0d04d3069306810034d0d0d4c0046130d034d0131a9a64d321804c68003400098344c13000991808c0001a00000000098004d3d4da4604c47a13012140aadf8d673c922c607ef6212a8c0403adea4b28aee578900e653b9cdeb8d11e6b838815f3ebaad5a01c5408d84a332170aff8734d4e06612d3c2889f31925fb89e33561f5100ae89b1f7047102e729373d3667e58d73aaa80fa7be368a1cc2dadd81d81ec8e1b504bd772ca31d03649269b01ceddaca07bf3d4eba24de141be3f86f93601e03714c0f64654671684f9f9528626fd4e1b76753dc0c54b842486b8d59d8ab314e86ca818e7a1f079463cbbd70d9b79b283c7edc419406311022e4be98c2c1374df9cdde2d008ce1d00e5f06ad1024baf555631f70831fc1023034e62be7c4bcb648caf276963ffa20e96bb50377fe1c113da0db4625b50741c35a058edb009c6ee5dbf93b8a6b060eec568180e8db791b82aab96cbf4326ca98361461379425ba8dcc347be670bdba7641883e5526ae3d833f6e9cb9bac9557747c79e206151072f7f0071dff3880411846f66bf4075c7462f302b53cb3400a74cf35652ad5641ed33572fd54e7ed7f85f58a0acba89327e7c6be5c58cb71528b99df2431f1d0358f8d28d81d95292da631fb06701decabb205fac59ff0fb1df536afc681eece6ea658c4d9eaa45f1342aa1ff70bdaff2ddaf25ec88c22f12829a0553db1ec2505554cb17d7b282e213a5a2aa30431ded2bce665bb199d023840832fedb2c0c350a27291407ff77440792872137df281592e82076a05c64c345ffb058c64f7f7c207ef78420b7010520610f17e302cc4dfcfaef72a0ed091aab4b541eb0531bbe941ca2f792bf7b31ca6162882b68054a8470115bc2c19f2df2023f7800432b39b04d3a304e8085ba3f1f0ca5b1ba4d38d339e6084de979cdea6d0e244c6c9fa0366bd890621e3d30846f5e8497e21597b8f29bbf52c961a485dfbea647600da0fc1f25ce4d203a8352ece310c39073525044e7ac46acf2ed9120bae1b4f6f02364abfe343f80b290983160c103557af1c68416480d024cc31b6c06cfec011456f1e95c420a12b48b1c3fe220c2879a982fb099948ac440db844b9a112a5188c7783fd3b19593290785f908d95c9db4b280bafe89c1313aeec24772046d9bc089645f0d182a21184e143823c5f52de50e5d7e98d3d7ab56f5413bbccd1415c9bcff707def475b643fb7f29842582104d4cc1dbaaca8f10a2f44273c339e0984f2b1e06ab2f0771db01fafa8142298345f3196f23e5847bda024034b6f59b11c29e981c881456e40d211929fd4f766200258aad8212016322bd5c605790dcfdf1bd2a93d99c9b8f498722d311d7eae7ff420496a31804c55f4759a7b13aaaf5f7ce006c3a8a998897d5e0a504398c2b627852545baf440798bcc5cc049357cf3f17d9771e4528a1af3d77dc794a11346e1bdf5efe37a405b127b4c43b616d61fbc5dc914e14240ef99a7400" +const rand3Hex = "1744b384d68c042371244e13500d4bfb98c6244e3d71a5b700224420b59c593553f33bd786e3d0ce31626f511bc985f59d1a88aa38ba8ad6218d306abee60dd9172540232b95be1af146c69e72e5fde667a090dc3f93bdc5c5af0ab80acdbaa7a505f628c59dc0247b31a439cacf5010a94376d71521df08c178b02fb96fdb1809144ea38c68536187c53201fea8631fb0a880b4451ccdca7cc61f6aafca21cc7449d920599db61789ac3b1e164b3390124f95022aeea39ccca3ec1053f4fa10de2978e2861ea58e477085c2220021a0927aa94c5d0006b5055abba340e4f9eba22e969978dfd18e278a8b89d877328ae34268bc0174cfe211954c0036f078025217d1269fac1932a03b05a0b616012271bbe1fb554171c7a59b196d8a4479f45a77931b5d97aaf6c0c673cbe597b79b96e2a0c1eae2e66e46ccc8c85798e23ffe972ebdaa3f6caea243c004e60321eb47cd79137d78fd0613be606feacc5b3637bdc96a89c13746db8cad886f3ccf912b2178c823bcac395f06d28080269bdca2debf3419c66c690fd1adcfbd53e32e79443d7a42511a84cb22ca94fffad9149275a075b2f8ae0b021dcde9bf62b102db920733b897560518b06e1ad7f4b03458493ddaa7f4fa2c1609f7a1735aeeb1b3e2cea3ab45fc376323cc91873b7e9c90d07c192e38d3f5dfc9bfab1fd821c854da9e607ea596c391c7ec4161c6c4493929a8176badaa5a5af7211c623f29643a937677d3df0da9266181b7c4da5dd40376db677fe8f4a1dc456adf6f33c1e37cec471dd318c2647644fe52f93707a77da7d1702380a80e14cc0fdce7bf2eed48a529090bae0388ee277ce6c7018c5fb00b88362554362205c641f0d0fab94fd5b8357b5ff08b207fee023709bc126ec90cfb17c006754638f8186aaeb1265e80be0c1189ec07d01d5f6f96cb9ce82744147d18490de7dc72862f42f024a16968891a356f5e7e0e695d8c933ba5b5e43ad4c4ade5399bc2cae9bb6189b7870d7f22956194d277f28b10e01c10c6ffe3e065f7e2d6d056aa790db5649ca84dc64c35566c0af1b68c32b5b7874aaa66467afa44f40e9a0846a07ae75360a641dd2acc69d93219b2891f190621511e62a27f5e4fbe641ece1fa234fc7e9a74f48d2a760d82160d9540f649256b169d1fed6fbefdc491126530f3cbad7913e19fbd7aa53b1e243fbf28d5f38c10ebd77c8b986775975cc1d619efb27cdcd733fa1ca36cffe9c0a33cc9f02463c91a886601fd349efee85ef1462065ef9bd2c8f533220ad93138b8382d5938103ab25b2d9af8ae106e1211eb9b18793fba033900c809c02cd6d17e2f3e6fc84dae873411f8e87c3f0a8f1765b7825d185ce3730f299c3028d4a62da9ee95c2b870fb70c79370d485f9d5d9acb78926d20444033d960524d2776dc31988ec7c0dbf23b9905d" diff --git a/libgo/go/compress/bzip2/huffman.go b/libgo/go/compress/bzip2/huffman.go new file mode 100644 index 00000000000..732bc4a21da --- /dev/null +++ b/libgo/go/compress/bzip2/huffman.go @@ -0,0 +1,223 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bzip2 + +import ( + "os" + "sort" +) + +// A huffmanTree is a binary tree which is navigated, bit-by-bit to reach a +// symbol. +type huffmanTree struct { + // nodes contains all the non-leaf nodes in the tree. nodes[0] is the + // root of the tree and nextNode contains the index of the next element + // of nodes to use when the tree is being constructed. + nodes []huffmanNode + nextNode int +} + +// A huffmanNode is a node in the tree. left and right contain indexes into the +// nodes slice of the tree. If left or right is invalidNodeValue then the child +// is a left node and its value is in leftValue/rightValue. +// +// The symbols are uint16s because bzip2 encodes not only MTF indexes in the +// tree, but also two magic values for run-length encoding and an EOF symbol. +// Thus there are more than 256 possible symbols. +type huffmanNode struct { + left, right uint16 + leftValue, rightValue uint16 +} + +// invalidNodeValue is an invalid index which marks a leaf node in the tree. +const invalidNodeValue = 0xffff + +// Decode reads bits from the given bitReader and navigates the tree until a +// symbol is found. +func (t huffmanTree) Decode(br *bitReader) (v uint16) { + nodeIndex := uint16(0) // node 0 is the root of the tree. + + for { + node := &t.nodes[nodeIndex] + bit := br.ReadBit() + // bzip2 encodes left as a true bit. + if bit { + // left + if node.left == invalidNodeValue { + return node.leftValue + } + nodeIndex = node.left + } else { + // right + if node.right == invalidNodeValue { + return node.rightValue + } + nodeIndex = node.right + } + } + + panic("unreachable") +} + +// newHuffmanTree builds a Huffman tree from a slice containing the code +// lengths of each symbol. The maximum code length is 32 bits. +func newHuffmanTree(lengths []uint8) (huffmanTree, os.Error) { + // There are many possible trees that assign the same code length to + // each symbol (consider reflecting a tree down the middle, for + // example). Since the code length assignments determine the + // efficiency of the tree, each of these trees is equally good. In + // order to minimise the amount of information needed to build a tree + // bzip2 uses a canonical tree so that it can be reconstructed given + // only the code length assignments. + + if len(lengths) < 2 { + panic("newHuffmanTree: too few symbols") + } + + var t huffmanTree + + // First we sort the code length assignments by ascending code length, + // using the symbol value to break ties. + pairs := huffmanSymbolLengthPairs(make([]huffmanSymbolLengthPair, len(lengths))) + for i, length := range lengths { + pairs[i].value = uint16(i) + pairs[i].length = length + } + + sort.Sort(pairs) + + // Now we assign codes to the symbols, starting with the longest code. + // We keep the codes packed into a uint32, at the most-significant end. + // So branches are taken from the MSB downwards. This makes it easy to + // sort them later. + code := uint32(0) + length := uint8(32) + + codes := huffmanCodes(make([]huffmanCode, len(lengths))) + for i := len(pairs) - 1; i >= 0; i-- { + if length > pairs[i].length { + // If the code length decreases we shift in order to + // zero any bits beyond the end of the code. + length >>= 32 - pairs[i].length + length <<= 32 - pairs[i].length + length = pairs[i].length + } + codes[i].code = code + codes[i].codeLen = length + codes[i].value = pairs[i].value + // We need to 'increment' the code, which means treating |code| + // like a |length| bit number. + code += 1 << (32 - length) + } + + // Now we can sort by the code so that the left half of each branch are + // grouped together, recursively. + sort.Sort(codes) + + t.nodes = make([]huffmanNode, len(codes)) + _, err := buildHuffmanNode(&t, codes, 0) + return t, err +} + +// huffmanSymbolLengthPair contains a symbol and its code length. +type huffmanSymbolLengthPair struct { + value uint16 + length uint8 +} + +// huffmanSymbolLengthPair is used to provide an interface for sorting. +type huffmanSymbolLengthPairs []huffmanSymbolLengthPair + +func (h huffmanSymbolLengthPairs) Len() int { + return len(h) +} + +func (h huffmanSymbolLengthPairs) Less(i, j int) bool { + if h[i].length < h[j].length { + return true + } + if h[i].length > h[j].length { + return false + } + if h[i].value < h[j].value { + return true + } + return false +} + +func (h huffmanSymbolLengthPairs) Swap(i, j int) { + h[i], h[j] = h[j], h[i] +} + +// huffmanCode contains a symbol, its code and code length. +type huffmanCode struct { + code uint32 + codeLen uint8 + value uint16 +} + +// huffmanCodes is used to provide an interface for sorting. +type huffmanCodes []huffmanCode + +func (n huffmanCodes) Len() int { + return len(n) +} + +func (n huffmanCodes) Less(i, j int) bool { + return n[i].code < n[j].code +} + +func (n huffmanCodes) Swap(i, j int) { + n[i], n[j] = n[j], n[i] +} + +// buildHuffmanNode takes a slice of sorted huffmanCodes and builds a node in +// the Huffman tree at the given level. It returns the index of the newly +// constructed node. +func buildHuffmanNode(t *huffmanTree, codes []huffmanCode, level uint32) (nodeIndex uint16, err os.Error) { + test := uint32(1) << (31 - level) + + // We have to search the list of codes to find the divide between the left and right sides. + firstRightIndex := len(codes) + for i, code := range codes { + if code.code&test != 0 { + firstRightIndex = i + break + } + } + + left := codes[:firstRightIndex] + right := codes[firstRightIndex:] + + if len(left) == 0 || len(right) == 0 { + return 0, StructuralError("superfluous level in Huffman tree") + } + + nodeIndex = uint16(t.nextNode) + node := &t.nodes[t.nextNode] + t.nextNode++ + + if len(left) == 1 { + // leaf node + node.left = invalidNodeValue + node.leftValue = left[0].value + } else { + node.left, err = buildHuffmanNode(t, left, level+1) + } + + if err != nil { + return + } + + if len(right) == 1 { + // leaf node + node.right = invalidNodeValue + node.rightValue = right[0].value + } else { + node.right, err = buildHuffmanNode(t, right, level+1) + } + + return +} diff --git a/libgo/go/compress/bzip2/move_to_front.go b/libgo/go/compress/bzip2/move_to_front.go new file mode 100644 index 00000000000..0ed19dec39c --- /dev/null +++ b/libgo/go/compress/bzip2/move_to_front.go @@ -0,0 +1,105 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bzip2 + +// moveToFrontDecoder implements a move-to-front list. Such a list is an +// efficient way to transform a string with repeating elements into one with +// many small valued numbers, which is suitable for entropy encoding. It works +// by starting with an initial list of symbols and references symbols by their +// index into that list. When a symbol is referenced, it's moved to the front +// of the list. Thus, a repeated symbol ends up being encoded with many zeros, +// as the symbol will be at the front of the list after the first access. +type moveToFrontDecoder struct { + // Rather than actually keep the list in memory, the symbols are stored + // as a circular, double linked list with the symbol indexed by head + // at the front of the list. + symbols []byte + next []uint8 + prev []uint8 + head uint8 +} + +// newMTFDecoder creates a move-to-front decoder with an explicit initial list +// of symbols. +func newMTFDecoder(symbols []byte) *moveToFrontDecoder { + if len(symbols) > 256 { + panic("too many symbols") + } + + m := &moveToFrontDecoder{ + symbols: symbols, + next: make([]uint8, len(symbols)), + prev: make([]uint8, len(symbols)), + } + + m.threadLinkedList() + return m +} + +// newMTFDecoderWithRange creates a move-to-front decoder with an initial +// symbol list of 0...n-1. +func newMTFDecoderWithRange(n int) *moveToFrontDecoder { + if n > 256 { + panic("newMTFDecoderWithRange: cannot have > 256 symbols") + } + + m := &moveToFrontDecoder{ + symbols: make([]uint8, n), + next: make([]uint8, n), + prev: make([]uint8, n), + } + + for i := 0; i < n; i++ { + m.symbols[i] = byte(i) + } + + m.threadLinkedList() + return m +} + +// threadLinkedList creates the initial linked-list pointers. +func (m *moveToFrontDecoder) threadLinkedList() { + if len(m.symbols) == 0 { + return + } + + m.prev[0] = uint8(len(m.symbols) - 1) + + for i := 0; i < len(m.symbols)-1; i++ { + m.next[i] = uint8(i + 1) + m.prev[i+1] = uint8(i) + } + + m.next[len(m.symbols)-1] = 0 +} + +func (m *moveToFrontDecoder) Decode(n int) (b byte) { + // Most of the time, n will be zero so it's worth dealing with this + // simple case. + if n == 0 { + return m.symbols[m.head] + } + + i := m.head + for j := 0; j < n; j++ { + i = m.next[i] + } + b = m.symbols[i] + + m.next[m.prev[i]] = m.next[i] + m.prev[m.next[i]] = m.prev[i] + m.next[i] = m.head + m.prev[i] = m.prev[m.head] + m.next[m.prev[m.head]] = i + m.prev[m.head] = i + m.head = i + + return +} + +// First returns the symbol at the front of the list. +func (m *moveToFrontDecoder) First() byte { + return m.symbols[m.head] +} diff --git a/libgo/go/compress/flate/deflate_test.go b/libgo/go/compress/flate/deflate_test.go index 3db955609d7..ff54164b2cc 100644 --- a/libgo/go/compress/flate/deflate_test.go +++ b/libgo/go/compress/flate/deflate_test.go @@ -116,9 +116,16 @@ func (b *syncBuffer) Read(p []byte) (n int, err os.Error) { panic("unreachable") } +func (b *syncBuffer) signal() { + select { + case b.ready <- true: + default: + } +} + func (b *syncBuffer) Write(p []byte) (n int, err os.Error) { n, err = b.buf.Write(p) - _ = b.ready <- true + b.signal() return } @@ -128,12 +135,12 @@ func (b *syncBuffer) WriteMode() { func (b *syncBuffer) ReadMode() { b.mu.Unlock() - _ = b.ready <- true + b.signal() } func (b *syncBuffer) Close() os.Error { b.closed = true - _ = b.ready <- true + b.signal() return nil } @@ -255,135 +262,9 @@ func TestReverseBits(t *testing.T) { } func TestDeflateInflateString(t *testing.T) { - gold := bytes.NewBufferString(getEdata()).Bytes() + gold, err := ioutil.ReadFile("../testdata/e.txt") + if err != nil { + t.Error(err) + } testToFromWithLevel(t, 1, gold, "2.718281828...") } - -func getEdata() string { - return "2.718281828459045235360287471352662497757247093699959574966967627724076630353547" + - "59457138217852516642742746639193200305992181741359662904357290033429526059563073" + - "81323286279434907632338298807531952510190115738341879307021540891499348841675092" + - "44761460668082264800168477411853742345442437107539077744992069551702761838606261" + - "33138458300075204493382656029760673711320070932870912744374704723069697720931014" + - "16928368190255151086574637721112523897844250569536967707854499699679468644549059" + - "87931636889230098793127736178215424999229576351482208269895193668033182528869398" + - "49646510582093923982948879332036250944311730123819706841614039701983767932068328" + - "23764648042953118023287825098194558153017567173613320698112509961818815930416903" + - "51598888519345807273866738589422879228499892086805825749279610484198444363463244" + - "96848756023362482704197862320900216099023530436994184914631409343173814364054625" + - "31520961836908887070167683964243781405927145635490613031072085103837505101157477" + - "04171898610687396965521267154688957035035402123407849819334321068170121005627880" + - "23519303322474501585390473041995777709350366041699732972508868769664035557071622" + - "68447162560798826517871341951246652010305921236677194325278675398558944896970964" + - "09754591856956380236370162112047742722836489613422516445078182442352948636372141" + - "74023889344124796357437026375529444833799801612549227850925778256209262264832627" + - "79333865664816277251640191059004916449982893150566047258027786318641551956532442" + - "58698294695930801915298721172556347546396447910145904090586298496791287406870504" + - "89585867174798546677575732056812884592054133405392200011378630094556068816674001" + - "69842055804033637953764520304024322566135278369511778838638744396625322498506549" + - "95886234281899707733276171783928034946501434558897071942586398772754710962953741" + - "52111513683506275260232648472870392076431005958411661205452970302364725492966693" + - "81151373227536450988890313602057248176585118063036442812314965507047510254465011" + - "72721155519486685080036853228183152196003735625279449515828418829478761085263981" + - "39559900673764829224437528718462457803619298197139914756448826260390338144182326" + - "25150974827987779964373089970388867782271383605772978824125611907176639465070633" + - "04527954661855096666185664709711344474016070462621568071748187784437143698821855" + - "96709591025968620023537185887485696522000503117343920732113908032936344797273559" + - "55277349071783793421637012050054513263835440001863239914907054797780566978533580" + - "48966906295119432473099587655236812859041383241160722602998330535370876138939639" + - "17795745401613722361878936526053815584158718692553860616477983402543512843961294" + - "60352913325942794904337299085731580290958631382683291477116396337092400316894586" + - "36060645845925126994655724839186564209752685082307544254599376917041977780085362" + - "73094171016343490769642372229435236612557250881477922315197477806056967253801718" + - "07763603462459278778465850656050780844211529697521890874019660906651803516501792" + - "50461950136658543663271254963990854914420001457476081930221206602433009641270489" + - "43903971771951806990869986066365832322787093765022601492910115171776359446020232" + - "49300280401867723910288097866605651183260043688508817157238669842242201024950551" + - "88169480322100251542649463981287367765892768816359831247788652014117411091360116" + - "49950766290779436460058519419985601626479076153210387275571269925182756879893027" + - "61761146162549356495903798045838182323368612016243736569846703785853305275833337" + - "93990752166069238053369887956513728559388349989470741618155012539706464817194670" + - "83481972144888987906765037959036696724949925452790337296361626589760394985767413" + - "97359441023744329709355477982629614591442936451428617158587339746791897571211956" + - "18738578364475844842355558105002561149239151889309946342841393608038309166281881" + - "15037152849670597416256282360921680751501777253874025642534708790891372917228286" + - "11515915683725241630772254406337875931059826760944203261924285317018781772960235" + - "41306067213604600038966109364709514141718577701418060644363681546444005331608778" + - "31431744408119494229755993140118886833148328027065538330046932901157441475631399" + - "97221703804617092894579096271662260740718749975359212756084414737823303270330168" + - "23719364800217328573493594756433412994302485023573221459784328264142168487872167" + - "33670106150942434569844018733128101079451272237378861260581656680537143961278887" + - "32527373890392890506865324138062796025930387727697783792868409325365880733988457" + - "21874602100531148335132385004782716937621800490479559795929059165547050577751430" + - "81751126989851884087185640260353055837378324229241856256442550226721559802740126" + - "17971928047139600689163828665277009752767069777036439260224372841840883251848770" + - "47263844037953016690546593746161932384036389313136432713768884102681121989127522" + - "30562567562547017250863497653672886059667527408686274079128565769963137897530346" + - "60616669804218267724560530660773899624218340859882071864682623215080288286359746" + - "83965435885668550377313129658797581050121491620765676995065971534476347032085321" + - "56036748286083786568030730626576334697742956346437167093971930608769634953288468" + - "33613038829431040800296873869117066666146800015121143442256023874474325250769387" + - "07777519329994213727721125884360871583483562696166198057252661220679754062106208" + - "06498829184543953015299820925030054982570433905535701686531205264956148572492573" + - "86206917403695213533732531666345466588597286659451136441370331393672118569553952" + - "10845840724432383558606310680696492485123263269951460359603729725319836842336390" + - "46321367101161928217111502828016044880588023820319814930963695967358327420249882" + - "45684941273860566491352526706046234450549227581151709314921879592718001940968866" + - "98683703730220047531433818109270803001720593553052070070607223399946399057131158" + - "70996357773590271962850611465148375262095653467132900259943976631145459026858989" + - "79115837093419370441155121920117164880566945938131183843765620627846310490346293" + - "95002945834116482411496975832601180073169943739350696629571241027323913874175492" + - "30718624545432220395527352952402459038057445028922468862853365422138157221311632" + - "88112052146489805180092024719391710555390113943316681515828843687606961102505171" + - "00739276238555338627255353883096067164466237092264680967125406186950214317621166" + - "81400975952814939072226011126811531083873176173232352636058381731510345957365382" + - "23534992935822836851007810884634349983518404451704270189381994243410090575376257" + - "76757111809008816418331920196262341628816652137471732547772778348877436651882875" + - "21566857195063719365653903894493664217640031215278702223664636357555035655769488" + - "86549500270853923617105502131147413744106134445544192101336172996285694899193369" + - "18472947858072915608851039678195942983318648075608367955149663644896559294818785" + - "17840387733262470519450504198477420141839477312028158868457072905440575106012852" + - "58056594703046836344592652552137008068752009593453607316226118728173928074623094" + - "68536782310609792159936001994623799343421068781349734695924646975250624695861690" + - "91785739765951993929939955675427146549104568607020990126068187049841780791739240" + - "71945996323060254707901774527513186809982284730860766536866855516467702911336827" + - "56310722334672611370549079536583453863719623585631261838715677411873852772292259" + - "47433737856955384562468010139057278710165129666367644518724656537304024436841408" + - "14488732957847348490003019477888020460324660842875351848364959195082888323206522" + - "12810419044804724794929134228495197002260131043006241071797150279343326340799596" + - "05314460532304885289729176598760166678119379323724538572096075822771784833616135" + - "82612896226118129455927462767137794487586753657544861407611931125958512655759734" + - "57301533364263076798544338576171533346232527057200530398828949903425956623297578" + - "24887350292591668258944568946559926584547626945287805165017206747854178879822768" + - "06536650641910973434528878338621726156269582654478205672987756426325321594294418" + - "03994321700009054265076309558846589517170914760743713689331946909098190450129030" + - "70995662266203031826493657336984195557769637876249188528656866076005660256054457" + - "11337286840205574416030837052312242587223438854123179481388550075689381124935386" + - "31863528708379984569261998179452336408742959118074745341955142035172618420084550" + - "91708456823682008977394558426792142734775608796442792027083121501564063413416171" + - "66448069815483764491573900121217041547872591998943825364950514771379399147205219" + - "52907939613762110723849429061635760459623125350606853765142311534966568371511660" + - "42207963944666211632551577290709784731562782775987881364919512574833287937715714" + - "59091064841642678309949723674420175862269402159407924480541255360431317992696739" + - "15754241929660731239376354213923061787675395871143610408940996608947141834069836" + - "29936753626215452472984642137528910798843813060955526227208375186298370667872244" + - "30195793793786072107254277289071732854874374355781966511716618330881129120245204" + - "04868220007234403502544820283425418788465360259150644527165770004452109773558589" + - "76226554849416217149895323834216001140629507184904277892585527430352213968356790" + - "18076406042138307308774460170842688272261177180842664333651780002171903449234264" + - "26629226145600433738386833555534345300426481847398921562708609565062934040526494" + - "32442614456659212912256488935696550091543064261342526684725949143142393988454324" + - "86327461842846655985332312210466259890141712103446084271616619001257195870793217" + - "56969854401339762209674945418540711844643394699016269835160784892451405894094639" + - "52678073545797003070511636825194877011897640028276484141605872061841852971891540" + - "19688253289309149665345753571427318482016384644832499037886069008072709327673127" + - "58196656394114896171683298045513972950668760474091542042842999354102582911350224" + - "16907694316685742425225090269390348148564513030699251995904363840284292674125734" + - "22447765584177886171737265462085498294498946787350929581652632072258992368768457" + - "01782303809656788311228930580914057261086588484587310165815116753332767488701482" + - "91674197015125597825727074064318086014281490241467804723275976842696339357735429" + - "30186739439716388611764209004068663398856841681003872389214483176070116684503887" + - "21236436704331409115573328018297798873659091665961240202177855885487617616198937" + - "07943800566633648843650891448055710397652146960276625835990519870423001794655367" + - "9" -} diff --git a/libgo/go/compress/lzw/reader.go b/libgo/go/compress/lzw/reader.go new file mode 100644 index 00000000000..8a540cbe6a1 --- /dev/null +++ b/libgo/go/compress/lzw/reader.go @@ -0,0 +1,210 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The lzw package implements the Lempel-Ziv-Welch compressed data format, +// described in T. A. Welch, ``A Technique for High-Performance Data +// Compression'', Computer, 17(6) (June 1984), pp 8-19. +// +// In particular, it implements LZW as used by the GIF, TIFF and PDF file +// formats, which means variable-width codes up to 12 bits and the first +// two non-literal codes are a clear code and an EOF code. +package lzw + +// TODO(nigeltao): check that TIFF and PDF use LZW in the same way as GIF, +// modulo LSB/MSB packing order. + +import ( + "bufio" + "fmt" + "io" + "os" +) + +// Order specifies the bit ordering in an LZW data stream. +type Order int + +const ( + // LSB means Least Significant Bits first, as used in the GIF file format. + LSB Order = iota + // MSB means Most Significant Bits first, as used in the TIFF and PDF + // file formats. + MSB +) + +// decoder is the state from which the readXxx method converts a byte +// stream into a code stream. +type decoder struct { + r io.ByteReader + bits uint32 + nBits uint + width uint +} + +// readLSB returns the next code for "Least Significant Bits first" data. +func (d *decoder) readLSB() (uint16, os.Error) { + for d.nBits < d.width { + x, err := d.r.ReadByte() + if err != nil { + return 0, err + } + d.bits |= uint32(x) << d.nBits + d.nBits += 8 + } + code := uint16(d.bits & (1<<d.width - 1)) + d.bits >>= d.width + d.nBits -= d.width + return code, nil +} + +// readMSB returns the next code for "Most Significant Bits first" data. +func (d *decoder) readMSB() (uint16, os.Error) { + for d.nBits < d.width { + x, err := d.r.ReadByte() + if err != nil { + return 0, err + } + d.bits |= uint32(x) << (24 - d.nBits) + d.nBits += 8 + } + code := uint16(d.bits >> (32 - d.width)) + d.bits <<= d.width + d.nBits -= d.width + return code, nil +} + +// decode decompresses bytes from r and writes them to pw. +// read specifies how to decode bytes into codes. +// litWidth is the width in bits of literal codes. +func decode(r io.Reader, read func(*decoder) (uint16, os.Error), litWidth int, pw *io.PipeWriter) { + br, ok := r.(io.ByteReader) + if !ok { + br = bufio.NewReader(r) + } + pw.CloseWithError(decode1(pw, br, read, uint(litWidth))) +} + +func decode1(pw *io.PipeWriter, r io.ByteReader, read func(*decoder) (uint16, os.Error), litWidth uint) os.Error { + const ( + maxWidth = 12 + invalidCode = 0xffff + ) + d := decoder{r, 0, 0, 1 + litWidth} + w := bufio.NewWriter(pw) + // The first 1<<litWidth codes are literal codes. + // The next two codes mean clear and EOF. + // Other valid codes are in the range [lo, hi] where lo := clear + 2, + // with the upper bound incrementing on each code seen. + clear := uint16(1) << litWidth + eof, hi := clear+1, clear+1 + // overflow is the code at which hi overflows the code width. + overflow := uint16(1) << d.width + var ( + // Each code c in [lo, hi] expands to two or more bytes. For c != hi: + // suffix[c] is the last of these bytes. + // prefix[c] is the code for all but the last byte. + // This code can either be a literal code or another code in [lo, c). + // The c == hi case is a special case. + suffix [1 << maxWidth]uint8 + prefix [1 << maxWidth]uint16 + // buf is a scratch buffer for reconstituting the bytes that a code expands to. + // Code suffixes are written right-to-left from the end of the buffer. + buf [1 << maxWidth]byte + ) + + // Loop over the code stream, converting codes into decompressed bytes. + last := uint16(invalidCode) + for { + code, err := read(&d) + if err != nil { + if err == os.EOF { + err = io.ErrUnexpectedEOF + } + return err + } + switch { + case code < clear: + // We have a literal code. + if err := w.WriteByte(uint8(code)); err != nil { + return err + } + if last != invalidCode { + // Save what the hi code expands to. + suffix[hi] = uint8(code) + prefix[hi] = last + } + case code == clear: + d.width = 1 + litWidth + hi = eof + overflow = 1 << d.width + last = invalidCode + continue + case code == eof: + return w.Flush() + case code <= hi: + c, i := code, len(buf)-1 + if code == hi { + // code == hi is a special case which expands to the last expansion + // followed by the head of the last expansion. To find the head, we walk + // the prefix chain until we find a literal code. + c = last + for c >= clear { + c = prefix[c] + } + buf[i] = uint8(c) + i-- + c = last + } + // Copy the suffix chain into buf and then write that to w. + for c >= clear { + buf[i] = suffix[c] + i-- + c = prefix[c] + } + buf[i] = uint8(c) + if _, err := w.Write(buf[i:]); err != nil { + return err + } + // Save what the hi code expands to. + suffix[hi] = uint8(c) + prefix[hi] = last + default: + return os.NewError("lzw: invalid code") + } + last, hi = code, hi+1 + if hi == overflow { + if d.width == maxWidth { + return os.NewError("lzw: missing clear code") + } + d.width++ + overflow <<= 1 + } + } + panic("unreachable") +} + +// NewReader creates a new io.ReadCloser that satisfies reads by decompressing +// the data read from r. +// It is the caller's responsibility to call Close on the ReadCloser when +// finished reading. +// The number of bits to use for literal codes, litWidth, must be in the +// range [2,8] and is typically 8. +func NewReader(r io.Reader, order Order, litWidth int) io.ReadCloser { + pr, pw := io.Pipe() + var read func(*decoder) (uint16, os.Error) + switch order { + case LSB: + read = (*decoder).readLSB + case MSB: + read = (*decoder).readMSB + default: + pw.CloseWithError(os.NewError("lzw: unknown order")) + return pr + } + if litWidth < 2 || 8 < litWidth { + pw.CloseWithError(fmt.Errorf("lzw: litWidth %d out of range", litWidth)) + return pr + } + go decode(r, read, litWidth, pw) + return pr +} diff --git a/libgo/go/compress/lzw/reader_test.go b/libgo/go/compress/lzw/reader_test.go new file mode 100644 index 00000000000..7795a4c1489 --- /dev/null +++ b/libgo/go/compress/lzw/reader_test.go @@ -0,0 +1,132 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzw + +import ( + "bytes" + "io" + "io/ioutil" + "os" + "strconv" + "strings" + "testing" +) + +type lzwTest struct { + desc string + raw string + compressed string + err os.Error +} + +var lzwTests = []lzwTest{ + { + "empty;LSB;8", + "", + "\x01\x01", + nil, + }, + { + "empty;MSB;8", + "", + "\x80\x80", + nil, + }, + { + "tobe;LSB;7", + "TOBEORNOTTOBEORTOBEORNOT", + "\x54\x4f\x42\x45\x4f\x52\x4e\x4f\x54\x82\x84\x86\x8b\x85\x87\x89\x81", + nil, + }, + { + "tobe;LSB;8", + "TOBEORNOTTOBEORTOBEORNOT", + "\x54\x9e\x08\x29\xf2\x44\x8a\x93\x27\x54\x04\x12\x34\xb8\xb0\xe0\xc1\x84\x01\x01", + nil, + }, + { + "tobe;MSB;7", + "TOBEORNOTTOBEORTOBEORNOT", + "\x54\x4f\x42\x45\x4f\x52\x4e\x4f\x54\x82\x84\x86\x8b\x85\x87\x89\x81", + nil, + }, + { + "tobe;MSB;8", + "TOBEORNOTTOBEORTOBEORNOT", + "\x2a\x13\xc8\x44\x52\x79\x48\x9c\x4f\x2a\x40\xa0\x90\x68\x5c\x16\x0f\x09\x80\x80", + nil, + }, + { + "tobe-truncated;LSB;8", + "TOBEORNOTTOBEORTOBEORNOT", + "\x54\x9e\x08\x29\xf2\x44\x8a\x93\x27\x54\x04", + io.ErrUnexpectedEOF, + }, + // This example comes from http://en.wikipedia.org/wiki/Graphics_Interchange_Format. + { + "gif;LSB;8", + "\x28\xff\xff\xff\x28\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\x00\x51\xfc\x1b\x28\x70\xa0\xc1\x83\x01\x01", + nil, + }, + // This example comes from http://compgroups.net/comp.lang.ruby/Decompressing-LZW-compression-from-PDF-file + { + "pdf;MSB;8", + "-----A---B", + "\x80\x0b\x60\x50\x22\x0c\x0c\x85\x01", + nil, + }, +} + +func TestReader(t *testing.T) { + b := bytes.NewBuffer(nil) + for _, tt := range lzwTests { + d := strings.Split(tt.desc, ";", -1) + var order Order + switch d[1] { + case "LSB": + order = LSB + case "MSB": + order = MSB + default: + t.Errorf("%s: bad order %q", tt.desc, d[1]) + } + litWidth, _ := strconv.Atoi(d[2]) + rc := NewReader(strings.NewReader(tt.compressed), order, litWidth) + defer rc.Close() + b.Reset() + n, err := io.Copy(b, rc) + if err != nil { + if err != tt.err { + t.Errorf("%s: io.Copy: %v want %v", tt.desc, err, tt.err) + } + continue + } + s := b.String() + if s != tt.raw { + t.Errorf("%s: got %d-byte %q want %d-byte %q", tt.desc, n, s, len(tt.raw), tt.raw) + } + } +} + +type devNull struct{} + +func (devNull) Write(p []byte) (int, os.Error) { + return len(p), nil +} + +func BenchmarkDecoder(b *testing.B) { + b.StopTimer() + buf0, _ := ioutil.ReadFile("../testdata/e.txt") + compressed := bytes.NewBuffer(nil) + w := NewWriter(compressed, LSB, 8) + io.Copy(w, bytes.NewBuffer(buf0)) + w.Close() + buf1 := compressed.Bytes() + b.StartTimer() + for i := 0; i < b.N; i++ { + io.Copy(devNull{}, NewReader(bytes.NewBuffer(buf1), LSB, 8)) + } +} diff --git a/libgo/go/compress/lzw/writer.go b/libgo/go/compress/lzw/writer.go new file mode 100644 index 00000000000..87143b7aa94 --- /dev/null +++ b/libgo/go/compress/lzw/writer.go @@ -0,0 +1,259 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzw + +import ( + "bufio" + "fmt" + "io" + "os" +) + +// A writer is a buffered, flushable writer. +type writer interface { + WriteByte(byte) os.Error + Flush() os.Error +} + +// An errWriteCloser is an io.WriteCloser that always returns a given error. +type errWriteCloser struct { + err os.Error +} + +func (e *errWriteCloser) Write([]byte) (int, os.Error) { + return 0, e.err +} + +func (e *errWriteCloser) Close() os.Error { + return e.err +} + +const ( + // A code is a 12 bit value, stored as a uint32 when encoding to avoid + // type conversions when shifting bits. + maxCode = 1<<12 - 1 + invalidCode = 1<<32 - 1 + // There are 1<<12 possible codes, which is an upper bound on the number of + // valid hash table entries at any given point in time. tableSize is 4x that. + tableSize = 4 * 1 << 12 + tableMask = tableSize - 1 + // A hash table entry is a uint32. Zero is an invalid entry since the + // lower 12 bits of a valid entry must be a non-literal code. + invalidEntry = 0 +) + +// encoder is LZW compressor. +type encoder struct { + // w is the writer that compressed bytes are written to. + w writer + // write, bits, nBits and width are the state for converting a code stream + // into a byte stream. + write func(*encoder, uint32) os.Error + bits uint32 + nBits uint + width uint + // litWidth is the width in bits of literal codes. + litWidth uint + // hi is the code implied by the next code emission. + // overflow is the code at which hi overflows the code width. + hi, overflow uint32 + // savedCode is the accumulated code at the end of the most recent Write + // call. It is equal to invalidCode if there was no such call. + savedCode uint32 + // err is the first error encountered during writing. Closing the encoder + // will make any future Write calls return os.EINVAL. + err os.Error + // table is the hash table from 20-bit keys to 12-bit values. Each table + // entry contains key<<12|val and collisions resolve by linear probing. + // The keys consist of a 12-bit code prefix and an 8-bit byte suffix. + // The values are a 12-bit code. + table [tableSize]uint32 +} + +// writeLSB writes the code c for "Least Significant Bits first" data. +func (e *encoder) writeLSB(c uint32) os.Error { + e.bits |= c << e.nBits + e.nBits += e.width + for e.nBits >= 8 { + if err := e.w.WriteByte(uint8(e.bits)); err != nil { + return err + } + e.bits >>= 8 + e.nBits -= 8 + } + return nil +} + +// writeMSB writes the code c for "Most Significant Bits first" data. +func (e *encoder) writeMSB(c uint32) os.Error { + e.bits |= c << (32 - e.width - e.nBits) + e.nBits += e.width + for e.nBits >= 8 { + if err := e.w.WriteByte(uint8(e.bits >> 24)); err != nil { + return err + } + e.bits <<= 8 + e.nBits -= 8 + } + return nil +} + +// errOutOfCodes is an internal error that means that the encoder has run out +// of unused codes and a clear code needs to be sent next. +var errOutOfCodes = os.NewError("lzw: out of codes") + +// incHi increments e.hi and checks for both overflow and running out of +// unused codes. In the latter case, incHi sends a clear code, resets the +// encoder state and returns errOutOfCodes. +func (e *encoder) incHi() os.Error { + e.hi++ + if e.hi == e.overflow { + e.width++ + e.overflow <<= 1 + } + if e.hi == maxCode { + clear := uint32(1) << e.litWidth + if err := e.write(e, clear); err != nil { + return err + } + e.width = uint(e.litWidth) + 1 + e.hi = clear + 1 + e.overflow = clear << 1 + for i := range e.table { + e.table[i] = invalidEntry + } + return errOutOfCodes + } + return nil +} + +// Write writes a compressed representation of p to e's underlying writer. +func (e *encoder) Write(p []byte) (int, os.Error) { + if e.err != nil { + return 0, e.err + } + if len(p) == 0 { + return 0, nil + } + litMask := uint32(1<<e.litWidth - 1) + code := e.savedCode + if code == invalidCode { + // The first code sent is always a literal code. + code, p = uint32(p[0])&litMask, p[1:] + } +loop: + for _, x := range p { + literal := uint32(x) & litMask + key := code<<8 | literal + // If there is a hash table hit for this key then we continue the loop + // and do not emit a code yet. + hash := (key>>12 ^ key) & tableMask + for h, t := hash, e.table[hash]; t != invalidEntry; { + if key == t>>12 { + code = t & maxCode + continue loop + } + h = (h + 1) & tableMask + t = e.table[h] + } + // Otherwise, write the current code, and literal becomes the start of + // the next emitted code. + if e.err = e.write(e, code); e.err != nil { + return 0, e.err + } + code = literal + // Increment e.hi, the next implied code. If we run out of codes, reset + // the encoder state (including clearing the hash table) and continue. + if err := e.incHi(); err != nil { + if err == errOutOfCodes { + continue + } + e.err = err + return 0, e.err + } + // Otherwise, insert key -> e.hi into the map that e.table represents. + for { + if e.table[hash] == invalidEntry { + e.table[hash] = (key << 12) | e.hi + break + } + hash = (hash + 1) & tableMask + } + } + e.savedCode = code + return len(p), nil +} + +// Close closes the encoder, flushing any pending output. It does not close or +// flush e's underlying writer. +func (e *encoder) Close() os.Error { + if e.err != nil { + if e.err == os.EINVAL { + return nil + } + return e.err + } + // Make any future calls to Write return os.EINVAL. + e.err = os.EINVAL + // Write the savedCode if valid. + if e.savedCode != invalidCode { + if err := e.write(e, e.savedCode); err != nil { + return err + } + if err := e.incHi(); err != nil && err != errOutOfCodes { + return err + } + } + // Write the eof code. + eof := uint32(1)<<e.litWidth + 1 + if err := e.write(e, eof); err != nil { + return err + } + // Write the final bits. + if e.nBits > 0 { + if e.write == (*encoder).writeMSB { + e.bits >>= 24 + } + if err := e.w.WriteByte(uint8(e.bits)); err != nil { + return err + } + } + return e.w.Flush() +} + +// NewWriter creates a new io.WriteCloser that satisfies writes by compressing +// the data and writing it to w. +// It is the caller's responsibility to call Close on the WriteCloser when +// finished writing. +// The number of bits to use for literal codes, litWidth, must be in the +// range [2,8] and is typically 8. +func NewWriter(w io.Writer, order Order, litWidth int) io.WriteCloser { + var write func(*encoder, uint32) os.Error + switch order { + case LSB: + write = (*encoder).writeLSB + case MSB: + write = (*encoder).writeMSB + default: + return &errWriteCloser{os.NewError("lzw: unknown order")} + } + if litWidth < 2 || 8 < litWidth { + return &errWriteCloser{fmt.Errorf("lzw: litWidth %d out of range", litWidth)} + } + bw, ok := w.(writer) + if !ok { + bw = bufio.NewWriter(w) + } + lw := uint(litWidth) + return &encoder{ + w: bw, + write: write, + width: 1 + lw, + litWidth: lw, + hi: 1<<lw + 1, + overflow: 1 << (lw + 1), + savedCode: invalidCode, + } +} diff --git a/libgo/go/compress/lzw/writer_test.go b/libgo/go/compress/lzw/writer_test.go new file mode 100644 index 00000000000..715b974aa1e --- /dev/null +++ b/libgo/go/compress/lzw/writer_test.go @@ -0,0 +1,111 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzw + +import ( + "io" + "io/ioutil" + "os" + "testing" +) + +var filenames = []string{ + "../testdata/e.txt", + "../testdata/pi.txt", +} + +// testFile tests that compressing and then decompressing the given file with +// the given options yields equivalent bytes to the original file. +func testFile(t *testing.T, fn string, order Order, litWidth int) { + // Read the file, as golden output. + golden, err := os.Open(fn, os.O_RDONLY, 0400) + if err != nil { + t.Errorf("%s (order=%d litWidth=%d): %v", fn, order, litWidth, err) + return + } + defer golden.Close() + + // Read the file again, and push it through a pipe that compresses at the write end, and decompresses at the read end. + raw, err := os.Open(fn, os.O_RDONLY, 0400) + if err != nil { + t.Errorf("%s (order=%d litWidth=%d): %v", fn, order, litWidth, err) + return + } + + piper, pipew := io.Pipe() + defer piper.Close() + go func() { + defer raw.Close() + defer pipew.Close() + lzww := NewWriter(pipew, order, litWidth) + defer lzww.Close() + var b [4096]byte + for { + n, err0 := raw.Read(b[:]) + if err0 != nil && err0 != os.EOF { + t.Errorf("%s (order=%d litWidth=%d): %v", fn, order, litWidth, err0) + return + } + _, err1 := lzww.Write(b[:n]) + if err1 == os.EPIPE { + // Fail, but do not report the error, as some other (presumably reportable) error broke the pipe. + return + } + if err1 != nil { + t.Errorf("%s (order=%d litWidth=%d): %v", fn, order, litWidth, err1) + return + } + if err0 == os.EOF { + break + } + } + }() + lzwr := NewReader(piper, order, litWidth) + defer lzwr.Close() + + // Compare the two. + b0, err0 := ioutil.ReadAll(golden) + b1, err1 := ioutil.ReadAll(lzwr) + if err0 != nil { + t.Errorf("%s (order=%d litWidth=%d): %v", fn, order, litWidth, err0) + return + } + if err1 != nil { + t.Errorf("%s (order=%d litWidth=%d): %v", fn, order, litWidth, err1) + return + } + if len(b0) != len(b1) { + t.Errorf("%s (order=%d litWidth=%d): length mismatch %d versus %d", fn, order, litWidth, len(b0), len(b1)) + return + } + for i := 0; i < len(b0); i++ { + if b0[i] != b1[i] { + t.Errorf("%s (order=%d litWidth=%d): mismatch at %d, 0x%02x versus 0x%02x\n", fn, order, litWidth, i, b0[i], b1[i]) + return + } + } +} + +func TestWriter(t *testing.T) { + for _, filename := range filenames { + for _, order := range [...]Order{LSB, MSB} { + // The test data "2.71828 etcetera" is ASCII text requiring at least 6 bits. + for _, litWidth := range [...]int{6, 7, 8} { + testFile(t, filename, order, litWidth) + } + } + } +} + +func BenchmarkEncoder(b *testing.B) { + b.StopTimer() + buf, _ := ioutil.ReadFile("../testdata/e.txt") + b.StartTimer() + for i := 0; i < b.N; i++ { + w := NewWriter(devNull{}, LSB, 8) + w.Write(buf) + w.Close() + } +} diff --git a/libgo/go/compress/testdata/e.txt b/libgo/go/compress/testdata/e.txt new file mode 100644 index 00000000000..76cf2a7b691 --- /dev/null +++ b/libgo/go/compress/testdata/e.txt @@ -0,0 +1 @@ +2.7182818284590452353602874713526624977572470936999595749669676277240766303535475945713821785251664274274663919320030599218174135966290435729003342952605956307381323286279434907632338298807531952510190115738341879307021540891499348841675092447614606680822648001684774118537423454424371075390777449920695517027618386062613313845830007520449338265602976067371132007093287091274437470472306969772093101416928368190255151086574637721112523897844250569536967707854499699679468644549059879316368892300987931277361782154249992295763514822082698951936680331825288693984964651058209392398294887933203625094431173012381970684161403970198376793206832823764648042953118023287825098194558153017567173613320698112509961818815930416903515988885193458072738667385894228792284998920868058257492796104841984443634632449684875602336248270419786232090021609902353043699418491463140934317381436405462531520961836908887070167683964243781405927145635490613031072085103837505101157477041718986106873969655212671546889570350354021234078498193343210681701210056278802351930332247450158539047304199577770935036604169973297250886876966403555707162268447162560798826517871341951246652010305921236677194325278675398558944896970964097545918569563802363701621120477427228364896134225164450781824423529486363721417402388934412479635743702637552944483379980161254922785092577825620926226483262779333865664816277251640191059004916449982893150566047258027786318641551956532442586982946959308019152987211725563475463964479101459040905862984967912874068705048958586717479854667757573205681288459205413340539220001137863009455606881667400169842055804033637953764520304024322566135278369511778838638744396625322498506549958862342818997077332761717839280349465014345588970719425863987727547109629537415211151368350627526023264847287039207643100595841166120545297030236472549296669381151373227536450988890313602057248176585118063036442812314965507047510254465011727211555194866850800368532281831521960037356252794495158284188294787610852639813955990067376482922443752871846245780361929819713991475644882626039033814418232625150974827987779964373089970388867782271383605772978824125611907176639465070633045279546618550966661856647097113444740160704626215680717481877844371436988218559670959102596862002353718588748569652200050311734392073211390803293634479727355955277349071783793421637012050054513263835440001863239914907054797780566978533580489669062951194324730995876552368128590413832411607226029983305353708761389396391779574540161372236187893652605381558415871869255386061647798340254351284396129460352913325942794904337299085731580290958631382683291477116396337092400316894586360606458459251269946557248391865642097526850823075442545993769170419777800853627309417101634349076964237222943523661255725088147792231519747780605696725380171807763603462459278778465850656050780844211529697521890874019660906651803516501792504619501366585436632712549639908549144200014574760819302212066024330096412704894390397177195180699086998606636583232278709376502260149291011517177635944602023249300280401867723910288097866605651183260043688508817157238669842242201024950551881694803221002515426494639812873677658927688163598312477886520141174110913601164995076629077943646005851941998560162647907615321038727557126992518275687989302761761146162549356495903798045838182323368612016243736569846703785853305275833337939907521660692380533698879565137285593883499894707416181550125397064648171946708348197214488898790676503795903669672494992545279033729636162658976039498576741397359441023744329709355477982629614591442936451428617158587339746791897571211956187385783644758448423555581050025611492391518893099463428413936080383091662818811503715284967059741625628236092168075150177725387402564253470879089137291722828611515915683725241630772254406337875931059826760944203261924285317018781772960235413060672136046000389661093647095141417185777014180606443636815464440053316087783143174440811949422975599314011888683314832802706553833004693290115744147563139997221703804617092894579096271662260740718749975359212756084414737823303270330168237193648002173285734935947564334129943024850235732214597843282641421684878721673367010615094243456984401873312810107945127223737886126058165668053714396127888732527373890392890506865324138062796025930387727697783792868409325365880733988457218746021005311483351323850047827169376218004904795597959290591655470505777514308175112698985188408718564026035305583737832422924185625644255022672155980274012617971928047139600689163828665277009752767069777036439260224372841840883251848770472638440379530166905465937461619323840363893131364327137688841026811219891275223056256756254701725086349765367288605966752740868627407912856576996313789753034660616669804218267724560530660773899624218340859882071864682623215080288286359746839654358856685503773131296587975810501214916207656769950659715344763470320853215603674828608378656803073062657633469774295634643716709397193060876963495328846833613038829431040800296873869117066666146800015121143442256023874474325250769387077775193299942137277211258843608715834835626961661980572526612206797540621062080649882918454395301529982092503005498257043390553570168653120526495614857249257386206917403695213533732531666345466588597286659451136441370331393672118569553952108458407244323835586063106806964924851232632699514603596037297253198368423363904632136710116192821711150282801604488058802382031981493096369596735832742024988245684941273860566491352526706046234450549227581151709314921879592718001940968866986837037302200475314338181092708030017205935530520700706072233999463990571311587099635777359027196285061146514837526209565346713290025994397663114545902685898979115837093419370441155121920117164880566945938131183843765620627846310490346293950029458341164824114969758326011800731699437393506966295712410273239138741754923071862454543222039552735295240245903805744502892246886285336542213815722131163288112052146489805180092024719391710555390113943316681515828843687606961102505171007392762385553386272553538830960671644662370922646809671254061869502143176211668140097595281493907222601112681153108387317617323235263605838173151034595736538223534992935822836851007810884634349983518404451704270189381994243410090575376257767571118090088164183319201962623416288166521374717325477727783488774366518828752156685719506371936565390389449366421764003121527870222366463635755503565576948886549500270853923617105502131147413744106134445544192101336172996285694899193369184729478580729156088510396781959429833186480756083679551496636448965592948187851784038773326247051945050419847742014183947731202815886845707290544057510601285258056594703046836344592652552137008068752009593453607316226118728173928074623094685367823106097921599360019946237993434210687813497346959246469752506246958616909178573976595199392993995567542714654910456860702099012606818704984178079173924071945996323060254707901774527513186809982284730860766536866855516467702911336827563107223346726113705490795365834538637196235856312618387156774118738527722922594743373785695538456246801013905727871016512966636764451872465653730402443684140814488732957847348490003019477888020460324660842875351848364959195082888323206522128104190448047247949291342284951970022601310430062410717971502793433263407995960531446053230488528972917659876016667811937932372453857209607582277178483361613582612896226118129455927462767137794487586753657544861407611931125958512655759734573015333642630767985443385761715333462325270572005303988289499034259566232975782488735029259166825894456894655992658454762694528780516501720674785417887982276806536650641910973434528878338621726156269582654478205672987756426325321594294418039943217000090542650763095588465895171709147607437136893319469090981904501290307099566226620303182649365733698419555776963787624918852865686607600566025605445711337286840205574416030837052312242587223438854123179481388550075689381124935386318635287083799845692619981794523364087429591180747453419551420351726184200845509170845682368200897739455842679214273477560879644279202708312150156406341341617166448069815483764491573900121217041547872591998943825364950514771379399147205219529079396137621107238494290616357604596231253506068537651423115349665683715116604220796394466621163255157729070978473156278277598788136491951257483328793771571459091064841642678309949723674420175862269402159407924480541255360431317992696739157542419296607312393763542139230617876753958711436104089409966089471418340698362993675362621545247298464213752891079884381306095552622720837518629837066787224430195793793786072107254277289071732854874374355781966511716618330881129120245204048682200072344035025448202834254187884653602591506445271657700044521097735585897622655484941621714989532383421600114062950718490427789258552743035221396835679018076406042138307308774460170842688272261177180842664333651780002171903449234264266292261456004337383868335555343453004264818473989215627086095650629340405264943244261445665921291225648893569655009154306426134252668472594914314239398845432486327461842846655985332312210466259890141712103446084271616619001257195870793217569698544013397622096749454185407118446433946990162698351607848924514058940946395267807354579700307051163682519487701189764002827648414160587206184185297189154019688253289309149665345753571427318482016384644832499037886069008072709327673127581966563941148961716832980455139729506687604740915420428429993541025829113502241690769431668574242522509026939034814856451303069925199590436384028429267412573422447765584177886171737265462085498294498946787350929581652632072258992368768457017823038096567883112289305809140572610865884845873101658151167533327674887014829167419701512559782572707406431808601428149024146780472327597684269633935773542930186739439716388611764209004068663398856841681003872389214483176070116684503887212364367043314091155733280182977988736590916659612402021778558854876176161989370794380056663364884365089144805571039765214696027662583599051987042300179465536788 diff --git a/libgo/go/compress/testdata/pi.txt b/libgo/go/compress/testdata/pi.txt new file mode 100644 index 00000000000..58d8f3b6dd4 --- /dev/null +++ b/libgo/go/compress/testdata/pi.txt @@ -0,0 +1 @@ +3.1415926535897932384626433832795028841971693993751058209749445923078164062862089986280348253421170679821480865132823066470938446095505822317253594081284811174502841027019385211055596446229489549303819644288109756659334461284756482337867831652712019091456485669234603486104543266482133936072602491412737245870066063155881748815209209628292540917153643678925903600113305305488204665213841469519415116094330572703657595919530921861173819326117931051185480744623799627495673518857527248912279381830119491298336733624406566430860213949463952247371907021798609437027705392171762931767523846748184676694051320005681271452635608277857713427577896091736371787214684409012249534301465495853710507922796892589235420199561121290219608640344181598136297747713099605187072113499999983729780499510597317328160963185950244594553469083026425223082533446850352619311881710100031378387528865875332083814206171776691473035982534904287554687311595628638823537875937519577818577805321712268066130019278766111959092164201989380952572010654858632788659361533818279682303019520353018529689957736225994138912497217752834791315155748572424541506959508295331168617278558890750983817546374649393192550604009277016711390098488240128583616035637076601047101819429555961989467678374494482553797747268471040475346462080466842590694912933136770289891521047521620569660240580381501935112533824300355876402474964732639141992726042699227967823547816360093417216412199245863150302861829745557067498385054945885869269956909272107975093029553211653449872027559602364806654991198818347977535663698074265425278625518184175746728909777727938000816470600161452491921732172147723501414419735685481613611573525521334757418494684385233239073941433345477624168625189835694855620992192221842725502542568876717904946016534668049886272327917860857843838279679766814541009538837863609506800642251252051173929848960841284886269456042419652850222106611863067442786220391949450471237137869609563643719172874677646575739624138908658326459958133904780275900994657640789512694683983525957098258226205224894077267194782684826014769909026401363944374553050682034962524517493996514314298091906592509372216964615157098583874105978859597729754989301617539284681382686838689427741559918559252459539594310499725246808459872736446958486538367362226260991246080512438843904512441365497627807977156914359977001296160894416948685558484063534220722258284886481584560285060168427394522674676788952521385225499546667278239864565961163548862305774564980355936345681743241125150760694794510965960940252288797108931456691368672287489405601015033086179286809208747609178249385890097149096759852613655497818931297848216829989487226588048575640142704775551323796414515237462343645428584447952658678210511413547357395231134271661021359695362314429524849371871101457654035902799344037420073105785390621983874478084784896833214457138687519435064302184531910484810053706146806749192781911979399520614196634287544406437451237181921799983910159195618146751426912397489409071864942319615679452080951465502252316038819301420937621378559566389377870830390697920773467221825625996615014215030680384477345492026054146659252014974428507325186660021324340881907104863317346496514539057962685610055081066587969981635747363840525714591028970641401109712062804390397595156771577004203378699360072305587631763594218731251471205329281918261861258673215791984148488291644706095752706957220917567116722910981690915280173506712748583222871835209353965725121083579151369882091444210067510334671103141267111369908658516398315019701651511685171437657618351556508849099898599823873455283316355076479185358932261854896321329330898570642046752590709154814165498594616371802709819943099244889575712828905923233260972997120844335732654893823911932597463667305836041428138830320382490375898524374417029132765618093773444030707469211201913020330380197621101100449293215160842444859637669838952286847831235526582131449576857262433441893039686426243410773226978028073189154411010446823252716201052652272111660396665573092547110557853763466820653109896526918620564769312570586356620185581007293606598764861179104533488503461136576867532494416680396265797877185560845529654126654085306143444318586769751456614068007002378776591344017127494704205622305389945613140711270004078547332699390814546646458807972708266830634328587856983052358089330657574067954571637752542021149557615814002501262285941302164715509792592309907965473761255176567513575178296664547791745011299614890304639947132962107340437518957359614589019389713111790429782856475032031986915140287080859904801094121472213179476477726224142548545403321571853061422881375850430633217518297986622371721591607716692547487389866549494501146540628433663937900397692656721463853067360965712091807638327166416274888800786925602902284721040317211860820419000422966171196377921337575114959501566049631862947265473642523081770367515906735023507283540567040386743513622224771589150495309844489333096340878076932599397805419341447377441842631298608099888687413260472156951623965864573021631598193195167353812974167729478672422924654366800980676928238280689964004824354037014163149658979409243237896907069779422362508221688957383798623001593776471651228935786015881617557829735233446042815126272037343146531977774160319906655418763979293344195215413418994854447345673831624993419131814809277771038638773431772075456545322077709212019051660962804909263601975988281613323166636528619326686336062735676303544776280350450777235547105859548702790814356240145171806246436267945612753181340783303362542327839449753824372058353114771199260638133467768796959703098339130771098704085913374641442822772634659470474587847787201927715280731767907707157213444730605700733492436931138350493163128404251219256517980694113528013147013047816437885185290928545201165839341965621349143415956258658655705526904965209858033850722426482939728584783163057777560688876446248246857926039535277348030480290058760758251047470916439613626760449256274204208320856611906254543372131535958450687724602901618766795240616342522577195429162991930645537799140373404328752628889639958794757291746426357455254079091451357111369410911939325191076020825202618798531887705842972591677813149699009019211697173727847684726860849003377024242916513005005168323364350389517029893922334517220138128069650117844087451960121228599371623130171144484640903890644954440061986907548516026327505298349187407866808818338510228334508504860825039302133219715518430635455007668282949304137765527939751754613953984683393638304746119966538581538420568533862186725233402830871123282789212507712629463229563989898935821167456270102183564622013496715188190973038119800497340723961036854066431939509790190699639552453005450580685501956730229219139339185680344903982059551002263535361920419947455385938102343955449597783779023742161727111723643435439478221818528624085140066604433258885698670543154706965747458550332323342107301545940516553790686627333799585115625784322988273723198987571415957811196358330059408730681216028764962867446047746491599505497374256269010490377819868359381465741268049256487985561453723478673303904688383436346553794986419270563872931748723320837601123029911367938627089438799362016295154133714248928307220126901475466847653576164773794675200490757155527819653621323926406160136358155907422020203187277605277219005561484255518792530343513984425322341576233610642506390497500865627109535919465897514131034822769306247435363256916078154781811528436679570611086153315044521274739245449454236828860613408414863776700961207151249140430272538607648236341433462351897576645216413767969031495019108575984423919862916421939949072362346468441173940326591840443780513338945257423995082965912285085558215725031071257012668302402929525220118726767562204154205161841634847565169998116141010029960783869092916030288400269104140792886215078424516709087000699282120660418371806535567252532567532861291042487761825829765157959847035622262934860034158722980534989650226291748788202734209222245339856264766914905562842503912757710284027998066365825488926488025456610172967026640765590429099456815065265305371829412703369313785178609040708667114965583434347693385781711386455873678123014587687126603489139095620099393610310291616152881384379099042317473363948045759314931405297634757481193567091101377517210080315590248530906692037671922033229094334676851422144773793937517034436619910403375111735471918550464490263655128162288244625759163330391072253837421821408835086573917715096828874782656995995744906617583441375223970968340800535598491754173818839994469748676265516582765848358845314277568790029095170283529716344562129640435231176006651012412006597558512761785838292041974844236080071930457618932349229279650198751872127267507981255470958904556357921221033346697499235630254947802490114195212382815309114079073860251522742995818072471625916685451333123948049470791191532673430282441860414263639548000448002670496248201792896476697583183271314251702969234889627668440323260927524960357996469256504936818360900323809293459588970695365349406034021665443755890045632882250545255640564482465151875471196218443965825337543885690941130315095261793780029741207665147939425902989695946995565761218656196733786236256125216320862869222103274889218654364802296780705765615144632046927906821207388377814233562823608963208068222468012248261177185896381409183903673672220888321513755600372798394004152970028783076670944474560134556417254370906979396122571429894671543578468788614445812314593571984922528471605049221242470141214780573455105008019086996033027634787081081754501193071412233908663938339529425786905076431006383519834389341596131854347546495569781038293097164651438407007073604112373599843452251610507027056235266012764848308407611830130527932054274628654036036745328651057065874882256981579367897669742205750596834408697350201410206723585020072452256326513410559240190274216248439140359989535394590944070469120914093870012645600162374288021092764579310657922955249887275846101264836999892256959688159205600101655256375678 diff --git a/libgo/go/compress/zlib/writer_test.go b/libgo/go/compress/zlib/writer_test.go index fa9e78e8e78..5a13ba82573 100644 --- a/libgo/go/compress/zlib/writer_test.go +++ b/libgo/go/compress/zlib/writer_test.go @@ -12,8 +12,8 @@ import ( ) var filenames = []string{ - "testdata/e.txt", - "testdata/pi.txt", + "../testdata/e.txt", + "../testdata/pi.txt", } // Tests that compressing and then decompressing the given file at the given compression level diff --git a/libgo/go/container/ring/ring.go b/libgo/go/container/ring/ring.go index 335afbc3cc5..5925164e9db 100644 --- a/libgo/go/container/ring/ring.go +++ b/libgo/go/container/ring/ring.go @@ -138,16 +138,13 @@ func (r *Ring) Len() int { } -func (r *Ring) Iter() <-chan interface{} { - c := make(chan interface{}) - go func() { - if r != nil { - c <- r.Value - for p := r.Next(); p != r; p = p.next { - c <- p.Value - } +// Do calls function f on each element of the ring, in forward order. +// The behavior of Do is undefined if f changes *r. +func (r *Ring) Do(f func(interface{})) { + if r != nil { + f(r.Value) + for p := r.Next(); p != r; p = p.next { + f(p.Value) } - close(c) - }() - return c + } } diff --git a/libgo/go/container/ring/ring_test.go b/libgo/go/container/ring/ring_test.go index ee3c411283c..778c083d02b 100644 --- a/libgo/go/container/ring/ring_test.go +++ b/libgo/go/container/ring/ring_test.go @@ -35,12 +35,12 @@ func verify(t *testing.T, r *Ring, N int, sum int) { // iteration n = 0 s := 0 - for p := range r.Iter() { + r.Do(func(p interface{}) { n++ if p != nil { s += p.(int) } - } + }) if n != N { t.Errorf("number of forward iterations == %d; expected %d", n, N) } @@ -128,16 +128,6 @@ func makeN(n int) *Ring { return r } - -func sum(r *Ring) int { - s := 0 - for p := range r.Iter() { - s += p.(int) - } - return s -} - - func sumN(n int) int { return (n*n + n) / 2 } diff --git a/libgo/go/crypto/cipher/ocfb.go b/libgo/go/crypto/cipher/ocfb.go index 43cb5a53108..b2d87759115 100644 --- a/libgo/go/crypto/cipher/ocfb.go +++ b/libgo/go/crypto/cipher/ocfb.go @@ -12,11 +12,21 @@ type ocfbEncrypter struct { outUsed int } +// An OCFBResyncOption determines if the "resynchronization step" of OCFB is +// performed. +type OCFBResyncOption bool + +const ( + OCFBResync OCFBResyncOption = true + OCFBNoResync OCFBResyncOption = false +) + // NewOCFBEncrypter returns a Stream which encrypts data with OpenPGP's cipher // feedback mode using the given Block, and an initial amount of ciphertext. // randData must be random bytes and be the same length as the Block's block -// size. -func NewOCFBEncrypter(block Block, randData []byte) (Stream, []byte) { +// size. Resync determines if the "resynchronization step" from RFC 4880, 13.9 +// step 7 is performed. Different parts of OpenPGP vary on this point. +func NewOCFBEncrypter(block Block, randData []byte, resync OCFBResyncOption) (Stream, []byte) { blockSize := block.BlockSize() if len(randData) != blockSize { return nil, nil @@ -38,7 +48,13 @@ func NewOCFBEncrypter(block Block, randData []byte) (Stream, []byte) { prefix[blockSize] = x.fre[0] ^ randData[blockSize-2] prefix[blockSize+1] = x.fre[1] ^ randData[blockSize-1] - block.Encrypt(x.fre, prefix[2:]) + if resync { + block.Encrypt(x.fre, prefix[2:]) + } else { + x.fre[0] = prefix[blockSize] + x.fre[1] = prefix[blockSize+1] + x.outUsed = 2 + } return x, prefix } @@ -64,8 +80,10 @@ type ocfbDecrypter struct { // NewOCFBDecrypter returns a Stream which decrypts data with OpenPGP's cipher // feedback mode using the given Block. Prefix must be the first blockSize + 2 // bytes of the ciphertext, where blockSize is the Block's block size. If an -// incorrect key is detected then nil is returned. -func NewOCFBDecrypter(block Block, prefix []byte) Stream { +// incorrect key is detected then nil is returned. Resync determines if the +// "resynchronization step" from RFC 4880, 13.9 step 7 is performed. Different +// parts of OpenPGP vary on this point. +func NewOCFBDecrypter(block Block, prefix []byte, resync OCFBResyncOption) Stream { blockSize := block.BlockSize() if len(prefix) != blockSize+2 { return nil @@ -93,7 +111,13 @@ func NewOCFBDecrypter(block Block, prefix []byte) Stream { return nil } - block.Encrypt(x.fre, prefix[2:]) + if resync { + block.Encrypt(x.fre, prefix[2:]) + } else { + x.fre[0] = prefix[blockSize] + x.fre[1] = prefix[blockSize+1] + x.outUsed = 2 + } return x } diff --git a/libgo/go/crypto/cipher/ocfb_test.go b/libgo/go/crypto/cipher/ocfb_test.go index 289bb7c91e5..40938b58923 100644 --- a/libgo/go/crypto/cipher/ocfb_test.go +++ b/libgo/go/crypto/cipher/ocfb_test.go @@ -11,29 +11,34 @@ import ( "testing" ) -func TestOCFB(t *testing.T) { +func testOCFB(t *testing.T, resync OCFBResyncOption) { block, err := aes.NewCipher(commonKey128) if err != nil { t.Error(err) return } - plaintext := []byte("this is the plaintext") + plaintext := []byte("this is the plaintext, which is long enough to span several blocks.") randData := make([]byte, block.BlockSize()) rand.Reader.Read(randData) - ocfb, prefix := NewOCFBEncrypter(block, randData) + ocfb, prefix := NewOCFBEncrypter(block, randData, resync) ciphertext := make([]byte, len(plaintext)) ocfb.XORKeyStream(ciphertext, plaintext) - ocfbdec := NewOCFBDecrypter(block, prefix) + ocfbdec := NewOCFBDecrypter(block, prefix, resync) if ocfbdec == nil { - t.Error("NewOCFBDecrypter failed") + t.Errorf("NewOCFBDecrypter failed (resync: %t)", resync) return } plaintextCopy := make([]byte, len(plaintext)) ocfbdec.XORKeyStream(plaintextCopy, ciphertext) if !bytes.Equal(plaintextCopy, plaintext) { - t.Errorf("got: %x, want: %x", plaintextCopy, plaintext) + t.Errorf("got: %x, want: %x (resync: %t)", plaintextCopy, plaintext, resync) } } + +func TestOCFB(t *testing.T) { + testOCFB(t, OCFBNoResync) + testOCFB(t, OCFBResync) +} diff --git a/libgo/go/crypto/crypto.go b/libgo/go/crypto/crypto.go new file mode 100644 index 00000000000..be6b34adf28 --- /dev/null +++ b/libgo/go/crypto/crypto.go @@ -0,0 +1,73 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The crypto package collects common cryptographic constants. +package crypto + +import ( + "hash" +) + +// Hash identifies a cryptographic hash function that is implemented in another +// package. +type Hash uint + +const ( + MD4 Hash = 1 + iota // in package crypto/md4 + MD5 // in package crypto/md5 + SHA1 // in package crypto/sha1 + SHA224 // in package crypto/sha256 + SHA256 // in package crypto/sha256 + SHA384 // in package crypto/sha512 + SHA512 // in package crypto/sha512 + MD5SHA1 // no implementation; MD5+SHA1 used for TLS RSA + RIPEMD160 // in package crypto/ripemd160 + maxHash +) + +var digestSizes = []uint8{ + MD4: 16, + MD5: 16, + SHA1: 20, + SHA224: 28, + SHA256: 32, + SHA384: 48, + SHA512: 64, + MD5SHA1: 36, + RIPEMD160: 20, +} + +// Size returns the length, in bytes, of a digest resulting from the given hash +// function. It doesn't require that the hash function in question be linked +// into the program. +func (h Hash) Size() int { + if h > 0 && h < maxHash { + return int(digestSizes[h]) + } + panic("crypto: Size of unknown hash function") +} + +var hashes = make([]func() hash.Hash, maxHash) + +// New returns a new hash.Hash calculating the given hash function. If the +// hash function is not linked into the binary, New returns nil. +func (h Hash) New() hash.Hash { + if h > 0 && h < maxHash { + f := hashes[h] + if f != nil { + return f() + } + } + return nil +} + +// RegisterHash registers a function that returns a new instance of the given +// hash function. This is intended to be called from the init function in +// packages that implement hash functions. +func RegisterHash(h Hash, f func() hash.Hash) { + if h >= maxHash { + panic("crypto: RegisterHash of unknown hash function") + } + hashes[h] = f +} diff --git a/libgo/go/crypto/dsa/dsa.go b/libgo/go/crypto/dsa/dsa.go new file mode 100644 index 00000000000..f0af8bb427e --- /dev/null +++ b/libgo/go/crypto/dsa/dsa.go @@ -0,0 +1,276 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package dsa implements the Digital Signature Algorithm, as defined in FIPS 186-3 +package dsa + +import ( + "big" + "io" + "os" +) + +// Parameters represents the domain parameters for a key. These parameters can +// be shared across many keys. The bit length of Q must be a multiple of 8. +type Parameters struct { + P, Q, G *big.Int +} + +// PublicKey represents a DSA public key. +type PublicKey struct { + Parameters + Y *big.Int +} + +// PrivateKey represents a DSA private key. +type PrivateKey struct { + PublicKey + X *big.Int +} + +type invalidPublicKeyError int + +func (invalidPublicKeyError) String() string { + return "crypto/dsa: invalid public key" +} + +// InvalidPublicKeyError results when a public key is not usable by this code. +// FIPS is quite strict about the format of DSA keys, but other code may be +// less so. Thus, when using keys which may have been generated by other code, +// this error must be handled. +var InvalidPublicKeyError = invalidPublicKeyError(0) + +// ParameterSizes is a enumeration of the acceptable bit lengths of the primes +// in a set of DSA parameters. See FIPS 186-3, section 4.2. +type ParameterSizes int + +const ( + L1024N160 ParameterSizes = iota + L2048N224 + L2048N256 + L3072N256 +) + +// numMRTests is the number of Miller-Rabin primality tests that we perform. We +// pick the largest recommended number from table C.1 of FIPS 186-3. +const numMRTests = 64 + +// GenerateParameters puts a random, valid set of DSA parameters into params. +// This function takes many seconds, even on fast machines. +func GenerateParameters(params *Parameters, rand io.Reader, sizes ParameterSizes) (err os.Error) { + // This function doesn't follow FIPS 186-3 exactly in that it doesn't + // use a verification seed to generate the primes. The verification + // seed doesn't appear to be exported or used by other code and + // omitting it makes the code cleaner. + + var L, N int + switch sizes { + case L1024N160: + L = 1024 + N = 160 + case L2048N224: + L = 2048 + N = 224 + case L2048N256: + L = 2048 + N = 256 + case L3072N256: + L = 3072 + N = 256 + default: + return os.ErrorString("crypto/dsa: invalid ParameterSizes") + } + + qBytes := make([]byte, N/8) + pBytes := make([]byte, L/8) + + q := new(big.Int) + p := new(big.Int) + rem := new(big.Int) + one := new(big.Int) + one.SetInt64(1) + +GeneratePrimes: + for { + _, err = io.ReadFull(rand, qBytes) + if err != nil { + return + } + + qBytes[len(qBytes)-1] |= 1 + qBytes[0] |= 0x80 + q.SetBytes(qBytes) + + if !big.ProbablyPrime(q, numMRTests) { + continue + } + + for i := 0; i < 4*L; i++ { + _, err = io.ReadFull(rand, pBytes) + if err != nil { + return + } + + pBytes[len(pBytes)-1] |= 1 + pBytes[0] |= 0x80 + + p.SetBytes(pBytes) + rem.Mod(p, q) + rem.Sub(rem, one) + p.Sub(p, rem) + if p.BitLen() < L { + continue + } + + if !big.ProbablyPrime(p, numMRTests) { + continue + } + + params.P = p + params.Q = q + break GeneratePrimes + } + } + + h := new(big.Int) + h.SetInt64(2) + g := new(big.Int) + + pm1 := new(big.Int).Sub(p, one) + e := new(big.Int).Div(pm1, q) + + for { + g.Exp(h, e, p) + if g.Cmp(one) == 0 { + h.Add(h, one) + continue + } + + params.G = g + return + } + + panic("unreachable") +} + +// GenerateKey generates a public&private key pair. The Parameters of the +// PrivateKey must already be valid (see GenerateParameters). +func GenerateKey(priv *PrivateKey, rand io.Reader) os.Error { + if priv.P == nil || priv.Q == nil || priv.G == nil { + return os.ErrorString("crypto/dsa: parameters not set up before generating key") + } + + x := new(big.Int) + xBytes := make([]byte, priv.Q.BitLen()/8) + + for { + _, err := io.ReadFull(rand, xBytes) + if err != nil { + return err + } + x.SetBytes(xBytes) + if x.Sign() != 0 && x.Cmp(priv.Q) < 0 { + break + } + } + + priv.X = x + priv.Y = new(big.Int) + priv.Y.Exp(priv.G, x, priv.P) + return nil +} + +// Sign signs an arbitrary length hash (which should be the result of hashing a +// larger message) using the private key, priv. It returns the signature as a +// pair of integers. The security of the private key depends on the entropy of +// rand. +func Sign(rand io.Reader, priv *PrivateKey, hash []byte) (r, s *big.Int, err os.Error) { + // FIPS 186-3, section 4.6 + + n := priv.Q.BitLen() + if n&7 != 0 { + err = InvalidPublicKeyError + return + } + n >>= 3 + + for { + k := new(big.Int) + buf := make([]byte, n) + for { + _, err = io.ReadFull(rand, buf) + if err != nil { + return + } + k.SetBytes(buf) + if k.Sign() > 0 && k.Cmp(priv.Q) < 0 { + break + } + } + + kInv := new(big.Int).ModInverse(k, priv.Q) + + r = new(big.Int).Exp(priv.G, k, priv.P) + r.Mod(r, priv.Q) + + if r.Sign() == 0 { + continue + } + + if n > len(hash) { + n = len(hash) + } + z := k.SetBytes(hash[:n]) + + s = new(big.Int).Mul(priv.X, r) + s.Add(s, z) + s.Mod(s, priv.Q) + s.Mul(s, kInv) + s.Mod(s, priv.Q) + + if s.Sign() != 0 { + break + } + } + + return +} + +// Verify verifies the signature in r, s of hash using the public key, pub. It +// returns true iff the signature is valid. +func Verify(pub *PublicKey, hash []byte, r, s *big.Int) bool { + // FIPS 186-3, section 4.7 + + if r.Sign() < 1 || r.Cmp(pub.Q) >= 0 { + return false + } + if s.Sign() < 1 || s.Cmp(pub.Q) >= 0 { + return false + } + + w := new(big.Int).ModInverse(s, pub.Q) + + n := pub.Q.BitLen() + if n&7 != 0 { + return false + } + n >>= 3 + + if n > len(hash) { + n = len(hash) + } + z := new(big.Int).SetBytes(hash[:n]) + + u1 := new(big.Int).Mul(z, w) + u1.Mod(u1, pub.Q) + u2 := w.Mul(r, w) + u2.Mod(u2, pub.Q) + v := u1.Exp(pub.G, u1, pub.P) + u2.Exp(pub.Y, u2, pub.P) + v.Mul(v, u2) + v.Mod(v, pub.P) + v.Mod(v, pub.Q) + + return v.Cmp(r) == 0 +} diff --git a/libgo/go/crypto/dsa/dsa_test.go b/libgo/go/crypto/dsa/dsa_test.go new file mode 100644 index 00000000000..deec08dfd8a --- /dev/null +++ b/libgo/go/crypto/dsa/dsa_test.go @@ -0,0 +1,84 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package dsa + +import ( + "big" + "crypto/rand" + "testing" +) + +func testSignAndVerify(t *testing.T, i int, priv *PrivateKey) { + hashed := []byte("testing") + r, s, err := Sign(rand.Reader, priv, hashed) + if err != nil { + t.Errorf("%d: error signing: %s", i, err) + return + } + + if !Verify(&priv.PublicKey, hashed, r, s) { + t.Errorf("%d: Verify failed", i) + } +} + +func testParameterGeneration(t *testing.T, sizes ParameterSizes, L, N int) { + var priv PrivateKey + params := &priv.Parameters + + err := GenerateParameters(params, rand.Reader, sizes) + if err != nil { + t.Errorf("%d: %s", int(sizes), err) + return + } + + if params.P.BitLen() != L { + t.Errorf("%d: params.BitLen got:%d want:%d", int(sizes), params.P.BitLen(), L) + } + + if params.Q.BitLen() != N { + t.Errorf("%d: q.BitLen got:%d want:%d", int(sizes), params.Q.BitLen(), L) + } + + one := new(big.Int) + one.SetInt64(1) + pm1 := new(big.Int).Sub(params.P, one) + quo, rem := new(big.Int).DivMod(pm1, params.Q, new(big.Int)) + if rem.Sign() != 0 { + t.Errorf("%d: p-1 mod q != 0", int(sizes)) + } + x := new(big.Int).Exp(params.G, quo, params.P) + if x.Cmp(one) == 0 { + t.Errorf("%d: invalid generator", int(sizes)) + } + + err = GenerateKey(&priv, rand.Reader) + if err != nil { + t.Errorf("error generating key: %s", err) + return + } + + testSignAndVerify(t, int(sizes), &priv) +} + +func TestParameterGeneration(t *testing.T) { + // This test is too slow to run all the time. + return + + testParameterGeneration(t, L1024N160, 1024, 160) + testParameterGeneration(t, L2048N224, 2048, 224) + testParameterGeneration(t, L2048N256, 2048, 256) + testParameterGeneration(t, L3072N256, 3072, 256) +} + +func TestSignAndVerify(t *testing.T) { + var priv PrivateKey + priv.P, _ = new(big.Int).SetString("A9B5B793FB4785793D246BAE77E8FF63CA52F442DA763C440259919FE1BC1D6065A9350637A04F75A2F039401D49F08E066C4D275A5A65DA5684BC563C14289D7AB8A67163BFBF79D85972619AD2CFF55AB0EE77A9002B0EF96293BDD0F42685EBB2C66C327079F6C98000FBCB79AACDE1BC6F9D5C7B1A97E3D9D54ED7951FEF", 16) + priv.Q, _ = new(big.Int).SetString("E1D3391245933D68A0714ED34BBCB7A1F422B9C1", 16) + priv.G, _ = new(big.Int).SetString("634364FC25248933D01D1993ECABD0657CC0CB2CEED7ED2E3E8AECDFCDC4A25C3B15E9E3B163ACA2984B5539181F3EFF1A5E8903D71D5B95DA4F27202B77D2C44B430BB53741A8D59A8F86887525C9F2A6A5980A195EAA7F2FF910064301DEF89D3AA213E1FAC7768D89365318E370AF54A112EFBA9246D9158386BA1B4EEFDA", 16) + priv.Y, _ = new(big.Int).SetString("32969E5780CFE1C849A1C276D7AEB4F38A23B591739AA2FE197349AEEBD31366AEE5EB7E6C6DDB7C57D02432B30DB5AA66D9884299FAA72568944E4EEDC92EA3FBC6F39F53412FBCC563208F7C15B737AC8910DBC2D9C9B8C001E72FDC40EB694AB1F06A5A2DBD18D9E36C66F31F566742F11EC0A52E9F7B89355C02FB5D32D2", 16) + priv.X, _ = new(big.Int).SetString("5078D4D29795CBE76D3AACFE48C9AF0BCDBEE91A", 16) + + testSignAndVerify(t, 0, &priv) +} diff --git a/libgo/go/crypto/md4/md4.go b/libgo/go/crypto/md4/md4.go index e13c986e686..ee46544a920 100644 --- a/libgo/go/crypto/md4/md4.go +++ b/libgo/go/crypto/md4/md4.go @@ -6,10 +6,15 @@ package md4 import ( + "crypto" "hash" "os" ) +func init() { + crypto.RegisterHash(crypto.MD4, New) +} + // The size of an MD4 checksum in bytes. const Size = 16 diff --git a/libgo/go/crypto/md5/md5.go b/libgo/go/crypto/md5/md5.go index 54fddb63b93..8f93fc4b354 100644 --- a/libgo/go/crypto/md5/md5.go +++ b/libgo/go/crypto/md5/md5.go @@ -6,10 +6,15 @@ package md5 import ( + "crypto" "hash" "os" ) +func init() { + crypto.RegisterHash(crypto.MD5, New) +} + // The size of an MD5 checksum in bytes. const Size = 16 diff --git a/libgo/go/crypto/ocsp/ocsp.go b/libgo/go/crypto/ocsp/ocsp.go index f3fa3bc834c..f42d8088884 100644 --- a/libgo/go/crypto/ocsp/ocsp.go +++ b/libgo/go/crypto/ocsp/ocsp.go @@ -9,8 +9,9 @@ package ocsp import ( "asn1" + "crypto" "crypto/rsa" - "crypto/sha1" + _ "crypto/sha1" "crypto/x509" "os" "time" @@ -168,8 +169,8 @@ func ParseResponse(bytes []byte) (*Response, os.Error) { return nil, x509.UnsupportedAlgorithmError{} } - h := sha1.New() - hashType := rsa.HashSHA1 + hashType := crypto.SHA1 + h := hashType.New() pub := ret.Certificate.PublicKey.(*rsa.PublicKey) h.Write(basicResp.TBSResponseData.Raw) diff --git a/libgo/go/crypto/openpgp/armor/armor.go b/libgo/go/crypto/openpgp/armor/armor.go index 97080f6c6d1..0c5ae9d716c 100644 --- a/libgo/go/crypto/openpgp/armor/armor.go +++ b/libgo/go/crypto/openpgp/armor/armor.go @@ -112,7 +112,7 @@ func (l *lineReader) Read(p []byte) (n int, err os.Error) { return 0, os.EOF } - if len(line) != 64 { + if len(line) > 64 { return 0, ArmorCorrupt } diff --git a/libgo/go/crypto/openpgp/armor/armor_test.go b/libgo/go/crypto/openpgp/armor/armor_test.go index e4ffd414b6a..9334e94e96c 100644 --- a/libgo/go/crypto/openpgp/armor/armor_test.go +++ b/libgo/go/crypto/openpgp/armor/armor_test.go @@ -34,7 +34,7 @@ func TestDecodeEncode(t *testing.T) { t.Error(err) } - if adler32.Checksum(contents) != 0x789d7f00 { + if adler32.Checksum(contents) != 0x27b144be { t.Errorf("contents: got: %x", contents) } @@ -73,13 +73,11 @@ func TestLongHeader(t *testing.T) { const armorExample1 = `-----BEGIN PGP SIGNATURE----- Version: GnuPG v1.4.10 (GNU/Linux) -iQEcBAABAgAGBQJMtFESAAoJEKsQXJGvOPsVj40H/1WW6jaMXv4BW+1ueDSMDwM8 -kx1fLOXbVM5/Kn5LStZNt1jWWnpxdz7eq3uiqeCQjmqUoRde3YbB2EMnnwRbAhpp -cacnAvy9ZQ78OTxUdNW1mhX5bS6q1MTEJnl+DcyigD70HG/yNNQD7sOPMdYQw0TA -byQBwmLwmTsuZsrYqB68QyLHI+DUugn+kX6Hd2WDB62DKa2suoIUIHQQCd/ofwB3 -WfCYInXQKKOSxu2YOg2Eb4kLNhSMc1i9uKUWAH+sdgJh7NBgdoE4MaNtBFkHXRvv -okWuf3+xA9ksp1npSY/mDvgHijmjvtpRDe6iUeqfCn8N9u9CBg8geANgaG8+QA4= -=wfQG +iJwEAAECAAYFAk1Fv/0ACgkQo01+GMIMMbsYTwQAiAw+QAaNfY6WBdplZ/uMAccm +4g+81QPmTSGHnetSb6WBiY13kVzK4HQiZH8JSkmmroMLuGeJwsRTEL4wbjRyUKEt +p1xwUZDECs234F1xiG5enc5SGlRtP7foLBz9lOsjx+LEcA4sTl5/2eZR9zyFZqWW +TxRjs+fJCIFuo71xb1g= +=/teI -----END PGP SIGNATURE-----` const armorLongLine = `-----BEGIN PGP SIGNATURE----- diff --git a/libgo/go/crypto/openpgp/armor/encode.go b/libgo/go/crypto/openpgp/armor/encode.go index 410e734602f..0f7de024127 100644 --- a/libgo/go/crypto/openpgp/armor/encode.go +++ b/libgo/go/crypto/openpgp/armor/encode.go @@ -116,6 +116,7 @@ func (e *encoding) Close() (err os.Error) { if err != nil { return } + e.breaker.Close() var checksumBytes [3]byte checksumBytes[0] = byte(e.crc >> 16) @@ -144,11 +145,9 @@ func Encode(out io.Writer, blockType string, headers map[string]string) (w io.Wr } } - if len(headers) > 0 { - _, err := out.Write(newline) - if err != nil { - return - } + _, err = out.Write(newline) + if err != nil { + return } e := &encoding{ diff --git a/libgo/go/crypto/openpgp/canonical_text.go b/libgo/go/crypto/openpgp/canonical_text.go new file mode 100644 index 00000000000..293eff3542d --- /dev/null +++ b/libgo/go/crypto/openpgp/canonical_text.go @@ -0,0 +1,58 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package openpgp + +import ( + "hash" + "os" +) + +// NewCanonicalTextHash reformats text written to it into the canonical +// form and then applies the hash h. See RFC 4880, section 5.2.1. +func NewCanonicalTextHash(h hash.Hash) hash.Hash { + return &canonicalTextHash{h, 0} +} + +type canonicalTextHash struct { + h hash.Hash + s int +} + +var newline = []byte{'\r', '\n'} + +func (cth *canonicalTextHash) Write(buf []byte) (int, os.Error) { + start := 0 + + for i, c := range buf { + switch cth.s { + case 0: + if c == '\r' { + cth.s = 1 + } else if c == '\n' { + cth.h.Write(buf[start:i]) + cth.h.Write(newline) + start = i + 1 + } + case 1: + cth.s = 0 + } + } + + cth.h.Write(buf[start:]) + return len(buf), nil +} + +func (cth *canonicalTextHash) Sum() []byte { + return cth.h.Sum() +} + +func (cth *canonicalTextHash) Reset() { + cth.h.Reset() + cth.s = 0 +} + +func (cth *canonicalTextHash) Size() int { + return cth.h.Size() +} diff --git a/libgo/go/crypto/openpgp/canonical_text_test.go b/libgo/go/crypto/openpgp/canonical_text_test.go new file mode 100644 index 00000000000..69ecf91a835 --- /dev/null +++ b/libgo/go/crypto/openpgp/canonical_text_test.go @@ -0,0 +1,50 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package openpgp + +import ( + "bytes" + "os" + "testing" +) + +type recordingHash struct { + buf *bytes.Buffer +} + +func (r recordingHash) Write(b []byte) (n int, err os.Error) { + return r.buf.Write(b) +} + +func (r recordingHash) Sum() []byte { + return r.buf.Bytes() +} + +func (r recordingHash) Reset() { + panic("shouldn't be called") +} + +func (r recordingHash) Size() int { + panic("shouldn't be called") +} + + +func testCanonicalText(t *testing.T, input, expected string) { + r := recordingHash{bytes.NewBuffer(nil)} + c := NewCanonicalTextHash(r) + c.Write([]byte(input)) + result := c.Sum() + if expected != string(result) { + t.Errorf("input: %x got: %x want: %x", input, result, expected) + } +} + +func TestCanonicalText(t *testing.T) { + testCanonicalText(t, "foo\n", "foo\r\n") + testCanonicalText(t, "foo", "foo") + testCanonicalText(t, "foo\r\n", "foo\r\n") + testCanonicalText(t, "foo\r\nbar", "foo\r\nbar") + testCanonicalText(t, "foo\r\nbar\n\n", "foo\r\nbar\r\n\r\n") +} diff --git a/libgo/go/crypto/openpgp/error/error.go b/libgo/go/crypto/openpgp/error/error.go index 2d80ce3734e..053d1596726 100644 --- a/libgo/go/crypto/openpgp/error/error.go +++ b/libgo/go/crypto/openpgp/error/error.go @@ -5,6 +5,10 @@ // This package contains common error types for the OpenPGP packages. package error +import ( + "strconv" +) + // A StructuralError is returned when OpenPGP data is found to be syntactically // invalid. type StructuralError string @@ -44,3 +48,17 @@ func (ki keyIncorrect) String() string { } var KeyIncorrectError = keyIncorrect(0) + +type unknownIssuer int + +func (unknownIssuer) String() string { + return "signature make by unknown entity" +} + +var UnknownIssuerError = unknownIssuer(0) + +type UnknownPacketTypeError uint8 + +func (upte UnknownPacketTypeError) String() string { + return "unknown OpenPGP packet type: " + strconv.Itoa(int(upte)) +} diff --git a/libgo/go/crypto/openpgp/keys.go b/libgo/go/crypto/openpgp/keys.go new file mode 100644 index 00000000000..ecaa86f2828 --- /dev/null +++ b/libgo/go/crypto/openpgp/keys.go @@ -0,0 +1,280 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package openpgp + +import ( + "crypto/openpgp/error" + "crypto/openpgp/packet" + "io" + "os" +) + +// PublicKeyType is the armor type for a PGP public key. +var PublicKeyType = "PGP PUBLIC KEY BLOCK" + +// An Entity represents the components of an OpenPGP key: a primary public key +// (which must be a signing key), one or more identities claimed by that key, +// and zero or more subkeys, which may be encryption keys. +type Entity struct { + PrimaryKey *packet.PublicKey + PrivateKey *packet.PrivateKey + Identities map[string]*Identity // indexed by Identity.Name + Subkeys []Subkey +} + +// An Identity represents an identity claimed by an Entity and zero or more +// assertions by other entities about that claim. +type Identity struct { + Name string // by convention, has the form "Full Name (comment) <email@example.com>" + UserId *packet.UserId + SelfSignature *packet.Signature + Signatures []*packet.Signature +} + +// A Subkey is an additional public key in an Entity. Subkeys can be used for +// encryption. +type Subkey struct { + PublicKey *packet.PublicKey + PrivateKey *packet.PrivateKey + Sig *packet.Signature +} + +// A Key identifies a specific public key in an Entity. This is either the +// Entity's primary key or a subkey. +type Key struct { + Entity *Entity + PublicKey *packet.PublicKey + PrivateKey *packet.PrivateKey + SelfSignature *packet.Signature +} + +// A KeyRing provides access to public and private keys. +type KeyRing interface { + // KeysById returns the set of keys that have the given key id. + KeysById(id uint64) []Key + // DecryptionKeys returns all private keys that are valid for + // decryption. + DecryptionKeys() []Key +} + +// An EntityList contains one or more Entities. +type EntityList []*Entity + +// KeysById returns the set of keys that have the given key id. +func (el EntityList) KeysById(id uint64) (keys []Key) { + for _, e := range el { + if e.PrimaryKey.KeyId == id { + var selfSig *packet.Signature + for _, ident := range e.Identities { + if selfSig == nil { + selfSig = ident.SelfSignature + } else if ident.SelfSignature.IsPrimaryId != nil && *ident.SelfSignature.IsPrimaryId { + selfSig = ident.SelfSignature + break + } + } + keys = append(keys, Key{e, e.PrimaryKey, e.PrivateKey, selfSig}) + } + + for _, subKey := range e.Subkeys { + if subKey.PublicKey.KeyId == id { + keys = append(keys, Key{e, subKey.PublicKey, subKey.PrivateKey, subKey.Sig}) + } + } + } + return +} + +// DecryptionKeys returns all private keys that are valid for decryption. +func (el EntityList) DecryptionKeys() (keys []Key) { + for _, e := range el { + for _, subKey := range e.Subkeys { + if subKey.PrivateKey != nil && (!subKey.Sig.FlagsValid || subKey.Sig.FlagEncryptStorage || subKey.Sig.FlagEncryptCommunications) { + keys = append(keys, Key{e, subKey.PublicKey, subKey.PrivateKey, subKey.Sig}) + } + } + } + return +} + +// ReadArmoredKeyRing reads one or more public/private keys from an armor keyring file. +func ReadArmoredKeyRing(r io.Reader) (EntityList, os.Error) { + body, err := readArmored(r, PublicKeyType) + if err != nil { + return nil, err + } + + return ReadKeyRing(body) +} + +// ReadKeyRing reads one or more public/private keys, ignoring unsupported keys. +func ReadKeyRing(r io.Reader) (el EntityList, err os.Error) { + packets := packet.NewReader(r) + + for { + var e *Entity + e, err = readEntity(packets) + if err != nil { + if _, ok := err.(error.UnsupportedError); ok { + err = readToNextPublicKey(packets) + } + if err == os.EOF { + err = nil + return + } + if err != nil { + el = nil + return + } + } else { + el = append(el, e) + } + } + return +} + +// readToNextPublicKey reads packets until the start of the entity and leaves +// the first packet of the new entity in the Reader. +func readToNextPublicKey(packets *packet.Reader) (err os.Error) { + var p packet.Packet + for { + p, err = packets.Next() + if err == os.EOF { + return + } else if err != nil { + if _, ok := err.(error.UnsupportedError); ok { + err = nil + continue + } + return + } + + if pk, ok := p.(*packet.PublicKey); ok && !pk.IsSubkey { + packets.Unread(p) + return + } + } + + panic("unreachable") +} + +// readEntity reads an entity (public key, identities, subkeys etc) from the +// given Reader. +func readEntity(packets *packet.Reader) (*Entity, os.Error) { + e := new(Entity) + e.Identities = make(map[string]*Identity) + + p, err := packets.Next() + if err != nil { + return nil, err + } + + var ok bool + if e.PrimaryKey, ok = p.(*packet.PublicKey); !ok { + if e.PrivateKey, ok = p.(*packet.PrivateKey); !ok { + packets.Unread(p) + return nil, error.StructuralError("first packet was not a public/private key") + } else { + e.PrimaryKey = &e.PrivateKey.PublicKey + } + } + + var current *Identity +EachPacket: + for { + p, err := packets.Next() + if err == os.EOF { + break + } else if err != nil { + return nil, err + } + + switch pkt := p.(type) { + case *packet.UserId: + current = new(Identity) + current.Name = pkt.Id + current.UserId = pkt + e.Identities[pkt.Id] = current + p, err = packets.Next() + if err == os.EOF { + err = io.ErrUnexpectedEOF + } + if err != nil { + if _, ok := err.(error.UnsupportedError); ok { + return nil, err + } + return nil, error.StructuralError("identity self-signature invalid: " + err.String()) + } + current.SelfSignature, ok = p.(*packet.Signature) + if !ok { + return nil, error.StructuralError("user ID packet not followed by self signature") + } + if current.SelfSignature.SigType != packet.SigTypePositiveCert { + return nil, error.StructuralError("user ID self-signature with wrong type") + } + if err = e.PrimaryKey.VerifyUserIdSignature(pkt.Id, current.SelfSignature); err != nil { + return nil, error.StructuralError("user ID self-signature invalid: " + err.String()) + } + case *packet.Signature: + if current == nil { + return nil, error.StructuralError("signature packet found before user id packet") + } + current.Signatures = append(current.Signatures, pkt) + case *packet.PrivateKey: + if pkt.IsSubkey == false { + packets.Unread(p) + break EachPacket + } + err = addSubkey(e, packets, &pkt.PublicKey, pkt) + if err != nil { + return nil, err + } + case *packet.PublicKey: + if pkt.IsSubkey == false { + packets.Unread(p) + break EachPacket + } + err = addSubkey(e, packets, pkt, nil) + if err != nil { + return nil, err + } + default: + // we ignore unknown packets + } + } + + if len(e.Identities) == 0 { + return nil, error.StructuralError("entity without any identities") + } + + return e, nil +} + +func addSubkey(e *Entity, packets *packet.Reader, pub *packet.PublicKey, priv *packet.PrivateKey) os.Error { + var subKey Subkey + subKey.PublicKey = pub + subKey.PrivateKey = priv + p, err := packets.Next() + if err == os.EOF { + return io.ErrUnexpectedEOF + } + if err != nil { + return error.StructuralError("subkey signature invalid: " + err.String()) + } + var ok bool + subKey.Sig, ok = p.(*packet.Signature) + if !ok { + return error.StructuralError("subkey packet not followed by signature") + } + if subKey.Sig.SigType != packet.SigTypeSubkeyBinding { + return error.StructuralError("subkey signature with wrong type") + } + err = e.PrimaryKey.VerifyKeySignature(subKey.PublicKey, subKey.Sig) + if err != nil { + return error.StructuralError("subkey signature invalid: " + err.String()) + } + e.Subkeys = append(e.Subkeys, subKey) + return nil +} diff --git a/libgo/go/crypto/openpgp/packet/compressed.go b/libgo/go/crypto/openpgp/packet/compressed.go new file mode 100644 index 00000000000..1c15c24c4b6 --- /dev/null +++ b/libgo/go/crypto/openpgp/packet/compressed.go @@ -0,0 +1,39 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "compress/flate" + "compress/zlib" + "crypto/openpgp/error" + "io" + "os" + "strconv" +) + +// Compressed represents a compressed OpenPGP packet. The decompressed contents +// will contain more OpenPGP packets. See RFC 4880, section 5.6. +type Compressed struct { + Body io.Reader +} + +func (c *Compressed) parse(r io.Reader) os.Error { + var buf [1]byte + _, err := readFull(r, buf[:]) + if err != nil { + return err + } + + switch buf[0] { + case 1: + c.Body = flate.NewReader(r) + case 2: + c.Body, err = zlib.NewReader(r) + default: + err = error.UnsupportedError("unknown compression algorithm: " + strconv.Itoa(int(buf[0]))) + } + + return err +} diff --git a/libgo/go/crypto/openpgp/packet/compressed_test.go b/libgo/go/crypto/openpgp/packet/compressed_test.go new file mode 100644 index 00000000000..24fe501edba --- /dev/null +++ b/libgo/go/crypto/openpgp/packet/compressed_test.go @@ -0,0 +1,41 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "bytes" + "encoding/hex" + "os" + "io/ioutil" + "testing" +) + +func TestCompressed(t *testing.T) { + packet, err := Read(readerFromHex(compressedHex)) + if err != nil { + t.Errorf("failed to read Compressed: %s", err) + return + } + + c, ok := packet.(*Compressed) + if !ok { + t.Error("didn't find Compressed packet") + return + } + + contents, err := ioutil.ReadAll(c.Body) + if err != nil && err != os.EOF { + t.Error(err) + return + } + + expected, _ := hex.DecodeString(compressedExpectedHex) + if !bytes.Equal(expected, contents) { + t.Errorf("got:%x want:%x", contents, expected) + } +} + +const compressedHex = "a3013b2d90c4e02b72e25f727e5e496a5e49b11e1700" +const compressedExpectedHex = "cb1062004d14c8fe636f6e74656e74732e0a" diff --git a/libgo/go/crypto/openpgp/packet/encrypted_key.go b/libgo/go/crypto/openpgp/packet/encrypted_key.go new file mode 100644 index 00000000000..b11a9b8301a --- /dev/null +++ b/libgo/go/crypto/openpgp/packet/encrypted_key.go @@ -0,0 +1,66 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "crypto/openpgp/error" + "crypto/rand" + "crypto/rsa" + "encoding/binary" + "io" + "os" + "strconv" +) + +// EncryptedKey represents a public-key encrypted session key. See RFC 4880, +// section 5.1. +type EncryptedKey struct { + KeyId uint64 + Algo PublicKeyAlgorithm + Encrypted []byte + CipherFunc CipherFunction // only valid after a successful Decrypt + Key []byte // only valid after a successful Decrypt +} + +func (e *EncryptedKey) parse(r io.Reader) (err os.Error) { + var buf [10]byte + _, err = readFull(r, buf[:]) + if err != nil { + return + } + if buf[0] != 3 { + return error.UnsupportedError("unknown EncryptedKey version " + strconv.Itoa(int(buf[0]))) + } + e.KeyId = binary.BigEndian.Uint64(buf[1:9]) + e.Algo = PublicKeyAlgorithm(buf[9]) + if e.Algo == PubKeyAlgoRSA || e.Algo == PubKeyAlgoRSAEncryptOnly { + e.Encrypted, _, err = readMPI(r) + } + _, err = consumeAll(r) + return +} + +// DecryptRSA decrypts an RSA encrypted session key with the given private key. +func (e *EncryptedKey) DecryptRSA(priv *rsa.PrivateKey) (err os.Error) { + if e.Algo != PubKeyAlgoRSA && e.Algo != PubKeyAlgoRSAEncryptOnly { + return error.InvalidArgumentError("EncryptedKey not RSA encrypted") + } + b, err := rsa.DecryptPKCS1v15(rand.Reader, priv, e.Encrypted) + if err != nil { + return + } + e.CipherFunc = CipherFunction(b[0]) + e.Key = b[1 : len(b)-2] + expectedChecksum := uint16(b[len(b)-2])<<8 | uint16(b[len(b)-1]) + var checksum uint16 + for _, v := range e.Key { + checksum += uint16(v) + } + if checksum != expectedChecksum { + return error.StructuralError("EncryptedKey checksum incorrect") + } + + return +} diff --git a/libgo/go/crypto/openpgp/packet/encrypted_key_test.go b/libgo/go/crypto/openpgp/packet/encrypted_key_test.go new file mode 100644 index 00000000000..755ae7a3074 --- /dev/null +++ b/libgo/go/crypto/openpgp/packet/encrypted_key_test.go @@ -0,0 +1,67 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "big" + "crypto/rsa" + "fmt" + "testing" +) + +func bigFromBase10(s string) *big.Int { + b, ok := new(big.Int).SetString(s, 10) + if !ok { + panic("bigFromBase10 failed") + } + return b +} + +func TestEncryptedKey(t *testing.T) { + p, err := Read(readerFromHex(encryptedKeyHex)) + if err != nil { + t.Errorf("error from Read: %s", err) + return + } + ek, ok := p.(*EncryptedKey) + if !ok { + t.Errorf("didn't parse an EncryptedKey, got %#v", p) + return + } + + if ek.KeyId != 0x2a67d68660df41c7 || ek.Algo != PubKeyAlgoRSA { + t.Errorf("unexpected EncryptedKey contents: %#v", ek) + return + } + + pub := rsa.PublicKey{ + E: 65537, + N: bigFromBase10("115804063926007623305902631768113868327816898845124614648849934718568541074358183759250136204762053879858102352159854352727097033322663029387610959884180306668628526686121021235757016368038585212410610742029286439607686208110250133174279811431933746643015923132833417396844716207301518956640020862630546868823"), + } + + priv := &rsa.PrivateKey{ + PublicKey: pub, + D: bigFromBase10("32355588668219869544751561565313228297765464314098552250409557267371233892496951383426602439009993875125222579159850054973310859166139474359774543943714622292329487391199285040721944491839695981199720170366763547754915493640685849961780092241140181198779299712578774460837139360803883139311171713302987058393"), + } + + err = ek.DecryptRSA(priv) + if err != nil { + t.Errorf("error from DecryptRSA: %s", err) + return + } + + if ek.CipherFunc != CipherAES256 { + t.Errorf("unexpected EncryptedKey contents: %#v", ek) + return + } + + keyHex := fmt.Sprintf("%x", ek.Key) + if keyHex != expectedKeyHex { + t.Errorf("bad key, got %s want %x", keyHex, expectedKeyHex) + } +} + +const encryptedKeyHex = "c18c032a67d68660df41c70104005789d0de26b6a50c985a02a13131ca829c413a35d0e6fa8d6842599252162808ac7439c72151c8c6183e76923fe3299301414d0c25a2f06a2257db3839e7df0ec964773f6e4c4ac7ff3b48c444237166dd46ba8ff443a5410dc670cb486672fdbe7c9dfafb75b4fea83af3a204fe2a7dfa86bd20122b4f3d2646cbeecb8f7be8" +const expectedKeyHex = "d930363f7e0308c333b9618617ea728963d8df993665ae7be1092d4926fd864b" diff --git a/libgo/go/crypto/openpgp/packet/literal.go b/libgo/go/crypto/openpgp/packet/literal.go new file mode 100644 index 00000000000..04f50e53e13 --- /dev/null +++ b/libgo/go/crypto/openpgp/packet/literal.go @@ -0,0 +1,53 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "encoding/binary" + "io" + "os" +) + +// LiteralData represents an encrypted file. See RFC 4880, section 5.9. +type LiteralData struct { + IsBinary bool + FileName string + Time uint32 // Unix epoch time. Either creation time or modification time. 0 means undefined. + Body io.Reader +} + +// ForEyesOnly returns whether the contents of the LiteralData have been marked +// as especially sensitive. +func (l *LiteralData) ForEyesOnly() bool { + return l.FileName == "_CONSOLE" +} + +func (l *LiteralData) parse(r io.Reader) (err os.Error) { + var buf [256]byte + + _, err = readFull(r, buf[:2]) + if err != nil { + return + } + + l.IsBinary = buf[0] == 'b' + fileNameLen := int(buf[1]) + + _, err = readFull(r, buf[:fileNameLen]) + if err != nil { + return + } + + l.FileName = string(buf[:fileNameLen]) + + _, err = readFull(r, buf[:4]) + if err != nil { + return + } + + l.Time = binary.BigEndian.Uint32(buf[:4]) + l.Body = r + return +} diff --git a/libgo/go/crypto/openpgp/packet/one_pass_signature.go b/libgo/go/crypto/openpgp/packet/one_pass_signature.go new file mode 100644 index 00000000000..acbf58bbefb --- /dev/null +++ b/libgo/go/crypto/openpgp/packet/one_pass_signature.go @@ -0,0 +1,49 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "crypto" + "crypto/openpgp/error" + "crypto/openpgp/s2k" + "encoding/binary" + "io" + "os" + "strconv" +) + +// OnePassSignature represents a one-pass signature packet. See RFC 4880, +// section 5.4. +type OnePassSignature struct { + SigType SignatureType + Hash crypto.Hash + PubKeyAlgo PublicKeyAlgorithm + KeyId uint64 + IsLast bool +} + +func (ops *OnePassSignature) parse(r io.Reader) (err os.Error) { + var buf [13]byte + + _, err = readFull(r, buf[:]) + if err != nil { + return + } + if buf[0] != 3 { + err = error.UnsupportedError("one-pass-signature packet version " + strconv.Itoa(int(buf[0]))) + } + + var ok bool + ops.Hash, ok = s2k.HashIdToHash(buf[2]) + if !ok { + return error.UnsupportedError("hash function: " + strconv.Itoa(int(buf[2]))) + } + + ops.SigType = SignatureType(buf[1]) + ops.PubKeyAlgo = PublicKeyAlgorithm(buf[3]) + ops.KeyId = binary.BigEndian.Uint64(buf[4:12]) + ops.IsLast = buf[12] != 0 + return +} diff --git a/libgo/go/crypto/openpgp/packet/packet.go b/libgo/go/crypto/openpgp/packet/packet.go new file mode 100644 index 00000000000..269603ba498 --- /dev/null +++ b/libgo/go/crypto/openpgp/packet/packet.go @@ -0,0 +1,395 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This package implements parsing and serialisation of OpenPGP packets, as +// specified in RFC 4880. +package packet + +import ( + "crypto/aes" + "crypto/cast5" + "crypto/cipher" + "crypto/openpgp/error" + "io" + "os" +) + +// readFull is the same as io.ReadFull except that reading zero bytes returns +// ErrUnexpectedEOF rather than EOF. +func readFull(r io.Reader, buf []byte) (n int, err os.Error) { + n, err = io.ReadFull(r, buf) + if err == os.EOF { + err = io.ErrUnexpectedEOF + } + return +} + +// readLength reads an OpenPGP length from r. See RFC 4880, section 4.2.2. +func readLength(r io.Reader) (length int64, isPartial bool, err os.Error) { + var buf [4]byte + _, err = readFull(r, buf[:1]) + if err != nil { + return + } + switch { + case buf[0] < 192: + length = int64(buf[0]) + case buf[0] < 224: + length = int64(buf[0]-192) << 8 + _, err = readFull(r, buf[0:1]) + if err != nil { + return + } + length += int64(buf[0]) + 192 + case buf[0] < 255: + length = int64(1) << (buf[0] & 0x1f) + isPartial = true + default: + _, err = readFull(r, buf[0:4]) + if err != nil { + return + } + length = int64(buf[0])<<24 | + int64(buf[1])<<16 | + int64(buf[2])<<8 | + int64(buf[3]) + } + return +} + +// partialLengthReader wraps an io.Reader and handles OpenPGP partial lengths. +// The continuation lengths are parsed and removed from the stream and EOF is +// returned at the end of the packet. See RFC 4880, section 4.2.2.4. +type partialLengthReader struct { + r io.Reader + remaining int64 + isPartial bool +} + +func (r *partialLengthReader) Read(p []byte) (n int, err os.Error) { + for r.remaining == 0 { + if !r.isPartial { + return 0, os.EOF + } + r.remaining, r.isPartial, err = readLength(r.r) + if err != nil { + return 0, err + } + } + + toRead := int64(len(p)) + if toRead > r.remaining { + toRead = r.remaining + } + + n, err = r.r.Read(p[:int(toRead)]) + r.remaining -= int64(n) + if n < int(toRead) && err == os.EOF { + err = io.ErrUnexpectedEOF + } + return +} + +// A spanReader is an io.LimitReader, but it returns ErrUnexpectedEOF if the +// underlying Reader returns EOF before the limit has been reached. +type spanReader struct { + r io.Reader + n int64 +} + +func (l *spanReader) Read(p []byte) (n int, err os.Error) { + if l.n <= 0 { + return 0, os.EOF + } + if int64(len(p)) > l.n { + p = p[0:l.n] + } + n, err = l.r.Read(p) + l.n -= int64(n) + if l.n > 0 && err == os.EOF { + err = io.ErrUnexpectedEOF + } + return +} + +// readHeader parses a packet header and returns an io.Reader which will return +// the contents of the packet. See RFC 4880, section 4.2. +func readHeader(r io.Reader) (tag packetType, length int64, contents io.Reader, err os.Error) { + var buf [4]byte + _, err = io.ReadFull(r, buf[:1]) + if err != nil { + return + } + if buf[0]&0x80 == 0 { + err = error.StructuralError("tag byte does not have MSB set") + return + } + if buf[0]&0x40 == 0 { + // Old format packet + tag = packetType((buf[0] & 0x3f) >> 2) + lengthType := buf[0] & 3 + if lengthType == 3 { + length = -1 + contents = r + return + } + lengthBytes := 1 << lengthType + _, err = readFull(r, buf[0:lengthBytes]) + if err != nil { + return + } + for i := 0; i < lengthBytes; i++ { + length <<= 8 + length |= int64(buf[i]) + } + contents = &spanReader{r, length} + return + } + + // New format packet + tag = packetType(buf[0] & 0x3f) + length, isPartial, err := readLength(r) + if err != nil { + return + } + if isPartial { + contents = &partialLengthReader{ + remaining: length, + isPartial: true, + r: r, + } + length = -1 + } else { + contents = &spanReader{r, length} + } + return +} + +// serialiseHeader writes an OpenPGP packet header to w. See RFC 4880, section +// 4.2. +func serialiseHeader(w io.Writer, ptype packetType, length int) (err os.Error) { + var buf [5]byte + var n int + + buf[0] = 0x80 | 0x40 | byte(ptype) + if length < 192 { + buf[1] = byte(length) + n = 2 + } else if length < 8384 { + length -= 192 + buf[1] = byte(length >> 8) + buf[2] = byte(length) + n = 3 + } else { + buf[0] = 255 + buf[1] = byte(length >> 24) + buf[2] = byte(length >> 16) + buf[3] = byte(length >> 8) + buf[4] = byte(length) + n = 5 + } + + _, err = w.Write(buf[:n]) + return +} + +// Packet represents an OpenPGP packet. Users are expected to try casting +// instances of this interface to specific packet types. +type Packet interface { + parse(io.Reader) os.Error +} + +// consumeAll reads from the given Reader until error, returning the number of +// bytes read. +func consumeAll(r io.Reader) (n int64, err os.Error) { + var m int + var buf [1024]byte + + for { + m, err = r.Read(buf[:]) + n += int64(m) + if err == os.EOF { + err = nil + return + } + if err != nil { + return + } + } + + panic("unreachable") +} + +// packetType represents the numeric ids of the different OpenPGP packet types. See +// http://www.iana.org/assignments/pgp-parameters/pgp-parameters.xhtml#pgp-parameters-2 +type packetType uint8 + +const ( + packetTypeEncryptedKey packetType = 1 + packetTypeSignature packetType = 2 + packetTypeSymmetricKeyEncrypted packetType = 3 + packetTypeOnePassSignature packetType = 4 + packetTypePrivateKey packetType = 5 + packetTypePublicKey packetType = 6 + packetTypePrivateSubkey packetType = 7 + packetTypeCompressed packetType = 8 + packetTypeSymmetricallyEncrypted packetType = 9 + packetTypeLiteralData packetType = 11 + packetTypeUserId packetType = 13 + packetTypePublicSubkey packetType = 14 + packetTypeSymmetricallyEncryptedMDC packetType = 18 +) + +// Read reads a single OpenPGP packet from the given io.Reader. If there is an +// error parsing a packet, the whole packet is consumed from the input. +func Read(r io.Reader) (p Packet, err os.Error) { + tag, _, contents, err := readHeader(r) + if err != nil { + return + } + + switch tag { + case packetTypeEncryptedKey: + p = new(EncryptedKey) + case packetTypeSignature: + p = new(Signature) + case packetTypeSymmetricKeyEncrypted: + p = new(SymmetricKeyEncrypted) + case packetTypeOnePassSignature: + p = new(OnePassSignature) + case packetTypePrivateKey, packetTypePrivateSubkey: + pk := new(PrivateKey) + if tag == packetTypePrivateSubkey { + pk.IsSubkey = true + } + p = pk + case packetTypePublicKey, packetTypePublicSubkey: + pk := new(PublicKey) + if tag == packetTypePublicSubkey { + pk.IsSubkey = true + } + p = pk + case packetTypeCompressed: + p = new(Compressed) + case packetTypeSymmetricallyEncrypted: + p = new(SymmetricallyEncrypted) + case packetTypeLiteralData: + p = new(LiteralData) + case packetTypeUserId: + p = new(UserId) + case packetTypeSymmetricallyEncryptedMDC: + se := new(SymmetricallyEncrypted) + se.MDC = true + p = se + default: + err = error.UnknownPacketTypeError(tag) + } + if p != nil { + err = p.parse(contents) + } + if err != nil { + consumeAll(contents) + } + return +} + +// SignatureType represents the different semantic meanings of an OpenPGP +// signature. See RFC 4880, section 5.2.1. +type SignatureType uint8 + +const ( + SigTypeBinary SignatureType = 0 + SigTypeText = 1 + SigTypeGenericCert = 0x10 + SigTypePersonaCert = 0x11 + SigTypeCasualCert = 0x12 + SigTypePositiveCert = 0x13 + SigTypeSubkeyBinding = 0x18 +) + +// PublicKeyAlgorithm represents the different public key system specified for +// OpenPGP. See +// http://www.iana.org/assignments/pgp-parameters/pgp-parameters.xhtml#pgp-parameters-12 +type PublicKeyAlgorithm uint8 + +const ( + PubKeyAlgoRSA PublicKeyAlgorithm = 1 + PubKeyAlgoRSAEncryptOnly PublicKeyAlgorithm = 2 + PubKeyAlgoRSASignOnly PublicKeyAlgorithm = 3 + PubKeyAlgoElgamal PublicKeyAlgorithm = 16 + PubKeyAlgoDSA PublicKeyAlgorithm = 17 +) + +// CipherFunction represents the different block ciphers specified for OpenPGP. See +// http://www.iana.org/assignments/pgp-parameters/pgp-parameters.xhtml#pgp-parameters-13 +type CipherFunction uint8 + +const ( + CipherCAST5 = 3 + CipherAES128 = 7 + CipherAES192 = 8 + CipherAES256 = 9 +) + +// keySize returns the key size, in bytes, of cipher. +func (cipher CipherFunction) keySize() int { + switch cipher { + case CipherCAST5: + return cast5.KeySize + case CipherAES128: + return 16 + case CipherAES192: + return 24 + case CipherAES256: + return 32 + } + return 0 +} + +// blockSize returns the block size, in bytes, of cipher. +func (cipher CipherFunction) blockSize() int { + switch cipher { + case CipherCAST5: + return 8 + case CipherAES128, CipherAES192, CipherAES256: + return 16 + } + return 0 +} + +// new returns a fresh instance of the given cipher. +func (cipher CipherFunction) new(key []byte) (block cipher.Block) { + switch cipher { + case CipherCAST5: + block, _ = cast5.NewCipher(key) + case CipherAES128, CipherAES192, CipherAES256: + block, _ = aes.NewCipher(key) + } + return +} + +// readMPI reads a big integer from r. The bit length returned is the bit +// length that was specified in r. This is preserved so that the integer can be +// reserialised exactly. +func readMPI(r io.Reader) (mpi []byte, bitLength uint16, err os.Error) { + var buf [2]byte + _, err = readFull(r, buf[0:]) + if err != nil { + return + } + bitLength = uint16(buf[0])<<8 | uint16(buf[1]) + numBytes := (int(bitLength) + 7) / 8 + mpi = make([]byte, numBytes) + _, err = readFull(r, mpi) + return +} + +// writeMPI serialises a big integer to r. +func writeMPI(w io.Writer, bitLength uint16, mpiBytes []byte) (err os.Error) { + _, err = w.Write([]byte{byte(bitLength >> 8), byte(bitLength)}) + if err == nil { + _, err = w.Write(mpiBytes) + } + return +} diff --git a/libgo/go/crypto/openpgp/packet/packet_test.go b/libgo/go/crypto/openpgp/packet/packet_test.go new file mode 100644 index 00000000000..6789d2abc79 --- /dev/null +++ b/libgo/go/crypto/openpgp/packet/packet_test.go @@ -0,0 +1,192 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "bytes" + "crypto/openpgp/error" + "encoding/hex" + "fmt" + "io" + "io/ioutil" + "os" + "testing" +) + +func TestReadFull(t *testing.T) { + var out [4]byte + + b := bytes.NewBufferString("foo") + n, err := readFull(b, out[:3]) + if n != 3 || err != nil { + t.Errorf("full read failed n:%d err:%s", n, err) + } + + b = bytes.NewBufferString("foo") + n, err = readFull(b, out[:4]) + if n != 3 || err != io.ErrUnexpectedEOF { + t.Errorf("partial read failed n:%d err:%s", n, err) + } + + b = bytes.NewBuffer(nil) + n, err = readFull(b, out[:3]) + if n != 0 || err != io.ErrUnexpectedEOF { + t.Errorf("empty read failed n:%d err:%s", n, err) + } +} + +func readerFromHex(s string) io.Reader { + data, err := hex.DecodeString(s) + if err != nil { + panic("readerFromHex: bad input") + } + return bytes.NewBuffer(data) +} + +var readLengthTests = []struct { + hexInput string + length int64 + isPartial bool + err os.Error +}{ + {"", 0, false, io.ErrUnexpectedEOF}, + {"1f", 31, false, nil}, + {"c0", 0, false, io.ErrUnexpectedEOF}, + {"c101", 256 + 1 + 192, false, nil}, + {"e0", 1, true, nil}, + {"e1", 2, true, nil}, + {"e2", 4, true, nil}, + {"ff", 0, false, io.ErrUnexpectedEOF}, + {"ff00", 0, false, io.ErrUnexpectedEOF}, + {"ff0000", 0, false, io.ErrUnexpectedEOF}, + {"ff000000", 0, false, io.ErrUnexpectedEOF}, + {"ff00000000", 0, false, nil}, + {"ff01020304", 16909060, false, nil}, +} + +func TestReadLength(t *testing.T) { + for i, test := range readLengthTests { + length, isPartial, err := readLength(readerFromHex(test.hexInput)) + if test.err != nil { + if err != test.err { + t.Errorf("%d: expected different error got:%s want:%s", i, err, test.err) + } + continue + } + if err != nil { + t.Errorf("%d: unexpected error: %s", i, err) + continue + } + if length != test.length || isPartial != test.isPartial { + t.Errorf("%d: bad result got:(%d,%t) want:(%d,%t)", i, length, isPartial, test.length, test.isPartial) + } + } +} + +var partialLengthReaderTests = []struct { + hexInput string + err os.Error + hexOutput string +}{ + {"e0", io.ErrUnexpectedEOF, ""}, + {"e001", io.ErrUnexpectedEOF, ""}, + {"e0010102", nil, "0102"}, + {"ff00000000", nil, ""}, + {"e10102e1030400", nil, "01020304"}, + {"e101", io.ErrUnexpectedEOF, ""}, +} + +func TestPartialLengthReader(t *testing.T) { + for i, test := range partialLengthReaderTests { + r := &partialLengthReader{readerFromHex(test.hexInput), 0, true} + out, err := ioutil.ReadAll(r) + if test.err != nil { + if err != test.err { + t.Errorf("%d: expected different error got:%s want:%s", i, err, test.err) + } + continue + } + if err != nil { + t.Errorf("%d: unexpected error: %s", i, err) + continue + } + + got := fmt.Sprintf("%x", out) + if got != test.hexOutput { + t.Errorf("%d: got:%s want:%s", i, test.hexOutput, got) + } + } +} + +var readHeaderTests = []struct { + hexInput string + structuralError bool + unexpectedEOF bool + tag int + length int64 + hexOutput string +}{ + {"", false, false, 0, 0, ""}, + {"7f", true, false, 0, 0, ""}, + + // Old format headers + {"80", false, true, 0, 0, ""}, + {"8001", false, true, 0, 1, ""}, + {"800102", false, false, 0, 1, "02"}, + {"81000102", false, false, 0, 1, "02"}, + {"820000000102", false, false, 0, 1, "02"}, + {"860000000102", false, false, 1, 1, "02"}, + {"83010203", false, false, 0, -1, "010203"}, + + // New format headers + {"c0", false, true, 0, 0, ""}, + {"c000", false, false, 0, 0, ""}, + {"c00102", false, false, 0, 1, "02"}, + {"c0020203", false, false, 0, 2, "0203"}, + {"c00202", false, true, 0, 2, ""}, + {"c3020203", false, false, 3, 2, "0203"}, +} + +func TestReadHeader(t *testing.T) { + for i, test := range readHeaderTests { + tag, length, contents, err := readHeader(readerFromHex(test.hexInput)) + if test.structuralError { + if _, ok := err.(error.StructuralError); ok { + continue + } + t.Errorf("%d: expected StructuralError, got:%s", i, err) + continue + } + if err != nil { + if len(test.hexInput) == 0 && err == os.EOF { + continue + } + if !test.unexpectedEOF || err != io.ErrUnexpectedEOF { + t.Errorf("%d: unexpected error from readHeader: %s", i, err) + } + continue + } + if int(tag) != test.tag || length != test.length { + t.Errorf("%d: got:(%d,%d) want:(%d,%d)", i, int(tag), length, test.tag, test.length) + continue + } + + body, err := ioutil.ReadAll(contents) + if err != nil { + if !test.unexpectedEOF || err != io.ErrUnexpectedEOF { + t.Errorf("%d: unexpected error from contents: %s", i, err) + } + continue + } + if test.unexpectedEOF { + t.Errorf("%d: expected ErrUnexpectedEOF from contents but got no error", i) + continue + } + got := fmt.Sprintf("%x", body) + if got != test.hexOutput { + t.Errorf("%d: got:%s want:%s", i, got, test.hexOutput) + } + } +} diff --git a/libgo/go/crypto/openpgp/packet/private_key.go b/libgo/go/crypto/openpgp/packet/private_key.go new file mode 100644 index 00000000000..b22891755e3 --- /dev/null +++ b/libgo/go/crypto/openpgp/packet/private_key.go @@ -0,0 +1,164 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "big" + "bytes" + "crypto/cipher" + "crypto/openpgp/error" + "crypto/openpgp/s2k" + "crypto/rsa" + "crypto/sha1" + "io" + "io/ioutil" + "os" + "strconv" +) + +// PrivateKey represents a possibly encrypted private key. See RFC 4880, +// section 5.5.3. +type PrivateKey struct { + PublicKey + Encrypted bool // if true then the private key is unavailable until Decrypt has been called. + encryptedData []byte + cipher CipherFunction + s2k func(out, in []byte) + PrivateKey interface{} // An *rsa.PrivateKey. + sha1Checksum bool + iv []byte +} + +func (pk *PrivateKey) parse(r io.Reader) (err os.Error) { + err = (&pk.PublicKey).parse(r) + if err != nil { + return + } + var buf [1]byte + _, err = readFull(r, buf[:]) + if err != nil { + return + } + + s2kType := buf[0] + + switch s2kType { + case 0: + pk.s2k = nil + pk.Encrypted = false + case 254, 255: + _, err = readFull(r, buf[:]) + if err != nil { + return + } + pk.cipher = CipherFunction(buf[0]) + pk.Encrypted = true + pk.s2k, err = s2k.Parse(r) + if err != nil { + return + } + if s2kType == 254 { + pk.sha1Checksum = true + } + default: + return error.UnsupportedError("deprecated s2k function in private key") + } + + if pk.Encrypted { + blockSize := pk.cipher.blockSize() + if blockSize == 0 { + return error.UnsupportedError("unsupported cipher in private key: " + strconv.Itoa(int(pk.cipher))) + } + pk.iv = make([]byte, blockSize) + _, err = readFull(r, pk.iv) + if err != nil { + return + } + } + + pk.encryptedData, err = ioutil.ReadAll(r) + if err != nil { + return + } + + if !pk.Encrypted { + return pk.parsePrivateKey(pk.encryptedData) + } + + return +} + +// Decrypt decrypts an encrypted private key using a passphrase. +func (pk *PrivateKey) Decrypt(passphrase []byte) os.Error { + if !pk.Encrypted { + return nil + } + + key := make([]byte, pk.cipher.keySize()) + pk.s2k(key, passphrase) + block := pk.cipher.new(key) + cfb := cipher.NewCFBDecrypter(block, pk.iv) + + data := pk.encryptedData + cfb.XORKeyStream(data, data) + + if pk.sha1Checksum { + if len(data) < sha1.Size { + return error.StructuralError("truncated private key data") + } + h := sha1.New() + h.Write(data[:len(data)-sha1.Size]) + sum := h.Sum() + if !bytes.Equal(sum, data[len(data)-sha1.Size:]) { + return error.StructuralError("private key checksum failure") + } + data = data[:len(data)-sha1.Size] + } else { + if len(data) < 2 { + return error.StructuralError("truncated private key data") + } + var sum uint16 + for i := 0; i < len(data)-2; i++ { + sum += uint16(data[i]) + } + if data[len(data)-2] != uint8(sum>>8) || + data[len(data)-1] != uint8(sum) { + return error.StructuralError("private key checksum failure") + } + data = data[:len(data)-2] + } + + return pk.parsePrivateKey(data) +} + +func (pk *PrivateKey) parsePrivateKey(data []byte) (err os.Error) { + // TODO(agl): support DSA and ECDSA private keys. + rsaPub := pk.PublicKey.PublicKey.(*rsa.PublicKey) + rsaPriv := new(rsa.PrivateKey) + rsaPriv.PublicKey = *rsaPub + + buf := bytes.NewBuffer(data) + d, _, err := readMPI(buf) + if err != nil { + return + } + p, _, err := readMPI(buf) + if err != nil { + return + } + q, _, err := readMPI(buf) + if err != nil { + return + } + + rsaPriv.D = new(big.Int).SetBytes(d) + rsaPriv.P = new(big.Int).SetBytes(p) + rsaPriv.Q = new(big.Int).SetBytes(q) + pk.PrivateKey = rsaPriv + pk.Encrypted = false + pk.encryptedData = nil + + return nil +} diff --git a/libgo/go/crypto/openpgp/packet/private_key_test.go b/libgo/go/crypto/openpgp/packet/private_key_test.go new file mode 100644 index 00000000000..e941cc735cf --- /dev/null +++ b/libgo/go/crypto/openpgp/packet/private_key_test.go @@ -0,0 +1,37 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "testing" +) + +func TestPrivateKeyRead(t *testing.T) { + packet, err := Read(readerFromHex(privKeyHex)) + if err != nil { + t.Error(err) + return + } + + privKey := packet.(*PrivateKey) + + if !privKey.Encrypted { + t.Error("private key isn't encrypted") + return + } + + err = privKey.Decrypt([]byte("testing")) + if err != nil { + t.Error(err) + return + } + + if privKey.CreationTime != 0x4cc349a8 || privKey.Encrypted { + t.Errorf("failed to parse, got: %#v", privKey) + } +} + +// Generated with `gpg --export-secret-keys "Test Key 2"` +const privKeyHex = "9501fe044cc349a8010400b70ca0010e98c090008d45d1ee8f9113bd5861fd57b88bacb7c68658747663f1e1a3b5a98f32fda6472373c024b97359cd2efc88ff60f77751adfbf6af5e615e6a1408cfad8bf0cea30b0d5f53aa27ad59089ba9b15b7ebc2777a25d7b436144027e3bcd203909f147d0e332b240cf63d3395f5dfe0df0a6c04e8655af7eacdf0011010001fe0303024a252e7d475fd445607de39a265472aa74a9320ba2dac395faa687e9e0336aeb7e9a7397e511b5afd9dc84557c80ac0f3d4d7bfec5ae16f20d41c8c84a04552a33870b930420e230e179564f6d19bb153145e76c33ae993886c388832b0fa042ddda7f133924f3854481533e0ede31d51278c0519b29abc3bf53da673e13e3e1214b52413d179d7f66deee35cac8eacb060f78379d70ef4af8607e68131ff529439668fc39c9ce6dfef8a5ac234d234802cbfb749a26107db26406213ae5c06d4673253a3cbee1fcbae58d6ab77e38d6e2c0e7c6317c48e054edadb5a40d0d48acb44643d998139a8a66bb820be1f3f80185bc777d14b5954b60effe2448a036d565c6bc0b915fcea518acdd20ab07bc1529f561c58cd044f723109b93f6fd99f876ff891d64306b5d08f48bab59f38695e9109c4dec34013ba3153488ce070268381ba923ee1eb77125b36afcb4347ec3478c8f2735b06ef17351d872e577fa95d0c397c88c71b59629a36aec" diff --git a/libgo/go/crypto/openpgp/packet/public_key.go b/libgo/go/crypto/openpgp/packet/public_key.go new file mode 100644 index 00000000000..8866bdaaa94 --- /dev/null +++ b/libgo/go/crypto/openpgp/packet/public_key.go @@ -0,0 +1,260 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "big" + "crypto/dsa" + "crypto/openpgp/error" + "crypto/rsa" + "crypto/sha1" + "encoding/binary" + "hash" + "io" + "os" +) + +// PublicKey represents an OpenPGP public key. See RFC 4880, section 5.5.2. +type PublicKey struct { + CreationTime uint32 // seconds since the epoch + PubKeyAlgo PublicKeyAlgorithm + PublicKey interface{} // Either a *rsa.PublicKey or *dsa.PublicKey + Fingerprint [20]byte + KeyId uint64 + IsSubkey bool + + n, e, p, q, g, y parsedMPI +} + +func (pk *PublicKey) parse(r io.Reader) (err os.Error) { + // RFC 4880, section 5.5.2 + var buf [6]byte + _, err = readFull(r, buf[:]) + if err != nil { + return + } + if buf[0] != 4 { + return error.UnsupportedError("public key version") + } + pk.CreationTime = uint32(buf[1])<<24 | uint32(buf[2])<<16 | uint32(buf[3])<<8 | uint32(buf[4]) + pk.PubKeyAlgo = PublicKeyAlgorithm(buf[5]) + switch pk.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSAEncryptOnly, PubKeyAlgoRSASignOnly: + err = pk.parseRSA(r) + case PubKeyAlgoDSA: + err = pk.parseDSA(r) + default: + err = error.UnsupportedError("public key type") + } + if err != nil { + return + } + + // RFC 4880, section 12.2 + fingerPrint := sha1.New() + pk.SerializeSignaturePrefix(fingerPrint) + pk.Serialize(fingerPrint) + copy(pk.Fingerprint[:], fingerPrint.Sum()) + pk.KeyId = binary.BigEndian.Uint64(pk.Fingerprint[12:20]) + + return +} + +// parseRSA parses RSA public key material from the given Reader. See RFC 4880, +// section 5.5.2. +func (pk *PublicKey) parseRSA(r io.Reader) (err os.Error) { + pk.n.bytes, pk.n.bitLength, err = readMPI(r) + if err != nil { + return + } + pk.e.bytes, pk.e.bitLength, err = readMPI(r) + if err != nil { + return + } + + if len(pk.e.bytes) > 3 { + err = error.UnsupportedError("large public exponent") + return + } + rsa := &rsa.PublicKey{ + N: new(big.Int).SetBytes(pk.n.bytes), + E: 0, + } + for i := 0; i < len(pk.e.bytes); i++ { + rsa.E <<= 8 + rsa.E |= int(pk.e.bytes[i]) + } + pk.PublicKey = rsa + return +} + +// parseRSA parses DSA public key material from the given Reader. See RFC 4880, +// section 5.5.2. +func (pk *PublicKey) parseDSA(r io.Reader) (err os.Error) { + pk.p.bytes, pk.p.bitLength, err = readMPI(r) + if err != nil { + return + } + pk.q.bytes, pk.q.bitLength, err = readMPI(r) + if err != nil { + return + } + pk.g.bytes, pk.g.bitLength, err = readMPI(r) + if err != nil { + return + } + pk.y.bytes, pk.y.bitLength, err = readMPI(r) + if err != nil { + return + } + + dsa := new(dsa.PublicKey) + dsa.P = new(big.Int).SetBytes(pk.p.bytes) + dsa.Q = new(big.Int).SetBytes(pk.q.bytes) + dsa.G = new(big.Int).SetBytes(pk.g.bytes) + dsa.Y = new(big.Int).SetBytes(pk.y.bytes) + pk.PublicKey = dsa + return +} + +// SerializeSignaturePrefix writes the prefix for this public key to the given Writer. +// The prefix is used when calculating a signature over this public key. See +// RFC 4880, section 5.2.4. +func (pk *PublicKey) SerializeSignaturePrefix(h hash.Hash) { + var pLength uint16 + switch pk.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSAEncryptOnly, PubKeyAlgoRSASignOnly: + pLength += 2 + uint16(len(pk.n.bytes)) + pLength += 2 + uint16(len(pk.e.bytes)) + case PubKeyAlgoDSA: + pLength += 2 + uint16(len(pk.p.bytes)) + pLength += 2 + uint16(len(pk.q.bytes)) + pLength += 2 + uint16(len(pk.g.bytes)) + pLength += 2 + uint16(len(pk.y.bytes)) + default: + panic("unknown public key algorithm") + } + pLength += 6 + h.Write([]byte{0x99, byte(pLength >> 8), byte(pLength)}) + return +} + +// Serialize marshals the PublicKey to w in the form of an OpenPGP public key +// packet, not including the packet header. +func (pk *PublicKey) Serialize(w io.Writer) (err os.Error) { + var buf [6]byte + buf[0] = 4 + buf[1] = byte(pk.CreationTime >> 24) + buf[2] = byte(pk.CreationTime >> 16) + buf[3] = byte(pk.CreationTime >> 8) + buf[4] = byte(pk.CreationTime) + buf[5] = byte(pk.PubKeyAlgo) + + _, err = w.Write(buf[:]) + if err != nil { + return + } + + switch pk.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSAEncryptOnly, PubKeyAlgoRSASignOnly: + return writeMPIs(w, pk.n, pk.e) + case PubKeyAlgoDSA: + return writeMPIs(w, pk.p, pk.q, pk.g, pk.y) + } + return error.InvalidArgumentError("bad public-key algorithm") +} + +// CanSign returns true iff this public key can generate signatures +func (pk *PublicKey) CanSign() bool { + return pk.PubKeyAlgo != PubKeyAlgoRSAEncryptOnly && pk.PubKeyAlgo != PubKeyAlgoElgamal +} + +// VerifySignature returns nil iff sig is a valid signature, made by this +// public key, of the data hashed into signed. signed is mutated by this call. +func (pk *PublicKey) VerifySignature(signed hash.Hash, sig *Signature) (err os.Error) { + if !pk.CanSign() { + return error.InvalidArgumentError("public key cannot generate signatures") + } + + rsaPublicKey, ok := pk.PublicKey.(*rsa.PublicKey) + if !ok { + // TODO(agl): support DSA and ECDSA keys. + return error.UnsupportedError("non-RSA public key") + } + + signed.Write(sig.HashSuffix) + hashBytes := signed.Sum() + + if hashBytes[0] != sig.HashTag[0] || hashBytes[1] != sig.HashTag[1] { + return error.SignatureError("hash tag doesn't match") + } + + err = rsa.VerifyPKCS1v15(rsaPublicKey, sig.Hash, hashBytes, sig.Signature) + if err != nil { + return error.SignatureError("RSA verification failure") + } + return nil +} + +// VerifyKeySignature returns nil iff sig is a valid signature, make by this +// public key, of the public key in signed. +func (pk *PublicKey) VerifyKeySignature(signed *PublicKey, sig *Signature) (err os.Error) { + h := sig.Hash.New() + if h == nil { + return error.UnsupportedError("hash function") + } + + // RFC 4880, section 5.2.4 + pk.SerializeSignaturePrefix(h) + pk.Serialize(h) + signed.SerializeSignaturePrefix(h) + signed.Serialize(h) + + return pk.VerifySignature(h, sig) +} + +// VerifyUserIdSignature returns nil iff sig is a valid signature, make by this +// public key, of the given user id. +func (pk *PublicKey) VerifyUserIdSignature(id string, sig *Signature) (err os.Error) { + h := sig.Hash.New() + if h == nil { + return error.UnsupportedError("hash function") + } + + // RFC 4880, section 5.2.4 + pk.SerializeSignaturePrefix(h) + pk.Serialize(h) + + var buf [5]byte + buf[0] = 0xb4 + buf[1] = byte(len(id) >> 24) + buf[2] = byte(len(id) >> 16) + buf[3] = byte(len(id) >> 8) + buf[4] = byte(len(id)) + h.Write(buf[:]) + h.Write([]byte(id)) + + return pk.VerifySignature(h, sig) +} + +// A parsedMPI is used to store the contents of a big integer, along with the +// bit length that was specified in the original input. This allows the MPI to +// be reserialised exactly. +type parsedMPI struct { + bytes []byte + bitLength uint16 +} + +// writeMPIs is a utility function for serialising several big integers to the +// given Writer. +func writeMPIs(w io.Writer, mpis ...parsedMPI) (err os.Error) { + for _, mpi := range mpis { + err = writeMPI(w, mpi.bitLength, mpi.bytes) + if err != nil { + return + } + } + return +} diff --git a/libgo/go/crypto/openpgp/packet/public_key_test.go b/libgo/go/crypto/openpgp/packet/public_key_test.go new file mode 100644 index 00000000000..c015f64aec9 --- /dev/null +++ b/libgo/go/crypto/openpgp/packet/public_key_test.go @@ -0,0 +1,58 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "bytes" + "encoding/hex" + "testing" +) + +var pubKeyTests = []struct { + hexData string + hexFingerprint string + creationTime uint32 + pubKeyAlgo PublicKeyAlgorithm + keyId uint64 +}{ + {rsaPkDataHex, rsaFingerprintHex, 0x4d3c5c10, PubKeyAlgoRSA, 0xa34d7e18c20c31bb}, + {dsaPkDataHex, dsaFingerprintHex, 0x4d432f89, PubKeyAlgoDSA, 0x8e8fbe54062f19ed}, +} + +func TestPublicKeyRead(t *testing.T) { + for i, test := range pubKeyTests { + packet, err := Read(readerFromHex(test.hexData)) + if err != nil { + t.Errorf("#%d: Read error: %s", i, err) + return + } + pk, ok := packet.(*PublicKey) + if !ok { + t.Errorf("#%d: failed to parse, got: %#v", i, packet) + return + } + if pk.PubKeyAlgo != test.pubKeyAlgo { + t.Errorf("#%d: bad public key algorithm got:%x want:%x", i, pk.PubKeyAlgo, test.pubKeyAlgo) + } + if pk.CreationTime != test.creationTime { + t.Errorf("#%d: bad creation time got:%x want:%x", i, pk.CreationTime, test.creationTime) + } + expectedFingerprint, _ := hex.DecodeString(test.hexFingerprint) + if !bytes.Equal(expectedFingerprint, pk.Fingerprint[:]) { + t.Errorf("#%d: bad fingerprint got:%x want:%x", i, pk.Fingerprint[:], expectedFingerprint) + } + if pk.KeyId != test.keyId { + t.Errorf("#%d: bad keyid got:%x want:%x", i, pk.KeyId, test.keyId) + } + } +} + +const rsaFingerprintHex = "5fb74b1d03b1e3cb31bc2f8aa34d7e18c20c31bb" + +const rsaPkDataHex = "988d044d3c5c10010400b1d13382944bd5aba23a4312968b5095d14f947f600eb478e14a6fcb16b0e0cac764884909c020bc495cfcc39a935387c661507bdb236a0612fb582cac3af9b29cc2c8c70090616c41b662f4da4c1201e195472eb7f4ae1ccbcbf9940fe21d985e379a5563dde5b9a23d35f1cfaa5790da3b79db26f23695107bfaca8e7b5bcd0011010001" + +const dsaFingerprintHex = "eece4c094db002103714c63c8e8fbe54062f19ed" + +const dsaPkDataHex = "9901a2044d432f89110400cd581334f0d7a1e1bdc8b9d6d8c0baf68793632735d2bb0903224cbaa1dfbf35a60ee7a13b92643421e1eb41aa8d79bea19a115a677f6b8ba3c7818ce53a6c2a24a1608bd8b8d6e55c5090cbde09dd26e356267465ae25e69ec8bdd57c7bbb2623e4d73336f73a0a9098f7f16da2e25252130fd694c0e8070c55a812a423ae7f00a0ebf50e70c2f19c3520a551bd4b08d30f23530d3d03ff7d0bf4a53a64a09dc5e6e6e35854b7d70c882b0c60293401958b1bd9e40abec3ea05ba87cf64899299d4bd6aa7f459c201d3fbbd6c82004bdc5e8a9eb8082d12054cc90fa9d4ec251a843236a588bf49552441817436c4f43326966fe85447d4e6d0acf8fa1ef0f014730770603ad7634c3088dc52501c237328417c31c89ed70400b2f1a98b0bf42f11fefc430704bebbaa41d9f355600c3facee1e490f64208e0e094ea55e3a598a219a58500bf78ac677b670a14f4e47e9cf8eab4f368cc1ddcaa18cc59309d4cc62dd4f680e73e6cc3e1ce87a84d0925efbcb26c575c093fc42eecf45135fabf6403a25c2016e1774c0484e440a18319072c617cc97ac0a3bb0" diff --git a/libgo/go/crypto/openpgp/packet/reader.go b/libgo/go/crypto/openpgp/packet/reader.go new file mode 100644 index 00000000000..5febc3bc8dc --- /dev/null +++ b/libgo/go/crypto/openpgp/packet/reader.go @@ -0,0 +1,63 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "crypto/openpgp/error" + "io" + "os" +) + +// Reader reads packets from an io.Reader and allows packets to be 'unread' so +// that they result from the next call to Next. +type Reader struct { + q []Packet + readers []io.Reader +} + +// Next returns the most recently unread Packet, or reads another packet from +// the top-most io.Reader. Unknown packet types are skipped. +func (r *Reader) Next() (p Packet, err os.Error) { + if len(r.q) > 0 { + p = r.q[len(r.q)-1] + r.q = r.q[:len(r.q)-1] + return + } + + for len(r.readers) > 0 { + p, err = Read(r.readers[len(r.readers)-1]) + if err == nil { + return + } + if err == os.EOF { + r.readers = r.readers[:len(r.readers)-1] + continue + } + if _, ok := err.(error.UnknownPacketTypeError); !ok { + return nil, err + } + } + + return nil, os.EOF +} + +// Push causes the Reader to start reading from a new io.Reader. When an EOF +// error is seen from the new io.Reader, it is popped and the Reader continues +// to read from the next most recent io.Reader. +func (r *Reader) Push(reader io.Reader) { + r.readers = append(r.readers, reader) +} + +// Unread causes the given Packet to be returned from the next call to Next. +func (r *Reader) Unread(p Packet) { + r.q = append(r.q, p) +} + +func NewReader(r io.Reader) *Reader { + return &Reader{ + q: nil, + readers: []io.Reader{r}, + } +} diff --git a/libgo/go/crypto/openpgp/packet/signature.go b/libgo/go/crypto/openpgp/packet/signature.go new file mode 100644 index 00000000000..fd2518ab41e --- /dev/null +++ b/libgo/go/crypto/openpgp/packet/signature.go @@ -0,0 +1,468 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "crypto" + "crypto/openpgp/error" + "crypto/openpgp/s2k" + "crypto/rand" + "crypto/rsa" + "encoding/binary" + "hash" + "io" + "os" + "strconv" +) + +// Signature represents a signature. See RFC 4880, section 5.2. +type Signature struct { + SigType SignatureType + PubKeyAlgo PublicKeyAlgorithm + Hash crypto.Hash + + // HashSuffix is extra data that is hashed in after the signed data. + HashSuffix []byte + // HashTag contains the first two bytes of the hash for fast rejection + // of bad signed data. + HashTag [2]byte + CreationTime uint32 // Unix epoch time + Signature []byte + + // The following are optional so are nil when not included in the + // signature. + + SigLifetimeSecs, KeyLifetimeSecs *uint32 + PreferredSymmetric, PreferredHash, PreferredCompression []uint8 + IssuerKeyId *uint64 + IsPrimaryId *bool + + // FlagsValid is set if any flags were given. See RFC 4880, section + // 5.2.3.21 for details. + FlagsValid bool + FlagCertify, FlagSign, FlagEncryptCommunications, FlagEncryptStorage bool + + outSubpackets []outputSubpacket +} + +func (sig *Signature) parse(r io.Reader) (err os.Error) { + // RFC 4880, section 5.2.3 + var buf [5]byte + _, err = readFull(r, buf[:1]) + if err != nil { + return + } + if buf[0] != 4 { + err = error.UnsupportedError("signature packet version " + strconv.Itoa(int(buf[0]))) + return + } + + _, err = readFull(r, buf[:5]) + if err != nil { + return + } + sig.SigType = SignatureType(buf[0]) + sig.PubKeyAlgo = PublicKeyAlgorithm(buf[1]) + switch sig.PubKeyAlgo { + case PubKeyAlgoRSA, PubKeyAlgoRSASignOnly: + default: + err = error.UnsupportedError("public key algorithm " + strconv.Itoa(int(sig.PubKeyAlgo))) + return + } + + var ok bool + sig.Hash, ok = s2k.HashIdToHash(buf[2]) + if !ok { + return error.UnsupportedError("hash function " + strconv.Itoa(int(buf[2]))) + } + + hashedSubpacketsLength := int(buf[3])<<8 | int(buf[4]) + l := 6 + hashedSubpacketsLength + sig.HashSuffix = make([]byte, l+6) + sig.HashSuffix[0] = 4 + copy(sig.HashSuffix[1:], buf[:5]) + hashedSubpackets := sig.HashSuffix[6:l] + _, err = readFull(r, hashedSubpackets) + if err != nil { + return + } + // See RFC 4880, section 5.2.4 + trailer := sig.HashSuffix[l:] + trailer[0] = 4 + trailer[1] = 0xff + trailer[2] = uint8(l >> 24) + trailer[3] = uint8(l >> 16) + trailer[4] = uint8(l >> 8) + trailer[5] = uint8(l) + + err = parseSignatureSubpackets(sig, hashedSubpackets, true) + if err != nil { + return + } + + _, err = readFull(r, buf[:2]) + if err != nil { + return + } + unhashedSubpacketsLength := int(buf[0])<<8 | int(buf[1]) + unhashedSubpackets := make([]byte, unhashedSubpacketsLength) + _, err = readFull(r, unhashedSubpackets) + if err != nil { + return + } + err = parseSignatureSubpackets(sig, unhashedSubpackets, false) + if err != nil { + return + } + + _, err = readFull(r, sig.HashTag[:2]) + if err != nil { + return + } + + // We have already checked that the public key algorithm is RSA. + sig.Signature, _, err = readMPI(r) + return +} + +// parseSignatureSubpackets parses subpackets of the main signature packet. See +// RFC 4880, section 5.2.3.1. +func parseSignatureSubpackets(sig *Signature, subpackets []byte, isHashed bool) (err os.Error) { + for len(subpackets) > 0 { + subpackets, err = parseSignatureSubpacket(sig, subpackets, isHashed) + if err != nil { + return + } + } + + if sig.CreationTime == 0 { + err = error.StructuralError("no creation time in signature") + } + + return +} + +type signatureSubpacketType uint8 + +const ( + creationTimeSubpacket signatureSubpacketType = 2 + signatureExpirationSubpacket signatureSubpacketType = 3 + keyExpirySubpacket signatureSubpacketType = 9 + prefSymmetricAlgosSubpacket signatureSubpacketType = 11 + issuerSubpacket signatureSubpacketType = 16 + prefHashAlgosSubpacket signatureSubpacketType = 21 + prefCompressionSubpacket signatureSubpacketType = 22 + primaryUserIdSubpacket signatureSubpacketType = 25 + keyFlagsSubpacket signatureSubpacketType = 27 +) + +// parseSignatureSubpacket parses a single subpacket. len(subpacket) is >= 1. +func parseSignatureSubpacket(sig *Signature, subpacket []byte, isHashed bool) (rest []byte, err os.Error) { + // RFC 4880, section 5.2.3.1 + var length uint32 + switch { + case subpacket[0] < 192: + length = uint32(subpacket[0]) + subpacket = subpacket[1:] + case subpacket[0] < 255: + if len(subpacket) < 2 { + goto Truncated + } + length = uint32(subpacket[0]-192)<<8 + uint32(subpacket[1]) + 192 + subpacket = subpacket[2:] + default: + if len(subpacket) < 5 { + goto Truncated + } + length = uint32(subpacket[1])<<24 | + uint32(subpacket[2])<<16 | + uint32(subpacket[3])<<8 | + uint32(subpacket[4]) + subpacket = subpacket[5:] + } + if length > uint32(len(subpacket)) { + goto Truncated + } + rest = subpacket[length:] + subpacket = subpacket[:length] + if len(subpacket) == 0 { + err = error.StructuralError("zero length signature subpacket") + return + } + packetType := subpacket[0] & 0x7f + isCritial := subpacket[0]&0x80 == 0x80 + subpacket = subpacket[1:] + switch signatureSubpacketType(packetType) { + case creationTimeSubpacket: + if !isHashed { + err = error.StructuralError("signature creation time in non-hashed area") + return + } + if len(subpacket) != 4 { + err = error.StructuralError("signature creation time not four bytes") + return + } + sig.CreationTime = binary.BigEndian.Uint32(subpacket) + case signatureExpirationSubpacket: + // Signature expiration time, section 5.2.3.10 + if !isHashed { + return + } + if len(subpacket) != 4 { + err = error.StructuralError("expiration subpacket with bad length") + return + } + sig.SigLifetimeSecs = new(uint32) + *sig.SigLifetimeSecs = binary.BigEndian.Uint32(subpacket) + case keyExpirySubpacket: + // Key expiration time, section 5.2.3.6 + if !isHashed { + return + } + if len(subpacket) != 4 { + err = error.StructuralError("key expiration subpacket with bad length") + return + } + sig.KeyLifetimeSecs = new(uint32) + *sig.KeyLifetimeSecs = binary.BigEndian.Uint32(subpacket) + case prefSymmetricAlgosSubpacket: + // Preferred symmetric algorithms, section 5.2.3.7 + if !isHashed { + return + } + sig.PreferredSymmetric = make([]byte, len(subpacket)) + copy(sig.PreferredSymmetric, subpacket) + case issuerSubpacket: + // Issuer, section 5.2.3.5 + if len(subpacket) != 8 { + err = error.StructuralError("issuer subpacket with bad length") + return + } + sig.IssuerKeyId = new(uint64) + *sig.IssuerKeyId = binary.BigEndian.Uint64(subpacket) + case prefHashAlgosSubpacket: + // Preferred hash algorithms, section 5.2.3.8 + if !isHashed { + return + } + sig.PreferredHash = make([]byte, len(subpacket)) + copy(sig.PreferredHash, subpacket) + case prefCompressionSubpacket: + // Preferred compression algorithms, section 5.2.3.9 + if !isHashed { + return + } + sig.PreferredCompression = make([]byte, len(subpacket)) + copy(sig.PreferredCompression, subpacket) + case primaryUserIdSubpacket: + // Primary User ID, section 5.2.3.19 + if !isHashed { + return + } + if len(subpacket) != 1 { + err = error.StructuralError("primary user id subpacket with bad length") + return + } + sig.IsPrimaryId = new(bool) + if subpacket[0] > 0 { + *sig.IsPrimaryId = true + } + case keyFlagsSubpacket: + // Key flags, section 5.2.3.21 + if !isHashed { + return + } + if len(subpacket) == 0 { + err = error.StructuralError("empty key flags subpacket") + return + } + sig.FlagsValid = true + if subpacket[0]&1 != 0 { + sig.FlagCertify = true + } + if subpacket[0]&2 != 0 { + sig.FlagSign = true + } + if subpacket[0]&4 != 0 { + sig.FlagEncryptCommunications = true + } + if subpacket[0]&8 != 0 { + sig.FlagEncryptStorage = true + } + + default: + if isCritial { + err = error.UnsupportedError("unknown critical signature subpacket type " + strconv.Itoa(int(packetType))) + return + } + } + return + +Truncated: + err = error.StructuralError("signature subpacket truncated") + return +} + +// subpacketLengthLength returns the length, in bytes, of an encoded length value. +func subpacketLengthLength(length int) int { + if length < 192 { + return 1 + } + if length < 16320 { + return 2 + } + return 5 +} + +// serialiseSubpacketLength marshals the given length into to. +func serialiseSubpacketLength(to []byte, length int) int { + if length < 192 { + to[0] = byte(length) + return 1 + } + if length < 16320 { + length -= 192 + to[0] = byte(length >> 8) + to[1] = byte(length) + return 2 + } + to[0] = 255 + to[1] = byte(length >> 24) + to[2] = byte(length >> 16) + to[3] = byte(length >> 8) + to[4] = byte(length) + return 5 +} + +// subpacketsLength returns the serialised length, in bytes, of the given +// subpackets. +func subpacketsLength(subpackets []outputSubpacket, hashed bool) (length int) { + for _, subpacket := range subpackets { + if subpacket.hashed == hashed { + length += subpacketLengthLength(len(subpacket.contents) + 1) + length += 1 // type byte + length += len(subpacket.contents) + } + } + return +} + +// serialiseSubpackets marshals the given subpackets into to. +func serialiseSubpackets(to []byte, subpackets []outputSubpacket, hashed bool) { + for _, subpacket := range subpackets { + if subpacket.hashed == hashed { + n := serialiseSubpacketLength(to, len(subpacket.contents)+1) + to[n] = byte(subpacket.subpacketType) + to = to[1+n:] + n = copy(to, subpacket.contents) + to = to[n:] + } + } + return +} + +// buildHashSuffix constructs the HashSuffix member of sig in preparation for signing. +func (sig *Signature) buildHashSuffix() (err os.Error) { + sig.outSubpackets = sig.buildSubpackets() + hashedSubpacketsLen := subpacketsLength(sig.outSubpackets, true) + + var ok bool + l := 6 + hashedSubpacketsLen + sig.HashSuffix = make([]byte, l+6) + sig.HashSuffix[0] = 4 + sig.HashSuffix[1] = uint8(sig.SigType) + sig.HashSuffix[2] = uint8(sig.PubKeyAlgo) + sig.HashSuffix[3], ok = s2k.HashToHashId(sig.Hash) + if !ok { + sig.HashSuffix = nil + return error.InvalidArgumentError("hash cannot be repesented in OpenPGP: " + strconv.Itoa(int(sig.Hash))) + } + sig.HashSuffix[4] = byte(hashedSubpacketsLen >> 8) + sig.HashSuffix[5] = byte(hashedSubpacketsLen) + serialiseSubpackets(sig.HashSuffix[6:l], sig.outSubpackets, true) + trailer := sig.HashSuffix[l:] + trailer[0] = 4 + trailer[1] = 0xff + trailer[2] = byte(l >> 24) + trailer[3] = byte(l >> 16) + trailer[4] = byte(l >> 8) + trailer[5] = byte(l) + return +} + +// SignRSA signs a message with an RSA private key. The hash, h, must contain +// the hash of message to be signed and will be mutated by this function. +func (sig *Signature) SignRSA(h hash.Hash, priv *rsa.PrivateKey) (err os.Error) { + err = sig.buildHashSuffix() + if err != nil { + return + } + + h.Write(sig.HashSuffix) + digest := h.Sum() + copy(sig.HashTag[:], digest) + sig.Signature, err = rsa.SignPKCS1v15(rand.Reader, priv, sig.Hash, digest) + return +} + +// Serialize marshals sig to w. SignRSA must have been called first. +func (sig *Signature) Serialize(w io.Writer) (err os.Error) { + if sig.Signature == nil { + return error.InvalidArgumentError("Signature: need to call SignRSA before Serialize") + } + + unhashedSubpacketsLen := subpacketsLength(sig.outSubpackets, false) + length := len(sig.HashSuffix) - 6 /* trailer not included */ + + 2 /* length of unhashed subpackets */ + unhashedSubpacketsLen + + 2 /* hash tag */ + 2 /* length of signature MPI */ + len(sig.Signature) + err = serialiseHeader(w, packetTypeSignature, length) + if err != nil { + return + } + + _, err = w.Write(sig.HashSuffix[:len(sig.HashSuffix)-6]) + if err != nil { + return + } + + unhashedSubpackets := make([]byte, 2+unhashedSubpacketsLen) + unhashedSubpackets[0] = byte(unhashedSubpacketsLen >> 8) + unhashedSubpackets[1] = byte(unhashedSubpacketsLen) + serialiseSubpackets(unhashedSubpackets[2:], sig.outSubpackets, false) + + _, err = w.Write(unhashedSubpackets) + if err != nil { + return + } + _, err = w.Write(sig.HashTag[:]) + if err != nil { + return + } + return writeMPI(w, 8*uint16(len(sig.Signature)), sig.Signature) +} + +// outputSubpacket represents a subpacket to be marshaled. +type outputSubpacket struct { + hashed bool // true if this subpacket is in the hashed area. + subpacketType signatureSubpacketType + contents []byte +} + +func (sig *Signature) buildSubpackets() (subpackets []outputSubpacket) { + creationTime := make([]byte, 4) + creationTime[0] = byte(sig.CreationTime >> 24) + creationTime[1] = byte(sig.CreationTime >> 16) + creationTime[2] = byte(sig.CreationTime >> 8) + creationTime[3] = byte(sig.CreationTime) + subpackets = append(subpackets, outputSubpacket{true, creationTimeSubpacket, creationTime}) + + if sig.IssuerKeyId != nil { + keyId := make([]byte, 8) + binary.BigEndian.PutUint64(keyId, *sig.IssuerKeyId) + subpackets = append(subpackets, outputSubpacket{true, issuerSubpacket, keyId}) + } + + return +} diff --git a/libgo/go/crypto/openpgp/packet/signature_test.go b/libgo/go/crypto/openpgp/packet/signature_test.go new file mode 100644 index 00000000000..1305548b2ae --- /dev/null +++ b/libgo/go/crypto/openpgp/packet/signature_test.go @@ -0,0 +1,28 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "bytes" + "crypto" + "encoding/hex" + "testing" +) + +func TestSignatureRead(t *testing.T) { + signatureData, _ := hex.DecodeString(signatureDataHex) + buf := bytes.NewBuffer(signatureData) + packet, err := Read(buf) + if err != nil { + t.Error(err) + return + } + sig, ok := packet.(*Signature) + if !ok || sig.SigType != SigTypeBinary || sig.PubKeyAlgo != PubKeyAlgoRSA || sig.Hash != crypto.SHA1 { + t.Errorf("failed to parse, got: %#v", packet) + } +} + +const signatureDataHex = "89011c04000102000605024cb45112000a0910ab105c91af38fb158f8d07ff5596ea368c5efe015bed6e78348c0f033c931d5f2ce5db54ce7f2a7e4b4ad64db758d65a7a71773edeab7ba2a9e0908e6a94a1175edd86c1d843279f045b021a6971a72702fcbd650efc393c5474d5b59a15f96d2eaad4c4c426797e0dcca2803ef41c6ff234d403eec38f31d610c344c06f2401c262f0993b2e66cad8a81ebc4322c723e0d4ba09fe917e8777658307ad8329adacba821420741009dfe87f007759f0982275d028a392c6ed983a0d846f890b36148c7358bdb8a516007fac760261ecd06076813831a36d0459075d1befa245ae7f7fb103d92ca759e9498fe60ef8078a39a3beda510deea251ea9f0a7f0df6ef42060f20780360686f3e400e" diff --git a/libgo/go/crypto/openpgp/packet/symmetric_key_encrypted.go b/libgo/go/crypto/openpgp/packet/symmetric_key_encrypted.go new file mode 100644 index 00000000000..d9010f88a3d --- /dev/null +++ b/libgo/go/crypto/openpgp/packet/symmetric_key_encrypted.go @@ -0,0 +1,102 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "crypto/cipher" + "crypto/openpgp/error" + "crypto/openpgp/s2k" + "io" + "os" + "strconv" +) + +// This is the largest session key that we'll support. Since no 512-bit cipher +// has even been seriously used, this is comfortably large. +const maxSessionKeySizeInBytes = 64 + +// SymmetricKeyEncrypted represents a passphrase protected session key. See RFC +// 4880, section 5.3. +type SymmetricKeyEncrypted struct { + CipherFunc CipherFunction + Encrypted bool + Key []byte // Empty unless Encrypted is false. + s2k func(out, in []byte) + encryptedKey []byte +} + +func (ske *SymmetricKeyEncrypted) parse(r io.Reader) (err os.Error) { + // RFC 4880, section 5.3. + var buf [2]byte + _, err = readFull(r, buf[:]) + if err != nil { + return + } + if buf[0] != 4 { + return error.UnsupportedError("SymmetricKeyEncrypted version") + } + ske.CipherFunc = CipherFunction(buf[1]) + + if ske.CipherFunc.keySize() == 0 { + return error.UnsupportedError("unknown cipher: " + strconv.Itoa(int(buf[1]))) + } + + ske.s2k, err = s2k.Parse(r) + if err != nil { + return + } + + encryptedKey := make([]byte, maxSessionKeySizeInBytes) + // The session key may follow. We just have to try and read to find + // out. If it exists then we limit it to maxSessionKeySizeInBytes. + n, err := readFull(r, encryptedKey) + if err != nil && err != io.ErrUnexpectedEOF { + return + } + err = nil + if n != 0 { + if n == maxSessionKeySizeInBytes { + return error.UnsupportedError("oversized encrypted session key") + } + ske.encryptedKey = encryptedKey[:n] + } + + ske.Encrypted = true + + return +} + +// Decrypt attempts to decrypt an encrypted session key. If it returns nil, +// ske.Key will contain the session key. +func (ske *SymmetricKeyEncrypted) Decrypt(passphrase []byte) os.Error { + if !ske.Encrypted { + return nil + } + + key := make([]byte, ske.CipherFunc.keySize()) + ske.s2k(key, passphrase) + + if len(ske.encryptedKey) == 0 { + ske.Key = key + } else { + // the IV is all zeros + iv := make([]byte, ske.CipherFunc.blockSize()) + c := cipher.NewCFBDecrypter(ske.CipherFunc.new(key), iv) + c.XORKeyStream(ske.encryptedKey, ske.encryptedKey) + ske.CipherFunc = CipherFunction(ske.encryptedKey[0]) + if ske.CipherFunc.blockSize() == 0 { + return error.UnsupportedError("unknown cipher: " + strconv.Itoa(int(ske.CipherFunc))) + } + ske.CipherFunc = CipherFunction(ske.encryptedKey[0]) + ske.Key = ske.encryptedKey[1:] + if len(ske.Key)%ske.CipherFunc.blockSize() != 0 { + ske.Key = nil + return error.StructuralError("length of decrypted key not a multiple of block size") + } + } + + ske.Encrypted = false + return nil +} diff --git a/libgo/go/crypto/openpgp/packet/symmetric_key_encrypted_test.go b/libgo/go/crypto/openpgp/packet/symmetric_key_encrypted_test.go new file mode 100644 index 00000000000..717c8ffa6d6 --- /dev/null +++ b/libgo/go/crypto/openpgp/packet/symmetric_key_encrypted_test.go @@ -0,0 +1,62 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "bytes" + "encoding/hex" + "io/ioutil" + "os" + "testing" +) + +func TestSymmetricKeyEncrypted(t *testing.T) { + buf := readerFromHex(symmetricallyEncryptedHex) + packet, err := Read(buf) + if err != nil { + t.Errorf("failed to read SymmetricKeyEncrypted: %s", err) + return + } + ske, ok := packet.(*SymmetricKeyEncrypted) + if !ok { + t.Error("didn't find SymmetricKeyEncrypted packet") + return + } + err = ske.Decrypt([]byte("password")) + if err != nil { + t.Error(err) + return + } + + packet, err = Read(buf) + if err != nil { + t.Errorf("failed to read SymmetricallyEncrypted: %s", err) + return + } + se, ok := packet.(*SymmetricallyEncrypted) + if !ok { + t.Error("didn't find SymmetricallyEncrypted packet") + return + } + r, err := se.Decrypt(ske.CipherFunc, ske.Key) + if err != nil { + t.Error(err) + return + } + + contents, err := ioutil.ReadAll(r) + if err != nil && err != os.EOF { + t.Error(err) + return + } + + expectedContents, _ := hex.DecodeString(symmetricallyEncryptedContentsHex) + if !bytes.Equal(expectedContents, contents) { + t.Errorf("bad contents got:%x want:%x", contents, expectedContents) + } +} + +const symmetricallyEncryptedHex = "8c0d04030302371a0b38d884f02060c91cf97c9973b8e58e028e9501708ccfe618fb92afef7fa2d80ddadd93cf" +const symmetricallyEncryptedContentsHex = "cb1062004d14c4df636f6e74656e74732e0a" diff --git a/libgo/go/crypto/openpgp/packet/symmetrically_encrypted.go b/libgo/go/crypto/openpgp/packet/symmetrically_encrypted.go new file mode 100644 index 00000000000..fc19ffe809a --- /dev/null +++ b/libgo/go/crypto/openpgp/packet/symmetrically_encrypted.go @@ -0,0 +1,206 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "crypto/cipher" + "crypto/openpgp/error" + "crypto/sha1" + "crypto/subtle" + "hash" + "io" + "os" + "strconv" +) + +// SymmetricallyEncrypted represents a symmetrically encrypted byte string. The +// encrypted contents will consist of more OpenPGP packets. See RFC 4880, +// sections 5.7 and 5.13. +type SymmetricallyEncrypted struct { + MDC bool // true iff this is a type 18 packet and thus has an embedded MAC. + contents io.Reader + prefix []byte +} + +func (se *SymmetricallyEncrypted) parse(r io.Reader) os.Error { + if se.MDC { + // See RFC 4880, section 5.13. + var buf [1]byte + _, err := readFull(r, buf[:]) + if err != nil { + return err + } + if buf[0] != 1 { + return error.UnsupportedError("unknown SymmetricallyEncrypted version") + } + } + se.contents = r + return nil +} + +// Decrypt returns a ReadCloser, from which the decrypted contents of the +// packet can be read. An incorrect key can, with high probability, be detected +// immediately and this will result in a KeyIncorrect error being returned. +func (se *SymmetricallyEncrypted) Decrypt(c CipherFunction, key []byte) (io.ReadCloser, os.Error) { + keySize := c.keySize() + if keySize == 0 { + return nil, error.UnsupportedError("unknown cipher: " + strconv.Itoa(int(c))) + } + if len(key) != keySize { + return nil, error.InvalidArgumentError("SymmetricallyEncrypted: incorrect key length") + } + + if se.prefix == nil { + se.prefix = make([]byte, c.blockSize()+2) + _, err := readFull(se.contents, se.prefix) + if err != nil { + return nil, err + } + } else if len(se.prefix) != c.blockSize()+2 { + return nil, error.InvalidArgumentError("can't try ciphers with different block lengths") + } + + ocfbResync := cipher.OCFBResync + if se.MDC { + // MDC packets use a different form of OCFB mode. + ocfbResync = cipher.OCFBNoResync + } + + s := cipher.NewOCFBDecrypter(c.new(key), se.prefix, ocfbResync) + if s == nil { + return nil, error.KeyIncorrectError + } + + plaintext := cipher.StreamReader{S: s, R: se.contents} + + if se.MDC { + // MDC packets have an embedded hash that we need to check. + h := sha1.New() + h.Write(se.prefix) + return &seMDCReader{in: plaintext, h: h}, nil + } + + // Otherwise, we just need to wrap plaintext so that it's a valid ReadCloser. + return seReader{plaintext}, nil +} + +// seReader wraps an io.Reader with a no-op Close method. +type seReader struct { + in io.Reader +} + +func (ser seReader) Read(buf []byte) (int, os.Error) { + return ser.in.Read(buf) +} + +func (ser seReader) Close() os.Error { + return nil +} + +const mdcTrailerSize = 1 /* tag byte */ + 1 /* length byte */ + sha1.Size + +// An seMDCReader wraps an io.Reader, maintains a running hash and keeps hold +// of the most recent 22 bytes (mdcTrailerSize). Upon EOF, those bytes form an +// MDC packet containing a hash of the previous contents which is checked +// against the running hash. See RFC 4880, section 5.13. +type seMDCReader struct { + in io.Reader + h hash.Hash + trailer [mdcTrailerSize]byte + scratch [mdcTrailerSize]byte + trailerUsed int + error bool + eof bool +} + +func (ser *seMDCReader) Read(buf []byte) (n int, err os.Error) { + if ser.error { + err = io.ErrUnexpectedEOF + return + } + if ser.eof { + err = os.EOF + return + } + + // If we haven't yet filled the trailer buffer then we must do that + // first. + for ser.trailerUsed < mdcTrailerSize { + n, err = ser.in.Read(ser.trailer[ser.trailerUsed:]) + ser.trailerUsed += n + if err == os.EOF { + if ser.trailerUsed != mdcTrailerSize { + n = 0 + err = io.ErrUnexpectedEOF + ser.error = true + return + } + ser.eof = true + n = 0 + return + } + + if err != nil { + n = 0 + return + } + } + + // If it's a short read then we read into a temporary buffer and shift + // the data into the caller's buffer. + if len(buf) <= mdcTrailerSize { + n, err = readFull(ser.in, ser.scratch[:len(buf)]) + copy(buf, ser.trailer[:n]) + ser.h.Write(buf[:n]) + copy(ser.trailer[:], ser.trailer[n:]) + copy(ser.trailer[mdcTrailerSize-n:], ser.scratch[:]) + if n < len(buf) { + ser.eof = true + err = os.EOF + } + return + } + + n, err = ser.in.Read(buf[mdcTrailerSize:]) + copy(buf, ser.trailer[:]) + ser.h.Write(buf[:n]) + copy(ser.trailer[:], buf[n:]) + + if err == os.EOF { + ser.eof = true + } + return +} + +func (ser *seMDCReader) Close() os.Error { + if ser.error { + return error.SignatureError("error during reading") + } + + for !ser.eof { + // We haven't seen EOF so we need to read to the end + var buf [1024]byte + _, err := ser.Read(buf[:]) + if err == os.EOF { + break + } + if err != nil { + return error.SignatureError("error during reading") + } + } + + // This is a new-format packet tag byte for a type 19 (MDC) packet. + const mdcPacketTagByte = byte(0x80) | 0x40 | 19 + if ser.trailer[0] != mdcPacketTagByte || ser.trailer[1] != sha1.Size { + return error.SignatureError("MDC packet not found") + } + ser.h.Write(ser.trailer[:2]) + + final := ser.h.Sum() + if subtle.ConstantTimeCompare(final, ser.trailer[2:]) == 1 { + return error.SignatureError("hash mismatch") + } + return nil +} diff --git a/libgo/go/crypto/openpgp/packet/symmetrically_encrypted_test.go b/libgo/go/crypto/openpgp/packet/symmetrically_encrypted_test.go new file mode 100644 index 00000000000..5543b20297a --- /dev/null +++ b/libgo/go/crypto/openpgp/packet/symmetrically_encrypted_test.go @@ -0,0 +1,78 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "bytes" + "crypto/openpgp/error" + "crypto/sha1" + "encoding/hex" + "io/ioutil" + "os" + "testing" +) + +// TestReader wraps a []byte and returns reads of a specific length. +type testReader struct { + data []byte + stride int +} + +func (t *testReader) Read(buf []byte) (n int, err os.Error) { + n = t.stride + if n > len(t.data) { + n = len(t.data) + } + if n > len(buf) { + n = len(buf) + } + copy(buf, t.data) + t.data = t.data[n:] + if len(t.data) == 0 { + err = os.EOF + } + return +} + +func testMDCReader(t *testing.T) { + mdcPlaintext, _ := hex.DecodeString(mdcPlaintextHex) + + for stride := 1; stride < len(mdcPlaintext)/2; stride++ { + r := &testReader{data: mdcPlaintext, stride: stride} + mdcReader := &seMDCReader{in: r, h: sha1.New()} + body, err := ioutil.ReadAll(mdcReader) + if err != nil { + t.Errorf("stride: %d, error: %s", stride, err) + continue + } + if !bytes.Equal(body, mdcPlaintext[:len(mdcPlaintext)-22]) { + t.Errorf("stride: %d: bad contents %x", stride, body) + continue + } + + err = mdcReader.Close() + if err != nil { + t.Errorf("stride: %d, error on Close: %s", stride, err) + } + } + + mdcPlaintext[15] ^= 80 + + r := &testReader{data: mdcPlaintext, stride: 2} + mdcReader := &seMDCReader{in: r, h: sha1.New()} + _, err := ioutil.ReadAll(mdcReader) + if err != nil { + t.Errorf("corruption test, error: %s", err) + return + } + err = mdcReader.Close() + if err == nil { + t.Error("corruption: no error") + } else if _, ok := err.(*error.SignatureError); !ok { + t.Errorf("corruption: expected SignatureError, got: %s", err) + } +} + +const mdcPlaintextHex = "a302789c3b2d93c4e0eb9aba22283539b3203335af44a134afb800c849cb4c4de10200aff40b45d31432c80cb384299a0655966d6939dfdeed1dddf980" diff --git a/libgo/go/crypto/openpgp/packet/userid.go b/libgo/go/crypto/openpgp/packet/userid.go new file mode 100644 index 00000000000..ed2ad777486 --- /dev/null +++ b/libgo/go/crypto/openpgp/packet/userid.go @@ -0,0 +1,105 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "io" + "io/ioutil" + "os" + "strings" +) + +// UserId contains text that is intended to represent the name and email +// address of the key holder. See RFC 4880, section 5.11. By convention, this +// takes the form "Full Name (Comment) <email@example.com>" +type UserId struct { + Id string // By convention, this takes the form "Full Name (Comment) <email@example.com>" which is split out in the fields below. + + Name, Comment, Email string +} + +func (uid *UserId) parse(r io.Reader) (err os.Error) { + // RFC 4880, section 5.11 + b, err := ioutil.ReadAll(r) + if err != nil { + return + } + uid.Id = string(b) + uid.Name, uid.Comment, uid.Email = parseUserId(uid.Id) + return +} + +// parseUserId extracts the name, comment and email from a user id string that +// is formatted as "Full Name (Comment) <email@example.com>". +func parseUserId(id string) (name, comment, email string) { + var n, c, e struct { + start, end int + } + var state int + + for offset, rune := range id { + switch state { + case 0: + // Entering name + n.start = offset + state = 1 + fallthrough + case 1: + // In name + if rune == '(' { + state = 2 + n.end = offset + } else if rune == '<' { + state = 5 + n.end = offset + } + case 2: + // Entering comment + c.start = offset + state = 3 + fallthrough + case 3: + // In comment + if rune == ')' { + state = 4 + c.end = offset + } + case 4: + // Between comment and email + if rune == '<' { + state = 5 + } + case 5: + // Entering email + e.start = offset + state = 6 + fallthrough + case 6: + // In email + if rune == '>' { + state = 7 + e.end = offset + } + default: + // After email + } + } + switch state { + case 1: + // ended in the name + n.end = len(id) + case 3: + // ended in comment + c.end = len(id) + case 6: + // ended in email + e.end = len(id) + } + + name = strings.TrimSpace(id[n.start:n.end]) + comment = strings.TrimSpace(id[c.start:c.end]) + email = strings.TrimSpace(id[e.start:e.end]) + return +} diff --git a/libgo/go/crypto/openpgp/packet/userid_test.go b/libgo/go/crypto/openpgp/packet/userid_test.go new file mode 100644 index 00000000000..394873dc38c --- /dev/null +++ b/libgo/go/crypto/openpgp/packet/userid_test.go @@ -0,0 +1,42 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "testing" +) + +var userIdTests = []struct { + id string + name, comment, email string +}{ + {"", "", "", ""}, + {"John Smith", "John Smith", "", ""}, + {"John Smith ()", "John Smith", "", ""}, + {"John Smith () <>", "John Smith", "", ""}, + {"(comment", "", "comment", ""}, + {"(comment)", "", "comment", ""}, + {"<email", "", "", "email"}, + {"<email> sdfk", "", "", "email"}, + {" John Smith ( Comment ) asdkflj < email > lksdfj", "John Smith", "Comment", "email"}, + {" John Smith < email > lksdfj", "John Smith", "", "email"}, + {"(<foo", "", "<foo", ""}, + {"René Descartes (العربي)", "René Descartes", "العربي", ""}, +} + +func TestParseUserId(t *testing.T) { + for i, test := range userIdTests { + name, comment, email := parseUserId(test.id) + if name != test.name { + t.Errorf("%d: name mismatch got:%s want:%s", i, name, test.name) + } + if comment != test.comment { + t.Errorf("%d: comment mismatch got:%s want:%s", i, comment, test.comment) + } + if email != test.email { + t.Errorf("%d: email mismatch got:%s want:%s", i, email, test.email) + } + } +} diff --git a/libgo/go/crypto/openpgp/read.go b/libgo/go/crypto/openpgp/read.go new file mode 100644 index 00000000000..ac6998f0d24 --- /dev/null +++ b/libgo/go/crypto/openpgp/read.go @@ -0,0 +1,413 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This openpgp package implements high level operations on OpenPGP messages. +package openpgp + +import ( + "crypto" + "crypto/openpgp/armor" + "crypto/openpgp/error" + "crypto/openpgp/packet" + "crypto/rsa" + _ "crypto/sha256" + "hash" + "io" + "os" + "strconv" +) + +// SignatureType is the armor type for a PGP signature. +var SignatureType = "PGP SIGNATURE" + +// readArmored reads an armored block with the given type. +func readArmored(r io.Reader, expectedType string) (body io.Reader, err os.Error) { + block, err := armor.Decode(r) + if err != nil { + return + } + + if block.Type != expectedType { + return nil, error.InvalidArgumentError("expected '" + expectedType + "', got: " + block.Type) + } + + return block.Body, nil +} + +// MessageDetails contains the result of parsing an OpenPGP encrypted and/or +// signed message. +type MessageDetails struct { + IsEncrypted bool // true if the message was encrypted. + EncryptedToKeyIds []uint64 // the list of recipient key ids. + IsSymmetricallyEncrypted bool // true if a passphrase could have decrypted the message. + DecryptedWith Key // the private key used to decrypt the message, if any. + IsSigned bool // true if the message is signed. + SignedByKeyId uint64 // the key id of the signer, if any. + SignedBy *Key // the key of the signer, if availible. + LiteralData *packet.LiteralData // the metadata of the contents + UnverifiedBody io.Reader // the contents of the message. + + // If IsSigned is true and SignedBy is non-zero then the signature will + // be verified as UnverifiedBody is read. The signature cannot be + // checked until the whole of UnverifiedBody is read so UnverifiedBody + // must be consumed until EOF before the data can trusted. Even if a + // message isn't signed (or the signer is unknown) the data may contain + // an authentication code that is only checked once UnverifiedBody has + // been consumed. Once EOF has been seen, the following fields are + // valid. (An authentication code failure is reported as a + // SignatureError error when reading from UnverifiedBody.) + + SignatureError os.Error // nil if the signature is good. + Signature *packet.Signature // the signature packet itself. + + decrypted io.ReadCloser +} + +// A PromptFunction is used as a callback by functions that may need to decrypt +// a private key, or prompt for a passphrase. It is called with a list of +// acceptable, encrypted private keys and a boolean that indicates whether a +// passphrase is usable. It should either decrypt a private key or return a +// passphrase to try. If the decrypted private key or given passphrase isn't +// correct, the function will be called again, forever. Any error returned will +// be passed up. +type PromptFunction func(keys []Key, symmetric bool) ([]byte, os.Error) + +// A keyEnvelopePair is used to store a private key with the envelope that +// contains a symmetric key, encrypted with that key. +type keyEnvelopePair struct { + key Key + encryptedKey *packet.EncryptedKey +} + +// ReadMessage parses an OpenPGP message that may be signed and/or encrypted. +// The given KeyRing should contain both public keys (for signature +// verification) and, possibly encrypted, private keys for decrypting. +func ReadMessage(r io.Reader, keyring KeyRing, prompt PromptFunction) (md *MessageDetails, err os.Error) { + var p packet.Packet + + var symKeys []*packet.SymmetricKeyEncrypted + var pubKeys []keyEnvelopePair + var se *packet.SymmetricallyEncrypted + + packets := packet.NewReader(r) + md = new(MessageDetails) + md.IsEncrypted = true + + // The message, if encrypted, starts with a number of packets + // containing an encrypted decryption key. The decryption key is either + // encrypted to a public key, or with a passphrase. This loop + // collects these packets. +ParsePackets: + for { + p, err = packets.Next() + if err != nil { + return nil, err + } + switch p := p.(type) { + case *packet.SymmetricKeyEncrypted: + // This packet contains the decryption key encrypted with a passphrase. + md.IsSymmetricallyEncrypted = true + symKeys = append(symKeys, p) + case *packet.EncryptedKey: + // This packet contains the decryption key encrypted to a public key. + md.EncryptedToKeyIds = append(md.EncryptedToKeyIds, p.KeyId) + if p.Algo != packet.PubKeyAlgoRSA && p.Algo != packet.PubKeyAlgoRSAEncryptOnly { + continue + } + var keys []Key + if p.KeyId == 0 { + keys = keyring.DecryptionKeys() + } else { + keys = keyring.KeysById(p.KeyId) + } + for _, k := range keys { + pubKeys = append(pubKeys, keyEnvelopePair{k, p}) + } + case *packet.SymmetricallyEncrypted: + se = p + break ParsePackets + case *packet.Compressed, *packet.LiteralData, *packet.OnePassSignature: + // This message isn't encrypted. + if len(symKeys) != 0 || len(pubKeys) != 0 { + return nil, error.StructuralError("key material not followed by encrypted message") + } + packets.Unread(p) + return readSignedMessage(packets, nil, keyring) + } + } + + var candidates []Key + var decrypted io.ReadCloser + + // Now that we have the list of encrypted keys we need to decrypt at + // least one of them or, if we cannot, we need to call the prompt + // function so that it can decrypt a key or give us a passphrase. +FindKey: + for { + // See if any of the keys already have a private key availible + candidates = candidates[:0] + candidateFingerprints := make(map[string]bool) + + for _, pk := range pubKeys { + if pk.key.PrivateKey == nil { + continue + } + if !pk.key.PrivateKey.Encrypted { + if len(pk.encryptedKey.Key) == 0 { + pk.encryptedKey.DecryptRSA(pk.key.PrivateKey.PrivateKey.(*rsa.PrivateKey)) + } + if len(pk.encryptedKey.Key) == 0 { + continue + } + decrypted, err = se.Decrypt(pk.encryptedKey.CipherFunc, pk.encryptedKey.Key) + if err != nil && err != error.KeyIncorrectError { + return nil, err + } + if decrypted != nil { + md.DecryptedWith = pk.key + break FindKey + } + } else { + fpr := string(pk.key.PublicKey.Fingerprint[:]) + if v := candidateFingerprints[fpr]; v { + continue + } + candidates = append(candidates, pk.key) + candidateFingerprints[fpr] = true + } + } + + if len(candidates) == 0 && len(symKeys) == 0 { + return nil, error.KeyIncorrectError + } + + if prompt == nil { + return nil, error.KeyIncorrectError + } + + passphrase, err := prompt(candidates, len(symKeys) != 0) + if err != nil { + return nil, err + } + + // Try the symmetric passphrase first + if len(symKeys) != 0 && passphrase != nil { + for _, s := range symKeys { + err = s.Decrypt(passphrase) + if err == nil && !s.Encrypted { + decrypted, err = se.Decrypt(s.CipherFunc, s.Key) + if err != nil && err != error.KeyIncorrectError { + return nil, err + } + if decrypted != nil { + break FindKey + } + } + + } + } + } + + md.decrypted = decrypted + packets.Push(decrypted) + return readSignedMessage(packets, md, keyring) +} + +// readSignedMessage reads a possibily signed message if mdin is non-zero then +// that structure is updated and returned. Otherwise a fresh MessageDetails is +// used. +func readSignedMessage(packets *packet.Reader, mdin *MessageDetails, keyring KeyRing) (md *MessageDetails, err os.Error) { + if mdin == nil { + mdin = new(MessageDetails) + } + md = mdin + + var p packet.Packet + var h hash.Hash + var wrappedHash hash.Hash +FindLiteralData: + for { + p, err = packets.Next() + if err != nil { + return nil, err + } + switch p := p.(type) { + case *packet.Compressed: + packets.Push(p.Body) + case *packet.OnePassSignature: + if !p.IsLast { + return nil, error.UnsupportedError("nested signatures") + } + + h, wrappedHash, err = hashForSignature(p.Hash, p.SigType) + if err != nil { + md = nil + return + } + + md.IsSigned = true + md.SignedByKeyId = p.KeyId + keys := keyring.KeysById(p.KeyId) + for _, key := range keys { + if key.SelfSignature.FlagsValid && !key.SelfSignature.FlagSign { + continue + } + md.SignedBy = &key + } + case *packet.LiteralData: + md.LiteralData = p + break FindLiteralData + } + } + + if md.SignedBy != nil { + md.UnverifiedBody = &signatureCheckReader{packets, h, wrappedHash, md} + } else if md.decrypted != nil { + md.UnverifiedBody = checkReader{md} + } else { + md.UnverifiedBody = md.LiteralData.Body + } + + return md, nil +} + +// hashForSignature returns a pair of hashes that can be used to verify a +// signature. The signature may specify that the contents of the signed message +// should be preprocessed (i.e. to normalise line endings). Thus this function +// returns two hashes. The second should be used to hash the message itself and +// performs any needed preprocessing. +func hashForSignature(hashId crypto.Hash, sigType packet.SignatureType) (hash.Hash, hash.Hash, os.Error) { + h := hashId.New() + if h == nil { + return nil, nil, error.UnsupportedError("hash not availible: " + strconv.Itoa(int(hashId))) + } + + switch sigType { + case packet.SigTypeBinary: + return h, h, nil + case packet.SigTypeText: + return h, NewCanonicalTextHash(h), nil + } + + return nil, nil, error.UnsupportedError("unsupported signature type: " + strconv.Itoa(int(sigType))) +} + +// checkReader wraps an io.Reader from a LiteralData packet. When it sees EOF +// it closes the ReadCloser from any SymmetricallyEncrypted packet to trigger +// MDC checks. +type checkReader struct { + md *MessageDetails +} + +func (cr checkReader) Read(buf []byte) (n int, err os.Error) { + n, err = cr.md.LiteralData.Body.Read(buf) + if err == os.EOF { + mdcErr := cr.md.decrypted.Close() + if mdcErr != nil { + err = mdcErr + } + } + return +} + +// signatureCheckReader wraps an io.Reader from a LiteralData packet and hashes +// the data as it is read. When it sees an EOF from the underlying io.Reader +// it parses and checks a trailing Signature packet and triggers any MDC checks. +type signatureCheckReader struct { + packets *packet.Reader + h, wrappedHash hash.Hash + md *MessageDetails +} + +func (scr *signatureCheckReader) Read(buf []byte) (n int, err os.Error) { + n, err = scr.md.LiteralData.Body.Read(buf) + scr.wrappedHash.Write(buf[:n]) + if err == os.EOF { + var p packet.Packet + p, scr.md.SignatureError = scr.packets.Next() + if scr.md.SignatureError != nil { + return + } + + var ok bool + if scr.md.Signature, ok = p.(*packet.Signature); !ok { + scr.md.SignatureError = error.StructuralError("LiteralData not followed by Signature") + return + } + + scr.md.SignatureError = scr.md.SignedBy.PublicKey.VerifySignature(scr.h, scr.md.Signature) + + // The SymmetricallyEncrypted packet, if any, might have an + // unsigned hash of its own. In order to check this we need to + // close that Reader. + if scr.md.decrypted != nil { + mdcErr := scr.md.decrypted.Close() + if mdcErr != nil { + err = mdcErr + } + } + } + return +} + +// CheckDetachedSignature takes a signed file and a detached signature and +// returns the signer if the signature is valid. If the signer isn't know, +// UnknownIssuerError is returned. +func CheckDetachedSignature(keyring KeyRing, signed, signature io.Reader) (signer *Entity, err os.Error) { + p, err := packet.Read(signature) + if err != nil { + return + } + + sig, ok := p.(*packet.Signature) + if !ok { + return nil, error.StructuralError("non signature packet found") + } + + if sig.IssuerKeyId == nil { + return nil, error.StructuralError("signature doesn't have an issuer") + } + + keys := keyring.KeysById(*sig.IssuerKeyId) + if len(keys) == 0 { + return nil, error.UnknownIssuerError + } + + h, wrappedHash, err := hashForSignature(sig.Hash, sig.SigType) + if err != nil { + return + } + + _, err = io.Copy(wrappedHash, signed) + if err != nil && err != os.EOF { + return + } + + for _, key := range keys { + if key.SelfSignature.FlagsValid && !key.SelfSignature.FlagSign { + continue + } + err = key.PublicKey.VerifySignature(h, sig) + if err == nil { + return key.Entity, nil + } + } + + if err != nil { + return + } + + return nil, error.UnknownIssuerError +} + +// CheckArmoredDetachedSignature performs the same actions as +// CheckDetachedSignature but expects the signature to be armored. +func CheckArmoredDetachedSignature(keyring KeyRing, signed, signature io.Reader) (signer *Entity, err os.Error) { + body, err := readArmored(signature, SignatureType) + if err != nil { + return + } + + return CheckDetachedSignature(keyring, signed, body) +} diff --git a/libgo/go/crypto/openpgp/read_test.go b/libgo/go/crypto/openpgp/read_test.go new file mode 100644 index 00000000000..58199e13297 --- /dev/null +++ b/libgo/go/crypto/openpgp/read_test.go @@ -0,0 +1,237 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package openpgp + +import ( + "bytes" + "crypto/openpgp/error" + "encoding/hex" + "io" + "io/ioutil" + "os" + "testing" +) + +func readerFromHex(s string) io.Reader { + data, err := hex.DecodeString(s) + if err != nil { + panic("readerFromHex: bad input") + } + return bytes.NewBuffer(data) +} + +func TestReadKeyRing(t *testing.T) { + kring, err := ReadKeyRing(readerFromHex(testKeys1And2Hex)) + if err != nil { + t.Error(err) + return + } + if len(kring) != 2 || uint32(kring[0].PrimaryKey.KeyId) != 0xC20C31BB || uint32(kring[1].PrimaryKey.KeyId) != 0x1E35246B { + t.Errorf("bad keyring: %#v", kring) + } +} + +func TestReadPrivateKeyRing(t *testing.T) { + kring, err := ReadKeyRing(readerFromHex(testKeys1And2PrivateHex)) + if err != nil { + t.Error(err) + return + } + if len(kring) != 2 || uint32(kring[0].PrimaryKey.KeyId) != 0xC20C31BB || uint32(kring[1].PrimaryKey.KeyId) != 0x1E35246B || kring[0].PrimaryKey == nil { + t.Errorf("bad keyring: %#v", kring) + } +} + +func TestGetKeyById(t *testing.T) { + kring, _ := ReadKeyRing(readerFromHex(testKeys1And2Hex)) + + keys := kring.KeysById(0xa34d7e18c20c31bb) + if len(keys) != 1 || keys[0].Entity != kring[0] { + t.Errorf("bad result for 0xa34d7e18c20c31bb: %#v", keys) + } + + keys = kring.KeysById(0xfd94408d4543314f) + if len(keys) != 1 || keys[0].Entity != kring[0] { + t.Errorf("bad result for 0xa34d7e18c20c31bb: %#v", keys) + } +} + +func checkSignedMessage(t *testing.T, signedHex, expected string) { + kring, _ := ReadKeyRing(readerFromHex(testKeys1And2Hex)) + + md, err := ReadMessage(readerFromHex(signedHex), kring, nil) + if err != nil { + t.Error(err) + return + } + + if !md.IsSigned || md.SignedByKeyId != 0xa34d7e18c20c31bb || md.SignedBy == nil || md.IsEncrypted || md.IsSymmetricallyEncrypted || len(md.EncryptedToKeyIds) != 0 || md.IsSymmetricallyEncrypted { + t.Errorf("bad MessageDetails: %#v", md) + } + + contents, err := ioutil.ReadAll(md.UnverifiedBody) + if err != nil { + t.Errorf("error reading UnverifiedBody: %s", err) + } + if string(contents) != expected { + t.Errorf("bad UnverifiedBody got:%s want:%s", string(contents), expected) + } + if md.SignatureError != nil || md.Signature == nil { + t.Errorf("failed to validate: %s", md.SignatureError) + } +} + +func TestSignedMessage(t *testing.T) { + checkSignedMessage(t, signedMessageHex, signedInput) +} + +func TestTextSignedMessage(t *testing.T) { + checkSignedMessage(t, signedTextMessageHex, signedTextInput) +} + +func TestSignedEncryptedMessage(t *testing.T) { + expected := "Signed and encrypted message\n" + kring, _ := ReadKeyRing(readerFromHex(testKeys1And2PrivateHex)) + prompt := func(keys []Key, symmetric bool) ([]byte, os.Error) { + if symmetric { + t.Errorf("prompt: message was marked as symmetrically encrypted") + return nil, error.KeyIncorrectError + } + + if len(keys) == 0 { + t.Error("prompt: no keys requested") + return nil, error.KeyIncorrectError + } + + err := keys[0].PrivateKey.Decrypt([]byte("passphrase")) + if err != nil { + t.Errorf("prompt: error decrypting key: %s", err) + return nil, error.KeyIncorrectError + } + + return nil, nil + } + + md, err := ReadMessage(readerFromHex(signedEncryptedMessageHex), kring, prompt) + if err != nil { + t.Errorf("error reading message: %s", err) + return + } + + if !md.IsSigned || md.SignedByKeyId != 0xa34d7e18c20c31bb || md.SignedBy == nil || !md.IsEncrypted || md.IsSymmetricallyEncrypted || len(md.EncryptedToKeyIds) == 0 || md.EncryptedToKeyIds[0] != 0x2a67d68660df41c7 { + t.Errorf("bad MessageDetails: %#v", md) + } + + contents, err := ioutil.ReadAll(md.UnverifiedBody) + if err != nil { + t.Errorf("error reading UnverifiedBody: %s", err) + } + if string(contents) != expected { + t.Errorf("bad UnverifiedBody got:%s want:%s", string(contents), expected) + } + + if md.SignatureError != nil || md.Signature == nil { + t.Errorf("failed to validate: %s", md.SignatureError) + } +} + +func TestUnspecifiedRecipient(t *testing.T) { + expected := "Recipient unspecified\n" + kring, _ := ReadKeyRing(readerFromHex(testKeys1And2PrivateHex)) + + md, err := ReadMessage(readerFromHex(recipientUnspecifiedHex), kring, nil) + if err != nil { + t.Errorf("error reading message: %s", err) + return + } + + contents, err := ioutil.ReadAll(md.UnverifiedBody) + if err != nil { + t.Errorf("error reading UnverifiedBody: %s", err) + } + if string(contents) != expected { + t.Errorf("bad UnverifiedBody got:%s want:%s", string(contents), expected) + } +} + +func TestSymmetricallyEncrypted(t *testing.T) { + expected := "Symmetrically encrypted.\n" + + prompt := func(keys []Key, symmetric bool) ([]byte, os.Error) { + if len(keys) != 0 { + t.Errorf("prompt: len(keys) = %d (want 0)", len(keys)) + } + + if !symmetric { + t.Errorf("symmetric is not set") + } + + return []byte("password"), nil + } + + md, err := ReadMessage(readerFromHex(symmetricallyEncryptedCompressedHex), nil, prompt) + if err != nil { + t.Errorf("ReadMessage: %s", err) + return + } + + contents, err := ioutil.ReadAll(md.UnverifiedBody) + if err != nil { + t.Errorf("ReadAll: %s", err) + } + + expectedCreatationTime := uint32(1295992998) + if md.LiteralData.Time != expectedCreatationTime { + t.Errorf("LiteralData.Time is %d, want %d", md.LiteralData.Time, expectedCreatationTime) + } + + if string(contents) != expected { + t.Errorf("contents got: %s want: %s", string(contents), expected) + } +} + +func testDetachedSignature(t *testing.T, kring KeyRing, signature io.Reader, sigInput, tag string) { + signed := bytes.NewBufferString(sigInput) + signer, err := CheckDetachedSignature(kring, signed, signature) + if err != nil { + t.Errorf("%s: signature error: %s", tag, err) + return + } + if signer == nil { + t.Errorf("%s: signer is nil", tag) + return + } + expectedSignerKeyId := uint64(0xa34d7e18c20c31bb) + if signer.PrimaryKey.KeyId != expectedSignerKeyId { + t.Errorf("%s: wrong signer got:%x want:%x", tag, signer.PrimaryKey.KeyId, expectedSignerKeyId) + } +} + +func TestDetachedSignature(t *testing.T) { + kring, _ := ReadKeyRing(readerFromHex(testKeys1And2Hex)) + testDetachedSignature(t, kring, readerFromHex(detachedSignatureHex), signedInput, "binary") + testDetachedSignature(t, kring, readerFromHex(detachedSignatureTextHex), signedInput, "text") +} + +const signedInput = "Signed message\nline 2\nline 3\n" +const signedTextInput = "Signed message\r\nline 2\r\nline 3\r\n" + +const recipientUnspecifiedHex = "848c0300000000000000000103ff62d4d578d03cf40c3da998dfe216c074fa6ddec5e31c197c9666ba292830d91d18716a80f699f9d897389a90e6d62d0238f5f07a5248073c0f24920e4bc4a30c2d17ee4e0cae7c3d4aaa4e8dced50e3010a80ee692175fa0385f62ecca4b56ee6e9980aa3ec51b61b077096ac9e800edaf161268593eedb6cc7027ff5cb32745d250010d407a6221ae22ef18469b444f2822478c4d190b24d36371a95cb40087cdd42d9399c3d06a53c0673349bfb607927f20d1e122bde1e2bf3aa6cae6edf489629bcaa0689539ae3b718914d88ededc3b" + +const detachedSignatureHex = "889c04000102000605024d449cd1000a0910a34d7e18c20c31bb167603ff57718d09f28a519fdc7b5a68b6a3336da04df85e38c5cd5d5bd2092fa4629848a33d85b1729402a2aab39c3ac19f9d573f773cc62c264dc924c067a79dfd8a863ae06c7c8686120760749f5fd9b1e03a64d20a7df3446ddc8f0aeadeaeba7cbaee5c1e366d65b6a0c6cc749bcb912d2f15013f812795c2e29eb7f7b77f39ce77" + +const detachedSignatureTextHex = "889c04010102000605024d449d21000a0910a34d7e18c20c31bbc8c60400a24fbef7342603a41cb1165767bd18985d015fb72fe05db42db36cfb2f1d455967f1e491194fbf6cf88146222b23bf6ffbd50d17598d976a0417d3192ff9cc0034fd00f287b02e90418bbefe609484b09231e4e7a5f3562e199bf39909ab5276c4d37382fe088f6b5c3426fc1052865da8b3ab158672d58b6264b10823dc4b39" + +const testKeys1And2Hex = "988d044d3c5c10010400b1d13382944bd5aba23a4312968b5095d14f947f600eb478e14a6fcb16b0e0cac764884909c020bc495cfcc39a935387c661507bdb236a0612fb582cac3af9b29cc2c8c70090616c41b662f4da4c1201e195472eb7f4ae1ccbcbf9940fe21d985e379a5563dde5b9a23d35f1cfaa5790da3b79db26f23695107bfaca8e7b5bcd0011010001b41054657374204b6579203120285253412988b804130102002205024d3c5c10021b03060b090807030206150802090a0b0416020301021e01021780000a0910a34d7e18c20c31bbb5b304009cc45fe610b641a2c146331be94dade0a396e73ca725e1b25c21708d9cab46ecca5ccebc23055879df8f99eea39b377962a400f2ebdc36a7c99c333d74aeba346315137c3ff9d0a09b0273299090343048afb8107cf94cbd1400e3026f0ccac7ecebbc4d78588eb3e478fe2754d3ca664bcf3eac96ca4a6b0c8d7df5102f60f6b0020003b88d044d3c5c10010400b201df61d67487301f11879d514f4248ade90c8f68c7af1284c161098de4c28c2850f1ec7b8e30f959793e571542ffc6532189409cb51c3d30dad78c4ad5165eda18b20d9826d8707d0f742e2ab492103a85bbd9ddf4f5720f6de7064feb0d39ee002219765bb07bcfb8b877f47abe270ddeda4f676108cecb6b9bb2ad484a4f0011010001889f04180102000905024d3c5c10021b0c000a0910a34d7e18c20c31bb1a03040085c8d62e16d05dc4e9dad64953c8a2eed8b6c12f92b1575eeaa6dcf7be9473dd5b24b37b6dffbb4e7c99ed1bd3cb11634be19b3e6e207bed7505c7ca111ccf47cb323bf1f8851eb6360e8034cbff8dd149993c959de89f8f77f38e7e98b8e3076323aa719328e2b408db5ec0d03936efd57422ba04f925cdc7b4c1af7590e40ab0020003988d044d3c5c33010400b488c3e5f83f4d561f317817538d9d0397981e9aef1321ca68ebfae1cf8b7d388e19f4b5a24a82e2fbbf1c6c26557a6c5845307a03d815756f564ac7325b02bc83e87d5480a8fae848f07cb891f2d51ce7df83dcafdc12324517c86d472cc0ee10d47a68fd1d9ae49a6c19bbd36d82af597a0d88cc9c49de9df4e696fc1f0b5d0011010001b42754657374204b6579203220285253412c20656e637279707465642070726976617465206b65792988b804130102002205024d3c5c33021b03060b090807030206150802090a0b0416020301021e01021780000a0910d4984f961e35246b98940400908a73b6a6169f700434f076c6c79015a49bee37130eaf23aaa3cfa9ce60bfe4acaa7bc95f1146ada5867e0079babb38804891f4f0b8ebca57a86b249dee786161a755b7a342e68ccf3f78ed6440a93a6626beb9a37aa66afcd4f888790cb4bb46d94a4ae3eb3d7d3e6b00f6bfec940303e89ec5b32a1eaaacce66497d539328b0020003b88d044d3c5c33010400a4e913f9442abcc7f1804ccab27d2f787ffa592077ca935a8bb23165bd8d57576acac647cc596b2c3f814518cc8c82953c7a4478f32e0cf645630a5ba38d9618ef2bc3add69d459ae3dece5cab778938d988239f8c5ae437807075e06c828019959c644ff05ef6a5a1dab72227c98e3a040b0cf219026640698d7a13d8538a570011010001889f04180102000905024d3c5c33021b0c000a0910d4984f961e35246b26c703ff7ee29ef53bc1ae1ead533c408fa136db508434e233d6e62be621e031e5940bbd4c08142aed0f82217e7c3e1ec8de574bc06ccf3c36633be41ad78a9eacd209f861cae7b064100758545cc9dd83db71806dc1cfd5fb9ae5c7474bba0c19c44034ae61bae5eca379383339dece94ff56ff7aa44a582f3e5c38f45763af577c0934b0020003" + +const testKeys1And2PrivateHex = "9501d8044d3c5c10010400b1d13382944bd5aba23a4312968b5095d14f947f600eb478e14a6fcb16b0e0cac764884909c020bc495cfcc39a935387c661507bdb236a0612fb582cac3af9b29cc2c8c70090616c41b662f4da4c1201e195472eb7f4ae1ccbcbf9940fe21d985e379a5563dde5b9a23d35f1cfaa5790da3b79db26f23695107bfaca8e7b5bcd00110100010003ff4d91393b9a8e3430b14d6209df42f98dc927425b881f1209f319220841273a802a97c7bdb8b3a7740b3ab5866c4d1d308ad0d3a79bd1e883aacf1ac92dfe720285d10d08752a7efe3c609b1d00f17f2805b217be53999a7da7e493bfc3e9618fd17018991b8128aea70a05dbce30e4fbe626aa45775fa255dd9177aabf4df7cf0200c1ded12566e4bc2bb590455e5becfb2e2c9796482270a943343a7835de41080582c2be3caf5981aa838140e97afa40ad652a0b544f83eb1833b0957dce26e47b0200eacd6046741e9ce2ec5beb6fb5e6335457844fb09477f83b050a96be7da043e17f3a9523567ed40e7a521f818813a8b8a72209f1442844843ccc7eb9805442570200bdafe0438d97ac36e773c7162028d65844c4d463e2420aa2228c6e50dc2743c3d6c72d0d782a5173fe7be2169c8a9f4ef8a7cf3e37165e8c61b89c346cdc6c1799d2b41054657374204b6579203120285253412988b804130102002205024d3c5c10021b03060b090807030206150802090a0b0416020301021e01021780000a0910a34d7e18c20c31bbb5b304009cc45fe610b641a2c146331be94dade0a396e73ca725e1b25c21708d9cab46ecca5ccebc23055879df8f99eea39b377962a400f2ebdc36a7c99c333d74aeba346315137c3ff9d0a09b0273299090343048afb8107cf94cbd1400e3026f0ccac7ecebbc4d78588eb3e478fe2754d3ca664bcf3eac96ca4a6b0c8d7df5102f60f6b00200009d01d8044d3c5c10010400b201df61d67487301f11879d514f4248ade90c8f68c7af1284c161098de4c28c2850f1ec7b8e30f959793e571542ffc6532189409cb51c3d30dad78c4ad5165eda18b20d9826d8707d0f742e2ab492103a85bbd9ddf4f5720f6de7064feb0d39ee002219765bb07bcfb8b877f47abe270ddeda4f676108cecb6b9bb2ad484a4f00110100010003fd17a7490c22a79c59281fb7b20f5e6553ec0c1637ae382e8adaea295f50241037f8997cf42c1ce26417e015091451b15424b2c59eb8d4161b0975630408e394d3b00f88d4b4e18e2cc85e8251d4753a27c639c83f5ad4a571c4f19d7cd460b9b73c25ade730c99df09637bd173d8e3e981ac64432078263bb6dc30d3e974150dd0200d0ee05be3d4604d2146fb0457f31ba17c057560785aa804e8ca5530a7cd81d3440d0f4ba6851efcfd3954b7e68908fc0ba47f7ac37bf559c6c168b70d3a7c8cd0200da1c677c4bce06a068070f2b3733b0a714e88d62aa3f9a26c6f5216d48d5c2b5624144f3807c0df30be66b3268eeeca4df1fbded58faf49fc95dc3c35f134f8b01fd1396b6c0fc1b6c4f0eb8f5e44b8eace1e6073e20d0b8bc5385f86f1cf3f050f66af789f3ef1fc107b7f4421e19e0349c730c68f0a226981f4e889054fdb4dc149e8e889f04180102000905024d3c5c10021b0c000a0910a34d7e18c20c31bb1a03040085c8d62e16d05dc4e9dad64953c8a2eed8b6c12f92b1575eeaa6dcf7be9473dd5b24b37b6dffbb4e7c99ed1bd3cb11634be19b3e6e207bed7505c7ca111ccf47cb323bf1f8851eb6360e8034cbff8dd149993c959de89f8f77f38e7e98b8e3076323aa719328e2b408db5ec0d03936efd57422ba04f925cdc7b4c1af7590e40ab00200009501fe044d3c5c33010400b488c3e5f83f4d561f317817538d9d0397981e9aef1321ca68ebfae1cf8b7d388e19f4b5a24a82e2fbbf1c6c26557a6c5845307a03d815756f564ac7325b02bc83e87d5480a8fae848f07cb891f2d51ce7df83dcafdc12324517c86d472cc0ee10d47a68fd1d9ae49a6c19bbd36d82af597a0d88cc9c49de9df4e696fc1f0b5d0011010001fe030302e9030f3c783e14856063f16938530e148bc57a7aa3f3e4f90df9dceccdc779bc0835e1ad3d006e4a8d7b36d08b8e0de5a0d947254ecfbd22037e6572b426bcfdc517796b224b0036ff90bc574b5509bede85512f2eefb520fb4b02aa523ba739bff424a6fe81c5041f253f8d757e69a503d3563a104d0d49e9e890b9d0c26f96b55b743883b472caa7050c4acfd4a21f875bdf1258d88bd61224d303dc9df77f743137d51e6d5246b88c406780528fd9a3e15bab5452e5b93970d9dcc79f48b38651b9f15bfbcf6da452837e9cc70683d1bdca94507870f743e4ad902005812488dd342f836e72869afd00ce1850eea4cfa53ce10e3608e13d3c149394ee3cbd0e23d018fcbcb6e2ec5a1a22972d1d462ca05355d0d290dd2751e550d5efb38c6c89686344df64852bf4ff86638708f644e8ec6bd4af9b50d8541cb91891a431326ab2e332faa7ae86cfb6e0540aa63160c1e5cdd5a4add518b303fff0a20117c6bc77f7cfbaf36b04c865c6c2b42754657374204b6579203220285253412c20656e637279707465642070726976617465206b65792988b804130102002205024d3c5c33021b03060b090807030206150802090a0b0416020301021e01021780000a0910d4984f961e35246b98940400908a73b6a6169f700434f076c6c79015a49bee37130eaf23aaa3cfa9ce60bfe4acaa7bc95f1146ada5867e0079babb38804891f4f0b8ebca57a86b249dee786161a755b7a342e68ccf3f78ed6440a93a6626beb9a37aa66afcd4f888790cb4bb46d94a4ae3eb3d7d3e6b00f6bfec940303e89ec5b32a1eaaacce66497d539328b00200009d01fe044d3c5c33010400a4e913f9442abcc7f1804ccab27d2f787ffa592077ca935a8bb23165bd8d57576acac647cc596b2c3f814518cc8c82953c7a4478f32e0cf645630a5ba38d9618ef2bc3add69d459ae3dece5cab778938d988239f8c5ae437807075e06c828019959c644ff05ef6a5a1dab72227c98e3a040b0cf219026640698d7a13d8538a570011010001fe030302e9030f3c783e148560f936097339ae381d63116efcf802ff8b1c9360767db5219cc987375702a4123fd8657d3e22700f23f95020d1b261eda5257e9a72f9a918e8ef22dd5b3323ae03bbc1923dd224db988cadc16acc04b120a9f8b7e84da9716c53e0334d7b66586ddb9014df604b41be1e960dcfcbc96f4ed150a1a0dd070b9eb14276b9b6be413a769a75b519a53d3ecc0c220e85cd91ca354d57e7344517e64b43b6e29823cbd87eae26e2b2e78e6dedfbb76e3e9f77bcb844f9a8932eb3db2c3f9e44316e6f5d60e9e2a56e46b72abe6b06dc9a31cc63f10023d1f5e12d2a3ee93b675c96f504af0001220991c88db759e231b3320dcedf814dcf723fd9857e3d72d66a0f2af26950b915abdf56c1596f46a325bf17ad4810d3535fb02a259b247ac3dbd4cc3ecf9c51b6c07cebb009c1506fba0a89321ec8683e3fd009a6e551d50243e2d5092fefb3321083a4bad91320dc624bd6b5dddf93553e3d53924c05bfebec1fb4bd47e89a1a889f04180102000905024d3c5c33021b0c000a0910d4984f961e35246b26c703ff7ee29ef53bc1ae1ead533c408fa136db508434e233d6e62be621e031e5940bbd4c08142aed0f82217e7c3e1ec8de574bc06ccf3c36633be41ad78a9eacd209f861cae7b064100758545cc9dd83db71806dc1cfd5fb9ae5c7474bba0c19c44034ae61bae5eca379383339dece94ff56ff7aa44a582f3e5c38f45763af577c0934b0020000" + +const signedMessageHex = "a3019bc0cbccc0c4b8d8b74ee2108fe16ec6d3ca490cbe362d3f8333d3f352531472538b8b13d353b97232f352158c20943157c71c16064626063656269052062e4e01987e9b6fccff4b7df3a34c534b23e679cbec3bc0f8f6e64dfb4b55fe3f8efa9ce110ddb5cd79faf1d753c51aecfa669f7e7aa043436596cccc3359cb7dd6bbe9ecaa69e5989d9e57209571edc0b2fa7f57b9b79a64ee6e99ce1371395fee92fec2796f7b15a77c386ff668ee27f6d38f0baa6c438b561657377bf6acff3c5947befd7bf4c196252f1d6e5c524d0300" + +const signedTextMessageHex = "a3019bc0cbccc8c4b8d8b74ee2108fe16ec6d36a250cbece0c178233d3f352531472538b8b13d35379b97232f352158ca0b4312f57c71c1646462606365626906a062e4e019811591798ff99bf8afee860b0d8a8c2a85c3387e3bcf0bb3b17987f2bbcfab2aa526d930cbfd3d98757184df3995c9f3e7790e36e3e9779f06089d4c64e9e47dd6202cb6e9bc73c5d11bb59fbaf89d22d8dc7cf199ddf17af96e77c5f65f9bbed56f427bd8db7af37f6c9984bf9385efaf5f184f986fb3e6adb0ecfe35bbf92d16a7aa2a344fb0bc52fb7624f0200" + +const signedEncryptedMessageHex = "848c032a67d68660df41c70103ff5789d0de26b6a50c985a02a13131ca829c413a35d0e6fa8d6842599252162808ac7439c72151c8c6183e76923fe3299301414d0c25a2f06a2257db3839e7df0ec964773f6e4c4ac7ff3b48c444237166dd46ba8ff443a5410dc670cb486672fdbe7c9dfafb75b4fea83af3a204fe2a7dfa86bd20122b4f3d2646cbeecb8f7be8d2c03b018bd210b1d3791e1aba74b0f1034e122ab72e760492c192383cf5e20b5628bd043272d63df9b923f147eb6091cd897553204832aba48fec54aa447547bb16305a1024713b90e77fd0065f1918271947549205af3c74891af22ee0b56cd29bfec6d6e351901cd4ab3ece7c486f1e32a792d4e474aed98ee84b3f591c7dff37b64e0ecd68fd036d517e412dcadf85840ce184ad7921ad446c4ee28db80447aea1ca8d4f574db4d4e37688158ddd19e14ee2eab4873d46947d65d14a23e788d912cf9a19624ca7352469b72a83866b7c23cb5ace3deab3c7018061b0ba0f39ed2befe27163e5083cf9b8271e3e3d52cc7ad6e2a3bd81d4c3d7022f8d" + +const symmetricallyEncryptedCompressedHex = "8c0d04030302eb4a03808145d0d260c92f714339e13de5a79881216431925bf67ee2898ea61815f07894cd0703c50d0a76ef64d482196f47a8bc729af9b80bb6" diff --git a/libgo/go/crypto/openpgp/s2k/s2k.go b/libgo/go/crypto/openpgp/s2k/s2k.go index f369d7ed4fc..873b33dc0d5 100644 --- a/libgo/go/crypto/openpgp/s2k/s2k.go +++ b/libgo/go/crypto/openpgp/s2k/s2k.go @@ -7,15 +7,12 @@ package s2k import ( - "crypto/md5" + "crypto" "crypto/openpgp/error" - "crypto/ripemd160" - "crypto/sha1" - "crypto/sha256" - "crypto/sha512" "hash" "io" "os" + "strconv" ) // Simple writes to out the result of computing the Simple S2K function (RFC @@ -87,9 +84,13 @@ func Parse(r io.Reader) (f func(out, in []byte), err os.Error) { return } - h := hashFuncFromType(buf[1]) + hash, ok := HashIdToHash(buf[1]) + if !ok { + return nil, error.UnsupportedError("hash for S2K function: " + strconv.Itoa(int(buf[1]))) + } + h := hash.New() if h == nil { - return nil, error.UnsupportedError("hash for S2K function") + return nil, error.UnsupportedError("hash not availible: " + strconv.Itoa(int(hash))) } switch buf[0] { @@ -122,25 +123,38 @@ func Parse(r io.Reader) (f func(out, in []byte), err os.Error) { return nil, error.UnsupportedError("S2K function") } -// hashFuncFromType returns a hash.Hash which corresponds to the given hash -// type byte. See RFC 4880, section 9.4. -func hashFuncFromType(hashType byte) hash.Hash { - switch hashType { - case 1: - return md5.New() - case 2: - return sha1.New() - case 3: - return ripemd160.New() - case 8: - return sha256.New() - case 9: - return sha512.New384() - case 10: - return sha512.New() - case 11: - return sha256.New224() +// hashToHashIdMapping contains pairs relating OpenPGP's hash identifier with +// Go's crypto.Hash type. See RFC 4880, section 9.4. +var hashToHashIdMapping = []struct { + id byte + hash crypto.Hash +}{ + {1, crypto.MD5}, + {2, crypto.SHA1}, + {3, crypto.RIPEMD160}, + {8, crypto.SHA256}, + {9, crypto.SHA384}, + {10, crypto.SHA512}, + {11, crypto.SHA224}, +} + +// HashIdToHash returns a crypto.Hash which corresponds to the given OpenPGP +// hash id. +func HashIdToHash(id byte) (h crypto.Hash, ok bool) { + for _, m := range hashToHashIdMapping { + if m.id == id { + return m.hash, true + } } + return 0, false +} - return nil +// HashIdToHash returns an OpenPGP hash id which corresponds the given Hash. +func HashToHashId(h crypto.Hash) (id byte, ok bool) { + for _, m := range hashToHashIdMapping { + if m.hash == h { + return m.id, true + } + } + return 0, false } diff --git a/libgo/go/crypto/openpgp/write.go b/libgo/go/crypto/openpgp/write.go new file mode 100644 index 00000000000..1a2e2bf0407 --- /dev/null +++ b/libgo/go/crypto/openpgp/write.go @@ -0,0 +1,92 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package openpgp + +import ( + "crypto" + "crypto/openpgp/armor" + "crypto/openpgp/error" + "crypto/openpgp/packet" + "crypto/rsa" + _ "crypto/sha256" + "io" + "os" + "strconv" + "time" +) + +// DetachSign signs message with the private key from signer (which must +// already have been decrypted) and writes the signature to w. +func DetachSign(w io.Writer, signer *Entity, message io.Reader) os.Error { + return detachSign(w, signer, message, packet.SigTypeBinary) +} + +// ArmoredDetachSign signs message with the private key from signer (which +// must already have been decrypted) and writes an armored signature to w. +func ArmoredDetachSign(w io.Writer, signer *Entity, message io.Reader) (err os.Error) { + return armoredDetachSign(w, signer, message, packet.SigTypeBinary) +} + +// DetachSignText signs message (after canonicalising the line endings) with +// the private key from signer (which must already have been decrypted) and +// writes the signature to w. +func DetachSignText(w io.Writer, signer *Entity, message io.Reader) os.Error { + return detachSign(w, signer, message, packet.SigTypeText) +} + +// ArmoredDetachSignText signs message (after canonicalising the line endings) +// with the private key from signer (which must already have been decrypted) +// and writes an armored signature to w. +func SignTextDetachedArmored(w io.Writer, signer *Entity, message io.Reader) os.Error { + return armoredDetachSign(w, signer, message, packet.SigTypeText) +} + +func armoredDetachSign(w io.Writer, signer *Entity, message io.Reader, sigType packet.SignatureType) (err os.Error) { + out, err := armor.Encode(w, SignatureType, nil) + if err != nil { + return + } + err = detachSign(out, signer, message, sigType) + if err != nil { + return + } + return out.Close() +} + +func detachSign(w io.Writer, signer *Entity, message io.Reader, sigType packet.SignatureType) (err os.Error) { + if signer.PrivateKey == nil { + return error.InvalidArgumentError("signing key doesn't have a private key") + } + if signer.PrivateKey.Encrypted { + return error.InvalidArgumentError("signing key is encrypted") + } + + sig := new(packet.Signature) + sig.SigType = sigType + sig.PubKeyAlgo = signer.PrivateKey.PubKeyAlgo + sig.Hash = crypto.SHA256 + sig.CreationTime = uint32(time.Seconds()) + sig.IssuerKeyId = &signer.PrivateKey.KeyId + + h, wrappedHash, err := hashForSignature(sig.Hash, sig.SigType) + if err != nil { + return + } + io.Copy(wrappedHash, message) + + switch signer.PrivateKey.PubKeyAlgo { + case packet.PubKeyAlgoRSA, packet.PubKeyAlgoRSASignOnly: + priv := signer.PrivateKey.PrivateKey.(*rsa.PrivateKey) + err = sig.SignRSA(h, priv) + default: + err = error.UnsupportedError("public key algorithm: " + strconv.Itoa(int(sig.PubKeyAlgo))) + } + + if err != nil { + return + } + + return sig.Serialize(w) +} diff --git a/libgo/go/crypto/openpgp/write_test.go b/libgo/go/crypto/openpgp/write_test.go new file mode 100644 index 00000000000..33e8809f224 --- /dev/null +++ b/libgo/go/crypto/openpgp/write_test.go @@ -0,0 +1,34 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package openpgp + +import ( + "bytes" + "testing" +) + +func TestSignDetached(t *testing.T) { + kring, _ := ReadKeyRing(readerFromHex(testKeys1And2PrivateHex)) + out := bytes.NewBuffer(nil) + message := bytes.NewBufferString(signedInput) + err := DetachSign(out, kring[0], message) + if err != nil { + t.Error(err) + } + + testDetachedSignature(t, kring, out, signedInput, "check") +} + +func TestSignTextDetached(t *testing.T) { + kring, _ := ReadKeyRing(readerFromHex(testKeys1And2PrivateHex)) + out := bytes.NewBuffer(nil) + message := bytes.NewBufferString(signedInput) + err := DetachSignText(out, kring[0], message) + if err != nil { + t.Error(err) + } + + testDetachedSignature(t, kring, out, signedInput, "check") +} diff --git a/libgo/go/crypto/rand/rand_unix.go b/libgo/go/crypto/rand/rand_unix.go index ff16f25547f..66b72c07662 100644 --- a/libgo/go/crypto/rand/rand_unix.go +++ b/libgo/go/crypto/rand/rand_unix.go @@ -8,6 +8,7 @@ package rand import ( + "bufio" "crypto/aes" "io" "os" @@ -23,21 +24,20 @@ func init() { Reader = &devReader{name: "/dev/urandom"} } // A devReader satisfies reads by reading the file named name. type devReader struct { name string - f *os.File + f io.Reader mu sync.Mutex } func (r *devReader) Read(b []byte) (n int, err os.Error) { r.mu.Lock() + defer r.mu.Unlock() if r.f == nil { f, err := os.Open(r.name, os.O_RDONLY, 0) if f == nil { - r.mu.Unlock() return 0, err } - r.f = f + r.f = bufio.NewReader(f) } - r.mu.Unlock() return r.f.Read(b) } diff --git a/libgo/go/crypto/rand/rand_windows.go b/libgo/go/crypto/rand/rand_windows.go index 4b2b7a26f37..281d6dc6aaf 100644 --- a/libgo/go/crypto/rand/rand_windows.go +++ b/libgo/go/crypto/rand/rand_windows.go @@ -28,15 +28,15 @@ func (r *rngReader) Read(b []byte) (n int, err os.Error) { if r.prov == 0 { const provType = syscall.PROV_RSA_FULL const flags = syscall.CRYPT_VERIFYCONTEXT | syscall.CRYPT_SILENT - ok, errno := syscall.CryptAcquireContext(&r.prov, nil, nil, provType, flags) - if !ok { + errno := syscall.CryptAcquireContext(&r.prov, nil, nil, provType, flags) + if errno != 0 { r.mu.Unlock() return 0, os.NewSyscallError("CryptAcquireContext", errno) } } r.mu.Unlock() - ok, errno := syscall.CryptGenRandom(r.prov, uint32(len(b)), &b[0]) - if !ok { + errno := syscall.CryptGenRandom(r.prov, uint32(len(b)), &b[0]) + if errno != 0 { return 0, os.NewSyscallError("CryptGenRandom", errno) } return len(b), nil diff --git a/libgo/go/crypto/ripemd160/ripemd160.go b/libgo/go/crypto/ripemd160/ripemd160.go index 5614f1360eb..6e88521c3f6 100644 --- a/libgo/go/crypto/ripemd160/ripemd160.go +++ b/libgo/go/crypto/ripemd160/ripemd160.go @@ -10,10 +10,15 @@ package ripemd160 // http://homes.esat.kuleuven.be/~cosicart/pdf/AB-9601/AB-9601.pdf. import ( + "crypto" "hash" "os" ) +func init() { + crypto.RegisterHash(crypto.RIPEMD160, New) +} + // The size of the checksum in bytes. const Size = 20 diff --git a/libgo/go/crypto/rsa/pkcs1v15.go b/libgo/go/crypto/rsa/pkcs1v15.go index 7140462509c..2eaadee24c2 100644 --- a/libgo/go/crypto/rsa/pkcs1v15.go +++ b/libgo/go/crypto/rsa/pkcs1v15.go @@ -6,6 +6,7 @@ package rsa import ( "big" + "crypto" "crypto/subtle" "io" "os" @@ -139,19 +140,6 @@ func nonZeroRandomBytes(s []byte, rand io.Reader) (err os.Error) { return } -// Due to the design of PKCS#1 v1.5, we need to know the exact hash function in -// use. A generic hash.Hash will not do. -type PKCS1v15Hash int - -const ( - HashMD5 PKCS1v15Hash = iota - HashSHA1 - HashSHA256 - HashSHA384 - HashSHA512 - HashMD5SHA1 // combined MD5 and SHA1 hash used for RSA signing in TLS. -) - // These are ASN1 DER structures: // DigestInfo ::= SEQUENCE { // digestAlgorithm AlgorithmIdentifier, @@ -160,25 +148,20 @@ const ( // For performance, we don't use the generic ASN1 encoder. Rather, we // precompute a prefix of the digest value that makes a valid ASN1 DER string // with the correct contents. -var hashPrefixes = [][]byte{ - // HashMD5 - {0x30, 0x20, 0x30, 0x0c, 0x06, 0x08, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x02, 0x05, 0x05, 0x00, 0x04, 0x10}, - // HashSHA1 - {0x30, 0x21, 0x30, 0x09, 0x06, 0x05, 0x2b, 0x0e, 0x03, 0x02, 0x1a, 0x05, 0x00, 0x04, 0x14}, - // HashSHA256 - {0x30, 0x31, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x01, 0x05, 0x00, 0x04, 0x20}, - // HashSHA384 - {0x30, 0x41, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x02, 0x05, 0x00, 0x04, 0x30}, - // HashSHA512 - {0x30, 0x51, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x03, 0x05, 0x00, 0x04, 0x40}, - // HashMD5SHA1 - {}, // A special TLS case which doesn't use an ASN1 prefix. +var hashPrefixes = map[crypto.Hash][]byte{ + crypto.MD5: []byte{0x30, 0x20, 0x30, 0x0c, 0x06, 0x08, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x02, 0x05, 0x05, 0x00, 0x04, 0x10}, + crypto.SHA1: []byte{0x30, 0x21, 0x30, 0x09, 0x06, 0x05, 0x2b, 0x0e, 0x03, 0x02, 0x1a, 0x05, 0x00, 0x04, 0x14}, + crypto.SHA256: []byte{0x30, 0x31, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x01, 0x05, 0x00, 0x04, 0x20}, + crypto.SHA384: []byte{0x30, 0x41, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x02, 0x05, 0x00, 0x04, 0x30}, + crypto.SHA512: {0x30, 0x51, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x03, 0x05, 0x00, 0x04, 0x40}, + crypto.MD5SHA1: {}, // A special TLS case which doesn't use an ASN1 prefix. + crypto.RIPEMD160: {0x30, 0x20, 0x30, 0x08, 0x06, 0x06, 0x28, 0xcf, 0x06, 0x03, 0x00, 0x31, 0x04, 0x14}, } -// SignPKCS1v15 calcuates the signature of hashed using RSASSA-PSS-SIGN from RSA PKCS#1 v1.5. +// SignPKCS1v15 calculates the signature of hashed using RSASSA-PKCS1-V1_5-SIGN from RSA PKCS#1 v1.5. // Note that hashed must be the result of hashing the input message using the // given hash function. -func SignPKCS1v15(rand io.Reader, priv *PrivateKey, hash PKCS1v15Hash, hashed []byte) (s []byte, err os.Error) { +func SignPKCS1v15(rand io.Reader, priv *PrivateKey, hash crypto.Hash, hashed []byte) (s []byte, err os.Error) { hashLen, prefix, err := pkcs1v15HashInfo(hash, len(hashed)) if err != nil { return @@ -211,7 +194,7 @@ func SignPKCS1v15(rand io.Reader, priv *PrivateKey, hash PKCS1v15Hash, hashed [] // hashed is the result of hashing the input message using the given hash // function and sig is the signature. A valid signature is indicated by // returning a nil error. -func VerifyPKCS1v15(pub *PublicKey, hash PKCS1v15Hash, hashed []byte, sig []byte) (err os.Error) { +func VerifyPKCS1v15(pub *PublicKey, hash crypto.Hash, hashed []byte, sig []byte) (err os.Error) { hashLen, prefix, err := pkcs1v15HashInfo(hash, len(hashed)) if err != nil { return @@ -246,28 +229,14 @@ func VerifyPKCS1v15(pub *PublicKey, hash PKCS1v15Hash, hashed []byte, sig []byte return nil } -func pkcs1v15HashInfo(hash PKCS1v15Hash, inLen int) (hashLen int, prefix []byte, err os.Error) { - switch hash { - case HashMD5: - hashLen = 16 - case HashSHA1: - hashLen = 20 - case HashSHA256: - hashLen = 32 - case HashSHA384: - hashLen = 48 - case HashSHA512: - hashLen = 64 - case HashMD5SHA1: - hashLen = 36 - default: - return 0, nil, os.ErrorString("unknown hash function") - } - +func pkcs1v15HashInfo(hash crypto.Hash, inLen int) (hashLen int, prefix []byte, err os.Error) { + hashLen = hash.Size() if inLen != hashLen { return 0, nil, os.ErrorString("input must be hashed message") } - - prefix = hashPrefixes[int(hash)] + prefix, ok := hashPrefixes[hash] + if !ok { + return 0, nil, os.ErrorString("unsupported hash function") + } return } diff --git a/libgo/go/crypto/rsa/pkcs1v15_test.go b/libgo/go/crypto/rsa/pkcs1v15_test.go index bf6306dc29b..7b2ce08cb0b 100644 --- a/libgo/go/crypto/rsa/pkcs1v15_test.go +++ b/libgo/go/crypto/rsa/pkcs1v15_test.go @@ -7,6 +7,7 @@ package rsa import ( "big" "bytes" + "crypto" "crypto/rand" "crypto/sha1" "encoding/base64" @@ -165,7 +166,7 @@ func TestSignPKCS1v15(t *testing.T) { h.Write([]byte(test.in)) digest := h.Sum() - s, err := SignPKCS1v15(nil, rsaPrivateKey, HashSHA1, digest) + s, err := SignPKCS1v15(nil, rsaPrivateKey, crypto.SHA1, digest) if err != nil { t.Errorf("#%d %s", i, err) } @@ -185,7 +186,7 @@ func TestVerifyPKCS1v15(t *testing.T) { sig, _ := hex.DecodeString(test.out) - err := VerifyPKCS1v15(&rsaPrivateKey.PublicKey, HashSHA1, digest, sig) + err := VerifyPKCS1v15(&rsaPrivateKey.PublicKey, crypto.SHA1, digest, sig) if err != nil { t.Errorf("#%d %s", i, err) } diff --git a/libgo/go/crypto/rsa/rsa.go b/libgo/go/crypto/rsa/rsa.go index c7a8d2053d8..faf914991d0 100644 --- a/libgo/go/crypto/rsa/rsa.go +++ b/libgo/go/crypto/rsa/rsa.go @@ -274,6 +274,14 @@ func EncryptOAEP(hash hash.Hash, rand io.Reader, pub *PublicKey, msg []byte, lab m.SetBytes(em) c := encrypt(new(big.Int), pub, m) out = c.Bytes() + + if len(out) < k { + // If the output is too small, we need to left-pad with zeros. + t := make([]byte, k) + copy(t[k-len(out):], out) + out = t + } + return } diff --git a/libgo/go/crypto/rsa/rsa_test.go b/libgo/go/crypto/rsa/rsa_test.go index df1f17f17ce..22d4576e8dd 100644 --- a/libgo/go/crypto/rsa/rsa_test.go +++ b/libgo/go/crypto/rsa/rsa_test.go @@ -66,7 +66,7 @@ func TestEncryptOAEP(t *testing.T) { t.Errorf("#%d,%d error: %s", i, j, err) } if bytes.Compare(out, message.out) != 0 { - t.Errorf("#%d,%d bad result: %s (want %s)", i, j, out, message.out) + t.Errorf("#%d,%d bad result: %x (want %x)", i, j, out, message.out) } } } diff --git a/libgo/go/crypto/sha1/sha1.go b/libgo/go/crypto/sha1/sha1.go index 8716c359107..e6aa096e2a6 100644 --- a/libgo/go/crypto/sha1/sha1.go +++ b/libgo/go/crypto/sha1/sha1.go @@ -6,10 +6,15 @@ package sha1 import ( + "crypto" "hash" "os" ) +func init() { + crypto.RegisterHash(crypto.SHA1, New) +} + // The size of a SHA1 checksum in bytes. const Size = 20 diff --git a/libgo/go/crypto/sha256/sha256.go b/libgo/go/crypto/sha256/sha256.go index 57a8ffa0d76..69b356b4e51 100644 --- a/libgo/go/crypto/sha256/sha256.go +++ b/libgo/go/crypto/sha256/sha256.go @@ -6,10 +6,16 @@ package sha256 import ( + "crypto" "hash" "os" ) +func init() { + crypto.RegisterHash(crypto.SHA224, New224) + crypto.RegisterHash(crypto.SHA256, New) +} + // The size of a SHA256 checksum in bytes. const Size = 32 diff --git a/libgo/go/crypto/sha512/sha512.go b/libgo/go/crypto/sha512/sha512.go index c3cda97d968..7e9f330e594 100644 --- a/libgo/go/crypto/sha512/sha512.go +++ b/libgo/go/crypto/sha512/sha512.go @@ -6,10 +6,16 @@ package sha512 import ( + "crypto" "hash" "os" ) +func init() { + crypto.RegisterHash(crypto.SHA384, New384) + crypto.RegisterHash(crypto.SHA512, New) +} + // The size of a SHA512 checksum in bytes. const Size = 64 diff --git a/libgo/go/crypto/tls/handshake_client.go b/libgo/go/crypto/tls/handshake_client.go index 1ca33f59dd0..a325a9b953c 100644 --- a/libgo/go/crypto/tls/handshake_client.go +++ b/libgo/go/crypto/tls/handshake_client.go @@ -5,6 +5,7 @@ package tls import ( + "crypto" "crypto/rsa" "crypto/subtle" "crypto/x509" @@ -56,7 +57,7 @@ func (c *Conn) clientHandshake() os.Error { vers, ok := mutualVersion(serverHello.vers) if !ok { - c.sendAlert(alertProtocolVersion) + return c.sendAlert(alertProtocolVersion) } c.vers = vers c.haveVers = true @@ -248,7 +249,7 @@ func (c *Conn) clientHandshake() os.Error { var digest [36]byte copy(digest[0:16], finishedHash.serverMD5.Sum()) copy(digest[16:36], finishedHash.serverSHA1.Sum()) - signed, err := rsa.SignPKCS1v15(c.config.rand(), c.config.Certificates[0].PrivateKey, rsa.HashMD5SHA1, digest[0:]) + signed, err := rsa.SignPKCS1v15(c.config.rand(), c.config.Certificates[0].PrivateKey, crypto.MD5SHA1, digest[0:]) if err != nil { return c.sendAlert(alertInternalError) } diff --git a/libgo/go/crypto/tls/handshake_client_test.go b/libgo/go/crypto/tls/handshake_client_test.go index e5c9684b977..fd1f145cfc4 100644 --- a/libgo/go/crypto/tls/handshake_client_test.go +++ b/libgo/go/crypto/tls/handshake_client_test.go @@ -61,7 +61,7 @@ func TestRunClient(t *testing.T) { // Script of interaction with gnutls implementation. // The values for this test are obtained by building and running in client mode: -// % gotest -match "TestRunClient" -connect +// % gotest -test.run "TestRunClient" -connect // and then: // % gnutls-serv -p 10443 --debug 100 --x509keyfile key.pem --x509certfile cert.pem -a > /tmp/log 2>&1 // % python parse-gnutls-cli-debug-log.py < /tmp/log diff --git a/libgo/go/crypto/tls/handshake_server.go b/libgo/go/crypto/tls/handshake_server.go index 955811ada35..809c8c15e5d 100644 --- a/libgo/go/crypto/tls/handshake_server.go +++ b/libgo/go/crypto/tls/handshake_server.go @@ -5,6 +5,7 @@ package tls import ( + "crypto" "crypto/rsa" "crypto/subtle" "crypto/x509" @@ -56,6 +57,7 @@ Curves: var suite *cipherSuite var suiteId uint16 +FindCipherSuite: for _, id := range clientHello.cipherSuites { for _, supported := range config.cipherSuites() { if id == supported { @@ -66,7 +68,7 @@ Curves: continue } suiteId = id - break + break FindCipherSuite } } } @@ -213,7 +215,7 @@ Curves: digest := make([]byte, 36) copy(digest[0:16], finishedHash.serverMD5.Sum()) copy(digest[16:36], finishedHash.serverSHA1.Sum()) - err = rsa.VerifyPKCS1v15(pub, rsa.HashMD5SHA1, digest, certVerify.signature) + err = rsa.VerifyPKCS1v15(pub, crypto.MD5SHA1, digest, certVerify.signature) if err != nil { c.sendAlert(alertBadCertificate) return os.ErrorString("could not validate signature of connection nonces: " + err.String()) diff --git a/libgo/go/crypto/tls/handshake_server_test.go b/libgo/go/crypto/tls/handshake_server_test.go index 5cf3ae0499d..6beb6a9f62b 100644 --- a/libgo/go/crypto/tls/handshake_server_test.go +++ b/libgo/go/crypto/tls/handshake_server_test.go @@ -194,7 +194,7 @@ var testPrivateKey = &rsa.PrivateKey{ // Script of interaction with gnutls implementation. // The values for this test are obtained by building and running in server mode: -// % gotest -match "TestRunServer" -serve +// % gotest -test.run "TestRunServer" -serve // and then: // % gnutls-cli --insecure --debug 100 -p 10443 localhost > /tmp/log 2>&1 // % python parse-gnutls-cli-debug-log.py < /tmp/log diff --git a/libgo/go/crypto/tls/key_agreement.go b/libgo/go/crypto/tls/key_agreement.go index 861c64f04bb..8edbb11900c 100644 --- a/libgo/go/crypto/tls/key_agreement.go +++ b/libgo/go/crypto/tls/key_agreement.go @@ -6,6 +6,7 @@ package tls import ( "big" + "crypto" "crypto/elliptic" "crypto/md5" "crypto/rsa" @@ -143,7 +144,7 @@ Curve: copy(serverECDHParams[4:], ecdhePublic) md5sha1 := md5SHA1Hash(clientHello.random, hello.random, serverECDHParams) - sig, err := rsa.SignPKCS1v15(config.rand(), config.Certificates[0].PrivateKey, rsa.HashMD5SHA1, md5sha1) + sig, err := rsa.SignPKCS1v15(config.rand(), config.Certificates[0].PrivateKey, crypto.MD5SHA1, md5sha1) if err != nil { return nil, os.ErrorString("failed to sign ECDHE parameters: " + err.String()) } @@ -216,7 +217,7 @@ func (ka *ecdheRSAKeyAgreement) processServerKeyExchange(config *Config, clientH sig = sig[2:] md5sha1 := md5SHA1Hash(clientHello.random, serverHello.random, serverECDHParams) - return rsa.VerifyPKCS1v15(cert.PublicKey.(*rsa.PublicKey), rsa.HashMD5SHA1, md5sha1, sig) + return rsa.VerifyPKCS1v15(cert.PublicKey.(*rsa.PublicKey), crypto.MD5SHA1, md5sha1, sig) Error: return os.ErrorString("invalid ServerKeyExchange") diff --git a/libgo/go/crypto/tls/tls.go b/libgo/go/crypto/tls/tls.go index b11d3225daa..e8290d728dd 100644 --- a/libgo/go/crypto/tls/tls.go +++ b/libgo/go/crypto/tls/tls.go @@ -124,14 +124,22 @@ func LoadX509KeyPair(certFile string, keyFile string) (cert Certificate, err os. return } - certDERBlock, _ := pem.Decode(certPEMBlock) - if certDERBlock == nil { + var certDERBlock *pem.Block + for { + certDERBlock, certPEMBlock = pem.Decode(certPEMBlock) + if certDERBlock == nil { + break + } + if certDERBlock.Type == "CERTIFICATE" { + cert.Certificate = append(cert.Certificate, certDERBlock.Bytes) + } + } + + if len(cert.Certificate) == 0 { err = os.ErrorString("crypto/tls: failed to parse certificate PEM data") return } - cert.Certificate = [][]byte{certDERBlock.Bytes} - keyPEMBlock, err := ioutil.ReadFile(keyFile) if err != nil { return @@ -153,7 +161,7 @@ func LoadX509KeyPair(certFile string, keyFile string) (cert Certificate, err os. // We don't need to parse the public key for TLS, but we so do anyway // to check that it looks sane and matches the private key. - x509Cert, err := x509.ParseCertificate(certDERBlock.Bytes) + x509Cert, err := x509.ParseCertificate(cert.Certificate[0]) if err != nil { return } diff --git a/libgo/go/crypto/x509/x509.go b/libgo/go/crypto/x509/x509.go index 6199e8db9f5..3af8ba8ca24 100644 --- a/libgo/go/crypto/x509/x509.go +++ b/libgo/go/crypto/x509/x509.go @@ -9,6 +9,7 @@ import ( "asn1" "big" "container/vector" + "crypto" "crypto/rsa" "crypto/sha1" "hash" @@ -330,6 +331,10 @@ type Certificate struct { DNSNames []string EmailAddresses []string + // Name constraints + PermittedDNSDomainsCritical bool // if true then the name constraints are marked critical. + PermittedDNSDomains []string + PolicyIdentifiers []asn1.ObjectIdentifier } @@ -374,12 +379,12 @@ func (c *Certificate) CheckSignatureFrom(parent *Certificate) (err os.Error) { // TODO(agl): don't ignore the path length constraint. var h hash.Hash - var hashType rsa.PKCS1v15Hash + var hashType crypto.Hash switch c.SignatureAlgorithm { case SHA1WithRSA: h = sha1.New() - hashType = rsa.HashSHA1 + hashType = crypto.SHA1 default: return UnsupportedAlgorithmError{} } @@ -474,6 +479,18 @@ type policyInformation struct { // policyQualifiers omitted } +// RFC 5280, 4.2.1.10 +type nameConstraints struct { + Permitted []generalSubtree "optional,tag:0" + Excluded []generalSubtree "optional,tag:1" +} + +type generalSubtree struct { + Name string "tag:2,optional,ia5" + Min int "optional,tag:0" + Max int "optional,tag:1" +} + func parsePublicKey(algo PublicKeyAlgorithm, asn1Data []byte) (interface{}, os.Error) { switch algo { case RSA: @@ -602,6 +619,43 @@ func parseCertificate(in *certificate) (*Certificate, os.Error) { // If we didn't parse any of the names then we // fall through to the critical check below. + case 30: + // RFC 5280, 4.2.1.10 + + // NameConstraints ::= SEQUENCE { + // permittedSubtrees [0] GeneralSubtrees OPTIONAL, + // excludedSubtrees [1] GeneralSubtrees OPTIONAL } + // + // GeneralSubtrees ::= SEQUENCE SIZE (1..MAX) OF GeneralSubtree + // + // GeneralSubtree ::= SEQUENCE { + // base GeneralName, + // minimum [0] BaseDistance DEFAULT 0, + // maximum [1] BaseDistance OPTIONAL } + // + // BaseDistance ::= INTEGER (0..MAX) + + var constraints nameConstraints + _, err := asn1.Unmarshal(e.Value, &constraints) + if err != nil { + return nil, err + } + + if len(constraints.Excluded) > 0 && e.Critical { + return out, UnhandledCriticalExtension{} + } + + for _, subtree := range constraints.Permitted { + if subtree.Min > 0 || subtree.Max > 0 || len(subtree.Name) == 0 { + if e.Critical { + return out, UnhandledCriticalExtension{} + } + continue + } + out.PermittedDNSDomains = append(out.PermittedDNSDomains, subtree.Name) + } + continue + case 35: // RFC 5280, 4.2.1.1 var a authKeyId @@ -698,10 +752,11 @@ var ( oidExtensionBasicConstraints = []int{2, 5, 29, 19} oidExtensionSubjectAltName = []int{2, 5, 29, 17} oidExtensionCertificatePolicies = []int{2, 5, 29, 32} + oidExtensionNameConstraints = []int{2, 5, 29, 30} ) func buildExtensions(template *Certificate) (ret []extension, err os.Error) { - ret = make([]extension, 6 /* maximum number of elements. */ ) + ret = make([]extension, 7 /* maximum number of elements. */ ) n := 0 if template.KeyUsage != 0 { @@ -778,6 +833,22 @@ func buildExtensions(template *Certificate) (ret []extension, err os.Error) { n++ } + if len(template.PermittedDNSDomains) > 0 { + ret[n].Id = oidExtensionNameConstraints + ret[n].Critical = template.PermittedDNSDomainsCritical + + var out nameConstraints + out.Permitted = make([]generalSubtree, len(template.PermittedDNSDomains)) + for i, permitted := range template.PermittedDNSDomains { + out.Permitted[i] = generalSubtree{Name: permitted} + } + ret[n].Value, err = asn1.Marshal(out) + if err != nil { + return + } + n++ + } + // Adding another extension here? Remember to update the maximum number // of elements in the make() at the top of the function. @@ -792,7 +863,8 @@ var ( // CreateSelfSignedCertificate creates a new certificate based on // a template. The following members of template are used: SerialNumber, // Subject, NotBefore, NotAfter, KeyUsage, BasicConstraintsValid, IsCA, -// MaxPathLen, SubjectKeyId, DNSNames. +// MaxPathLen, SubjectKeyId, DNSNames, PermittedDNSDomainsCritical, +// PermittedDNSDomains. // // The certificate is signed by parent. If parent is equal to template then the // certificate is self-signed. The parameter pub is the public key of the @@ -840,7 +912,7 @@ func CreateCertificate(rand io.Reader, template, parent *Certificate, pub *rsa.P h.Write(tbsCertContents) digest := h.Sum() - signature, err := rsa.SignPKCS1v15(rand, priv, rsa.HashSHA1, digest) + signature, err := rsa.SignPKCS1v15(rand, priv, crypto.SHA1, digest) if err != nil { return } diff --git a/libgo/go/crypto/x509/x509_test.go b/libgo/go/crypto/x509/x509_test.go index 2fe47fdbe59..57889e7e122 100644 --- a/libgo/go/crypto/x509/x509_test.go +++ b/libgo/go/crypto/x509/x509_test.go @@ -171,7 +171,8 @@ func TestCreateSelfSignedCertificate(t *testing.T) { IsCA: true, DNSNames: []string{"test.example.com"}, - PolicyIdentifiers: []asn1.ObjectIdentifier{[]int{1, 2, 3}}, + PolicyIdentifiers: []asn1.ObjectIdentifier{[]int{1, 2, 3}}, + PermittedDNSDomains: []string{".example.com", "example.com"}, } derBytes, err := CreateCertificate(random, &template, &template, &priv.PublicKey, priv) @@ -190,6 +191,10 @@ func TestCreateSelfSignedCertificate(t *testing.T) { t.Errorf("Failed to parse policy identifiers: got:%#v want:%#v", cert.PolicyIdentifiers, template.PolicyIdentifiers) } + if len(cert.PermittedDNSDomains) != 2 || cert.PermittedDNSDomains[0] != ".example.com" || cert.PermittedDNSDomains[1] != "example.com" { + t.Errorf("Failed to parse name constraints: %#v", cert.PermittedDNSDomains) + } + err = cert.CheckSignatureFrom(cert) if err != nil { t.Errorf("Signature verification failed: %s", err) diff --git a/libgo/go/debug/pe/file.go b/libgo/go/debug/pe/file.go index 82c02407bbe..1bcbdc5e9a5 100644 --- a/libgo/go/debug/pe/file.go +++ b/libgo/go/debug/pe/file.go @@ -57,7 +57,6 @@ type ImportDirectory struct { FirstThunk uint32 dll string - rva []uint32 } // Data reads and returns the contents of the PE section. @@ -267,34 +266,28 @@ func (f *File) ImportedSymbols() ([]string, os.Error) { } ida = append(ida, dt) } - for i, _ := range ida { + names, _ := ds.Data() + var all []string + for _, dt := range ida { + dt.dll, _ = getString(names, int(dt.Name-ds.VirtualAddress)) + d, _ = ds.Data() + // seek to OriginalFirstThunk + d = d[dt.OriginalFirstThunk-ds.VirtualAddress:] for len(d) > 0 { va := binary.LittleEndian.Uint32(d[0:4]) d = d[4:] if va == 0 { break } - ida[i].rva = append(ida[i].rva, va) - } - } - for _, _ = range ida { - for len(d) > 0 { - va := binary.LittleEndian.Uint32(d[0:4]) - d = d[4:] - if va == 0 { - break + if va&0x80000000 > 0 { // is Ordinal + // TODO add dynimport ordinal support. + //ord := va&0x0000FFFF + } else { + fn, _ := getString(names, int(va-ds.VirtualAddress+2)) + all = append(all, fn+":"+dt.dll) } } } - names, _ := ds.Data() - var all []string - for _, dt := range ida { - dt.dll, _ = getString(names, int(dt.Name-ds.VirtualAddress)) - for _, va := range dt.rva { - fn, _ := getString(names, int(va-ds.VirtualAddress+2)) - all = append(all, fn+":"+dt.dll) - } - } return all, nil } diff --git a/libgo/go/ebnf/ebnf_test.go b/libgo/go/ebnf/ebnf_test.go index bbe530c278f..69ad5fed1cf 100644 --- a/libgo/go/ebnf/ebnf_test.go +++ b/libgo/go/ebnf/ebnf_test.go @@ -15,31 +15,31 @@ var fset = token.NewFileSet() var grammars = []string{ - `Program = . - `, - - `Program = foo . - foo = "foo" . - `, - - `Program = "a" | "b" "c" . - `, - - `Program = "a" ... "z" . - `, - - `Program = Song . - Song = { Note } . - Note = Do | (Re | Mi | Fa | So | La) | Ti . - Do = "c" . - Re = "d" . - Mi = "e" . - Fa = "f" . - So = "g" . - La = "a" . - Ti = ti . - ti = "b" . - `, +`Program = . +`, + +`Program = foo . +foo = "foo" . +`, + +`Program = "a" | "b" "c" . +`, + +`Program = "a" ... "z" . +`, + +`Program = Song . + Song = { Note } . + Note = Do | (Re | Mi | Fa | So | La) | Ti . + Do = "c" . + Re = "d" . + Mi = "e" . + Fa = "f" . + So = "g" . + La = "a" . + Ti = ti . + ti = "b" . +`, } diff --git a/libgo/go/encoding/binary/binary.go b/libgo/go/encoding/binary/binary.go index 77ff3a9f3ef..ee2f23dbba2 100644 --- a/libgo/go/encoding/binary/binary.go +++ b/libgo/go/encoding/binary/binary.go @@ -2,8 +2,9 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// This package implements translation between -// unsigned integer values and byte sequences. +// Package binary implements translation between +// unsigned integer values and byte sequences +// and the reading and writing of fixed-size values. package binary import ( diff --git a/libgo/go/encoding/line/line.go b/libgo/go/encoding/line/line.go index 92dddcb996d..f46ce1c83a0 100644 --- a/libgo/go/encoding/line/line.go +++ b/libgo/go/encoding/line/line.go @@ -2,8 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// This package implements a Reader which handles reading \r and \r\n -// deliminated lines. +// The line package implements a Reader that reads lines delimited by '\n' or ' \r\n'. package line import ( @@ -11,8 +10,7 @@ import ( "os" ) -// Reader reads lines from an io.Reader (which may use either '\n' or -// '\r\n'). +// Reader reads lines, delimited by '\n' or \r\n', from an io.Reader. type Reader struct { buf []byte consumed int @@ -20,11 +18,33 @@ type Reader struct { err os.Error } -func NewReader(in io.Reader, maxLineLength int) *Reader { +// NewReader returns a new Reader that will read successive +// lines from the input Reader. +func NewReader(input io.Reader, maxLineLength int) *Reader { return &Reader{ buf: make([]byte, 0, maxLineLength), consumed: 0, - in: in, + in: input, + } +} + +// Read reads from any buffered data past the last line read, or from the underlying +// io.Reader if the buffer is empty. +func (l *Reader) Read(p []byte) (n int, err os.Error) { + l.removeConsumedFromBuffer() + if len(l.buf) > 0 { + n = copy(p, l.buf) + l.consumed += n + return + } + return l.in.Read(p) +} + +func (l *Reader) removeConsumedFromBuffer() { + if l.consumed > 0 { + n := copy(l.buf, l.buf[l.consumed:]) + l.buf = l.buf[:n] + l.consumed = 0 } } @@ -36,11 +56,7 @@ func NewReader(in io.Reader, maxLineLength int) *Reader { // the Reader and is only valid until the next call to ReadLine. ReadLine // either returns a non-nil line or it returns an error, never both. func (l *Reader) ReadLine() (line []byte, isPrefix bool, err os.Error) { - if l.consumed > 0 { - n := copy(l.buf, l.buf[l.consumed:]) - l.buf = l.buf[:n] - l.consumed = 0 - } + l.removeConsumedFromBuffer() if len(l.buf) == 0 && l.err != nil { err = l.err @@ -89,6 +105,9 @@ func (l *Reader) ReadLine() (line []byte, isPrefix bool, err os.Error) { l.buf = l.buf[:oldLen+n] if readErr != nil { l.err = readErr + if len(l.buf) == 0 { + return nil, false, readErr + } } } panic("unreachable") diff --git a/libgo/go/encoding/line/line_test.go b/libgo/go/encoding/line/line_test.go index 68d13b58616..ff3d51669b5 100644 --- a/libgo/go/encoding/line/line_test.go +++ b/libgo/go/encoding/line/line_test.go @@ -6,6 +6,8 @@ package line import ( "bytes" + "io" + "io/ioutil" "os" "testing" ) @@ -87,3 +89,45 @@ func TestLineTooLong(t *testing.T) { t.Errorf("bad result for third line: %x", line) } } + +func TestReadAfterLines(t *testing.T) { + line1 := "line1" + restData := "line2\nline 3\n" + inbuf := bytes.NewBuffer([]byte(line1 + "\n" + restData)) + outbuf := new(bytes.Buffer) + maxLineLength := len(line1) + len(restData)/2 + l := NewReader(inbuf, maxLineLength) + line, isPrefix, err := l.ReadLine() + if isPrefix || err != nil || string(line) != line1 { + t.Errorf("bad result for first line: isPrefix=%v err=%v line=%q", isPrefix, err, string(line)) + } + n, err := io.Copy(outbuf, l) + if int(n) != len(restData) || err != nil { + t.Errorf("bad result for Read: n=%d err=%v", n, err) + } + if outbuf.String() != restData { + t.Errorf("bad result for Read: got %q; expected %q", outbuf.String(), restData) + } +} + +func TestReadEmptyBuffer(t *testing.T) { + l := NewReader(bytes.NewBuffer(nil), 10) + line, isPrefix, err := l.ReadLine() + if err != os.EOF { + t.Errorf("expected EOF from ReadLine, got '%s' %t %s", line, isPrefix, err) + } +} + +func TestLinesAfterRead(t *testing.T) { + l := NewReader(bytes.NewBuffer([]byte("foo")), 10) + _, err := ioutil.ReadAll(l) + if err != nil { + t.Error(err) + return + } + + line, isPrefix, err := l.ReadLine() + if err != os.EOF { + t.Errorf("expected EOF from ReadLine, got '%s' %t %s", line, isPrefix, err) + } +} diff --git a/libgo/go/exec/exec.go b/libgo/go/exec/exec.go index ba9bd2472a4..80f6f3c7dd4 100644 --- a/libgo/go/exec/exec.go +++ b/libgo/go/exec/exec.go @@ -7,6 +7,7 @@ package exec import ( "os" + "strconv" ) // Arguments to Run. @@ -21,12 +22,22 @@ const ( // Stdin, Stdout, and Stderr are Files representing pipes // connected to the running command's standard input, output, and error, // or else nil, depending on the arguments to Run. -// Pid is the running command's operating system process ID. +// Process represents the underlying operating system process. type Cmd struct { - Stdin *os.File - Stdout *os.File - Stderr *os.File - Pid int + Stdin *os.File + Stdout *os.File + Stderr *os.File + Process *os.Process +} + +// PathError records the name of a binary that was not +// found on the current $PATH. +type PathError struct { + Name string +} + +func (e *PathError) String() string { + return "command " + strconv.Quote(e.Name) + " not found in $PATH" } // Given mode (DevNull, etc), return file for child @@ -77,24 +88,24 @@ func modeToFiles(mode, fd int) (*os.File, *os.File, os.Error) { // If a parameter is Pipe, then the corresponding field (Stdin, Stdout, Stderr) // of the returned Cmd is the other end of the pipe. // Otherwise the field in Cmd is nil. -func Run(name string, argv, envv []string, dir string, stdin, stdout, stderr int) (p *Cmd, err os.Error) { - p = new(Cmd) +func Run(name string, argv, envv []string, dir string, stdin, stdout, stderr int) (c *Cmd, err os.Error) { + c = new(Cmd) var fd [3]*os.File - if fd[0], p.Stdin, err = modeToFiles(stdin, 0); err != nil { + if fd[0], c.Stdin, err = modeToFiles(stdin, 0); err != nil { goto Error } - if fd[1], p.Stdout, err = modeToFiles(stdout, 1); err != nil { + if fd[1], c.Stdout, err = modeToFiles(stdout, 1); err != nil { goto Error } if stderr == MergeWithStdout { fd[2] = fd[1] - } else if fd[2], p.Stderr, err = modeToFiles(stderr, 2); err != nil { + } else if fd[2], c.Stderr, err = modeToFiles(stderr, 2); err != nil { goto Error } // Run command. - p.Pid, err = os.ForkExec(name, argv, envv, dir, fd[0:]) + c.Process, err = os.StartProcess(name, argv, envv, dir, fd[0:]) if err != nil { goto Error } @@ -107,7 +118,7 @@ func Run(name string, argv, envv []string, dir string, stdin, stdout, stderr int if fd[2] != os.Stderr && fd[2] != fd[1] { fd[2].Close() } - return p, nil + return c, nil Error: if fd[0] != os.Stdin && fd[0] != nil { @@ -119,63 +130,67 @@ Error: if fd[2] != os.Stderr && fd[2] != nil && fd[2] != fd[1] { fd[2].Close() } - if p.Stdin != nil { - p.Stdin.Close() + if c.Stdin != nil { + c.Stdin.Close() + } + if c.Stdout != nil { + c.Stdout.Close() } - if p.Stdout != nil { - p.Stdout.Close() + if c.Stderr != nil { + c.Stderr.Close() } - if p.Stderr != nil { - p.Stderr.Close() + if c.Process != nil { + c.Process.Release() } return nil, err } -// Wait waits for the running command p, -// returning the Waitmsg returned by os.Wait and an error. -// The options are passed through to os.Wait. -// Setting options to 0 waits for p to exit; +// Wait waits for the running command c, +// returning the Waitmsg returned when the process exits. +// The options are passed to the process's Wait method. +// Setting options to 0 waits for c to exit; // other options cause Wait to return for other // process events; see package os for details. -func (p *Cmd) Wait(options int) (*os.Waitmsg, os.Error) { - if p.Pid <= 0 { +func (c *Cmd) Wait(options int) (*os.Waitmsg, os.Error) { + if c.Process == nil { return nil, os.ErrorString("exec: invalid use of Cmd.Wait") } - w, err := os.Wait(p.Pid, options) + w, err := c.Process.Wait(options) if w != nil && (w.Exited() || w.Signaled()) { - p.Pid = -1 + c.Process.Release() + c.Process = nil } return w, err } -// Close waits for the running command p to exit, +// Close waits for the running command c to exit, // if it hasn't already, and then closes the non-nil file descriptors -// p.Stdin, p.Stdout, and p.Stderr. -func (p *Cmd) Close() os.Error { - if p.Pid > 0 { +// c.Stdin, c.Stdout, and c.Stderr. +func (c *Cmd) Close() os.Error { + if c.Process != nil { // Loop on interrupt, but // ignore other errors -- maybe // caller has already waited for pid. - _, err := p.Wait(0) + _, err := c.Wait(0) for err == os.EINTR { - _, err = p.Wait(0) + _, err = c.Wait(0) } } // Close the FDs that are still open. var err os.Error - if p.Stdin != nil && p.Stdin.Fd() >= 0 { - if err1 := p.Stdin.Close(); err1 != nil { + if c.Stdin != nil && c.Stdin.Fd() >= 0 { + if err1 := c.Stdin.Close(); err1 != nil { err = err1 } } - if p.Stdout != nil && p.Stdout.Fd() >= 0 { - if err1 := p.Stdout.Close(); err1 != nil && err != nil { + if c.Stdout != nil && c.Stdout.Fd() >= 0 { + if err1 := c.Stdout.Close(); err1 != nil && err != nil { err = err1 } } - if p.Stderr != nil && p.Stderr != p.Stdout && p.Stderr.Fd() >= 0 { - if err1 := p.Stderr.Close(); err1 != nil && err != nil { + if c.Stderr != nil && c.Stderr != c.Stdout && c.Stderr.Fd() >= 0 { + if err1 := c.Stderr.Close(); err1 != nil && err != nil { err = err1 } } diff --git a/libgo/go/exec/lp_test.go b/libgo/go/exec/lp_test.go new file mode 100644 index 00000000000..54081771ecc --- /dev/null +++ b/libgo/go/exec/lp_test.go @@ -0,0 +1,33 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package exec + +import ( + "testing" +) + +var nonExistentPaths = []string{ + "some-non-existent-path", + "non-existent-path/slashed", +} + +func TestLookPathNotFound(t *testing.T) { + for _, name := range nonExistentPaths { + path, err := LookPath(name) + if err == nil { + t.Fatalf("LookPath found %q in $PATH", name) + } + if path != "" { + t.Fatalf("LookPath path == %q when err != nil", path) + } + perr, ok := err.(*PathError) + if !ok { + t.Fatal("LookPath error is not a PathError") + } + if perr.Name != name { + t.Fatalf("want PathError name %q, got %q", name, perr.Name) + } + } +} diff --git a/libgo/go/exec/lp_unix.go b/libgo/go/exec/lp_unix.go index 292e24fccdd..44f84347b99 100644 --- a/libgo/go/exec/lp_unix.go +++ b/libgo/go/exec/lp_unix.go @@ -29,7 +29,7 @@ func LookPath(file string) (string, os.Error) { if canExec(file) { return file, nil } - return "", &os.PathError{"lookpath", file, os.ENOENT} + return "", &PathError{file} } pathenv := os.Getenv("PATH") for _, dir := range strings.Split(pathenv, ":", -1) { @@ -41,5 +41,5 @@ func LookPath(file string) (string, os.Error) { return dir + "/" + file, nil } } - return "", &os.PathError{"lookpath", file, os.ENOENT} + return "", &PathError{file} } diff --git a/libgo/go/exec/lp_windows.go b/libgo/go/exec/lp_windows.go index 7b56afa8566..d357575fdbe 100644 --- a/libgo/go/exec/lp_windows.go +++ b/libgo/go/exec/lp_windows.go @@ -49,7 +49,7 @@ func LookPath(file string) (string, os.Error) { if f, ok := canExec(file, exts); ok { return f, nil } - return ``, &os.PathError{"lookpath", file, os.ENOENT} + return ``, &PathError{file} } if pathenv := os.Getenv(`PATH`); pathenv == `` { if f, ok := canExec(`.\`+file, exts); ok { @@ -62,5 +62,5 @@ func LookPath(file string) (string, os.Error) { } } } - return ``, &os.PathError{"lookpath", file, os.ENOENT} + return ``, &PathError{file} } diff --git a/libgo/go/exp/draw/x11/conn.go b/libgo/go/exp/draw/x11/conn.go index da2181536fb..e28fb217065 100644 --- a/libgo/go/exp/draw/x11/conn.go +++ b/libgo/go/exp/draw/x11/conn.go @@ -122,10 +122,13 @@ func (c *conn) writeSocket() { func (c *conn) Screen() draw.Image { return c.img } func (c *conn) FlushImage() { - // We do the send (the <- operator) in an expression context, rather than in - // a statement context, so that it does not block, and fails if the buffered - // channel is full (in which case there already is a flush request pending). - _ = c.flush <- false + select { + case c.flush <- false: + // Flush notification sent. + default: + // Could not send. + // Flush notification must be pending already. + } } func (c *conn) Close() os.Error { diff --git a/libgo/go/exp/eval/stmt.go b/libgo/go/exp/eval/stmt.go index 77ff066d09c..5c5d4338a1d 100644 --- a/libgo/go/exp/eval/stmt.go +++ b/libgo/go/exp/eval/stmt.go @@ -908,7 +908,7 @@ func (a *stmtCompiler) compileBranchStmt(s *ast.BranchStmt) { return default: - log.Panic("Unexpected branch token %v", s.Tok) + log.Panicf("Unexpected branch token %v", s.Tok) } a.flow.put1(false, pc) diff --git a/libgo/go/exp/eval/stmt_test.go b/libgo/go/exp/eval/stmt_test.go index a14a288d936..4a883ef5ee7 100644 --- a/libgo/go/exp/eval/stmt_test.go +++ b/libgo/go/exp/eval/stmt_test.go @@ -217,7 +217,7 @@ var stmtTests = []test{ Val2("if false { i = 2 } else { i = 3 }; i2 = 4", "i", 3, "i2", 4), Val2("if i == i2 { i = 2 } else { i = 3 }; i2 = 4", "i", 3, "i2", 4), // Omit optional parts - Val2("if { i = 2 } else { i = 3 }; i2 = 4", "i", 2, "i2", 4), + Val2("if true { i = 2 } else { i = 3 }; i2 = 4", "i", 2, "i2", 4), Val2("if true { i = 2 }; i2 = 4", "i", 2, "i2", 4), Val2("if false { i = 2 }; i2 = 4", "i", 1, "i2", 4), // Init @@ -243,11 +243,11 @@ var stmtTests = []test{ CErr("fn1 := func() int { if true { return 1 } }", "return"), CErr("fn1 := func() int { if true { } }", "return"), Run("fn1 := func() int { if true { }; return 1 }"), - CErr("fn1 := func() int { if { } }", "return"), - CErr("fn1 := func() int { if { } else { return 2 } }", "return"), - Run("fn1 := func() int { if { return 1 } }"), - Run("fn1 := func() int { if { return 1 } else { } }"), - Run("fn1 := func() int { if { return 1 } else { } }"), + CErr("fn1 := func() int { if true { } }", "return"), + CErr("fn1 := func() int { if true { } else { return 2 } }", "return"), + Run("fn1 := func() int { if true { return 1 }; return 0 }"), + Run("fn1 := func() int { if true { return 1 } else { }; return 0 }"), + Run("fn1 := func() int { if true { return 1 } else { }; return 0 }"), // Switch Val1("switch { case false: i += 2; case true: i += 4; default: i += 8 }", "i", 1+4), diff --git a/libgo/go/exp/wingui/gui.go b/libgo/go/exp/wingui/gui.go new file mode 100644 index 00000000000..cf392934c5b --- /dev/null +++ b/libgo/go/exp/wingui/gui.go @@ -0,0 +1,153 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "fmt" + "syscall" + "os" + "unsafe" +) + +// some help functions + +func abortf(format string, a ...interface{}) { + fmt.Fprintf(os.Stdout, format, a...) + os.Exit(1) +} + +func abortErrNo(funcname string, err int) { + abortf("%s failed: %d %s\n", funcname, err, syscall.Errstr(err)) +} + +// global vars + +var ( + mh uint32 + bh uint32 +) + +// WinProc called by windows to notify us of all windows events we might be interested in. +func WndProc(hwnd, msg uint32, wparam, lparam int32) uintptr { + var rc int32 + switch msg { + case WM_CREATE: + var e int + // CreateWindowEx + bh, e = CreateWindowEx( + 0, + syscall.StringToUTF16Ptr("button"), + syscall.StringToUTF16Ptr("Quit"), + WS_CHILD|WS_VISIBLE|BS_DEFPUSHBUTTON, + 75, 70, 140, 25, + hwnd, 1, mh, 0) + if e != 0 { + abortErrNo("CreateWindowEx", e) + } + fmt.Printf("button handle is %x\n", bh) + rc = DefWindowProc(hwnd, msg, wparam, lparam) + case WM_COMMAND: + switch uint32(lparam) { + case bh: + e := PostMessage(hwnd, WM_CLOSE, 0, 0) + if e != 0 { + abortErrNo("PostMessage", e) + } + default: + rc = DefWindowProc(hwnd, msg, wparam, lparam) + } + case WM_CLOSE: + DestroyWindow(hwnd) + case WM_DESTROY: + PostQuitMessage(0) + default: + rc = DefWindowProc(hwnd, msg, wparam, lparam) + } + //fmt.Printf("WndProc(0x%08x, %d, 0x%08x, 0x%08x) (%d)\n", hwnd, msg, wparam, lparam, rc) + return uintptr(rc) +} + +func rungui() int { + var e int + + // GetModuleHandle + mh, e = GetModuleHandle(nil) + if e != 0 { + abortErrNo("GetModuleHandle", e) + } + + // Get icon we're going to use. + myicon, e := LoadIcon(0, IDI_APPLICATION) + if e != 0 { + abortErrNo("LoadIcon", e) + } + + // Get cursor we're going to use. + mycursor, e := LoadCursor(0, IDC_ARROW) + if e != 0 { + abortErrNo("LoadCursor", e) + } + + // Create callback + wproc := syscall.NewCallback(WndProc) + + // RegisterClassEx + wcname := syscall.StringToUTF16Ptr("myWindowClass") + var wc Wndclassex + wc.Size = uint32(unsafe.Sizeof(wc)) + wc.WndProc = wproc + wc.Instance = mh + wc.Icon = myicon + wc.Cursor = mycursor + wc.Background = COLOR_BTNFACE + 1 + wc.MenuName = nil + wc.ClassName = wcname + wc.IconSm = myicon + if _, e := RegisterClassEx(&wc); e != 0 { + abortErrNo("RegisterClassEx", e) + } + + // CreateWindowEx + wh, e := CreateWindowEx( + WS_EX_CLIENTEDGE, + wcname, + syscall.StringToUTF16Ptr("My window"), + WS_OVERLAPPEDWINDOW, + CW_USEDEFAULT, CW_USEDEFAULT, 300, 200, + 0, 0, mh, 0) + if e != 0 { + abortErrNo("CreateWindowEx", e) + } + fmt.Printf("main window handle is %x\n", wh) + + // ShowWindow + ShowWindow(wh, SW_SHOWDEFAULT) + + // UpdateWindow + if e := UpdateWindow(wh); e != 0 { + abortErrNo("UpdateWindow", e) + } + + // Process all windows messages until WM_QUIT. + var m Msg + for { + r, e := GetMessage(&m, 0, 0, 0) + if e != 0 { + abortErrNo("GetMessage", e) + } + if r == 0 { + // WM_QUIT received -> get out + break + } + TranslateMessage(&m) + DispatchMessage(&m) + } + return int(m.Wparam) +} + +func main() { + rc := rungui() + os.Exit(rc) +} diff --git a/libgo/go/exp/wingui/winapi.go b/libgo/go/exp/wingui/winapi.go new file mode 100644 index 00000000000..c96f452999f --- /dev/null +++ b/libgo/go/exp/wingui/winapi.go @@ -0,0 +1,148 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "syscall" + "unsafe" +) + +func loadDll(fname string) uint32 { + h, e := syscall.LoadLibrary(fname) + if e != 0 { + abortf("LoadLibrary(%s) failed with err=%d.\n", fname, e) + } + return h +} + +func getSysProcAddr(m uint32, pname string) uintptr { + p, e := syscall.GetProcAddress(m, pname) + if e != 0 { + abortf("GetProcAddress(%s) failed with err=%d.\n", pname, e) + } + return uintptr(p) +} + +type Wndclassex struct { + Size uint32 + Style uint32 + WndProc uintptr + ClsExtra int32 + WndExtra int32 + Instance uint32 + Icon uint32 + Cursor uint32 + Background uint32 + MenuName *uint16 + ClassName *uint16 + IconSm uint32 +} + +type Point struct { + X int32 + Y int32 +} + +type Msg struct { + Hwnd uint32 + Message uint32 + Wparam int32 + Lparam int32 + Time uint32 + Pt Point +} + +const ( + // Window styles + WS_OVERLAPPED = 0 + WS_POPUP = 0x80000000 + WS_CHILD = 0x40000000 + WS_MINIMIZE = 0x20000000 + WS_VISIBLE = 0x10000000 + WS_DISABLED = 0x8000000 + WS_CLIPSIBLINGS = 0x4000000 + WS_CLIPCHILDREN = 0x2000000 + WS_MAXIMIZE = 0x1000000 + WS_CAPTION = WS_BORDER | WS_DLGFRAME + WS_BORDER = 0x800000 + WS_DLGFRAME = 0x400000 + WS_VSCROLL = 0x200000 + WS_HSCROLL = 0x100000 + WS_SYSMENU = 0x80000 + WS_THICKFRAME = 0x40000 + WS_GROUP = 0x20000 + WS_TABSTOP = 0x10000 + WS_MINIMIZEBOX = 0x20000 + WS_MAXIMIZEBOX = 0x10000 + WS_TILED = WS_OVERLAPPED + WS_ICONIC = WS_MINIMIZE + WS_SIZEBOX = WS_THICKFRAME + // Common Window Styles + WS_OVERLAPPEDWINDOW = WS_OVERLAPPED | WS_CAPTION | WS_SYSMENU | WS_THICKFRAME | WS_MINIMIZEBOX | WS_MAXIMIZEBOX + WS_TILEDWINDOW = WS_OVERLAPPEDWINDOW + WS_POPUPWINDOW = WS_POPUP | WS_BORDER | WS_SYSMENU + WS_CHILDWINDOW = WS_CHILD + + WS_EX_CLIENTEDGE = 0x200 + + // Some windows messages + WM_CREATE = 1 + WM_DESTROY = 2 + WM_CLOSE = 16 + WM_COMMAND = 273 + + // Some button control styles + BS_DEFPUSHBUTTON = 1 + + // Some colour constants + COLOR_WINDOW = 5 + COLOR_BTNFACE = 15 + + // Default window position + CW_USEDEFAULT = 0x80000000 - 0x100000000 + + // Show window default style + SW_SHOWDEFAULT = 10 +) + +var ( + // Some globaly known cusrors + IDC_ARROW = MakeIntResource(32512) + IDC_IBEAM = MakeIntResource(32513) + IDC_WAIT = MakeIntResource(32514) + IDC_CROSS = MakeIntResource(32515) + + // Some globaly known icons + IDI_APPLICATION = MakeIntResource(32512) + IDI_HAND = MakeIntResource(32513) + IDI_QUESTION = MakeIntResource(32514) + IDI_EXCLAMATION = MakeIntResource(32515) + IDI_ASTERISK = MakeIntResource(32516) + IDI_WINLOGO = MakeIntResource(32517) + IDI_WARNING = IDI_EXCLAMATION + IDI_ERROR = IDI_HAND + IDI_INFORMATION = IDI_ASTERISK +) + +//sys GetModuleHandle(modname *uint16) (handle uint32, errno int) = GetModuleHandleW +//sys RegisterClassEx(wndclass *Wndclassex) (atom uint16, errno int) = user32.RegisterClassExW +//sys CreateWindowEx(exstyle uint32, classname *uint16, windowname *uint16, style uint32, x int32, y int32, width int32, height int32, wndparent uint32, menu uint32, instance uint32, param uintptr) (hwnd uint32, errno int) = user32.CreateWindowExW +//sys DefWindowProc(hwnd uint32, msg uint32, wparam int32, lparam int32) (lresult int32) = user32.DefWindowProcW +//sys DestroyWindow(hwnd uint32) (errno int) = user32.DestroyWindow +//sys PostQuitMessage(exitcode int32) = user32.PostQuitMessage +//sys ShowWindow(hwnd uint32, cmdshow int32) (wasvisible bool) = user32.ShowWindow +//sys UpdateWindow(hwnd uint32) (errno int) = user32.UpdateWindow +//sys GetMessage(msg *Msg, hwnd uint32, MsgFilterMin uint32, MsgFilterMax uint32) (ret int32, errno int) [failretval==-1] = user32.GetMessageW +//sys TranslateMessage(msg *Msg) (done bool) = user32.TranslateMessage +//sys DispatchMessage(msg *Msg) (ret int32) = user32.DispatchMessageW +//sys LoadIcon(instance uint32, iconname *uint16) (icon uint32, errno int) = user32.LoadIconW +//sys LoadCursor(instance uint32, cursorname *uint16) (cursor uint32, errno int) = user32.LoadCursorW +//sys SetCursor(cursor uint32) (precursor uint32, errno int) = user32.SetCursor +//sys SendMessage(hwnd uint32, msg uint32, wparam int32, lparam int32) (lresult int32) = user32.SendMessageW +//sys PostMessage(hwnd uint32, msg uint32, wparam int32, lparam int32) (errno int) = user32.PostMessageW + +func MakeIntResource(id uint16) *uint16 { + return (*uint16)(unsafe.Pointer(uintptr(id))) +} diff --git a/libgo/go/exp/wingui/zwinapi.go b/libgo/go/exp/wingui/zwinapi.go new file mode 100644 index 00000000000..60aaac6cf16 --- /dev/null +++ b/libgo/go/exp/wingui/zwinapi.go @@ -0,0 +1,211 @@ +// mksyscall_windows.sh winapi.go +// MACHINE GENERATED BY THE COMMAND ABOVE; DO NOT EDIT + +package main + +import "unsafe" +import "syscall" + +var ( + modkernel32 = loadDll("kernel32.dll") + moduser32 = loadDll("user32.dll") + + procGetModuleHandleW = getSysProcAddr(modkernel32, "GetModuleHandleW") + procRegisterClassExW = getSysProcAddr(moduser32, "RegisterClassExW") + procCreateWindowExW = getSysProcAddr(moduser32, "CreateWindowExW") + procDefWindowProcW = getSysProcAddr(moduser32, "DefWindowProcW") + procDestroyWindow = getSysProcAddr(moduser32, "DestroyWindow") + procPostQuitMessage = getSysProcAddr(moduser32, "PostQuitMessage") + procShowWindow = getSysProcAddr(moduser32, "ShowWindow") + procUpdateWindow = getSysProcAddr(moduser32, "UpdateWindow") + procGetMessageW = getSysProcAddr(moduser32, "GetMessageW") + procTranslateMessage = getSysProcAddr(moduser32, "TranslateMessage") + procDispatchMessageW = getSysProcAddr(moduser32, "DispatchMessageW") + procLoadIconW = getSysProcAddr(moduser32, "LoadIconW") + procLoadCursorW = getSysProcAddr(moduser32, "LoadCursorW") + procSetCursor = getSysProcAddr(moduser32, "SetCursor") + procSendMessageW = getSysProcAddr(moduser32, "SendMessageW") + procPostMessageW = getSysProcAddr(moduser32, "PostMessageW") +) + +func GetModuleHandle(modname *uint16) (handle uint32, errno int) { + r0, _, e1 := syscall.Syscall(procGetModuleHandleW, 1, uintptr(unsafe.Pointer(modname)), 0, 0) + handle = uint32(r0) + if handle == 0 { + if e1 != 0 { + errno = int(e1) + } else { + errno = syscall.EINVAL + } + } else { + errno = 0 + } + return +} + +func RegisterClassEx(wndclass *Wndclassex) (atom uint16, errno int) { + r0, _, e1 := syscall.Syscall(procRegisterClassExW, 1, uintptr(unsafe.Pointer(wndclass)), 0, 0) + atom = uint16(r0) + if atom == 0 { + if e1 != 0 { + errno = int(e1) + } else { + errno = syscall.EINVAL + } + } else { + errno = 0 + } + return +} + +func CreateWindowEx(exstyle uint32, classname *uint16, windowname *uint16, style uint32, x int32, y int32, width int32, height int32, wndparent uint32, menu uint32, instance uint32, param uintptr) (hwnd uint32, errno int) { + r0, _, e1 := syscall.Syscall12(procCreateWindowExW, 12, uintptr(exstyle), uintptr(unsafe.Pointer(classname)), uintptr(unsafe.Pointer(windowname)), uintptr(style), uintptr(x), uintptr(y), uintptr(width), uintptr(height), uintptr(wndparent), uintptr(menu), uintptr(instance), uintptr(param)) + hwnd = uint32(r0) + if hwnd == 0 { + if e1 != 0 { + errno = int(e1) + } else { + errno = syscall.EINVAL + } + } else { + errno = 0 + } + return +} + +func DefWindowProc(hwnd uint32, msg uint32, wparam int32, lparam int32) (lresult int32) { + r0, _, _ := syscall.Syscall6(procDefWindowProcW, 4, uintptr(hwnd), uintptr(msg), uintptr(wparam), uintptr(lparam), 0, 0) + lresult = int32(r0) + return +} + +func DestroyWindow(hwnd uint32) (errno int) { + r1, _, e1 := syscall.Syscall(procDestroyWindow, 1, uintptr(hwnd), 0, 0) + if int(r1) == 0 { + if e1 != 0 { + errno = int(e1) + } else { + errno = syscall.EINVAL + } + } else { + errno = 0 + } + return +} + +func PostQuitMessage(exitcode int32) { + syscall.Syscall(procPostQuitMessage, 1, uintptr(exitcode), 0, 0) + return +} + +func ShowWindow(hwnd uint32, cmdshow int32) (wasvisible bool) { + r0, _, _ := syscall.Syscall(procShowWindow, 2, uintptr(hwnd), uintptr(cmdshow), 0) + wasvisible = bool(r0 != 0) + return +} + +func UpdateWindow(hwnd uint32) (errno int) { + r1, _, e1 := syscall.Syscall(procUpdateWindow, 1, uintptr(hwnd), 0, 0) + if int(r1) == 0 { + if e1 != 0 { + errno = int(e1) + } else { + errno = syscall.EINVAL + } + } else { + errno = 0 + } + return +} + +func GetMessage(msg *Msg, hwnd uint32, MsgFilterMin uint32, MsgFilterMax uint32) (ret int32, errno int) { + r0, _, e1 := syscall.Syscall6(procGetMessageW, 4, uintptr(unsafe.Pointer(msg)), uintptr(hwnd), uintptr(MsgFilterMin), uintptr(MsgFilterMax), 0, 0) + ret = int32(r0) + if ret == -1 { + if e1 != 0 { + errno = int(e1) + } else { + errno = syscall.EINVAL + } + } else { + errno = 0 + } + return +} + +func TranslateMessage(msg *Msg) (done bool) { + r0, _, _ := syscall.Syscall(procTranslateMessage, 1, uintptr(unsafe.Pointer(msg)), 0, 0) + done = bool(r0 != 0) + return +} + +func DispatchMessage(msg *Msg) (ret int32) { + r0, _, _ := syscall.Syscall(procDispatchMessageW, 1, uintptr(unsafe.Pointer(msg)), 0, 0) + ret = int32(r0) + return +} + +func LoadIcon(instance uint32, iconname *uint16) (icon uint32, errno int) { + r0, _, e1 := syscall.Syscall(procLoadIconW, 2, uintptr(instance), uintptr(unsafe.Pointer(iconname)), 0) + icon = uint32(r0) + if icon == 0 { + if e1 != 0 { + errno = int(e1) + } else { + errno = syscall.EINVAL + } + } else { + errno = 0 + } + return +} + +func LoadCursor(instance uint32, cursorname *uint16) (cursor uint32, errno int) { + r0, _, e1 := syscall.Syscall(procLoadCursorW, 2, uintptr(instance), uintptr(unsafe.Pointer(cursorname)), 0) + cursor = uint32(r0) + if cursor == 0 { + if e1 != 0 { + errno = int(e1) + } else { + errno = syscall.EINVAL + } + } else { + errno = 0 + } + return +} + +func SetCursor(cursor uint32) (precursor uint32, errno int) { + r0, _, e1 := syscall.Syscall(procSetCursor, 1, uintptr(cursor), 0, 0) + precursor = uint32(r0) + if precursor == 0 { + if e1 != 0 { + errno = int(e1) + } else { + errno = syscall.EINVAL + } + } else { + errno = 0 + } + return +} + +func SendMessage(hwnd uint32, msg uint32, wparam int32, lparam int32) (lresult int32) { + r0, _, _ := syscall.Syscall6(procSendMessageW, 4, uintptr(hwnd), uintptr(msg), uintptr(wparam), uintptr(lparam), 0, 0) + lresult = int32(r0) + return +} + +func PostMessage(hwnd uint32, msg uint32, wparam int32, lparam int32) (errno int) { + r1, _, e1 := syscall.Syscall6(procPostMessageW, 4, uintptr(hwnd), uintptr(msg), uintptr(wparam), uintptr(lparam), 0, 0) + if int(r1) == 0 { + if e1 != 0 { + errno = int(e1) + } else { + errno = syscall.EINVAL + } + } else { + errno = 0 + } + return +} diff --git a/libgo/go/flag/flag.go b/libgo/go/flag/flag.go index 143a1061150..be972057ed7 100644 --- a/libgo/go/flag/flag.go +++ b/libgo/go/flag/flag.go @@ -96,7 +96,7 @@ func newIntValue(val int, p *int) *intValue { } func (i *intValue) Set(s string) bool { - v, err := strconv.Atoi(s) + v, err := strconv.Btoi64(s, 0) *i = intValue(v) return err == nil } @@ -112,7 +112,7 @@ func newInt64Value(val int64, p *int64) *int64Value { } func (i *int64Value) Set(s string) bool { - v, err := strconv.Atoi64(s) + v, err := strconv.Btoi64(s, 0) *i = int64Value(v) return err == nil } @@ -128,7 +128,7 @@ func newUintValue(val uint, p *uint) *uintValue { } func (i *uintValue) Set(s string) bool { - v, err := strconv.Atoui(s) + v, err := strconv.Btoui64(s, 0) *i = uintValue(v) return err == nil } @@ -144,7 +144,7 @@ func newUint64Value(val uint64, p *uint64) *uint64Value { } func (i *uint64Value) Set(s string) bool { - v, err := strconv.Atoui64(s) + v, err := strconv.Btoui64(s, 0) *i = uint64Value(v) return err == nil } diff --git a/libgo/go/flag/flag_test.go b/libgo/go/flag/flag_test.go index b91a8b56795..30a21e61ae2 100644 --- a/libgo/go/flag/flag_test.go +++ b/libgo/go/flag/flag_test.go @@ -106,7 +106,7 @@ func TestParse(t *testing.T) { "-bool", "-bool2=true", "--int", "22", - "--int64", "23", + "--int64", "0x23", "-uint", "24", "--uint64", "25", "-string", "hello", @@ -125,8 +125,8 @@ func TestParse(t *testing.T) { if *intFlag != 22 { t.Error("int flag should be 22, is ", *intFlag) } - if *int64Flag != 23 { - t.Error("int64 flag should be 23, is ", *int64Flag) + if *int64Flag != 0x23 { + t.Error("int64 flag should be 0x23, is ", *int64Flag) } if *uintFlag != 24 { t.Error("uint flag should be 24, is ", *uintFlag) diff --git a/libgo/go/fmt/doc.go b/libgo/go/fmt/doc.go index 191bf68b13b..77ee62bb1dd 100644 --- a/libgo/go/fmt/doc.go +++ b/libgo/go/fmt/doc.go @@ -16,6 +16,7 @@ when printing structs, the plus flag (%+v) adds field names %#v a Go-syntax representation of the value %T a Go-syntax representation of the type of the value + %% a literal percent sign; consumes no value Boolean: %t the word true or false @@ -26,8 +27,10 @@ %o base 8 %x base 16, with lower-case letters for a-f %X base 16, with upper-case letters for A-F - %U unicode format: U+1234; same as "U+%x" with 4 digits default + %U Unicode format: U+1234; same as "U+%x" with 4 digits default Floating-point and complex constituents: + %b decimalless scientific notation with exponent a power + of two, in the manner of strconv.Ftoa32, e.g. -123456p-78 %e scientific notation, e.g. -1234.456e+78 %E scientific notation, e.g. -1234.456E+78 %f decimal point but no exponent, e.g. 123.456 @@ -44,14 +47,19 @@ There is no 'u' flag. Integers are printed unsigned if they have unsigned type. Similarly, there is no need to specify the size of the operand (int8, int64). - For numeric values, the width and precision flags control - formatting; width sets the width of the field, precision the - number of places after the decimal, if appropriate. The - format %6.2f prints 123.45. The width of a field is the number - of Unicode code points in the string. This differs from C's printf where - the field width is the number of bytes. Either or both of the - flags may be replaced with the character '*', causing their values - to be obtained from the next operand, which must be of type int. + The width and precision control formatting and are in units of Unicode + code points. (This differs from C's printf where the units are numbers + of bytes.) Either or both of the flags may be replaced with the + character '*', causing their values to be obtained from the next + operand, which must be of type int. + + For numeric values, width sets the width of the field and precision + sets the number of places after the decimal, if appropriate. For + example, the format %6.2f prints 123.45. + + For strings, width is the minimum number of characters to output, + padding with spaces if necessary, and precision is the maximum + number of characters to output, truncating if necessary. Other flags: + always print a sign for numeric values @@ -112,7 +120,7 @@ An analogous set of functions scans formatted text to yield values. Scan, Scanf and Scanln read from os.Stdin; Fscan, Fscanf and Fscanln read from a specified os.Reader; Sscan, - Sscanf and Sscanln read from an argument string. Sscanln, + Sscanf and Sscanln read from an argument string. Scanln, Fscanln and Sscanln stop scanning at a newline and require that the items be followed by one; Sscanf, Fscanf and Sscanf require newlines in the input to match newlines in the format; the other @@ -131,6 +139,10 @@ %e %E %f %F %g %g are all equivalent and scan any floating point or complex value %s and %v on strings scan a space-delimited token + The familiar base-setting prefixes 0 (octal) and 0x + (hexadecimal) are accepted when scanning integers without a + format or with the %v verb. + Width is interpreted in the input text (%5s means at most five runes of input will be read to scan a string) but there is no syntax for scanning with a precision (no %5.2f, just @@ -152,13 +164,15 @@ All arguments to be scanned must be either pointers to basic types or implementations of the Scanner interface. - Note: Fscan etc. can read one character (rune) past the - input they return, which means that a loop calling a scan - routine may skip some of the input. This is usually a - problem only when there is no space between input values. - However, if the reader provided to Fscan implements UnreadRune, + Note: Fscan etc. can read one character (rune) past the input + they return, which means that a loop calling a scan routine + may skip some of the input. This is usually a problem only + when there is no space between input values. If the reader + provided to Fscan implements ReadRune, that method will be used + to read characters. If the reader also implements UnreadRune, that method will be used to save the character and successive - calls will not lose data. To attach an UnreadRune method - to a reader without that capability, use bufio.NewReader. + calls will not lose data. To attach ReadRune and UnreadRune + methods to a reader without that capability, use + bufio.NewReader. */ package fmt diff --git a/libgo/go/fmt/fmt_test.go b/libgo/go/fmt/fmt_test.go index 3f085b72245..c8aa6090bbf 100644 --- a/libgo/go/fmt/fmt_test.go +++ b/libgo/go/fmt/fmt_test.go @@ -311,9 +311,9 @@ var fmttests = []struct { // go syntax {"%#v", A{1, 2, "a", []int{1, 2}}, `fmt_test.A{i:1, j:0x2, s:"a", x:[]int{1, 2}}`}, - {"%#v", &b, "(*uint8)(PTR)"}, - {"%#v", TestFmtInterface, "(func(*testing.T))(PTR)"}, - {"%#v", make(chan int), "(chan int)(PTR)"}, + {"%#v", &b, "(*uint8)(0xPTR)"}, + {"%#v", TestFmtInterface, "(func(*testing.T))(0xPTR)"}, + {"%#v", make(chan int), "(chan int)(0xPTR)"}, {"%#v", uint64(1<<64 - 1), "0xffffffffffffffff"}, {"%#v", 1000000000, "1000000000"}, {"%#v", map[string]int{"a": 1, "b": 2}, `map[string] int{"a":1, "b":2}`}, @@ -365,14 +365,15 @@ var fmttests = []struct { {"%6T", &intVal, " *int"}, // %p - {"p0=%p", new(int), "p0=PTR"}, + {"p0=%p", new(int), "p0=0xPTR"}, {"p1=%s", &pValue, "p1=String(p)"}, // String method... - {"p2=%p", &pValue, "p2=PTR"}, // ... not called with %p + {"p2=%p", &pValue, "p2=0xPTR"}, // ... not called with %p + {"p4=%#p", new(int), "p4=PTR"}, // %p on non-pointers - {"%p", make(chan int), "PTR"}, - {"%p", make(map[int]int), "PTR"}, - {"%p", make([]int, 1), "PTR"}, + {"%p", make(chan int), "0xPTR"}, + {"%p", make(map[int]int), "0xPTR"}, + {"%p", make([]int, 1), "0xPTR"}, {"%p", 27, "%!p(int=27)"}, // not a pointer at all // erroneous things @@ -388,8 +389,8 @@ var fmttests = []struct { func TestSprintf(t *testing.T) { for _, tt := range fmttests { s := Sprintf(tt.fmt, tt.val) - if i := strings.Index(s, "0x"); i >= 0 && strings.Contains(tt.out, "PTR") { - j := i + 2 + if i := strings.Index(tt.out, "PTR"); i >= 0 { + j := i for ; j < len(s); j++ { c := s[j] if (c < '0' || c > '9') && (c < 'a' || c > 'f') && (c < 'A' || c > 'F') { diff --git a/libgo/go/fmt/print.go b/libgo/go/fmt/print.go index 96029a8789f..4e14fdaa4a7 100644 --- a/libgo/go/fmt/print.go +++ b/libgo/go/fmt/print.go @@ -74,15 +74,42 @@ type pp struct { fmt fmt } -// A leaky bucket of reusable pp structures. -var ppFree = make(chan *pp, 100) +// A cache holds a set of reusable objects. +// The buffered channel holds the currently available objects. +// If more are needed, the cache creates them by calling new. +type cache struct { + saved chan interface{} + new func() interface{} +} + +func (c *cache) put(x interface{}) { + select { + case c.saved <- x: + // saved in cache + default: + // discard + } +} -// Allocate a new pp struct. Probably can grab the previous one from ppFree. -func newPrinter() *pp { - p, ok := <-ppFree - if !ok { - p = new(pp) +func (c *cache) get() interface{} { + select { + case x := <-c.saved: + return x // reused from cache + default: + return c.new() } + panic("not reached") +} + +func newCache(f func() interface{}) *cache { + return &cache{make(chan interface{}, 100), f} +} + +var ppFree = newCache(func() interface{} { return new(pp) }) + +// Allocate a new pp struct or grab a cached one. +func newPrinter() *pp { + p := ppFree.get().(*pp) p.fmt.init(&p.buf) return p } @@ -94,7 +121,7 @@ func (p *pp) free() { return } p.buf.Reset() - _ = ppFree <- p + ppFree.put(p) } func (p *pp) Width() (wid int, ok bool) { return p.fmt.wid, p.fmt.widPresent } @@ -321,11 +348,11 @@ func (p *pp) fmtInt64(v int64, verb int, value interface{}) { } } -// fmt0x64 formats a uint64 in hexadecimal and prefixes it with 0x by -// temporarily turning on the sharp flag. -func (p *pp) fmt0x64(v uint64) { +// fmt0x64 formats a uint64 in hexadecimal and prefixes it with 0x or +// not, as requested, by temporarily setting the sharp flag. +func (p *pp) fmt0x64(v uint64, leading0x bool) { sharp := p.fmt.sharp - p.fmt.sharp = true // turn on 0x + p.fmt.sharp = leading0x p.fmt.integer(int64(v), 16, unsigned, ldigits) p.fmt.sharp = sharp } @@ -357,7 +384,7 @@ func (p *pp) fmtUint64(v uint64, verb int, goSyntax bool, value interface{}) { p.fmt.integer(int64(v), 10, unsigned, ldigits) case 'v': if goSyntax { - p.fmt0x64(v) + p.fmt0x64(v, true) } else { p.fmt.integer(int64(v), 10, unsigned, ldigits) } @@ -507,11 +534,11 @@ func (p *pp) fmtPointer(field interface{}, value reflect.Value, verb int, goSynt if u == 0 { p.buf.Write(nilBytes) } else { - p.fmt0x64(uint64(v.Get())) + p.fmt0x64(uint64(v.Get()), true) } p.add(')') } else { - p.fmt0x64(uint64(u)) + p.fmt0x64(uint64(u), !p.fmt.sharp) } } @@ -774,7 +801,7 @@ BigSwitch: if v == 0 { p.buf.Write(nilBytes) } else { - p.fmt0x64(uint64(v)) + p.fmt0x64(uint64(v), true) } p.buf.WriteByte(')') break @@ -783,7 +810,7 @@ BigSwitch: p.buf.Write(nilAngleBytes) break } - p.fmt0x64(uint64(v)) + p.fmt0x64(uint64(v), true) case uintptrGetter: p.fmtPointer(field, value, verb, goSyntax) default: diff --git a/libgo/go/fmt/scan.go b/libgo/go/fmt/scan.go index ebbb17155e4..c0f2bacb69b 100644 --- a/libgo/go/fmt/scan.go +++ b/libgo/go/fmt/scan.go @@ -7,6 +7,7 @@ package fmt import ( "bytes" "io" + "math" "os" "reflect" "strconv" @@ -15,18 +16,11 @@ import ( "utf8" ) -// readRuner is the interface to something that can read runes. If -// the object provided to Scan does not satisfy this interface, the -// object will be wrapped by a readRune object. -type readRuner interface { - ReadRune() (rune int, size int, err os.Error) -} - -// unreadRuner is the interface to something that can unread runes. +// runeUnreader is the interface to something that can unread runes. // If the object provided to Scan does not satisfy this interface, // a local buffer will be used to back up the input, but its contents // will be lost when Scan returns. -type unreadRuner interface { +type runeUnreader interface { UnreadRune() os.Error } @@ -34,23 +28,30 @@ type unreadRuner interface { // Scanners may do rune-at-a-time scanning or ask the ScanState // to discover the next space-delimited token. type ScanState interface { - // GetRune reads the next rune (Unicode code point) from the input. - GetRune() (rune int, err os.Error) - // UngetRune causes the next call to GetRune to return the rune. - UngetRune() - // Width returns the value of the width option and whether it has been set. - // The unit is Unicode code points. - Width() (wid int, ok bool) + // ReadRune reads the next rune (Unicode code point) from the input. + // If invoked during Scanln, Fscanln, or Sscanln, ReadRune() will + // return EOF after returning the first '\n' or when reading beyond + // the specified width. + ReadRune() (rune int, size int, err os.Error) + // UnreadRune causes the next call to ReadRune to return the same rune. + UnreadRune() os.Error // Token returns the next space-delimited token from the input. If // a width has been specified, the returned token will be no longer // than the width. Token() (token string, err os.Error) + // Width returns the value of the width option and whether it has been set. + // The unit is Unicode code points. + Width() (wid int, ok bool) + // Because ReadRune is implemented by the interface, Read should never be + // called by the scanning routines and a valid implementation of + // ScanState may choose always to return an error from Read. + Read(buf []byte) (n int, err os.Error) } // Scanner is implemented by any value that has a Scan method, which scans // the input for the representation of a value and stores the result in the // receiver, which must be a pointer to be useful. The Scan method is called -// for any argument to Scan or Scanln that implements it. +// for any argument to Scan, Scanf, or Scanln that implements it. type Scanner interface { Scan(state ScanState, verb int) os.Error } @@ -102,18 +103,18 @@ func Sscanf(str string, format string, a ...interface{}) (n int, err os.Error) { // returns the number of items successfully scanned. If that is less // than the number of arguments, err will report why. func Fscan(r io.Reader, a ...interface{}) (n int, err os.Error) { - s := newScanState(r, true) + s, old := newScanState(r, true, false) n, err = s.doScan(a) - s.free() + s.free(old) return } // Fscanln is similar to Fscan, but stops scanning at a newline and // after the final item there must be a newline or EOF. func Fscanln(r io.Reader, a ...interface{}) (n int, err os.Error) { - s := newScanState(r, false) + s, old := newScanState(r, false, true) n, err = s.doScan(a) - s.free() + s.free(old) return } @@ -121,9 +122,9 @@ func Fscanln(r io.Reader, a ...interface{}) (n int, err os.Error) { // values into successive arguments as determined by the format. It // returns the number of items successfully parsed. func Fscanf(r io.Reader, format string, a ...interface{}) (n int, err os.Error) { - s := newScanState(r, false) + s, old := newScanState(r, false, false) n, err = s.doScanf(format, a) - s.free() + s.free(old) return } @@ -137,53 +138,70 @@ const EOF = -1 // ss is the internal implementation of ScanState. type ss struct { - rr readRuner // where to read input - buf bytes.Buffer // token accumulator - nlIsSpace bool // whether newline counts as white space - peekRune int // one-rune lookahead - prevRune int // last rune returned by GetRune - atEOF bool // already read EOF - maxWid int // max width of field, in runes - widPresent bool // width was specified - wid int // width consumed so far; used in accept() -} - -func (s *ss) GetRune() (rune int, err os.Error) { + rr io.RuneReader // where to read input + buf bytes.Buffer // token accumulator + peekRune int // one-rune lookahead + prevRune int // last rune returned by ReadRune + count int // runes consumed so far. + atEOF bool // already read EOF + ssave +} + +// ssave holds the parts of ss that need to be +// saved and restored on recursive scans. +type ssave struct { + validSave bool // is or was a part of an actual ss. + nlIsEnd bool // whether newline terminates scan + nlIsSpace bool // whether newline counts as white space + fieldLimit int // max value of ss.count for this field; fieldLimit <= limit + limit int // max value of ss.count. + maxWid int // width of this field. +} + +// The Read method is only in ScanState so that ScanState +// satisfies io.Reader. It will never be called when used as +// intended, so there is no need to make it actually work. +func (s *ss) Read(buf []byte) (n int, err os.Error) { + return 0, os.ErrorString("ScanState's Read should not be called. Use ReadRune") +} + +func (s *ss) ReadRune() (rune int, size int, err os.Error) { if s.peekRune >= 0 { + s.count++ rune = s.peekRune + size = utf8.RuneLen(rune) s.prevRune = rune s.peekRune = -1 return } - rune, _, err = s.rr.ReadRune() + if s.atEOF || s.nlIsEnd && s.prevRune == '\n' || s.count >= s.fieldLimit { + err = os.EOF + return + } + + rune, size, err = s.rr.ReadRune() if err == nil { + s.count++ s.prevRune = rune + } else if err == os.EOF { + s.atEOF = true } return } func (s *ss) Width() (wid int, ok bool) { - return s.maxWid, s.widPresent + if s.maxWid == hugeWid { + return 0, false + } + return s.maxWid, true } // The public method returns an error; this private one panics. // If getRune reaches EOF, the return value is EOF (-1). func (s *ss) getRune() (rune int) { - if s.atEOF { - return EOF - } - if s.peekRune >= 0 { - rune = s.peekRune - s.prevRune = rune - s.peekRune = -1 - return - } - rune, _, err := s.rr.ReadRune() - if err == nil { - s.prevRune = rune - } else if err != nil { + rune, _, err := s.ReadRune() + if err != nil { if err == os.EOF { - s.atEOF = true return EOF } s.error(err) @@ -191,35 +209,25 @@ func (s *ss) getRune() (rune int) { return } -// mustGetRune turns os.EOF into a panic(io.ErrUnexpectedEOF). +// mustReadRune turns os.EOF into a panic(io.ErrUnexpectedEOF). // It is called in cases such as string scanning where an EOF is a // syntax error. -func (s *ss) mustGetRune() (rune int) { - if s.atEOF { +func (s *ss) mustReadRune() (rune int) { + rune = s.getRune() + if rune == EOF { s.error(io.ErrUnexpectedEOF) } - if s.peekRune >= 0 { - rune = s.peekRune - s.peekRune = -1 - return - } - rune, _, err := s.rr.ReadRune() - if err != nil { - if err == os.EOF { - err = io.ErrUnexpectedEOF - } - s.error(err) - } return } - -func (s *ss) UngetRune() { - if u, ok := s.rr.(unreadRuner); ok { +func (s *ss) UnreadRune() os.Error { + if u, ok := s.rr.(runeUnreader); ok { u.UnreadRune() } else { s.peekRune = s.prevRune } + s.count-- + return nil } func (s *ss) error(err os.Error) { @@ -246,7 +254,7 @@ func (s *ss) Token() (tok string, err os.Error) { // readRune is a structure to enable reading UTF-8 encoded code points // from an io.Reader. It is used if the Reader given to the scanner does -// not already implement ReadRuner. +// not already implement io.RuneReader. type readRune struct { reader io.Reader buf [utf8.UTFMax]byte // used only inside ReadRune @@ -303,37 +311,53 @@ func (r *readRune) ReadRune() (rune int, size int, err os.Error) { } -// A leaky bucket of reusable ss structures. -var ssFree = make(chan *ss, 100) +var ssFree = newCache(func() interface{} { return new(ss) }) -// Allocate a new ss struct. Probably can grab the previous one from ssFree. -func newScanState(r io.Reader, nlIsSpace bool) *ss { - s, ok := <-ssFree - if !ok { - s = new(ss) +// Allocate a new ss struct or grab a cached one. +func newScanState(r io.Reader, nlIsSpace, nlIsEnd bool) (s *ss, old ssave) { + // If the reader is a *ss, then we've got a recursive + // call to Scan, so re-use the scan state. + s, ok := r.(*ss) + if ok { + old = s.ssave + s.limit = s.fieldLimit + s.nlIsEnd = nlIsEnd || s.nlIsEnd + s.nlIsSpace = nlIsSpace + return } - if rr, ok := r.(readRuner); ok { + + s = ssFree.get().(*ss) + if rr, ok := r.(io.RuneReader); ok { s.rr = rr } else { s.rr = &readRune{reader: r} } s.nlIsSpace = nlIsSpace + s.nlIsEnd = nlIsEnd + s.prevRune = -1 s.peekRune = -1 s.atEOF = false - s.maxWid = 0 - s.widPresent = false - return s + s.limit = hugeWid + s.fieldLimit = hugeWid + s.maxWid = hugeWid + s.validSave = true + return } // Save used ss structs in ssFree; avoid an allocation per invocation. -func (s *ss) free() { +func (s *ss) free(old ssave) { + // If it was used recursively, just restore the old state. + if old.validSave { + s.ssave = old + return + } // Don't hold on to ss structs with large buffers. if cap(s.buf.Bytes()) > 1024 { return } s.buf.Reset() s.rr = nil - _ = ssFree <- s + ssFree.put(s) } // skipSpace skips spaces and maybe newlines. @@ -354,7 +378,7 @@ func (s *ss) skipSpace(stopAtNewline bool) { return } if !unicode.IsSpace(rune) { - s.UngetRune() + s.UnreadRune() break } } @@ -366,13 +390,13 @@ func (s *ss) skipSpace(stopAtNewline bool) { func (s *ss) token() string { s.skipSpace(false) // read until white space or newline - for nrunes := 0; !s.widPresent || nrunes < s.maxWid; nrunes++ { + for { rune := s.getRune() if rune == EOF { break } if unicode.IsSpace(rune) { - s.UngetRune() + s.UnreadRune() break } s.buf.WriteRune(rune) @@ -391,28 +415,31 @@ var boolError = os.ErrorString("syntax error scanning boolean") // consume reads the next rune in the input and reports whether it is in the ok string. // If accept is true, it puts the character into the input token. func (s *ss) consume(ok string, accept bool) bool { - if s.wid >= s.maxWid { - return false - } rune := s.getRune() if rune == EOF { return false } - for i := 0; i < len(ok); i++ { - if int(ok[i]) == rune { - if accept { - s.buf.WriteRune(rune) - s.wid++ - } - return true + if strings.IndexRune(ok, rune) >= 0 { + if accept { + s.buf.WriteRune(rune) } + return true } if rune != EOF && accept { - s.UngetRune() + s.UnreadRune() } return false } +// peek reports whether the next character is in the ok string, without consuming it. +func (s *ss) peek(ok string) bool { + rune := s.getRune() + if rune != EOF { + s.UnreadRune() + } + return strings.IndexRune(ok, rune) >= 0 +} + // accept checks the next rune in the input. If it's a byte (sic) in the string, it puts it in the // buffer and returns true. Otherwise it return false. func (s *ss) accept(ok string) bool { @@ -436,7 +463,7 @@ func (s *ss) scanBool(verb int) bool { return false } // Syntax-checking a boolean is annoying. We're not fastidious about case. - switch s.mustGetRune() { + switch s.mustReadRune() { case '0': return false case '1': @@ -463,7 +490,7 @@ const ( hexadecimalDigits = "0123456789aAbBcCdDeEfF" sign = "+-" period = "." - exponent = "eE" + exponent = "eEp" ) // getBase returns the numeric base represented by the verb and its digit string. @@ -486,8 +513,8 @@ func (s *ss) getBase(verb int) (base int, digits string) { } // scanNumber returns the numerical string with specified digits starting here. -func (s *ss) scanNumber(digits string) string { - if !s.accept(digits) { +func (s *ss) scanNumber(digits string, haveDigits bool) string { + if !haveDigits && !s.accept(digits) { s.errorString("expected integer") } for s.accept(digits) { @@ -497,7 +524,7 @@ func (s *ss) scanNumber(digits string) string { // scanRune returns the next rune value in the input. func (s *ss) scanRune(bitSize int) int64 { - rune := int64(s.mustGetRune()) + rune := int64(s.mustReadRune()) n := uint(bitSize) x := (rune << (64 - n)) >> (64 - n) if x != rune { @@ -506,22 +533,44 @@ func (s *ss) scanRune(bitSize int) int64 { return rune } +// scanBasePrefix reports whether the integer begins with a 0 or 0x, +// and returns the base, digit string, and whether a zero was found. +// It is called only if the verb is %v. +func (s *ss) scanBasePrefix() (base int, digits string, found bool) { + if !s.peek("0") { + return 10, decimalDigits, false + } + s.accept("0") + found = true // We've put a digit into the token buffer. + // Special cases for '0' && '0x' + base, digits = 8, octalDigits + if s.peek("xX") { + s.consume("xX", false) + base, digits = 16, hexadecimalDigits + } + return +} + // scanInt returns the value of the integer represented by the next // token, checking for overflow. Any error is stored in s.err. func (s *ss) scanInt(verb int, bitSize int) int64 { if verb == 'c' { return s.scanRune(bitSize) } - base, digits := s.getBase(verb) s.skipSpace(false) + base, digits := s.getBase(verb) + haveDigits := false if verb == 'U' { if !s.consume("U", false) || !s.consume("+", false) { s.errorString("bad unicode format ") } } else { s.accept(sign) // If there's a sign, it will be left in the token buffer. + if verb == 'v' { + base, digits, haveDigits = s.scanBasePrefix() + } } - tok := s.scanNumber(digits) + tok := s.scanNumber(digits, haveDigits) i, err := strconv.Btoi64(tok, base) if err != nil { s.error(err) @@ -540,14 +589,17 @@ func (s *ss) scanUint(verb int, bitSize int) uint64 { if verb == 'c' { return uint64(s.scanRune(bitSize)) } - base, digits := s.getBase(verb) s.skipSpace(false) + base, digits := s.getBase(verb) + haveDigits := false if verb == 'U' { if !s.consume("U", false) || !s.consume("+", false) { s.errorString("bad unicode format ") } + } else if verb == 'v' { + base, digits, haveDigits = s.scanBasePrefix() } - tok := s.scanNumber(digits) + tok := s.scanNumber(digits, haveDigits) i, err := strconv.Btoui64(tok, base) if err != nil { s.error(err) @@ -621,6 +673,27 @@ func (s *ss) complexTokens() (real, imag string) { // convertFloat converts the string to a float64value. func (s *ss) convertFloat(str string, n int) float64 { + if p := strings.Index(str, "p"); p >= 0 { + // Atof doesn't handle power-of-2 exponents, + // but they're easy to evaluate. + f, err := strconv.AtofN(str[:p], n) + if err != nil { + // Put full string into error. + if e, ok := err.(*strconv.NumError); ok { + e.Num = str + } + s.error(err) + } + n, err := strconv.Atoi(str[p+1:]) + if err != nil { + // Put full string into error. + if e, ok := err.(*strconv.NumError); ok { + e.Num = str + } + s.error(err) + } + return math.Ldexp(f, n) + } f, err := strconv.AtofN(str, n) if err != nil { s.error(err) @@ -667,12 +740,12 @@ func (s *ss) convertString(verb int) (str string) { // quotedString returns the double- or back-quoted string represented by the next input characters. func (s *ss) quotedString() string { - quote := s.mustGetRune() + quote := s.mustReadRune() switch quote { case '`': // Back-quoted: Anything goes until EOF or back quote. for { - rune := s.mustGetRune() + rune := s.mustReadRune() if rune == quote { break } @@ -683,13 +756,13 @@ func (s *ss) quotedString() string { // Double-quoted: Include the quotes and let strconv.Unquote do the backslash escapes. s.buf.WriteRune(quote) for { - rune := s.mustGetRune() + rune := s.mustReadRune() s.buf.WriteRune(rune) if rune == '\\' { // In a legal backslash escape, no matter how long, only the character // immediately after the escape can itself be a backslash or quote. // Thus we only need to protect the first character after the backslash. - rune := s.mustGetRune() + rune := s.mustReadRune() s.buf.WriteRune(rune) } else if rune == '"' { break @@ -728,10 +801,10 @@ func (s *ss) hexByte() (b byte, ok bool) { return } if unicode.IsSpace(rune1) { - s.UngetRune() + s.UnreadRune() return } - rune2 := s.mustGetRune() + rune2 := s.mustReadRune() return byte(s.hexDigit(rune1)<<4 | s.hexDigit(rune2)), true } @@ -751,7 +824,9 @@ func (s *ss) hexString() string { return s.buf.String() } -const floatVerbs = "eEfFgGv" +const floatVerbs = "beEfFgGv" + +const hugeWid = 1 << 30 // scanOne scans a single value, deriving the scanner from the type of the argument. func (s *ss) scanOne(verb int, field interface{}) { @@ -761,14 +836,13 @@ func (s *ss) scanOne(verb int, field interface{}) { if v, ok := field.(Scanner); ok { err = v.Scan(s, verb) if err != nil { + if err == os.EOF { + err = io.ErrUnexpectedEOF + } s.error(err) } return } - if !s.widPresent { - s.maxWid = 1 << 30 // Huge - } - s.wid = 0 switch v := field.(type) { case *bool: *v = s.scanBool(verb) @@ -869,7 +943,6 @@ func errorHandler(errp *os.Error) { } // doScan does the real work for scanning without a format string. -// At the moment, it handles only pointers to basic types. func (s *ss) doScan(a []interface{}) (numProcessed int, err os.Error) { defer errorHandler(&err) for _, field := range a { @@ -930,9 +1003,9 @@ func (s *ss) advance(format string) (i int) { s.skipSpace(true) continue } - inputc := s.mustGetRune() + inputc := s.mustReadRune() if fmtc != inputc { - s.UngetRune() + s.UnreadRune() return -1 } i += w @@ -964,7 +1037,15 @@ func (s *ss) doScanf(format string, a []interface{}) (numProcessed int, err os.E i++ // % is one byte // do we have 20 (width)? - s.maxWid, s.widPresent, i = parsenum(format, i, end) + var widPresent bool + s.maxWid, widPresent, i = parsenum(format, i, end) + if !widPresent { + s.maxWid = hugeWid + } + s.fieldLimit = s.limit + if f := s.count + s.maxWid; f < s.fieldLimit { + s.fieldLimit = f + } c, w := utf8.DecodeRuneInString(format[i:]) i += w @@ -977,6 +1058,7 @@ func (s *ss) doScanf(format string, a []interface{}) (numProcessed int, err os.E s.scanOne(c, field) numProcessed++ + s.fieldLimit = s.limit } if numProcessed < len(a) { s.errorString("too many operands") diff --git a/libgo/go/fmt/scan_test.go b/libgo/go/fmt/scan_test.go index 78b9fbb4ab0..65adb023686 100644 --- a/libgo/go/fmt/scan_test.go +++ b/libgo/go/fmt/scan_test.go @@ -6,6 +6,7 @@ package fmt_test import ( "bufio" + "bytes" . "fmt" "io" "math" @@ -87,21 +88,7 @@ type FloatTest struct { type Xs string func (x *Xs) Scan(state ScanState, verb int) os.Error { - var tok string - var c int - var err os.Error - wid, present := state.Width() - if !present { - tok, err = state.Token() - } else { - for i := 0; i < wid; i++ { - c, err = state.GetRune() - if err != nil { - break - } - tok += string(c) - } - } + tok, err := state.Token() if err != nil { return err } @@ -114,6 +101,26 @@ func (x *Xs) Scan(state ScanState, verb int) os.Error { var xVal Xs +// IntString accepts an integer followed immediately by a string. +// It tests the embedding of a scan within a scan. +type IntString struct { + i int + s string +} + +func (s *IntString) Scan(state ScanState, verb int) os.Error { + if _, err := Fscan(state, &s.i); err != nil { + return err + } + + if _, err := Fscan(state, &s.s); err != nil { + return err + } + return nil +} + +var intStringVal IntString + // myStringReader implements Read but not ReadRune, allowing us to test our readRune wrapper // type that creates something that can read runes given only Read(). type myStringReader struct { @@ -129,10 +136,20 @@ func newReader(s string) *myStringReader { } var scanTests = []ScanTest{ - // Numbers + // Basic types {"T\n", &boolVal, true}, // boolean test vals toggle to be sure they are written {"F\n", &boolVal, false}, // restored to zero value {"21\n", &intVal, 21}, + {"0\n", &intVal, 0}, + {"000\n", &intVal, 0}, + {"0x10\n", &intVal, 0x10}, + {"-0x10\n", &intVal, -0x10}, + {"0377\n", &intVal, 0377}, + {"-0377\n", &intVal, -0377}, + {"0\n", &uintVal, uint(0)}, + {"000\n", &uintVal, uint(0)}, + {"0x10\n", &uintVal, uint(0x10)}, + {"0377\n", &uintVal, uint(0377)}, {"22\n", &int8Val, int8(22)}, {"23\n", &int16Val, int16(23)}, {"24\n", &int32Val, int32(24)}, @@ -160,6 +177,10 @@ var scanTests = []ScanTest{ {"2.3\n", &float64Val, 2.3}, {"2.3e1\n", &float32Val, float32(2.3e1)}, {"2.3e2\n", &float64Val, 2.3e2}, + {"2.3p2\n", &float64Val, 2.3 * 4}, + {"2.3p+2\n", &float64Val, 2.3 * 4}, + {"2.3p+66\n", &float64Val, 2.3 * (1 << 32) * (1 << 32) * 4}, + {"2.3p-66\n", &float64Val, 2.3 / ((1 << 32) * (1 << 32) * 4)}, {"2.35\n", &stringVal, "2.35"}, {"2345678\n", &bytesVal, []byte("2345678")}, {"(3.4e1-2i)\n", &complex128Val, 3.4e1 - 2i}, @@ -186,8 +207,9 @@ var scanTests = []ScanTest{ {"114\n", &renamedStringVal, renamedString("114")}, {"115\n", &renamedBytesVal, renamedBytes([]byte("115"))}, - // Custom scanner. + // Custom scanners. {" vvv ", &xVal, Xs("vvv")}, + {" 1234hello", &intStringVal, IntString{1234, "hello"}}, // Fixed bugs {"2147483648\n", &int64Val, int64(2147483648)}, // was: integer overflow @@ -197,6 +219,8 @@ var scanfTests = []ScanfTest{ {"%v", "TRUE\n", &boolVal, true}, {"%t", "false\n", &boolVal, false}, {"%v", "-71\n", &intVal, -71}, + {"%v", "0377\n", &intVal, 0377}, + {"%v", "0x44\n", &intVal, 0x44}, {"%d", "72\n", &intVal, 72}, {"%c", "a\n", &intVal, 'a'}, {"%c", "\u5072\n", &intVal, 0x5072}, @@ -292,6 +316,7 @@ var f float64 var s, t string var c complex128 var x, y Xs +var z IntString var multiTests = []ScanfMultiTest{ {"", "", nil, nil, ""}, @@ -305,8 +330,9 @@ var multiTests = []ScanfMultiTest{ {"%d%s", "123abc", args(&i, &s), args(123, "abc"), ""}, {"%c%c%c", "2\u50c2X", args(&i, &j, &k), args('2', '\u50c2', 'X'), ""}, - // Custom scanner. + // Custom scanners. {"%2e%f", "eefffff", args(&x, &y), args(Xs("ee"), Xs("fffff")), ""}, + {"%4v%s", "12abcd", args(&z, &s), args(IntString{12, "ab"}, "cd"), ""}, // Errors {"%t", "23 18", args(&i), nil, "bad verb"}, @@ -329,7 +355,11 @@ func testScan(name string, t *testing.T, scan func(r io.Reader, a ...interface{} } n, err := scan(r, test.in) if err != nil { - t.Errorf("%s got error scanning %q: %s", name, test.text, err) + m := "" + if n > 0 { + m = Sprintf(" (%d fields ok)", n) + } + t.Errorf("%s got error scanning %q: %s%s", name, test.text, err, m) continue } if n != 1 { @@ -657,3 +687,178 @@ func TestUnreadRuneWithBufio(t *testing.T) { t.Errorf("expected αb; got %q", a) } } + +type TwoLines string + +// Attempt to read two lines into the object. Scanln should prevent this +// because it stops at newline; Scan and Scanf should be fine. +func (t *TwoLines) Scan(state ScanState, verb int) os.Error { + chars := make([]int, 0, 100) + for nlCount := 0; nlCount < 2; { + c, _, err := state.ReadRune() + if err != nil { + return err + } + chars = append(chars, c) + if c == '\n' { + nlCount++ + } + } + *t = TwoLines(string(chars)) + return nil +} + +func TestMultiLine(t *testing.T) { + input := "abc\ndef\n" + // Sscan should work + var tscan TwoLines + n, err := Sscan(input, &tscan) + if n != 1 { + t.Errorf("Sscan: expected 1 item; got %d", n) + } + if err != nil { + t.Errorf("Sscan: expected no error; got %s", err) + } + if string(tscan) != input { + t.Errorf("Sscan: expected %q; got %q", input, tscan) + } + // Sscanf should work + var tscanf TwoLines + n, err = Sscanf(input, "%s", &tscanf) + if n != 1 { + t.Errorf("Sscanf: expected 1 item; got %d", n) + } + if err != nil { + t.Errorf("Sscanf: expected no error; got %s", err) + } + if string(tscanf) != input { + t.Errorf("Sscanf: expected %q; got %q", input, tscanf) + } + // Sscanln should not work + var tscanln TwoLines + n, err = Sscanln(input, &tscanln) + if n != 0 { + t.Errorf("Sscanln: expected 0 items; got %d: %q", n, tscanln) + } + if err == nil { + t.Error("Sscanln: expected error; got none") + } else if err != io.ErrUnexpectedEOF { + t.Errorf("Sscanln: expected io.ErrUnexpectedEOF (ha!); got %s", err) + } +} + +// RecursiveInt accepts an string matching %d.%d.%d.... +// and parses it into a linked list. +// It allows us to benchmark recursive descent style scanners. +type RecursiveInt struct { + i int + next *RecursiveInt +} + +func (r *RecursiveInt) Scan(state ScanState, verb int) (err os.Error) { + _, err = Fscan(state, &r.i) + if err != nil { + return + } + next := new(RecursiveInt) + _, err = Fscanf(state, ".%v", next) + if err != nil { + if err == os.ErrorString("input does not match format") || err == io.ErrUnexpectedEOF { + err = nil + } + return + } + r.next = next + return +} + +// Perform the same scanning task as RecursiveInt.Scan +// but without recurring through scanner, so we can compare +// performance more directly. +func scanInts(r *RecursiveInt, b *bytes.Buffer) (err os.Error) { + r.next = nil + _, err = Fscan(b, &r.i) + if err != nil { + return + } + var c int + c, _, err = b.ReadRune() + if err != nil { + if err == os.EOF { + err = nil + } + return + } + if c != '.' { + return + } + next := new(RecursiveInt) + err = scanInts(next, b) + if err == nil { + r.next = next + } + return +} + +func makeInts(n int) []byte { + var buf bytes.Buffer + Fprintf(&buf, "1") + for i := 1; i < n; i++ { + Fprintf(&buf, ".%d", i+1) + } + return buf.Bytes() +} + +func TestScanInts(t *testing.T) { + testScanInts(t, scanInts) + testScanInts(t, func(r *RecursiveInt, b *bytes.Buffer) (err os.Error) { + _, err = Fscan(b, r) + return + }) +} + +const intCount = 1000 + +func testScanInts(t *testing.T, scan func(*RecursiveInt, *bytes.Buffer) os.Error) { + r := new(RecursiveInt) + ints := makeInts(intCount) + buf := bytes.NewBuffer(ints) + err := scan(r, buf) + if err != nil { + t.Error("unexpected error", err) + } + i := 1 + for ; r != nil; r = r.next { + if r.i != i { + t.Fatal("bad scan: expected %d got %d", i, r.i) + } + i++ + } + if i-1 != intCount { + t.Fatal("bad scan count: expected %d got %d", intCount, i-1) + } +} + +func BenchmarkScanInts(b *testing.B) { + b.ResetTimer() + ints := makeInts(intCount) + var r RecursiveInt + for i := b.N - 1; i >= 0; i-- { + buf := bytes.NewBuffer(ints) + b.StartTimer() + scanInts(&r, buf) + b.StopTimer() + } +} + +func BenchmarkScanRecursiveInt(b *testing.B) { + b.ResetTimer() + ints := makeInts(intCount) + var r RecursiveInt + for i := b.N - 1; i >= 0; i-- { + buf := bytes.NewBuffer(ints) + b.StartTimer() + Fscan(buf, &r) + b.StopTimer() + } +} diff --git a/libgo/go/go/ast/ast.go b/libgo/go/go/ast/ast.go index cf2ce36df88..abafb5663b3 100644 --- a/libgo/go/go/ast/ast.go +++ b/libgo/go/go/ast/ast.go @@ -535,6 +535,13 @@ type ( X Expr // expression } + // A SendStmt node represents a send statement. + SendStmt struct { + Chan Expr + Arrow token.Pos // position of "<-" + Value Expr + } + // An IncDecStmt node represents an increment or decrement statement. IncDecStmt struct { X Expr @@ -590,7 +597,7 @@ type ( IfStmt struct { If token.Pos // position of "if" keyword Init Stmt // initalization statement; or nil - Cond Expr // condition; or nil + Cond Expr // condition Body *BlockStmt Else Stmt // else branch; or nil } @@ -629,11 +636,10 @@ type ( // A CommClause node represents a case of a select statement. CommClause struct { - Case token.Pos // position of "case" or "default" keyword - Tok token.Token // ASSIGN or DEFINE (valid only if Lhs != nil) - Lhs, Rhs Expr // Rhs == nil means default case - Colon token.Pos // position of ":" - Body []Stmt // statement list; or nil + Case token.Pos // position of "case" or "default" keyword + Comm Stmt // send or receive statement; nil means default case + Colon token.Pos // position of ":" + Body []Stmt // statement list; or nil } // An SelectStmt node represents a select statement. @@ -670,6 +676,7 @@ func (s *DeclStmt) Pos() token.Pos { return s.Decl.Pos() } func (s *EmptyStmt) Pos() token.Pos { return s.Semicolon } func (s *LabeledStmt) Pos() token.Pos { return s.Label.Pos() } func (s *ExprStmt) Pos() token.Pos { return s.X.Pos() } +func (s *SendStmt) Pos() token.Pos { return s.Chan.Pos() } func (s *IncDecStmt) Pos() token.Pos { return s.X.Pos() } func (s *AssignStmt) Pos() token.Pos { return s.Lhs[0].Pos() } func (s *GoStmt) Pos() token.Pos { return s.Go } @@ -695,6 +702,7 @@ func (s *EmptyStmt) End() token.Pos { } func (s *LabeledStmt) End() token.Pos { return s.Stmt.End() } func (s *ExprStmt) End() token.Pos { return s.X.End() } +func (s *SendStmt) End() token.Pos { return s.Value.End() } func (s *IncDecStmt) End() token.Pos { return s.TokPos + 2 /* len("++") */ } @@ -753,6 +761,7 @@ func (s *DeclStmt) stmtNode() {} func (s *EmptyStmt) stmtNode() {} func (s *LabeledStmt) stmtNode() {} func (s *ExprStmt) stmtNode() {} +func (s *SendStmt) stmtNode() {} func (s *IncDecStmt) stmtNode() {} func (s *AssignStmt) stmtNode() {} func (s *GoStmt) stmtNode() {} diff --git a/libgo/go/go/ast/walk.go b/libgo/go/go/ast/walk.go index 875a92f3f49..20c337c3be9 100644 --- a/libgo/go/go/ast/walk.go +++ b/libgo/go/go/ast/walk.go @@ -195,6 +195,10 @@ func Walk(v Visitor, node Node) { case *ExprStmt: Walk(v, n.X) + case *SendStmt: + Walk(v, n.Chan) + Walk(v, n.Value) + case *IncDecStmt: Walk(v, n.X) @@ -223,9 +227,7 @@ func Walk(v Visitor, node Node) { if n.Init != nil { Walk(v, n.Init) } - if n.Cond != nil { - Walk(v, n.Cond) - } + Walk(v, n.Cond) Walk(v, n.Body) if n.Else != nil { Walk(v, n.Else) @@ -258,11 +260,8 @@ func Walk(v Visitor, node Node) { Walk(v, n.Body) case *CommClause: - if n.Lhs != nil { - Walk(v, n.Lhs) - } - if n.Rhs != nil { - Walk(v, n.Rhs) + if n.Comm != nil { + Walk(v, n.Comm) } walkStmtList(v, n.Body) diff --git a/libgo/go/go/parser/parser.go b/libgo/go/go/parser/parser.go index f1746e04055..7c5843f3637 100644 --- a/libgo/go/go/parser/parser.go +++ b/libgo/go/go/parser/parser.go @@ -1193,18 +1193,6 @@ func (p *parser) parseSimpleStmt(labelOk bool) ast.Stmt { x := p.parseExprList() switch p.tok { - case token.COLON: - // labeled statement - colon := p.pos - p.next() - if labelOk && len(x) == 1 { - if label, isIdent := x[0].(*ast.Ident); isIdent { - return &ast.LabeledStmt{label, colon, p.parseStmt()} - } - } - p.error(x[0].Pos(), "illegal label declaration") - return &ast.BadStmt{x[0].Pos(), colon + 1} - case token.DEFINE, token.ASSIGN, token.ADD_ASSIGN, token.SUB_ASSIGN, token.MUL_ASSIGN, token.QUO_ASSIGN, @@ -1218,11 +1206,29 @@ func (p *parser) parseSimpleStmt(labelOk bool) ast.Stmt { } if len(x) > 1 { - p.error(x[0].Pos(), "only one expression allowed") + p.errorExpected(x[0].Pos(), "1 expression") // continue with first expression } - if p.tok == token.INC || p.tok == token.DEC { + switch p.tok { + case token.COLON: + // labeled statement + colon := p.pos + p.next() + if label, isIdent := x[0].(*ast.Ident); labelOk && isIdent { + return &ast.LabeledStmt{label, colon, p.parseStmt()} + } + p.error(x[0].Pos(), "illegal label declaration") + return &ast.BadStmt{x[0].Pos(), colon + 1} + + case token.ARROW: + // send statement + arrow := p.pos + p.next() // consume "<-" + y := p.parseExpr() + return &ast.SendStmt{x[0], arrow, y} + + case token.INC, token.DEC: // increment or decrement s := &ast.IncDecStmt{x[0], p.pos, p.tok} p.next() // consume "++" or "--" @@ -1321,44 +1327,34 @@ func (p *parser) makeExpr(s ast.Stmt) ast.Expr { } -func (p *parser) parseControlClause(isForStmt bool) (s1, s2, s3 ast.Stmt) { - if p.tok != token.LBRACE { +func (p *parser) parseIfStmt() *ast.IfStmt { + if p.trace { + defer un(trace(p, "IfStmt")) + } + + pos := p.expect(token.IF) + + var s ast.Stmt + var x ast.Expr + { prevLev := p.exprLev p.exprLev = -1 - - if p.tok != token.SEMICOLON { - s1 = p.parseSimpleStmt(false) - } if p.tok == token.SEMICOLON { p.next() - if p.tok != token.LBRACE && p.tok != token.SEMICOLON { - s2 = p.parseSimpleStmt(false) - } - if isForStmt { - // for statements have a 3rd section - p.expectSemi() - if p.tok != token.LBRACE { - s3 = p.parseSimpleStmt(false) - } - } + x = p.parseExpr() } else { - s1, s2 = nil, s1 + s = p.parseSimpleStmt(false) + if p.tok == token.SEMICOLON { + p.next() + x = p.parseExpr() + } else { + x = p.makeExpr(s) + s = nil + } } - p.exprLev = prevLev } - return s1, s2, s3 -} - - -func (p *parser) parseIfStmt() *ast.IfStmt { - if p.trace { - defer un(trace(p, "IfStmt")) - } - - pos := p.expect(token.IF) - s1, s2, _ := p.parseControlClause(false) body := p.parseBlockStmt() var else_ ast.Stmt if p.tok == token.ELSE { @@ -1368,7 +1364,7 @@ func (p *parser) parseIfStmt() *ast.IfStmt { p.expectSemi() } - return &ast.IfStmt{pos, s1, p.makeExpr(s2), body, else_} + return &ast.IfStmt{pos, s, x, body, else_} } @@ -1451,7 +1447,24 @@ func (p *parser) parseSwitchStmt() ast.Stmt { } pos := p.expect(token.SWITCH) - s1, s2, _ := p.parseControlClause(false) + + var s1, s2 ast.Stmt + if p.tok != token.LBRACE { + prevLev := p.exprLev + p.exprLev = -1 + if p.tok != token.SEMICOLON { + s2 = p.parseSimpleStmt(false) + } + if p.tok == token.SEMICOLON { + p.next() + s1 = s2 + s2 = nil + if p.tok != token.LBRACE { + s2 = p.parseSimpleStmt(false) + } + } + p.exprLev = prevLev + } if isExprSwitch(s2) { lbrace := p.expect(token.LBRACE) @@ -1486,28 +1499,52 @@ func (p *parser) parseCommClause() *ast.CommClause { // CommCase pos := p.pos - var tok token.Token - var lhs, rhs ast.Expr + var comm ast.Stmt if p.tok == token.CASE { p.next() + lhs := p.parseExprList() if p.tok == token.ARROW { - // RecvExpr without assignment - rhs = p.parseExpr() + // SendStmt + if len(lhs) > 1 { + p.errorExpected(lhs[0].Pos(), "1 expression") + // continue with first expression + } + arrow := p.pos + p.next() + rhs := p.parseExpr() + comm = &ast.SendStmt{lhs[0], arrow, rhs} } else { - // SendExpr or RecvExpr - rhs = p.parseExpr() + // RecvStmt + pos := p.pos + tok := p.tok + var rhs ast.Expr if p.tok == token.ASSIGN || p.tok == token.DEFINE { - // RecvExpr with assignment - tok = p.tok + // RecvStmt with assignment + if len(lhs) > 2 { + p.errorExpected(lhs[0].Pos(), "1 or 2 expressions") + // continue with first two expressions + lhs = lhs[0:2] + } p.next() - lhs = rhs - if p.tok == token.ARROW { - rhs = p.parseExpr() - } else { - p.expect(token.ARROW) // use expect() error handling + rhs = p.parseExpr() + } else { + // rhs must be single receive operation + if len(lhs) > 1 { + p.errorExpected(lhs[0].Pos(), "1 expression") + // continue with first expression } + rhs = lhs[0] + lhs = nil // there is no lhs + } + if x, isUnary := rhs.(*ast.UnaryExpr); !isUnary || x.Op != token.ARROW { + p.errorExpected(rhs.Pos(), "send or receive operation") + rhs = &ast.BadExpr{rhs.Pos(), rhs.End()} + } + if lhs != nil { + comm = &ast.AssignStmt{lhs, pos, tok, []ast.Expr{rhs}} + } else { + comm = &ast.ExprStmt{rhs} } - // else SendExpr } } else { p.expect(token.DEFAULT) @@ -1516,7 +1553,7 @@ func (p *parser) parseCommClause() *ast.CommClause { colon := p.expect(token.COLON) body := p.parseStmtList() - return &ast.CommClause{pos, tok, lhs, rhs, colon, body} + return &ast.CommClause{pos, comm, colon, body} } @@ -1545,7 +1582,29 @@ func (p *parser) parseForStmt() ast.Stmt { } pos := p.expect(token.FOR) - s1, s2, s3 := p.parseControlClause(true) + + var s1, s2, s3 ast.Stmt + if p.tok != token.LBRACE { + prevLev := p.exprLev + p.exprLev = -1 + if p.tok != token.SEMICOLON { + s2 = p.parseSimpleStmt(false) + } + if p.tok == token.SEMICOLON { + p.next() + s1 = s2 + s2 = nil + if p.tok != token.SEMICOLON { + s2 = p.parseSimpleStmt(false) + } + p.expectSemi() + if p.tok != token.LBRACE { + s3 = p.parseSimpleStmt(false) + } + } + p.exprLev = prevLev + } + body := p.parseBlockStmt() p.expectSemi() @@ -1568,7 +1627,7 @@ func (p *parser) parseForStmt() ast.Stmt { } // check rhs if len(as.Rhs) != 1 { - p.errorExpected(as.Rhs[0].Pos(), "1 expressions") + p.errorExpected(as.Rhs[0].Pos(), "1 expression") return &ast.BadStmt{pos, body.End()} } if rhs, isUnary := as.Rhs[0].(*ast.UnaryExpr); isUnary && rhs.Op == token.RANGE { diff --git a/libgo/go/go/parser/parser_test.go b/libgo/go/go/parser/parser_test.go index 56bd80ef1fc..38535627a75 100644 --- a/libgo/go/go/parser/parser_test.go +++ b/libgo/go/go/parser/parser_test.go @@ -18,6 +18,9 @@ var illegalInputs = []interface{}{ 3.14, []byte(nil), "foo!", + `package p; func f() { if /* should have condition */ {} };`, + `package p; func f() { if ; /* should have condition */ {} };`, + `package p; func f() { if f(); /* should have condition */ {} };`, } @@ -32,20 +35,23 @@ func TestParseIllegalInputs(t *testing.T) { var validPrograms = []interface{}{ - "package main\n", - `package main;`, - `package main; import "fmt"; func main() { fmt.Println("Hello, World!") };`, - `package main; func main() { if f(T{}) {} };`, - `package main; func main() { _ = (<-chan int)(x) };`, - `package main; func main() { _ = (<-chan <-chan int)(x) };`, - `package main; func f(func() func() func());`, - `package main; func f(...T);`, - `package main; func f(float, ...int);`, - `package main; func f(x int, a ...int) { f(0, a...); f(1, a...,) };`, - `package main; type T []int; var a []bool; func f() { if a[T{42}[0]] {} };`, - `package main; type T []int; func g(int) bool { return true }; func f() { if g(T{42}[0]) {} };`, - `package main; type T []int; func f() { for _ = range []int{T{42}[0]} {} };`, - `package main; var a = T{{1, 2}, {3, 4}}`, + "package p\n", + `package p;`, + `package p; import "fmt"; func f() { fmt.Println("Hello, World!") };`, + `package p; func f() { if f(T{}) {} };`, + `package p; func f() { _ = (<-chan int)(x) };`, + `package p; func f() { _ = (<-chan <-chan int)(x) };`, + `package p; func f(func() func() func());`, + `package p; func f(...T);`, + `package p; func f(float, ...int);`, + `package p; func f(x int, a ...int) { f(0, a...); f(1, a...,) };`, + `package p; type T []int; var a []bool; func f() { if a[T{42}[0]] {} };`, + `package p; type T []int; func g(int) bool { return true }; func f() { if g(T{42}[0]) {} };`, + `package p; type T []int; func f() { for _ = range []int{T{42}[0]} {} };`, + `package p; var a = T{{1, 2}, {3, 4}}`, + `package p; func f() { select { case <- c: case c <- d: case c <- <- d: case <-c <- d: } };`, + `package p; func f() { if ; true {} };`, + `package p; func f() { switch ; {} };`, } diff --git a/libgo/go/go/printer/nodes.go b/libgo/go/go/printer/nodes.go index 8207996dcdc..7933c2f1820 100644 --- a/libgo/go/go/printer/nodes.go +++ b/libgo/go/go/printer/nodes.go @@ -506,12 +506,12 @@ const ( ) -func walkBinary(e *ast.BinaryExpr) (has5, has6 bool, maxProblem int) { +func walkBinary(e *ast.BinaryExpr) (has4, has5 bool, maxProblem int) { switch e.Op.Precedence() { + case 4: + has4 = true case 5: has5 = true - case 6: - has6 = true } switch l := e.X.(type) { @@ -521,9 +521,9 @@ func walkBinary(e *ast.BinaryExpr) (has5, has6 bool, maxProblem int) { // pretend this is an *ast.ParenExpr and do nothing. break } - h5, h6, mp := walkBinary(l) + h4, h5, mp := walkBinary(l) + has4 = has4 || h4 has5 = has5 || h5 - has6 = has6 || h6 if maxProblem < mp { maxProblem = mp } @@ -536,25 +536,25 @@ func walkBinary(e *ast.BinaryExpr) (has5, has6 bool, maxProblem int) { // pretend this is an *ast.ParenExpr and do nothing. break } - h5, h6, mp := walkBinary(r) + h4, h5, mp := walkBinary(r) + has4 = has4 || h4 has5 = has5 || h5 - has6 = has6 || h6 if maxProblem < mp { maxProblem = mp } case *ast.StarExpr: if e.Op.String() == "/" { - maxProblem = 6 + maxProblem = 5 } case *ast.UnaryExpr: switch e.Op.String() + r.Op.String() { case "/*", "&&", "&^": - maxProblem = 6 + maxProblem = 5 case "++", "--": - if maxProblem < 5 { - maxProblem = 5 + if maxProblem < 4 { + maxProblem = 4 } } } @@ -563,20 +563,20 @@ func walkBinary(e *ast.BinaryExpr) (has5, has6 bool, maxProblem int) { func cutoff(e *ast.BinaryExpr, depth int) int { - has5, has6, maxProblem := walkBinary(e) + has4, has5, maxProblem := walkBinary(e) if maxProblem > 0 { return maxProblem + 1 } - if has5 && has6 { + if has4 && has5 { if depth == 1 { - return 6 + return 5 } - return 5 + return 4 } if depth == 1 { - return 7 + return 6 } - return 5 + return 4 } @@ -603,15 +603,14 @@ func reduceDepth(depth int) int { // (Algorithm suggestion by Russ Cox.) // // The precedences are: -// 6 * / % << >> & &^ -// 5 + - | ^ -// 4 == != < <= > >= -// 3 <- +// 5 * / % << >> & &^ +// 4 + - | ^ +// 3 == != < <= > >= // 2 && // 1 || // -// The only decision is whether there will be spaces around levels 5 and 6. -// There are never spaces at level 7 (unary), and always spaces at levels 4 and below. +// The only decision is whether there will be spaces around levels 4 and 5. +// There are never spaces at level 6 (unary), and always spaces at levels 3 and below. // // To choose the cutoff, look at the whole expression but excluding primary // expressions (function calls, parenthesized exprs), and apply these rules: @@ -619,21 +618,21 @@ func reduceDepth(depth int) int { // 1) If there is a binary operator with a right side unary operand // that would clash without a space, the cutoff must be (in order): // -// /* 7 -// && 7 -// &^ 7 -// ++ 6 -// -- 6 +// /* 6 +// && 6 +// &^ 6 +// ++ 5 +// -- 5 // // (Comparison operators always have spaces around them.) // -// 2) If there is a mix of level 6 and level 5 operators, then the cutoff -// is 6 (use spaces to distinguish precedence) in Normal mode -// and 5 (never use spaces) in Compact mode. +// 2) If there is a mix of level 5 and level 4 operators, then the cutoff +// is 5 (use spaces to distinguish precedence) in Normal mode +// and 4 (never use spaces) in Compact mode. // -// 3) If there are no level 5 operators or no level 6 operators, then the -// cutoff is 7 (always use spaces) in Normal mode -// and 5 (never use spaces) in Compact mode. +// 3) If there are no level 4 operators or no level 5 operators, then the +// cutoff is 6 (always use spaces) in Normal mode +// and 4 (never use spaces) in Compact mode. // // Sets multiLine to true if the binary expression spans multiple lines. func (p *printer) binaryExpr(x *ast.BinaryExpr, prec1, cutoff, depth int, multiLine *bool) { @@ -1083,6 +1082,12 @@ func (p *printer) stmt(stmt ast.Stmt, nextIsRBrace bool, multiLine *bool) { const depth = 1 p.expr0(s.X, depth, multiLine) + case *ast.SendStmt: + const depth = 1 + p.expr0(s.Chan, depth, multiLine) + p.print(blank, s.Arrow, token.ARROW, blank) + p.expr0(s.Value, depth, multiLine) + case *ast.IncDecStmt: const depth = 1 p.expr0(s.X, depth+1, multiLine) @@ -1179,13 +1184,9 @@ func (p *printer) stmt(stmt ast.Stmt, nextIsRBrace bool, multiLine *bool) { *multiLine = true case *ast.CommClause: - if s.Rhs != nil { + if s.Comm != nil { p.print(token.CASE, blank) - if s.Lhs != nil { - p.expr(s.Lhs, multiLine) - p.print(blank, s.Tok, blank) - } - p.expr(s.Rhs, multiLine) + p.stmt(s.Comm, false, ignoreMultiLine) } else { p.print(token.DEFAULT) } diff --git a/libgo/go/go/printer/printer.go b/libgo/go/go/printer/printer.go index 34b0c4e2dc4..48e2af1b736 100644 --- a/libgo/go/go/printer/printer.go +++ b/libgo/go/go/printer/printer.go @@ -34,18 +34,18 @@ const ( ) +const ( + esc2 = '\xfe' // an escape byte that cannot occur in regular UTF-8 + _ = 1 / (esc2 - tabwriter.Escape) // cause compiler error if esc2 == tabwriter.Escape +) + + var ( esc = []byte{tabwriter.Escape} htab = []byte{'\t'} htabs = []byte("\t\t\t\t\t\t\t\t") newlines = []byte("\n\n\n\n\n\n\n\n") // more than the max determined by nlines formfeeds = []byte("\f\f\f\f\f\f\f\f") // more than the max determined by nlines - - esc_quot = []byte(""") // shorter than """ - esc_apos = []byte("'") // shorter than "'" - esc_amp = []byte("&") - esc_lt = []byte("<") - esc_gt = []byte(">") ) @@ -145,18 +145,20 @@ func (p *printer) nlines(n, min int) int { // write0 does not indent after newlines, and does not HTML-escape or update p.pos. // func (p *printer) write0(data []byte) { - n, err := p.output.Write(data) - p.written += n - if err != nil { - p.errors <- err - runtime.Goexit() + if len(data) > 0 { + n, err := p.output.Write(data) + p.written += n + if err != nil { + p.errors <- err + runtime.Goexit() + } } } // write interprets data and writes it to p.output. It inserts indentation -// after a line break unless in a tabwriter escape sequence, and it HTML- -// escapes characters if GenHTML is set. It updates p.pos as a side-effect. +// after a line break unless in a tabwriter escape sequence. +// It updates p.pos as a side-effect. // func (p *printer) write(data []byte) { i0 := 0 @@ -189,36 +191,6 @@ func (p *printer) write(data []byte) { // next segment start i0 = i + 1 - case '"', '\'', '&', '<', '>': - if p.Mode&GenHTML != 0 { - // write segment ending in b - p.write0(data[i0:i]) - - // write HTML-escaped b - var esc []byte - switch b { - case '"': - esc = esc_quot - case '\'': - esc = esc_apos - case '&': - esc = esc_amp - case '<': - esc = esc_lt - case '>': - esc = esc_gt - } - p.write0(esc) - - // update p.pos - d := i + 1 - i0 - p.pos.Offset += d - p.pos.Column += d - - // next segment start - i0 = i + 1 - } - case tabwriter.Escape: p.mode ^= inLiteral @@ -251,29 +223,13 @@ func (p *printer) writeNewlines(n int, useFF bool) { } -func (p *printer) writeTaggedItem(data []byte, tag HTMLTag) { - // write start tag, if any - // (no html-escaping and no p.pos update for tags - use write0) - if tag.Start != "" { - p.write0([]byte(tag.Start)) - } - p.write(data) - // write end tag, if any - if tag.End != "" { - p.write0([]byte(tag.End)) - } -} - - // writeItem writes data at position pos. data is the text corresponding to // a single lexical token, but may also be comment text. pos is the actual // (or at least very accurately estimated) position of the data in the original -// source text. If tags are present and GenHTML is set, the tags are written -// before and after the data. writeItem updates p.last to the position -// immediately following the data. +// source text. writeItem updates p.last to the position immediately following +// the data. // -func (p *printer) writeItem(pos token.Position, data []byte, tag HTMLTag) { - fileChanged := false +func (p *printer) writeItem(pos token.Position, data []byte) { if pos.IsValid() { // continue with previous position if we don't have a valid pos if p.last.IsValid() && p.last.Filename != pos.Filename { @@ -283,7 +239,6 @@ func (p *printer) writeItem(pos token.Position, data []byte, tag HTMLTag) { p.indent = 0 p.mode = 0 p.buffer = p.buffer[0:0] - fileChanged = true } p.pos = pos } @@ -292,18 +247,7 @@ func (p *printer) writeItem(pos token.Position, data []byte, tag HTMLTag) { _, filename := path.Split(pos.Filename) p.write0([]byte(fmt.Sprintf("[%s:%d:%d]", filename, pos.Line, pos.Column))) } - if p.Mode&GenHTML != 0 { - // write line tag if on a new line - // TODO(gri): should write line tags on each line at the start - // will be more useful (e.g. to show line numbers) - if p.Styler != nil && (pos.Line != p.lastTaggedLine || fileChanged) { - p.writeTaggedItem(p.Styler.LineTag(pos.Line)) - p.lastTaggedLine = pos.Line - } - p.writeTaggedItem(data, tag) - } else { - p.write(data) - } + p.write(data) p.last = p.pos } @@ -312,14 +256,13 @@ func (p *printer) writeItem(pos token.Position, data []byte, tag HTMLTag) { // If there is any pending whitespace, it consumes as much of // it as is likely to help position the comment nicely. // pos is the comment position, next the position of the item -// after all pending comments, isFirst indicates if this is the -// first comment in a group of comments, and isKeyword indicates -// if the next item is a keyword. +// after all pending comments, prev is the previous comment in +// a group of comments (or nil), and isKeyword indicates if the +// next item is a keyword. // -func (p *printer) writeCommentPrefix(pos, next token.Position, isFirst, isKeyword bool) { - if !p.last.IsValid() { - // there was no preceeding item and the comment is the - // first item to be printed - don't write any whitespace +func (p *printer) writeCommentPrefix(pos, next token.Position, prev *ast.Comment, isKeyword bool) { + if p.written == 0 { + // the comment is the first item to be printed - don't write any whitespace return } @@ -329,11 +272,12 @@ func (p *printer) writeCommentPrefix(pos, next token.Position, isFirst, isKeywor return } - if pos.IsValid() && pos.Line == p.last.Line { + if pos.Line == p.last.Line && (prev == nil || prev.Text[1] != '/') { // comment on the same line as last item: // separate with at least one separator hasSep := false - if isFirst { + if prev == nil { + // first comment of a comment group j := 0 for i, ch := range p.buffer { switch ch { @@ -370,7 +314,8 @@ func (p *printer) writeCommentPrefix(pos, next token.Position, isFirst, isKeywor } else { // comment on a different line: // separate with at least one line break - if isFirst { + if prev == nil { + // first comment of a comment group j := 0 for i, ch := range p.buffer { switch ch { @@ -402,10 +347,14 @@ func (p *printer) writeCommentPrefix(pos, next token.Position, isFirst, isKeywor } // use formfeeds to break columns before a comment; // this is analogous to using formfeeds to separate - // individual lines of /*-style comments - // (if !pos.IsValid(), pos.Line == 0, and this will - // print no newlines) - p.writeNewlines(pos.Line-p.last.Line, true) + // individual lines of /*-style comments - but make + // sure there is at least one line break if the previous + // comment was a line comment + n := pos.Line - p.last.Line // if !pos.IsValid(), pos.Line == 0, and n will be 0 + if n <= 0 && prev != nil && prev.Text[1] == '/' { + n = 1 + } + p.writeNewlines(n, true) } } @@ -413,21 +362,10 @@ func (p *printer) writeCommentPrefix(pos, next token.Position, isFirst, isKeywor func (p *printer) writeCommentLine(comment *ast.Comment, pos token.Position, line []byte) { // line must pass through unchanged, bracket it with tabwriter.Escape line = bytes.Join([][]byte{esc, line, esc}, nil) - - // apply styler, if any - var tag HTMLTag - if p.Styler != nil { - line, tag = p.Styler.Comment(comment, line) - } - - p.writeItem(pos, line, tag) + p.writeItem(pos, line) } -// TODO(gri): Similar (but not quite identical) functionality for -// comment processing can be found in go/doc/comment.go. -// Perhaps this can be factored eventually. - // Split comment text into lines func split(text []byte) [][]byte { // count lines (comment text never ends in a newline) @@ -680,7 +618,7 @@ func (p *printer) intersperseComments(next token.Position, tok token.Token) (dro var last *ast.Comment for ; p.commentBefore(next); p.cindex++ { for _, c := range p.comments[p.cindex].List { - p.writeCommentPrefix(p.fset.Position(c.Pos()), next, last == nil, tok.IsKeyword()) + p.writeCommentPrefix(p.fset.Position(c.Pos()), next, last, tok.IsKeyword()) p.writeComment(c) last = c } @@ -796,7 +734,6 @@ func (p *printer) print(args ...interface{}) { for _, f := range args { next := p.pos // estimated position of next item var data []byte - var tag HTMLTag var tok token.Token switch x := f.(type) { @@ -821,28 +758,31 @@ func (p *printer) print(args ...interface{}) { p.buffer = p.buffer[0 : i+1] p.buffer[i] = x case *ast.Ident: - if p.Styler != nil { - data, tag = p.Styler.Ident(x) - } else { - data = []byte(x.Name) - } + data = []byte(x.Name) tok = token.IDENT case *ast.BasicLit: - if p.Styler != nil { - data, tag = p.Styler.BasicLit(x) - } else { - data = x.Value - } // escape all literals so they pass through unchanged // (note that valid Go programs cannot contain // tabwriter.Escape bytes since they do not appear in // legal UTF-8 sequences) - escData := make([]byte, 0, len(data)+2) - escData = append(escData, tabwriter.Escape) - escData = append(escData, data...) - escData = append(escData, tabwriter.Escape) - data = escData + data = make([]byte, 0, len(x.Value)+2) + data = append(data, tabwriter.Escape) + data = append(data, x.Value...) + data = append(data, tabwriter.Escape) tok = x.Kind + // If we have a raw string that spans multiple lines and + // the opening quote (`) is on a line preceded only by + // indentation, we don't want to write that indentation + // because the following lines of the raw string are not + // indented. It's easiest to correct the output at the end + // via the trimmer (because of the complex handling of + // white space). + // Mark multi-line raw strings by replacing the opening + // quote with esc2 and have the trimmer take care of fixing + // it up. (Do this _after_ making a copy of data!) + if data[1] == '`' && bytes.IndexByte(data, '\n') > 0 { + data[1] = esc2 + } case token.Token: s := x.String() if mayCombine(p.lastTok, s[0]) { @@ -858,11 +798,7 @@ func (p *printer) print(args ...interface{}) { p.buffer = p.buffer[0:1] p.buffer[0] = ' ' } - if p.Styler != nil { - data, tag = p.Styler.Token(x) - } else { - data = []byte(s) - } + data = []byte(s) tok = x case token.Pos: if x.IsValid() { @@ -885,7 +821,7 @@ func (p *printer) print(args ...interface{}) { // before p.writeNewlines(next.Line-p.pos.Line, droppedFF) - p.writeItem(next, data, tag) + p.writeItem(next, data) } } } @@ -927,21 +863,26 @@ func (p *printer) flush(next token.Position, tok token.Token) (droppedFF bool) { // through unchanged. // type trimmer struct { - output io.Writer - space bytes.Buffer - state int + output io.Writer + state int + space bytes.Buffer + hasText bool } // trimmer is implemented as a state machine. // It can be in one of the following states: const ( - inSpace = iota - inEscape - inText + inSpace = iota // inside space + atEscape // inside space and the last char was an opening tabwriter.Escape + inEscape // inside text bracketed by tabwriter.Escapes + inText // inside text ) +var backquote = []byte{'`'} + + // Design note: It is tempting to eliminate extra blanks occurring in // whitespace in this function as it could simplify some // of the blanks logic in the node printing functions. @@ -949,7 +890,13 @@ const ( // the tabwriter. func (p *trimmer) Write(data []byte) (n int, err os.Error) { - m := 0 // if p.state != inSpace, data[m:n] is unwritten + // invariants: + // p.state == inSpace, atEscape: + // p.space is unwritten + // p.hasText indicates if there is any text on this line + // p.state == inEscape, inText: + // data[m:n] is unwritten + m := 0 var b byte for n, b = range data { if b == '\v' { @@ -960,37 +907,55 @@ func (p *trimmer) Write(data []byte) (n int, err os.Error) { switch b { case '\t', ' ': p.space.WriteByte(b) // WriteByte returns no errors - case '\f', '\n': + case '\n', '\f': p.space.Reset() // discard trailing space _, err = p.output.Write(newlines[0:1]) // write newline + p.hasText = false case tabwriter.Escape: - _, err = p.output.Write(p.space.Bytes()) - p.space.Reset() - p.state = inEscape - m = n + 1 // drop tabwriter.Escape + p.state = atEscape default: _, err = p.output.Write(p.space.Bytes()) - p.space.Reset() p.state = inText m = n } + case atEscape: + // discard indentation if we have a multi-line raw string + // (see printer.print for details) + if b != esc2 || p.hasText { + _, err = p.output.Write(p.space.Bytes()) + } + p.state = inEscape + m = n + if b == esc2 { + _, err = p.output.Write(backquote) // convert back + m++ + } case inEscape: if b == tabwriter.Escape { _, err = p.output.Write(data[m:n]) p.state = inSpace + p.space.Reset() + p.hasText = true } case inText: switch b { case '\t', ' ': _, err = p.output.Write(data[m:n]) p.state = inSpace + p.space.Reset() p.space.WriteByte(b) // WriteByte returns no errors - case '\f': - data[n] = '\n' // convert to newline + p.hasText = true + case '\n', '\f': + _, err = p.output.Write(data[m:n]) + p.state = inSpace + p.space.Reset() + _, err = p.output.Write(newlines[0:1]) // write newline + p.hasText = false case tabwriter.Escape: _, err = p.output.Write(data[m:n]) - p.state = inEscape - m = n + 1 // drop tabwriter.Escape + p.state = atEscape + p.space.Reset() + p.hasText = true } } if err != nil { @@ -999,9 +964,12 @@ func (p *trimmer) Write(data []byte) (n int, err os.Error) { } n = len(data) - if p.state != inSpace { + switch p.state { + case inEscape, inText: _, err = p.output.Write(data[m:n]) p.state = inSpace + p.space.Reset() + p.hasText = true } return @@ -1013,36 +981,16 @@ func (p *trimmer) Write(data []byte) (n int, err os.Error) { // General printing is controlled with these Config.Mode flags. const ( - GenHTML uint = 1 << iota // generate HTML - RawFormat // do not use a tabwriter; if set, UseSpaces is ignored + RawFormat uint = 1 << iota // do not use a tabwriter; if set, UseSpaces is ignored TabIndent // use tabs for indentation independent of UseSpaces UseSpaces // use spaces instead of tabs for alignment ) -// An HTMLTag specifies a start and end tag. -type HTMLTag struct { - Start, End string // empty if tags are absent -} - - -// A Styler specifies formatting of line tags and elementary Go words. -// A format consists of text and a (possibly empty) surrounding HTML tag. -// -type Styler interface { - LineTag(line int) ([]byte, HTMLTag) - Comment(c *ast.Comment, line []byte) ([]byte, HTMLTag) - BasicLit(x *ast.BasicLit) ([]byte, HTMLTag) - Ident(id *ast.Ident) ([]byte, HTMLTag) - Token(tok token.Token) ([]byte, HTMLTag) -} - - // A Config node controls the output of Fprint. type Config struct { - Mode uint // default: 0 - Tabwidth int // default: 8 - Styler Styler // default: nil + Mode uint // default: 0 + Tabwidth int // default: 8 } @@ -1070,9 +1018,6 @@ func (cfg *Config) Fprint(output io.Writer, fset *token.FileSet, node interface{ } twmode := tabwriter.DiscardEmptyColumns - if cfg.Mode&GenHTML != 0 { - twmode |= tabwriter.FilterHTML - } if cfg.Mode&TabIndent != 0 { minwidth = 0 twmode |= tabwriter.TabIndent diff --git a/libgo/go/go/printer/printer_test.go b/libgo/go/go/printer/printer_test.go index c66471b926a..565075aa20c 100644 --- a/libgo/go/go/printer/printer_test.go +++ b/libgo/go/go/printer/printer_test.go @@ -127,7 +127,7 @@ var data = []entry{ } -func Test(t *testing.T) { +func TestFiles(t *testing.T) { for _, e := range data { source := path.Join(dataDir, e.source) golden := path.Join(dataDir, e.golden) @@ -136,3 +136,38 @@ func Test(t *testing.T) { //check(t, golden, golden, e.mode); } } + + +// TestLineComments, using a simple test case, checks that consequtive line +// comments are properly terminated with a newline even if the AST position +// information is incorrect. +// +func TestLineComments(t *testing.T) { + const src = `// comment 1 + // comment 2 + // comment 3 + package main + ` + + fset := token.NewFileSet() + ast1, err1 := parser.ParseFile(fset, "", src, parser.ParseComments) + if err1 != nil { + panic(err1) + } + + var buf bytes.Buffer + fset = token.NewFileSet() // use the wrong file set + Fprint(&buf, fset, ast1) + + nlines := 0 + for _, ch := range buf.Bytes() { + if ch == '\n' { + nlines++ + } + } + + const expected = 3 + if nlines < expected { + t.Errorf("got %d, expected %d\n", nlines, expected) + } +} diff --git a/libgo/go/go/printer/testdata/expressions.golden b/libgo/go/go/printer/testdata/expressions.golden index 882c7624c01..7f18f338a63 100644 --- a/libgo/go/go/printer/testdata/expressions.golden +++ b/libgo/go/go/printer/testdata/expressions.golden @@ -248,6 +248,77 @@ they must not be removed` func _() { + // smart handling of indentation for multi-line raw strings + var _ = `` + var _ = `foo` + var _ = `foo +bar` + + var _ = `` + var _ = `foo` + var _ = + // the next line should not be indented +`foo +bar` + + var _ = // comment + `` + var _ = // comment + `foo` + var _ = // comment + // the next line should not be indented +`foo +bar` + + var _ = /* comment */ `` + var _ = /* comment */ `foo` + var _ = /* comment */ `foo +bar` + + var _ = /* comment */ + `` + var _ = /* comment */ + `foo` + var _ = /* comment */ + // the next line should not be indented +`foo +bar` + + var board = []int( +`........... +........... +....●●●.... +....●●●.... +..●●●●●●●.. +..●●●○●●●.. +..●●●●●●●.. +....●●●.... +....●●●.... +........... +........... +`) + + var state = S{ + "foo", + // the next line should not be indented +`........... +........... +....●●●.... +....●●●.... +..●●●●●●●.. +..●●●○●●●.. +..●●●●●●●.. +....●●●.... +....●●●.... +........... +........... +`, + "bar", + } +} + + +func _() { // one-line function literals (body is on a single line) _ = func() {} _ = func() int { return 0 } diff --git a/libgo/go/go/printer/testdata/expressions.input b/libgo/go/go/printer/testdata/expressions.input index 647706b0923..6bcd9b5f89e 100644 --- a/libgo/go/go/printer/testdata/expressions.input +++ b/libgo/go/go/printer/testdata/expressions.input @@ -244,6 +244,85 @@ they must not be removed` func _() { + // smart handling of indentation for multi-line raw strings + var _ = `` + var _ = `foo` + var _ = `foo +bar` + + +var _ = + `` +var _ = + `foo` +var _ = + // the next line should not be indented + `foo +bar` + + + var _ = // comment + `` + var _ = // comment + `foo` + var _ = // comment + // the next line should not be indented + `foo +bar` + + +var _ = /* comment */ `` +var _ = /* comment */ `foo` +var _ = /* comment */ `foo +bar` + + + var _ = /* comment */ + `` + var _ = /* comment */ + `foo` + var _ = /* comment */ + // the next line should not be indented + `foo +bar` + + +var board = []int( + `........... +........... +....●●●.... +....●●●.... +..●●●●●●●.. +..●●●○●●●.. +..●●●●●●●.. +....●●●.... +....●●●.... +........... +........... +`) + + + var state = S{ + "foo", + // the next line should not be indented + `........... +........... +....●●●.... +....●●●.... +..●●●●●●●.. +..●●●○●●●.. +..●●●●●●●.. +....●●●.... +....●●●.... +........... +........... +`, + "bar", + } +} + + +func _() { // one-line function literals (body is on a single line) _ = func() {} _ = func() int { return 0 } diff --git a/libgo/go/go/printer/testdata/expressions.raw b/libgo/go/go/printer/testdata/expressions.raw index 62be00cc301..f1944c94bb4 100644 --- a/libgo/go/go/printer/testdata/expressions.raw +++ b/libgo/go/go/printer/testdata/expressions.raw @@ -243,7 +243,77 @@ func _() { _ = `foo bar` _ = `three spaces before the end of the line starting here: -they must not be removed` +they must not be removed` } + + +func _() { + // smart handling of indentation for multi-line raw strings + var _ = `` + var _ = `foo` + var _ = `foo +bar` + + var _ = `` + var _ = `foo` + var _ = + // the next line should not be indented +`foo +bar` + + var _ = // comment + `` + var _ = // comment + `foo` + var _ = // comment + // the next line should not be indented +`foo +bar` + + var _ = /* comment */ `` + var _ = /* comment */ `foo` + var _ = /* comment */ `foo +bar` + + var _ = /* comment */ + `` + var _ = /* comment */ + `foo` + var _ = /* comment */ + // the next line should not be indented +`foo +bar` + + var board = []int( +`........... +........... +....●●●.... +....●●●.... +..●●●●●●●.. +..●●●○●●●.. +..●●●●●●●.. +....●●●.... +....●●●.... +........... +........... +`) + + var state = S{ + "foo", + // the next line should not be indented +`........... +........... +....●●●.... +....●●●.... +..●●●●●●●.. +..●●●○●●●.. +..●●●●●●●.. +....●●●.... +....●●●.... +........... +........... +`, + "bar", + } } diff --git a/libgo/go/go/printer/testdata/statements.golden b/libgo/go/go/printer/testdata/statements.golden index 5eceb7dd555..2900602699f 100644 --- a/libgo/go/go/printer/testdata/statements.golden +++ b/libgo/go/go/printer/testdata/statements.golden @@ -10,9 +10,9 @@ func use(x interface{}) {} // Formatting of if-statement headers. func _() { - if { + if true { } - if { + if true { } // no semicolon printed if expr { } @@ -22,7 +22,7 @@ func _() { } // no parens printed if expr { } // no semicolon and parens printed - if x := expr; { + if x := expr; true { use(x) } if x := expr; expr { @@ -354,14 +354,14 @@ func _() { func _() { - if { + if true { _ = 0 } _ = 0 // the indentation here should not be affected by the long label name AnOverlongLabel: _ = 0 - if { + if true { _ = 0 } _ = 0 diff --git a/libgo/go/go/printer/testdata/statements.input b/libgo/go/go/printer/testdata/statements.input index 7819820edef..21e61efc4f8 100644 --- a/libgo/go/go/printer/testdata/statements.input +++ b/libgo/go/go/printer/testdata/statements.input @@ -10,13 +10,13 @@ func use(x interface{}) {} // Formatting of if-statement headers. func _() { - if {} - if;{} // no semicolon printed + if true {} + if; true {} // no semicolon printed if expr{} if;expr{} // no semicolon printed if (expr){} // no parens printed if;((expr)){} // no semicolon and parens printed - if x:=expr;{ + if x:=expr;true{ use(x)} if x:=expr; expr {use(x)} } @@ -271,14 +271,14 @@ func _() { func _() { - if { + if true { _ = 0 } _ = 0 // the indentation here should not be affected by the long label name AnOverlongLabel: _ = 0 - if { + if true { _ = 0 } _ = 0 diff --git a/libgo/go/go/scanner/scanner.go b/libgo/go/go/scanner/scanner.go index 8c3205230e8..2ae296b3f15 100644 --- a/libgo/go/go/scanner/scanner.go +++ b/libgo/go/go/scanner/scanner.go @@ -8,7 +8,8 @@ // // var s Scanner // fset := token.NewFileSet() // position information is relative to fset -// s.Init(fset, filename, src, nil /* no error handler */, 0) +// file := fset.AddFile(filename, fset.Base(), len(src)) // register file +// s.Init(file, src, nil /* no error handler */, 0) // for { // pos, tok, lit := s.Scan() // if tok == token.EOF { diff --git a/libgo/go/go/scanner/scanner_test.go b/libgo/go/go/scanner/scanner_test.go index 1c3b6728c27..c622ff482f3 100644 --- a/libgo/go/go/scanner/scanner_test.go +++ b/libgo/go/go/scanner/scanner_test.go @@ -223,7 +223,7 @@ func TestScan(t *testing.T) { for _, e := range tokens { src += e.lit + whitespace } - src_linecount := newlineCount(src) + 1 + src_linecount := newlineCount(src) whitespace_linecount := newlineCount(whitespace) // verify scan @@ -241,7 +241,7 @@ func TestScan(t *testing.T) { if tok == token.EOF { lit = "<EOF>" epos.Line = src_linecount - epos.Column = 1 + epos.Column = 2 } checkPos(t, lit, pos, epos) if tok != e.tok { diff --git a/libgo/go/go/token/position.go b/libgo/go/go/token/position.go index 0044a0ed77d..809e53f0aa2 100644 --- a/libgo/go/go/token/position.go +++ b/libgo/go/go/token/position.go @@ -153,7 +153,7 @@ type lineInfo struct { // AddLineInfo adds alternative file and line number information for // a given file offset. The offset must be larger than the offset for -// the previously added alternative line info and not larger than the +// the previously added alternative line info and smaller than the // file size; otherwise the information is ignored. // // AddLineInfo is typically used to register alternative position @@ -161,7 +161,7 @@ type lineInfo struct { // func (f *File) AddLineInfo(offset int, filename string, line int) { f.set.mutex.Lock() - if i := len(f.infos); i == 0 || f.infos[i-1].offset < offset && offset <= f.size { + if i := len(f.infos); i == 0 || f.infos[i-1].offset < offset && offset < f.size { f.infos = append(f.infos, lineInfo{offset, filename, line}) } f.set.mutex.Unlock() @@ -212,27 +212,30 @@ func (f *File) LineCount() int { // AddLine adds the line offset for a new line. // The line offset must be larger than the offset for the previous line -// and not larger than the file size; otherwise the line offset is ignored. +// and smaller than the file size; otherwise the line offset is ignored. // func (f *File) AddLine(offset int) { f.set.mutex.Lock() - if i := len(f.lines); (i == 0 || f.lines[i-1] < offset) && offset <= f.size { + if i := len(f.lines); (i == 0 || f.lines[i-1] < offset) && offset < f.size { f.lines = append(f.lines, offset) } f.set.mutex.Unlock() } -// SetLines sets all line offsets for a file and returns true if successful. +// SetLines sets the line offsets for a file and returns true if successful. +// The line offsets are the offsets of the first character of each line; +// for instance for the content "ab\nc\n" the line offsets are {0, 3}. +// An empty file has an empty line offset table. // Each line offset must be larger than the offset for the previous line -// and not larger than the file size; otherwise the SetLines fails and returns +// and smaller than the file size; otherwise SetLines fails and returns // false. // func (f *File) SetLines(lines []int) bool { // verify validity of lines table size := f.size for i, offset := range lines { - if i > 0 && offset <= lines[i-1] || size < offset { + if i > 0 && offset <= lines[i-1] || size <= offset { return false } } @@ -245,6 +248,27 @@ func (f *File) SetLines(lines []int) bool { } +// SetLinesForContent sets the line offsets for the given file content. +func (f *File) SetLinesForContent(content []byte) { + var lines []int + line := 0 + for offset, b := range content { + if line >= 0 { + lines = append(lines, line) + } + line = -1 + if b == '\n' { + line = offset + 1 + } + } + + // set lines table + f.set.mutex.Lock() + f.lines = lines + f.set.mutex.Unlock() +} + + // Pos returns the Pos value for the given file offset; // the offset must be <= f.Size(). // f.Pos(f.Offset(p)) == p. diff --git a/libgo/go/go/token/position_test.go b/libgo/go/go/token/position_test.go index 1cffcc3c278..979c9b1e8e7 100644 --- a/libgo/go/go/token/position_test.go +++ b/libgo/go/go/token/position_test.go @@ -39,14 +39,18 @@ func TestNoPos(t *testing.T) { var tests = []struct { filename string + source []byte // may be nil size int lines []int }{ - {"a", 0, []int{}}, - {"b", 5, []int{0}}, - {"c", 10, []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}}, - {"d", 100, []int{0, 5, 10, 20, 30, 70, 71, 72, 80, 85, 90, 99}}, - {"e", 777, []int{0, 80, 100, 120, 130, 180, 267, 455, 500, 567, 620}}, + {"a", []byte{}, 0, []int{}}, + {"b", []byte("01234"), 5, []int{0}}, + {"c", []byte("\n\n\n\n\n\n\n\n\n"), 9, []int{0, 1, 2, 3, 4, 5, 6, 7, 8}}, + {"d", nil, 100, []int{0, 5, 10, 20, 30, 70, 71, 72, 80, 85, 90, 99}}, + {"e", nil, 777, []int{0, 80, 100, 120, 130, 180, 267, 455, 500, 567, 620}}, + {"f", []byte("package p\n\nimport \"fmt\""), 23, []int{0, 10, 11}}, + {"g", []byte("package p\n\nimport \"fmt\"\n"), 24, []int{0, 10, 11}}, + {"h", []byte("package p\n\nimport \"fmt\"\n "), 25, []int{0, 10, 11, 24}}, } @@ -77,10 +81,26 @@ func verifyPositions(t *testing.T, fset *FileSet, f *File, lines []int) { } +func makeTestSource(size int, lines []int) []byte { + src := make([]byte, size) + for _, offs := range lines { + if offs > 0 { + src[offs-1] = '\n' + } + } + return src +} + + func TestPositions(t *testing.T) { const delta = 7 // a non-zero base offset increment fset := NewFileSet() for _, test := range tests { + // verify consistency of test case + if test.source != nil && len(test.source) != test.size { + t.Errorf("%s: inconsistent test case: expected file size %d; got %d", test.filename, test.size, len(test.source)) + } + // add file and verify name and size f := fset.AddFile(test.filename, fset.Base()+delta, test.size) if f.Name() != test.filename { @@ -107,15 +127,26 @@ func TestPositions(t *testing.T) { verifyPositions(t, fset, f, test.lines[0:i+1]) } - // add lines at once and verify all positions - ok := f.SetLines(test.lines) - if !ok { + // add lines with SetLines and verify all positions + if ok := f.SetLines(test.lines); !ok { t.Errorf("%s: SetLines failed", f.Name()) } if f.LineCount() != len(test.lines) { t.Errorf("%s, SetLines: expected line count %d; got %d", f.Name(), len(test.lines), f.LineCount()) } verifyPositions(t, fset, f, test.lines) + + // add lines with SetLinesForContent and verify all positions + src := test.source + if src == nil { + // no test source available - create one from scratch + src = makeTestSource(test.size, test.lines) + } + f.SetLinesForContent(src) + if f.LineCount() != len(test.lines) { + t.Errorf("%s, SetLinesForContent: expected line count %d; got %d", f.Name(), len(test.lines), f.LineCount()) + } + verifyPositions(t, fset, f, test.lines) } } diff --git a/libgo/go/go/token/token.go b/libgo/go/go/token/token.go index 1bd81c1b143..2a2d3ecc4fb 100644 --- a/libgo/go/go/token/token.go +++ b/libgo/go/go/token/token.go @@ -252,8 +252,8 @@ func (tok Token) String() string { // const ( LowestPrec = 0 // non-operators - UnaryPrec = 7 - HighestPrec = 8 + UnaryPrec = 6 + HighestPrec = 7 ) @@ -267,14 +267,12 @@ func (op Token) Precedence() int { return 1 case LAND: return 2 - case ARROW: - return 3 case EQL, NEQ, LSS, LEQ, GTR, GEQ: - return 4 + return 3 case ADD, SUB, OR, XOR: - return 5 + return 4 case MUL, QUO, REM, SHL, SHR, AND, AND_NOT: - return 6 + return 5 } return LowestPrec } diff --git a/libgo/go/gob/codec_test.go b/libgo/go/gob/codec_test.go index af941c629cd..c822d6863ac 100644 --- a/libgo/go/gob/codec_test.go +++ b/libgo/go/gob/codec_test.go @@ -58,7 +58,7 @@ func TestUintCodec(t *testing.T) { t.Errorf("encodeUint: %#x encode: expected % x got % x", tt.x, tt.b, b.Bytes()) } } - decState := newDecodeState(nil, &b) + decState := newDecodeState(nil, b) for u := uint64(0); ; u = (u + 1) * 7 { b.Reset() encState.encodeUint(u) @@ -77,7 +77,7 @@ func verifyInt(i int64, t *testing.T) { var b = new(bytes.Buffer) encState := newEncoderState(nil, b) encState.encodeInt(i) - decState := newDecodeState(nil, &b) + decState := newDecodeState(nil, b) decState.buf = make([]byte, 8) j := decState.decodeInt() if i != j { @@ -315,7 +315,7 @@ func execDec(typ string, instr *decInstr, state *decodeState, t *testing.T, p un func newDecodeStateFromData(data []byte) *decodeState { b := bytes.NewBuffer(data) - state := newDecodeState(nil, &b) + state := newDecodeState(nil, b) state.fieldnum = -1 return state } @@ -342,7 +342,7 @@ func TestScalarDecInstructions(t *testing.T) { var data struct { a int } - instr := &decInstr{decOpMap[reflect.Int], 6, 0, 0, ovfl} + instr := &decInstr{decOpTable[reflect.Int], 6, 0, 0, ovfl} state := newDecodeStateFromData(signedResult) execDec("int", instr, state, t, unsafe.Pointer(&data)) if data.a != 17 { @@ -355,7 +355,7 @@ func TestScalarDecInstructions(t *testing.T) { var data struct { a uint } - instr := &decInstr{decOpMap[reflect.Uint], 6, 0, 0, ovfl} + instr := &decInstr{decOpTable[reflect.Uint], 6, 0, 0, ovfl} state := newDecodeStateFromData(unsignedResult) execDec("uint", instr, state, t, unsafe.Pointer(&data)) if data.a != 17 { @@ -446,7 +446,7 @@ func TestScalarDecInstructions(t *testing.T) { var data struct { a uintptr } - instr := &decInstr{decOpMap[reflect.Uintptr], 6, 0, 0, ovfl} + instr := &decInstr{decOpTable[reflect.Uintptr], 6, 0, 0, ovfl} state := newDecodeStateFromData(unsignedResult) execDec("uintptr", instr, state, t, unsafe.Pointer(&data)) if data.a != 17 { @@ -511,7 +511,7 @@ func TestScalarDecInstructions(t *testing.T) { var data struct { a complex64 } - instr := &decInstr{decOpMap[reflect.Complex64], 6, 0, 0, ovfl} + instr := &decInstr{decOpTable[reflect.Complex64], 6, 0, 0, ovfl} state := newDecodeStateFromData(complexResult) execDec("complex", instr, state, t, unsafe.Pointer(&data)) if data.a != 17+19i { @@ -524,7 +524,7 @@ func TestScalarDecInstructions(t *testing.T) { var data struct { a complex128 } - instr := &decInstr{decOpMap[reflect.Complex128], 6, 0, 0, ovfl} + instr := &decInstr{decOpTable[reflect.Complex128], 6, 0, 0, ovfl} state := newDecodeStateFromData(complexResult) execDec("complex", instr, state, t, unsafe.Pointer(&data)) if data.a != 17+19i { @@ -973,18 +973,32 @@ func TestIgnoredFields(t *testing.T) { } } + +func TestBadRecursiveType(t *testing.T) { + type Rec ***Rec + var rec Rec + b := new(bytes.Buffer) + err := NewEncoder(b).Encode(&rec) + if err == nil { + t.Error("expected error; got none") + } else if strings.Index(err.String(), "recursive") < 0 { + t.Error("expected recursive type error; got", err) + } + // Can't test decode easily because we can't encode one, so we can't pass one to a Decoder. +} + type Bad0 struct { - ch chan int - c float64 + CH chan int + C float64 } -var nilEncoder *Encoder func TestInvalidField(t *testing.T) { var bad0 Bad0 - bad0.ch = make(chan int) + bad0.CH = make(chan int) b := new(bytes.Buffer) - err := nilEncoder.encode(b, reflect.NewValue(&bad0)) + var nilEncoder *Encoder + err := nilEncoder.encode(b, reflect.NewValue(&bad0), userType(reflect.Typeof(&bad0))) if err == nil { t.Error("expected error; got none") } else if strings.Index(err.String(), "type") < 0 { @@ -1088,11 +1102,11 @@ func (v Vector) Square() int { } type Point struct { - a, b int + X, Y int } func (p Point) Square() int { - return p.a*p.a + p.b*p.b + return p.X*p.X + p.Y*p.Y } // A struct with interfaces in it. @@ -1162,7 +1176,6 @@ func TestInterface(t *testing.T) { } } } - } // A struct with all basic types, stored in interfaces. @@ -1182,7 +1195,7 @@ func TestInterfaceBasic(t *testing.T) { int(1), int8(1), int16(1), int32(1), int64(1), uint(1), uint8(1), uint16(1), uint32(1), uint64(1), float32(1), 1.0, - complex64(0i), complex128(0i), + complex64(1i), complex128(1i), true, "hello", []byte("sailor"), @@ -1307,6 +1320,31 @@ func TestUnexportedFields(t *testing.T) { } } +var singletons = []interface{}{ + true, + 7, + 3.2, + "hello", + [3]int{11, 22, 33}, + []float32{0.5, 0.25, 0.125}, + map[string]int{"one": 1, "two": 2}, +} + +func TestDebugSingleton(t *testing.T) { + if debugFunc == nil { + return + } + b := new(bytes.Buffer) + // Accumulate a number of values and print them out all at once. + for _, x := range singletons { + err := NewEncoder(b).Encode(x) + if err != nil { + t.Fatal("encode:", err) + } + } + debugFunc(b) +} + // A type that won't be defined in the gob until we send it in an interface value. type OnTheFly struct { A int @@ -1325,7 +1363,7 @@ type DT struct { S []string } -func TestDebug(t *testing.T) { +func TestDebugStruct(t *testing.T) { if debugFunc == nil { return } diff --git a/libgo/go/gob/decode.go b/libgo/go/gob/decode.go index 2db75215c19..8f599e10041 100644 --- a/libgo/go/gob/decode.go +++ b/libgo/go/gob/decode.go @@ -30,15 +30,17 @@ type decodeState struct { dec *Decoder // The buffer is stored with an extra indirection because it may be replaced // if we load a type during decode (when reading an interface value). - b **bytes.Buffer + b *bytes.Buffer fieldnum int // the last field number read. buf []byte } -func newDecodeState(dec *Decoder, b **bytes.Buffer) *decodeState { +// We pass the bytes.Buffer separately for easier testing of the infrastructure +// without requiring a full Decoder. +func newDecodeState(dec *Decoder, buf *bytes.Buffer) *decodeState { d := new(decodeState) d.dec = dec - d.b = b + d.b = buf d.buf = make([]byte, uint64Size) return d } @@ -49,14 +51,15 @@ func overflow(name string) os.ErrorString { // decodeUintReader reads an encoded unsigned integer from an io.Reader. // Used only by the Decoder to read the message length. -func decodeUintReader(r io.Reader, buf []byte) (x uint64, err os.Error) { - _, err = r.Read(buf[0:1]) +func decodeUintReader(r io.Reader, buf []byte) (x uint64, width int, err os.Error) { + width = 1 + _, err = r.Read(buf[0:width]) if err != nil { return } b := buf[0] if b <= 0x7f { - return uint64(b), nil + return uint64(b), width, nil } nb := -int(int8(b)) if nb > uint64Size { @@ -75,6 +78,7 @@ func decodeUintReader(r io.Reader, buf []byte) (x uint64, err os.Error) { for i := 0; i < n; i++ { x <<= 8 x |= uint64(buf[i]) + width++ } return } @@ -405,10 +409,9 @@ func allocate(rtyp reflect.Type, p uintptr, indir int) uintptr { return *(*uintptr)(up) } -func (dec *Decoder) decodeSingle(engine *decEngine, rtyp reflect.Type, b **bytes.Buffer, p uintptr, indir int) (err os.Error) { - defer catchError(&err) - p = allocate(rtyp, p, indir) - state := newDecodeState(dec, b) +func (dec *Decoder) decodeSingle(engine *decEngine, ut *userTypeInfo, p uintptr) (err os.Error) { + p = allocate(ut.base, p, ut.indir) + state := newDecodeState(dec, &dec.buf) state.fieldnum = singletonField basep := p delta := int(state.decodeUint()) @@ -424,10 +427,13 @@ func (dec *Decoder) decodeSingle(engine *decEngine, rtyp reflect.Type, b **bytes return nil } -func (dec *Decoder) decodeStruct(engine *decEngine, rtyp *reflect.StructType, b **bytes.Buffer, p uintptr, indir int) (err os.Error) { - defer catchError(&err) - p = allocate(rtyp, p, indir) - state := newDecodeState(dec, b) +// Indir is for the value, not the type. At the time of the call it may +// differ from ut.indir, which was computed when the engine was built. +// This state cannot arise for decodeSingle, which is called directly +// from the user's value, not from the innards of an engine. +func (dec *Decoder) decodeStruct(engine *decEngine, ut *userTypeInfo, p uintptr, indir int) (err os.Error) { + p = allocate(ut.base.(*reflect.StructType), p, indir) + state := newDecodeState(dec, &dec.buf) state.fieldnum = -1 basep := p for state.b.Len() > 0 { @@ -454,9 +460,8 @@ func (dec *Decoder) decodeStruct(engine *decEngine, rtyp *reflect.StructType, b return nil } -func (dec *Decoder) ignoreStruct(engine *decEngine, b **bytes.Buffer) (err os.Error) { - defer catchError(&err) - state := newDecodeState(dec, b) +func (dec *Decoder) ignoreStruct(engine *decEngine) (err os.Error) { + state := newDecodeState(dec, &dec.buf) state.fieldnum = -1 for state.b.Len() > 0 { delta := int(state.decodeUint()) @@ -477,6 +482,18 @@ func (dec *Decoder) ignoreStruct(engine *decEngine, b **bytes.Buffer) (err os.Er return nil } +func (dec *Decoder) ignoreSingle(engine *decEngine) (err os.Error) { + state := newDecodeState(dec, &dec.buf) + state.fieldnum = singletonField + delta := int(state.decodeUint()) + if delta != 0 { + errorf("gob decode: corrupted data: non-zero delta for singleton") + } + instr := &engine.instr[singletonField] + instr.op(instr, state, unsafe.Pointer(nil)) + return nil +} + func (dec *Decoder) decodeArrayHelper(state *decodeState, p uintptr, elemOp decOp, elemWid uintptr, length, elemIndir int, ovfl os.ErrorString) { instr := &decInstr{elemOp, 0, elemIndir, 0, ovfl} for i := 0; i < length; i++ { @@ -501,7 +518,7 @@ func (dec *Decoder) decodeArray(atyp *reflect.ArrayType, state *decodeState, p u func decodeIntoValue(state *decodeState, op decOp, indir int, v reflect.Value, ovfl os.ErrorString) reflect.Value { instr := &decInstr{op, 0, indir, 0, ovfl} - up := unsafe.Pointer(v.Addr()) + up := unsafe.Pointer(v.UnsafeAddr()) if indir > 1 { up = decIndirect(up, indir) } @@ -612,9 +629,17 @@ func (dec *Decoder) decodeInterface(ityp *reflect.InterfaceType, state *decodeSt if !ok { errorf("gob: name not registered for interface: %q", name) } + // Read the type id of the concrete value. + concreteId := dec.decodeTypeSequence(true) + if concreteId < 0 { + error(dec.err) + } + // Byte count of value is next; we don't care what it is (it's there + // in case we want to ignore the value by skipping it completely). + state.decodeUint() // Read the concrete value. value := reflect.MakeZero(typ) - dec.decodeValueFromBuffer(value, false, true) + dec.decodeValue(concreteId, value) if dec.err != nil { error(dec.err) } @@ -637,14 +662,16 @@ func (dec *Decoder) ignoreInterface(state *decodeState) { if err != nil { error(err) } - dec.decodeValueFromBuffer(nil, true, true) - if dec.err != nil { - error(err) + id := dec.decodeTypeSequence(true) + if id < 0 { + error(dec.err) } + // At this point, the decoder buffer contains a delimited value. Just toss it. + state.b.Next(int(state.decodeUint())) } // Index by Go types. -var decOpMap = []decOp{ +var decOpTable = [...]decOp{ reflect.Bool: decBool, reflect.Int8: decInt8, reflect.Int16: decInt16, @@ -674,35 +701,43 @@ var decIgnoreOpMap = map[typeId]decOp{ // Return the decoding op for the base type under rt and // the indirection count to reach it. -func (dec *Decoder) decOpFor(wireId typeId, rt reflect.Type, name string) (decOp, int) { - typ, indir := indirect(rt) +func (dec *Decoder) decOpFor(wireId typeId, rt reflect.Type, name string, inProgress map[reflect.Type]*decOp) (*decOp, int) { + ut := userType(rt) + // If this type is already in progress, it's a recursive type (e.g. map[string]*T). + // Return the pointer to the op we're already building. + if opPtr := inProgress[rt]; opPtr != nil { + return opPtr, ut.indir + } + typ := ut.base + indir := ut.indir var op decOp k := typ.Kind() - if int(k) < len(decOpMap) { - op = decOpMap[k] + if int(k) < len(decOpTable) { + op = decOpTable[k] } if op == nil { + inProgress[rt] = &op // Special cases switch t := typ.(type) { case *reflect.ArrayType: name = "element of " + name elemId := dec.wireType[wireId].ArrayT.Elem - elemOp, elemIndir := dec.decOpFor(elemId, t.Elem(), name) + elemOp, elemIndir := dec.decOpFor(elemId, t.Elem(), name, inProgress) ovfl := overflow(name) op = func(i *decInstr, state *decodeState, p unsafe.Pointer) { - state.dec.decodeArray(t, state, uintptr(p), elemOp, t.Elem().Size(), t.Len(), i.indir, elemIndir, ovfl) + state.dec.decodeArray(t, state, uintptr(p), *elemOp, t.Elem().Size(), t.Len(), i.indir, elemIndir, ovfl) } case *reflect.MapType: name = "element of " + name keyId := dec.wireType[wireId].MapT.Key elemId := dec.wireType[wireId].MapT.Elem - keyOp, keyIndir := dec.decOpFor(keyId, t.Key(), name) - elemOp, elemIndir := dec.decOpFor(elemId, t.Elem(), name) + keyOp, keyIndir := dec.decOpFor(keyId, t.Key(), name, inProgress) + elemOp, elemIndir := dec.decOpFor(elemId, t.Elem(), name, inProgress) ovfl := overflow(name) op = func(i *decInstr, state *decodeState, p unsafe.Pointer) { up := unsafe.Pointer(p) - state.dec.decodeMap(t, state, uintptr(up), keyOp, elemOp, i.indir, keyIndir, elemIndir, ovfl) + state.dec.decodeMap(t, state, uintptr(up), *keyOp, *elemOp, i.indir, keyIndir, elemIndir, ovfl) } case *reflect.SliceType: @@ -717,10 +752,10 @@ func (dec *Decoder) decOpFor(wireId typeId, rt reflect.Type, name string) (decOp } else { elemId = dec.wireType[wireId].SliceT.Elem } - elemOp, elemIndir := dec.decOpFor(elemId, t.Elem(), name) + elemOp, elemIndir := dec.decOpFor(elemId, t.Elem(), name, inProgress) ovfl := overflow(name) op = func(i *decInstr, state *decodeState, p unsafe.Pointer) { - state.dec.decodeSlice(t, state, uintptr(p), elemOp, t.Elem().Size(), i.indir, elemIndir, ovfl) + state.dec.decodeSlice(t, state, uintptr(p), *elemOp, t.Elem().Size(), i.indir, elemIndir, ovfl) } case *reflect.StructType: @@ -730,8 +765,8 @@ func (dec *Decoder) decOpFor(wireId typeId, rt reflect.Type, name string) (decOp error(err) } op = func(i *decInstr, state *decodeState, p unsafe.Pointer) { - // indirect through enginePtr to delay evaluation for recursive structs - err = dec.decodeStruct(*enginePtr, t, state.b, uintptr(p), i.indir) + // indirect through enginePtr to delay evaluation for recursive structs. + err = dec.decodeStruct(*enginePtr, userType(typ), uintptr(p), i.indir) if err != nil { error(err) } @@ -745,7 +780,7 @@ func (dec *Decoder) decOpFor(wireId typeId, rt reflect.Type, name string) (decOp if op == nil { errorf("gob: decode can't handle type %s", rt.String()) } - return op, indir + return &op, indir } // Return the decoding op for a field that has no destination. @@ -796,7 +831,7 @@ func (dec *Decoder) decIgnoreOpFor(wireId typeId) decOp { } op = func(i *decInstr, state *decodeState, p unsafe.Pointer) { // indirect through enginePtr to delay evaluation for recursive structs - state.dec.ignoreStruct(*enginePtr, state.b) + state.dec.ignoreStruct(*enginePtr) } } } @@ -809,11 +844,15 @@ func (dec *Decoder) decIgnoreOpFor(wireId typeId) decOp { // Are these two gob Types compatible? // Answers the question for basic types, arrays, and slices. // Structs are considered ok; fields will be checked later. -func (dec *Decoder) compatibleType(fr reflect.Type, fw typeId) bool { - fr, _ = indirect(fr) +func (dec *Decoder) compatibleType(fr reflect.Type, fw typeId, inProgress map[reflect.Type]typeId) bool { + if rhs, ok := inProgress[fr]; ok { + return rhs == fw + } + inProgress[fr] = fw + fr = userType(fr).base switch t := fr.(type) { default: - // map, chan, etc: cannot handle. + // chan, etc: cannot handle. return false case *reflect.BoolType: return fw == tBool @@ -835,14 +874,14 @@ func (dec *Decoder) compatibleType(fr reflect.Type, fw typeId) bool { return false } array := wire.ArrayT - return t.Len() == array.Len && dec.compatibleType(t.Elem(), array.Elem) + return t.Len() == array.Len && dec.compatibleType(t.Elem(), array.Elem, inProgress) case *reflect.MapType: wire, ok := dec.wireType[fw] if !ok || wire.MapT == nil { return false } MapType := wire.MapT - return dec.compatibleType(t.Key(), MapType.Key) && dec.compatibleType(t.Elem(), MapType.Elem) + return dec.compatibleType(t.Key(), MapType.Key, inProgress) && dec.compatibleType(t.Elem(), MapType.Elem, inProgress) case *reflect.SliceType: // Is it an array of bytes? if t.Elem().Kind() == reflect.Uint8 { @@ -855,8 +894,8 @@ func (dec *Decoder) compatibleType(fr reflect.Type, fw typeId) bool { } else { sw = dec.wireType[fw].SliceT } - elem, _ := indirect(t.Elem()) - return sw != nil && dec.compatibleType(elem, sw.Elem) + elem := userType(t.Elem()).base + return sw != nil && dec.compatibleType(elem, sw.Elem, inProgress) case *reflect.StructType: return true } @@ -877,12 +916,22 @@ func (dec *Decoder) compileSingle(remoteId typeId, rt reflect.Type) (engine *dec engine = new(decEngine) engine.instr = make([]decInstr, 1) // one item name := rt.String() // best we can do - if !dec.compatibleType(rt, remoteId) { + if !dec.compatibleType(rt, remoteId, make(map[reflect.Type]typeId)) { return nil, os.ErrorString("gob: wrong type received for local value " + name + ": " + dec.typeString(remoteId)) } - op, indir := dec.decOpFor(remoteId, rt, name) + op, indir := dec.decOpFor(remoteId, rt, name, make(map[reflect.Type]*decOp)) ovfl := os.ErrorString(`value for "` + name + `" out of range`) - engine.instr[singletonField] = decInstr{op, singletonField, indir, 0, ovfl} + engine.instr[singletonField] = decInstr{*op, singletonField, indir, 0, ovfl} + engine.numInstr = 1 + return +} + +func (dec *Decoder) compileIgnoreSingle(remoteId typeId) (engine *decEngine, err os.Error) { + engine = new(decEngine) + engine.instr = make([]decInstr, 1) // one item + op := dec.decIgnoreOpFor(remoteId) + ovfl := overflow(dec.typeString(remoteId)) + engine.instr[0] = decInstr{op, 0, 0, 0, ovfl} engine.numInstr = 1 return } @@ -894,7 +943,6 @@ func isExported(name string) bool { } func (dec *Decoder) compileDec(remoteId typeId, rt reflect.Type) (engine *decEngine, err os.Error) { - defer catchError(&err) srt, ok := rt.(*reflect.StructType) if !ok { return dec.compileSingle(remoteId, rt) @@ -905,13 +953,18 @@ func (dec *Decoder) compileDec(remoteId typeId, rt reflect.Type) (engine *decEng if t, ok := builtinIdToType[remoteId]; ok { wireStruct, _ = t.(*structType) } else { - wireStruct = dec.wireType[remoteId].StructT + wire := dec.wireType[remoteId] + if wire == nil { + error(errBadType) + } + wireStruct = wire.StructT } if wireStruct == nil { errorf("gob: type mismatch in decoder: want struct type %s; got non-struct", rt.String()) } engine = new(decEngine) engine.instr = make([]decInstr, len(wireStruct.Field)) + seen := make(map[reflect.Type]*decOp) // Loop over the fields of the wire type. for fieldnum := 0; fieldnum < len(wireStruct.Field); fieldnum++ { wireField := wireStruct.Field[fieldnum] @@ -927,11 +980,11 @@ func (dec *Decoder) compileDec(remoteId typeId, rt reflect.Type) (engine *decEng engine.instr[fieldnum] = decInstr{op, fieldnum, 0, 0, ovfl} continue } - if !dec.compatibleType(localField.Type, wireField.Id) { + if !dec.compatibleType(localField.Type, wireField.Id, make(map[reflect.Type]typeId)) { errorf("gob: wrong type (%s) for received field %s.%s", localField.Type, wireStruct.Name, wireField.Name) } - op, indir := dec.decOpFor(wireField.Id, localField.Type, localField.Name) - engine.instr[fieldnum] = decInstr{op, fieldnum, indir, uintptr(localField.Offset), ovfl} + op, indir := dec.decOpFor(wireField.Id, localField.Type, localField.Name, seen) + engine.instr[fieldnum] = decInstr{*op, fieldnum, indir, uintptr(localField.Offset), ovfl} engine.numInstr++ } return @@ -966,7 +1019,12 @@ func (dec *Decoder) getIgnoreEnginePtr(wireId typeId) (enginePtr **decEngine, er // To handle recursive types, mark this engine as underway before compiling. enginePtr = new(*decEngine) dec.ignorerCache[wireId] = enginePtr - *enginePtr, err = dec.compileDec(wireId, emptyStructType) + wire := dec.wireType[wireId] + if wire != nil && wire.StructT != nil { + *enginePtr, err = dec.compileDec(wireId, emptyStructType) + } else { + *enginePtr, err = dec.compileIgnoreSingle(wireId) + } if err != nil { dec.ignorerCache[wireId] = nil, false } @@ -974,22 +1032,41 @@ func (dec *Decoder) getIgnoreEnginePtr(wireId typeId) (enginePtr **decEngine, er return } -func (dec *Decoder) decode(wireId typeId, val reflect.Value) os.Error { +func (dec *Decoder) decodeValue(wireId typeId, val reflect.Value) (err os.Error) { + defer catchError(&err) + // If the value is nil, it means we should just ignore this item. + if val == nil { + return dec.decodeIgnoredValue(wireId) + } // Dereference down to the underlying struct type. - rt, indir := indirect(val.Type()) - enginePtr, err := dec.getDecEnginePtr(wireId, rt) + ut := userType(val.Type()) + base := ut.base + indir := ut.indir + enginePtr, err := dec.getDecEnginePtr(wireId, base) if err != nil { return err } engine := *enginePtr - if st, ok := rt.(*reflect.StructType); ok { + if st, ok := base.(*reflect.StructType); ok { if engine.numInstr == 0 && st.NumField() > 0 && len(dec.wireType[wireId].StructT.Field) > 0 { - name := rt.Name() + name := base.Name() return os.ErrorString("gob: type mismatch: no fields matched compiling decoder for " + name) } - return dec.decodeStruct(engine, st, dec.state.b, uintptr(val.Addr()), indir) + return dec.decodeStruct(engine, ut, uintptr(val.UnsafeAddr()), indir) + } + return dec.decodeSingle(engine, ut, uintptr(val.UnsafeAddr())) +} + +func (dec *Decoder) decodeIgnoredValue(wireId typeId) os.Error { + enginePtr, err := dec.getIgnoreEnginePtr(wireId) + if err != nil { + return err + } + wire := dec.wireType[wireId] + if wire != nil && wire.StructT != nil { + return dec.ignoreStruct(*enginePtr) } - return dec.decodeSingle(engine, rt, dec.state.b, uintptr(val.Addr()), indir) + return dec.ignoreSingle(*enginePtr) } func init() { @@ -1004,8 +1081,8 @@ func init() { default: panic("gob: unknown size of int/uint") } - decOpMap[reflect.Int] = iop - decOpMap[reflect.Uint] = uop + decOpTable[reflect.Int] = iop + decOpTable[reflect.Uint] = uop // Finally uintptr switch reflect.Typeof(uintptr(0)).Bits() { @@ -1016,5 +1093,5 @@ func init() { default: panic("gob: unknown size of uintptr") } - decOpMap[reflect.Uintptr] = uop + decOpTable[reflect.Uintptr] = uop } diff --git a/libgo/go/gob/decoder.go b/libgo/go/gob/decoder.go index 664001a4b21..f7c994ffa78 100644 --- a/libgo/go/gob/decoder.go +++ b/libgo/go/gob/decoder.go @@ -17,14 +17,13 @@ import ( type Decoder struct { mutex sync.Mutex // each item must be received atomically r io.Reader // source of the data + buf bytes.Buffer // buffer for more efficient i/o from r wireType map[typeId]*wireType // map from remote ID to local description decoderCache map[reflect.Type]map[typeId]**decEngine // cache of compiled engines ignorerCache map[typeId]**decEngine // ditto for ignored objects - state *decodeState // reads data from in-memory buffer countState *decodeState // reads counts from wire - buf []byte - countBuf [9]byte // counts may be uint64s (unlikely!), require 9 bytes - byteBuffer *bytes.Buffer + countBuf []byte // used for decoding integers while parsing messages + tmp []byte // temporary storage for i/o; saves reallocating err os.Error } @@ -33,128 +32,160 @@ func NewDecoder(r io.Reader) *Decoder { dec := new(Decoder) dec.r = r dec.wireType = make(map[typeId]*wireType) - dec.state = newDecodeState(dec, &dec.byteBuffer) // buffer set in Decode() dec.decoderCache = make(map[reflect.Type]map[typeId]**decEngine) dec.ignorerCache = make(map[typeId]**decEngine) + dec.countBuf = make([]byte, 9) // counts may be uint64s (unlikely!), require 9 bytes return dec } -// recvType loads the definition of a type and reloads the Decoder's buffer. +// recvType loads the definition of a type. func (dec *Decoder) recvType(id typeId) { // Have we already seen this type? That's an error - if dec.wireType[id] != nil { + if id < firstUserId || dec.wireType[id] != nil { dec.err = os.ErrorString("gob: duplicate type received") return } // Type: wire := new(wireType) - dec.err = dec.decode(tWireType, reflect.NewValue(wire)) + dec.err = dec.decodeValue(tWireType, reflect.NewValue(wire)) if dec.err != nil { return } // Remember we've seen this type. dec.wireType[id] = wire - - // Load the next parcel. - dec.recv() } -// Decode reads the next value from the connection and stores -// it in the data represented by the empty interface value. -// The value underlying e must be the correct type for the next -// data item received, and must be a pointer. -func (dec *Decoder) Decode(e interface{}) os.Error { - value := reflect.NewValue(e) - // If e represents a value as opposed to a pointer, the answer won't - // get back to the caller. Make sure it's a pointer. - if value.Type().Kind() != reflect.Ptr { - dec.err = os.ErrorString("gob: attempt to decode into a non-pointer") - return dec.err +// recvMessage reads the next count-delimited item from the input. It is the converse +// of Encoder.writeMessage. It returns false on EOF or other error reading the message. +func (dec *Decoder) recvMessage() bool { + // Read a count. + nbytes, _, err := decodeUintReader(dec.r, dec.countBuf) + if err != nil { + dec.err = err + return false } - return dec.DecodeValue(value) + dec.readMessage(int(nbytes)) + return dec.err == nil } -// recv reads the next count-delimited item from the input. It is the converse -// of Encoder.send. -func (dec *Decoder) recv() { - // Read a count. - var nbytes uint64 - nbytes, dec.err = decodeUintReader(dec.r, dec.countBuf[0:]) - if dec.err != nil { - return - } +// readMessage reads the next nbytes bytes from the input. +func (dec *Decoder) readMessage(nbytes int) { // Allocate the buffer. - if nbytes > uint64(len(dec.buf)) { - dec.buf = make([]byte, nbytes+1000) + if cap(dec.tmp) < nbytes { + dec.tmp = make([]byte, nbytes+100) // room to grow } - dec.byteBuffer = bytes.NewBuffer(dec.buf[0:nbytes]) + dec.tmp = dec.tmp[:nbytes] // Read the data - _, dec.err = io.ReadFull(dec.r, dec.buf[0:nbytes]) + _, dec.err = io.ReadFull(dec.r, dec.tmp) if dec.err != nil { if dec.err == os.EOF { dec.err = io.ErrUnexpectedEOF } return } + dec.buf.Write(dec.tmp) } -// decodeValueFromBuffer grabs the next value from the input. The Decoder's -// buffer already contains data. If the next item in the buffer is a type -// descriptor, it may be necessary to reload the buffer, but recvType does that. -func (dec *Decoder) decodeValueFromBuffer(value reflect.Value, ignoreInterfaceValue, countPresent bool) { - for dec.state.b.Len() > 0 { - // Receive a type id. - id := typeId(dec.state.decodeInt()) +// toInt turns an encoded uint64 into an int, according to the marshaling rules. +func toInt(x uint64) int64 { + i := int64(x >> 1) + if x&1 != 0 { + i = ^i + } + return i +} + +func (dec *Decoder) nextInt() int64 { + n, _, err := decodeUintReader(&dec.buf, dec.countBuf) + if err != nil { + dec.err = err + } + return toInt(n) +} - // Is it a new type? - if id < 0 { // 0 is the error state, handled above - // If the id is negative, we have a type. - dec.recvType(-id) - if dec.err != nil { +func (dec *Decoder) nextUint() uint64 { + n, _, err := decodeUintReader(&dec.buf, dec.countBuf) + if err != nil { + dec.err = err + } + return n +} + +// decodeTypeSequence parses: +// TypeSequence +// (TypeDefinition DelimitedTypeDefinition*)? +// and returns the type id of the next value. It returns -1 at +// EOF. Upon return, the remainder of dec.buf is the value to be +// decoded. If this is an interface value, it can be ignored by +// simply resetting that buffer. +func (dec *Decoder) decodeTypeSequence(isInterface bool) typeId { + for dec.err == nil { + if dec.buf.Len() == 0 { + if !dec.recvMessage() { break } - continue } - - // Make sure the type has been defined already or is a builtin type (for - // top-level singleton values). - if dec.wireType[id] == nil && builtinIdToType[id] == nil { - dec.err = errBadType - break + // Receive a type id. + id := typeId(dec.nextInt()) + if id >= 0 { + // Value follows. + return id } - // An interface value is preceded by a byte count. - if countPresent { - count := int(dec.state.decodeUint()) - if ignoreInterfaceValue { - // An interface value is preceded by a byte count. Just skip that many bytes. - dec.state.b.Next(int(count)) + // Type definition for (-id) follows. + dec.recvType(-id) + // When decoding an interface, after a type there may be a + // DelimitedValue still in the buffer. Skip its count. + // (Alternatively, the buffer is empty and the byte count + // will be absorbed by recvMessage.) + if dec.buf.Len() > 0 { + if !isInterface { + dec.err = os.ErrorString("extra data in buffer") break } - // Otherwise fall through and decode it. + dec.nextUint() } - dec.err = dec.decode(id, value) - break } + return -1 +} + +// Decode reads the next value from the connection and stores +// it in the data represented by the empty interface value. +// If e is nil, the value will be discarded. Otherwise, +// the value underlying e must either be the correct type for the next +// data item received, and must be a pointer. +func (dec *Decoder) Decode(e interface{}) os.Error { + if e == nil { + return dec.DecodeValue(nil) + } + value := reflect.NewValue(e) + // If e represents a value as opposed to a pointer, the answer won't + // get back to the caller. Make sure it's a pointer. + if value.Type().Kind() != reflect.Ptr { + dec.err = os.ErrorString("gob: attempt to decode into a non-pointer") + return dec.err + } + return dec.DecodeValue(value) } // DecodeValue reads the next value from the connection and stores // it in the data represented by the reflection value. // The value must be the correct type for the next -// data item received. +// data item received, or it may be nil, which means the +// value will be discarded. func (dec *Decoder) DecodeValue(value reflect.Value) os.Error { // Make sure we're single-threaded through here. dec.mutex.Lock() defer dec.mutex.Unlock() + dec.buf.Reset() // In case data lingers from previous invocation. dec.err = nil - dec.recv() - if dec.err != nil { - return dec.err + id := dec.decodeTypeSequence(false) + if dec.err == nil { + dec.err = dec.decodeValue(id, value) } - dec.decodeValueFromBuffer(value, false, false) return dec.err } diff --git a/libgo/go/gob/doc.go b/libgo/go/gob/doc.go index 31253f16d09..613974a000f 100644 --- a/libgo/go/gob/doc.go +++ b/libgo/go/gob/doc.go @@ -220,6 +220,54 @@ be predefined or be defined before the value in the stream. package gob /* +Grammar: + +Tokens starting with a lower case letter are terminals; int(n) +and uint(n) represent the signed/unsigned encodings of the value n. + +GobStream: + DelimitedMessage* +DelimitedMessage: + uint(lengthOfMessage) Message +Message: + TypeSequence TypedValue +TypeSequence + (TypeDefinition DelimitedTypeDefinition*)? +DelimitedTypeDefinition: + uint(lengthOfTypeDefinition) TypeDefinition +TypedValue: + int(typeId) Value +TypeDefinition: + int(-typeId) encodingOfWireType +Value: + SingletonValue | StructValue +SingletonValue: + uint(0) FieldValue +FieldValue: + builtinValue | ArrayValue | MapValue | SliceValue | StructValue | InterfaceValue +InterfaceValue: + NilInterfaceValue | NonNilInterfaceValue +NilInterfaceValue: + uint(0) +NonNilInterfaceValue: + ConcreteTypeName TypeSequence InterfaceContents +ConcreteTypeName: + uint(lengthOfName) [already read=n] name +InterfaceContents: + int(concreteTypeId) DelimitedValue +DelimitedValue: + uint(length) Value +ArrayValue: + uint(n) FieldValue*n [n elements] +MapValue: + uint(n) (FieldValue FieldValue)*n [n (key, value) pairs] +SliceValue: + uint(n) FieldValue*n [n elements] +StructValue: + (uint(fieldDelta) FieldValue)* +*/ + +/* For implementers and the curious, here is an encoded example. Given type Point struct {x, y int} and the value diff --git a/libgo/go/gob/encode.go b/libgo/go/gob/encode.go index d286a7e00b8..e92db74ffdd 100644 --- a/libgo/go/gob/encode.go +++ b/libgo/go/gob/encode.go @@ -264,9 +264,6 @@ func encComplex128(i *encInstr, state *encoderState, p unsafe.Pointer) { } } -func encNoOp(i *encInstr, state *encoderState, p unsafe.Pointer) { -} - // Byte arrays are encoded as an unsigned count followed by the raw bytes. func encUint8Array(i *encInstr, state *encoderState, p unsafe.Pointer) { b := *(*[]byte)(p) @@ -359,7 +356,7 @@ func encodeReflectValue(state *encoderState, v reflect.Value, op encOp, indir in if v == nil { errorf("gob: encodeReflectValue: nil element") } - op(nil, state, unsafe.Pointer(v.Addr())) + op(nil, state, unsafe.Pointer(v.UnsafeAddr())) } func (enc *Encoder) encodeMap(b *bytes.Buffer, mv *reflect.MapValue, keyOp, elemOp encOp, keyIndir, elemIndir int) { @@ -387,10 +384,10 @@ func (enc *Encoder) encodeInterface(b *bytes.Buffer, iv *reflect.InterfaceValue) return } - typ, _ := indirect(iv.Elem().Type()) - name, ok := concreteTypeToName[typ] + ut := userType(iv.Elem().Type()) + name, ok := concreteTypeToName[ut.base] if !ok { - errorf("gob: type not registered for interface: %s", typ) + errorf("gob: type not registered for interface: %s", ut.base) } // Send the name. state.encodeUint(uint64(len(name))) @@ -398,22 +395,26 @@ func (enc *Encoder) encodeInterface(b *bytes.Buffer, iv *reflect.InterfaceValue) if err != nil { error(err) } - // Send (and maybe first define) the type id. - enc.sendTypeDescriptor(typ) - // Encode the value into a new buffer. + // Define the type id if necessary. + enc.sendTypeDescriptor(enc.writer(), state, ut) + // Send the type id. + enc.sendTypeId(state, ut) + // Encode the value into a new buffer. Any nested type definitions + // should be written to b, before the encoded value. + enc.pushWriter(b) data := new(bytes.Buffer) - err = enc.encode(data, iv.Elem()) + err = enc.encode(data, iv.Elem(), ut) if err != nil { error(err) } - state.encodeUint(uint64(data.Len())) - _, err = state.b.Write(data.Bytes()) - if err != nil { + enc.popWriter() + enc.writeMessage(b, data) + if enc.err != nil { error(err) } } -var encOpMap = []encOp{ +var encOpTable = [...]encOp{ reflect.Bool: encBool, reflect.Int: encInt, reflect.Int8: encInt8, @@ -433,16 +434,24 @@ var encOpMap = []encOp{ reflect.String: encString, } -// Return the encoding op for the base type under rt and +// Return (a pointer to) the encoding op for the base type under rt and // the indirection count to reach it. -func (enc *Encoder) encOpFor(rt reflect.Type) (encOp, int) { - typ, indir := indirect(rt) - var op encOp +func (enc *Encoder) encOpFor(rt reflect.Type, inProgress map[reflect.Type]*encOp) (*encOp, int) { + ut := userType(rt) + // If this type is already in progress, it's a recursive type (e.g. map[string]*T). + // Return the pointer to the op we're already building. + if opPtr := inProgress[rt]; opPtr != nil { + return opPtr, ut.indir + } + typ := ut.base + indir := ut.indir k := typ.Kind() - if int(k) < len(encOpMap) { - op = encOpMap[k] + var op encOp + if int(k) < len(encOpTable) { + op = encOpTable[k] } if op == nil { + inProgress[rt] = &op // Special cases switch t := typ.(type) { case *reflect.SliceType: @@ -451,25 +460,25 @@ func (enc *Encoder) encOpFor(rt reflect.Type) (encOp, int) { break } // Slices have a header; we decode it to find the underlying array. - elemOp, indir := enc.encOpFor(t.Elem()) + elemOp, indir := enc.encOpFor(t.Elem(), inProgress) op = func(i *encInstr, state *encoderState, p unsafe.Pointer) { slice := (*reflect.SliceHeader)(p) if !state.sendZero && slice.Len == 0 { return } state.update(i) - state.enc.encodeArray(state.b, slice.Data, elemOp, t.Elem().Size(), indir, int(slice.Len)) + state.enc.encodeArray(state.b, slice.Data, *elemOp, t.Elem().Size(), indir, int(slice.Len)) } case *reflect.ArrayType: // True arrays have size in the type. - elemOp, indir := enc.encOpFor(t.Elem()) + elemOp, indir := enc.encOpFor(t.Elem(), inProgress) op = func(i *encInstr, state *encoderState, p unsafe.Pointer) { state.update(i) - state.enc.encodeArray(state.b, uintptr(p), elemOp, t.Elem().Size(), indir, t.Len()) + state.enc.encodeArray(state.b, uintptr(p), *elemOp, t.Elem().Size(), indir, t.Len()) } case *reflect.MapType: - keyOp, keyIndir := enc.encOpFor(t.Key()) - elemOp, elemIndir := enc.encOpFor(t.Elem()) + keyOp, keyIndir := enc.encOpFor(t.Key(), inProgress) + elemOp, elemIndir := enc.encOpFor(t.Elem(), inProgress) op = func(i *encInstr, state *encoderState, p unsafe.Pointer) { // Maps cannot be accessed by moving addresses around the way // that slices etc. can. We must recover a full reflection value for @@ -480,7 +489,7 @@ func (enc *Encoder) encOpFor(rt reflect.Type) (encOp, int) { return } state.update(i) - state.enc.encodeMap(state.b, mv, keyOp, elemOp, keyIndir, elemIndir) + state.enc.encodeMap(state.b, mv, *keyOp, *elemOp, keyIndir, elemIndir) } case *reflect.StructType: // Generate a closure that calls out to the engine for the nested type. @@ -508,28 +517,31 @@ func (enc *Encoder) encOpFor(rt reflect.Type) (encOp, int) { if op == nil { errorf("gob enc: can't happen: encode type %s", rt.String()) } - return op, indir + return &op, indir } // The local Type was compiled from the actual value, so we know it's compatible. func (enc *Encoder) compileEnc(rt reflect.Type) *encEngine { srt, isStruct := rt.(*reflect.StructType) engine := new(encEngine) + seen := make(map[reflect.Type]*encOp) if isStruct { - engine.instr = make([]encInstr, srt.NumField()+1) // +1 for terminator - for fieldnum := 0; fieldnum < srt.NumField(); fieldnum++ { - f := srt.Field(fieldnum) - op, indir := enc.encOpFor(f.Type) + for fieldNum := 0; fieldNum < srt.NumField(); fieldNum++ { + f := srt.Field(fieldNum) if !isExported(f.Name) { - op = encNoOp + continue } - engine.instr[fieldnum] = encInstr{op, fieldnum, indir, uintptr(f.Offset)} + op, indir := enc.encOpFor(f.Type, seen) + engine.instr = append(engine.instr, encInstr{*op, fieldNum, indir, uintptr(f.Offset)}) + } + if srt.NumField() > 0 && len(engine.instr) == 0 { + errorf("type %s has no exported fields", rt) } - engine.instr[srt.NumField()] = encInstr{encStructTerminator, 0, 0, 0} + engine.instr = append(engine.instr, encInstr{encStructTerminator, 0, 0, 0}) } else { engine.instr = make([]encInstr, 1) - op, indir := enc.encOpFor(rt) - engine.instr[0] = encInstr{op, singletonField, indir, 0} // offset is zero + op, indir := enc.encOpFor(rt, seen) + engine.instr[0] = encInstr{*op, singletonField, indir, 0} // offset is zero } return engine } @@ -556,18 +568,16 @@ func (enc *Encoder) lockAndGetEncEngine(rt reflect.Type) *encEngine { return enc.getEncEngine(rt) } -func (enc *Encoder) encode(b *bytes.Buffer, value reflect.Value) (err os.Error) { +func (enc *Encoder) encode(b *bytes.Buffer, value reflect.Value, ut *userTypeInfo) (err os.Error) { defer catchError(&err) - // Dereference down to the underlying object. - rt, indir := indirect(value.Type()) - for i := 0; i < indir; i++ { + for i := 0; i < ut.indir; i++ { value = reflect.Indirect(value) } - engine := enc.lockAndGetEncEngine(rt) + engine := enc.lockAndGetEncEngine(ut.base) if value.Type().Kind() == reflect.Struct { - enc.encodeStruct(b, engine, value.Addr()) + enc.encodeStruct(b, engine, value.UnsafeAddr()) } else { - enc.encodeSingle(b, engine, value.Addr()) + enc.encodeSingle(b, engine, value.UnsafeAddr()) } return nil } diff --git a/libgo/go/gob/encoder.go b/libgo/go/gob/encoder.go index 8869b262982..92d036c11c3 100644 --- a/libgo/go/gob/encoder.go +++ b/libgo/go/gob/encoder.go @@ -16,9 +16,8 @@ import ( // other side of a connection. type Encoder struct { mutex sync.Mutex // each item must be sent atomically - w io.Writer // where to send the data + w []io.Writer // where to send the data sent map[reflect.Type]typeId // which types we've already sent - state *encoderState // so we can encode integers, strings directly countState *encoderState // stage for writing counts buf []byte // for collecting the output. err os.Error @@ -27,13 +26,27 @@ type Encoder struct { // NewEncoder returns a new encoder that will transmit on the io.Writer. func NewEncoder(w io.Writer) *Encoder { enc := new(Encoder) - enc.w = w + enc.w = []io.Writer{w} enc.sent = make(map[reflect.Type]typeId) - enc.state = newEncoderState(enc, new(bytes.Buffer)) enc.countState = newEncoderState(enc, new(bytes.Buffer)) return enc } +// writer() returns the innermost writer the encoder is using +func (enc *Encoder) writer() io.Writer { + return enc.w[len(enc.w)-1] +} + +// pushWriter adds a writer to the encoder. +func (enc *Encoder) pushWriter(w io.Writer) { + enc.w = append(enc.w, w) +} + +// popWriter pops the innermost writer. +func (enc *Encoder) popWriter() { + enc.w = enc.w[0 : len(enc.w)-1] +} + func (enc *Encoder) badType(rt reflect.Type) { enc.setError(os.ErrorString("gob: can't encode type " + rt.String())) } @@ -42,16 +55,14 @@ func (enc *Encoder) setError(err os.Error) { if enc.err == nil { // remember the first. enc.err = err } - enc.state.b.Reset() } -// Send the data item preceded by a unsigned count of its length. -func (enc *Encoder) send() { - // Encode the length. - enc.countState.encodeUint(uint64(enc.state.b.Len())) +// writeMessage sends the data item preceded by a unsigned count of its length. +func (enc *Encoder) writeMessage(w io.Writer, b *bytes.Buffer) { + enc.countState.encodeUint(uint64(b.Len())) // Build the buffer. countLen := enc.countState.b.Len() - total := countLen + enc.state.b.Len() + total := countLen + b.Len() if total > len(enc.buf) { enc.buf = make([]byte, total+1000) // extra for growth } @@ -59,17 +70,18 @@ func (enc *Encoder) send() { // TODO(r): avoid the extra copy here. enc.countState.b.Read(enc.buf[0:countLen]) // Now the data. - enc.state.b.Read(enc.buf[countLen:total]) + b.Read(enc.buf[countLen:total]) // Write the data. - _, err := enc.w.Write(enc.buf[0:total]) + _, err := w.Write(enc.buf[0:total]) if err != nil { enc.setError(err) } } -func (enc *Encoder) sendType(origt reflect.Type) (sent bool) { +func (enc *Encoder) sendType(w io.Writer, state *encoderState, origt reflect.Type) (sent bool) { // Drill down to the base type. - rt, _ := indirect(origt) + ut := userType(origt) + rt := ut.base switch rt := rt.(type) { default: @@ -112,10 +124,10 @@ func (enc *Encoder) sendType(origt reflect.Type) (sent bool) { } // Send the pair (-id, type) // Id: - enc.state.encodeInt(-int64(info.id)) + state.encodeInt(-int64(info.id)) // Type: - enc.encode(enc.state.b, reflect.NewValue(info.wire)) - enc.send() + enc.encode(state.b, reflect.NewValue(info.wire), wireTypeUserInfo) + enc.writeMessage(w, state.b) if enc.err != nil { return } @@ -128,10 +140,10 @@ func (enc *Encoder) sendType(origt reflect.Type) (sent bool) { switch st := rt.(type) { case *reflect.StructType: for i := 0; i < st.NumField(); i++ { - enc.sendType(st.Field(i).Type) + enc.sendType(w, state, st.Field(i).Type) } case reflect.ArrayOrSliceType: - enc.sendType(st.Elem()) + enc.sendType(w, state, st.Elem()) } return true } @@ -142,15 +154,16 @@ func (enc *Encoder) Encode(e interface{}) os.Error { return enc.EncodeValue(reflect.NewValue(e)) } -// sendTypeId makes sure the remote side knows about this type. +// sendTypeDescriptor makes sure the remote side knows about this type. // It will send a descriptor if this is the first time the type has been -// sent. Regardless, it sends the id. -func (enc *Encoder) sendTypeDescriptor(rt reflect.Type) { +// sent. +func (enc *Encoder) sendTypeDescriptor(w io.Writer, state *encoderState, ut *userTypeInfo) { // Make sure the type is known to the other side. - // First, have we already sent this type? - if _, alreadySent := enc.sent[rt]; !alreadySent { + // First, have we already sent this (base) type? + base := ut.base + if _, alreadySent := enc.sent[base]; !alreadySent { // No, so send it. - sent := enc.sendType(rt) + sent := enc.sendType(w, state, base) if enc.err != nil { return } @@ -159,18 +172,21 @@ func (enc *Encoder) sendTypeDescriptor(rt reflect.Type) { // need to send the type info but we do need to update enc.sent. if !sent { typeLock.Lock() - info, err := getTypeInfo(rt) + info, err := getTypeInfo(base) typeLock.Unlock() if err != nil { enc.setError(err) return } - enc.sent[rt] = info.id + enc.sent[base] = info.id } } +} +// sendTypeId sends the id, which must have already been defined. +func (enc *Encoder) sendTypeId(state *encoderState, ut *userTypeInfo) { // Identify the type of this top-level value. - enc.state.encodeInt(int64(enc.sent[rt])) + state.encodeInt(int64(enc.sent[ut.base])) } // EncodeValue transmits the data item represented by the reflection value, @@ -181,26 +197,29 @@ func (enc *Encoder) EncodeValue(value reflect.Value) os.Error { enc.mutex.Lock() defer enc.mutex.Unlock() - enc.err = nil - rt, _ := indirect(value.Type()) + // Remove any nested writers remaining due to previous errors. + enc.w = enc.w[0:1] - // Sanity check only: encoder should never come in with data present. - if enc.state.b.Len() > 0 || enc.countState.b.Len() > 0 { - enc.err = os.ErrorString("encoder: buffer not empty") - return enc.err + ut, err := validUserType(value.Type()) + if err != nil { + return err } - enc.sendTypeDescriptor(rt) + enc.err = nil + state := newEncoderState(enc, new(bytes.Buffer)) + + enc.sendTypeDescriptor(enc.writer(), state, ut) + enc.sendTypeId(state, ut) if enc.err != nil { return enc.err } // Encode the object. - err := enc.encode(enc.state.b, value) + err = enc.encode(state.b, value, ut) if err != nil { enc.setError(err) } else { - enc.send() + enc.writeMessage(enc.writer(), state.b) } return enc.err diff --git a/libgo/go/gob/encoder_test.go b/libgo/go/gob/encoder_test.go index c2309352a09..a0c713b81df 100644 --- a/libgo/go/gob/encoder_test.go +++ b/libgo/go/gob/encoder_test.go @@ -6,6 +6,7 @@ package gob import ( "bytes" + "fmt" "io" "os" "reflect" @@ -120,7 +121,7 @@ func corruptDataCheck(s string, err os.Error, t *testing.T) { dec := NewDecoder(b) err1 := dec.Decode(new(ET2)) if err1 != err { - t.Error("expected error", err, "got", err1) + t.Errorf("from %q expected error %s; got %s", s, err, err1) } } @@ -220,7 +221,7 @@ func TestSlice(t *testing.T) { func TestValueError(t *testing.T) { // Encode a *T, decode a T type Type4 struct { - a int + A int } t4p := &Type4{3} var t4 Type4 // note: not a pointer. @@ -248,6 +249,24 @@ func TestArray(t *testing.T) { } } +func TestRecursiveMapType(t *testing.T) { + type recursiveMap map[string]recursiveMap + r1 := recursiveMap{"A": recursiveMap{"B": nil, "C": nil}, "D": nil} + r2 := make(recursiveMap) + if err := encAndDec(r1, &r2); err != nil { + t.Error(err) + } +} + +func TestRecursiveSliceType(t *testing.T) { + type recursiveSlice []recursiveSlice + r1 := recursiveSlice{0: recursiveSlice{0: nil}, 1: nil} + r2 := make(recursiveSlice, 0) + if err := encAndDec(r1, &r2); err != nil { + t.Error(err) + } +} + // Regression test for bug: must send zero values inside arrays func TestDefaultsInArray(t *testing.T) { type Type7 struct { @@ -383,3 +402,115 @@ func TestInterfaceIndirect(t *testing.T) { t.Fatal("decode error:", err) } } + +// Now follow various tests that decode into things that can't represent the +// encoded value, all of which should be legal. + +// Also, when the ignored object contains an interface value, it may define +// types. Make sure that skipping the value still defines the types by using +// the encoder/decoder pair to send a value afterwards. If an interface +// is sent, its type in the test is always NewType0, so this checks that the +// encoder and decoder don't skew with respect to type definitions. + +type Struct0 struct { + I interface{} +} + +type NewType0 struct { + S string +} + +type ignoreTest struct { + in, out interface{} +} + +var ignoreTests = []ignoreTest{ + // Decode normal struct into an empty struct + {&struct{ A int }{23}, &struct{}{}}, + // Decode normal struct into a nil. + {&struct{ A int }{23}, nil}, + // Decode singleton string into a nil. + {"hello, world", nil}, + // Decode singleton slice into a nil. + {[]int{1, 2, 3, 4}, nil}, + // Decode struct containing an interface into a nil. + {&Struct0{&NewType0{"value0"}}, nil}, + // Decode singleton slice of interfaces into a nil. + {[]interface{}{"hi", &NewType0{"value1"}, 23}, nil}, +} + +func TestDecodeIntoNothing(t *testing.T) { + Register(new(NewType0)) + for i, test := range ignoreTests { + b := new(bytes.Buffer) + enc := NewEncoder(b) + err := enc.Encode(test.in) + if err != nil { + t.Errorf("%d: encode error %s:", i, err) + continue + } + dec := NewDecoder(b) + err = dec.Decode(test.out) + if err != nil { + t.Errorf("%d: decode error: %s", i, err) + continue + } + // Now see if the encoder and decoder are in a consistent state. + str := fmt.Sprintf("Value %d", i) + err = enc.Encode(&NewType0{str}) + if err != nil { + t.Fatalf("%d: NewType0 encode error: %s", i, err) + } + ns := new(NewType0) + err = dec.Decode(ns) + if err != nil { + t.Fatalf("%d: NewType0 decode error: %s", i, err) + } + if ns.S != str { + t.Fatalf("%d: expected %q got %q", i, str, ns.S) + } + } +} + +// Another bug from golang-nuts, involving nested interfaces. +type Bug0Outer struct { + Bug0Field interface{} +} + +type Bug0Inner struct { + A int +} + +func TestNestedInterfaces(t *testing.T) { + var buf bytes.Buffer + e := NewEncoder(&buf) + d := NewDecoder(&buf) + Register(new(Bug0Outer)) + Register(new(Bug0Inner)) + f := &Bug0Outer{&Bug0Outer{&Bug0Inner{7}}} + var v interface{} = f + err := e.Encode(&v) + if err != nil { + t.Fatal("Encode:", err) + } + err = d.Decode(&v) + if err != nil { + t.Fatal("Decode:", err) + } + // Make sure it decoded correctly. + outer1, ok := v.(*Bug0Outer) + if !ok { + t.Fatalf("v not Bug0Outer: %T", v) + } + outer2, ok := outer1.Bug0Field.(*Bug0Outer) + if !ok { + t.Fatalf("v.Bug0Field not Bug0Outer: %T", outer1.Bug0Field) + } + inner, ok := outer2.Bug0Field.(*Bug0Inner) + if !ok { + t.Fatalf("v.Bug0Field.Bug0Field not Bug0Inner: %T", outer2.Bug0Field) + } + if inner.A != 7 { + t.Fatalf("final value %d; expected %d", inner.A, 7) + } +} diff --git a/libgo/go/gob/type.go b/libgo/go/gob/type.go index 22502a6e6b9..6e3f148b4e7 100644 --- a/libgo/go/gob/type.go +++ b/libgo/go/gob/type.go @@ -11,13 +11,76 @@ import ( "sync" ) -// Reflection types are themselves interface values holding structs -// describing the type. Each type has a different struct so that struct can -// be the kind. For example, if typ is the reflect type for an int8, typ is -// a pointer to a reflect.Int8Type struct; if typ is the reflect type for a -// function, typ is a pointer to a reflect.FuncType struct; we use the type -// of that pointer as the kind. +// userTypeInfo stores the information associated with a type the user has handed +// to the package. It's computed once and stored in a map keyed by reflection +// type. +type userTypeInfo struct { + user reflect.Type // the type the user handed us + base reflect.Type // the base type after all indirections + indir int // number of indirections to reach the base type +} + +var ( + // Protected by an RWMutex because we read it a lot and write + // it only when we see a new type, typically when compiling. + userTypeLock sync.RWMutex + userTypeCache = make(map[reflect.Type]*userTypeInfo) +) + +// validType returns, and saves, the information associated with user-provided type rt. +// If the user type is not valid, err will be non-nil. To be used when the error handler +// is not set up. +func validUserType(rt reflect.Type) (ut *userTypeInfo, err os.Error) { + userTypeLock.RLock() + ut = userTypeCache[rt] + userTypeLock.RUnlock() + if ut != nil { + return + } + // Now set the value under the write lock. + userTypeLock.Lock() + defer userTypeLock.Unlock() + if ut = userTypeCache[rt]; ut != nil { + // Lost the race; not a problem. + return + } + ut = new(userTypeInfo) + ut.base = rt + ut.user = rt + // A type that is just a cycle of pointers (such as type T *T) cannot + // be represented in gobs, which need some concrete data. We use a + // cycle detection algorithm from Knuth, Vol 2, Section 3.1, Ex 6, + // pp 539-540. As we step through indirections, run another type at + // half speed. If they meet up, there's a cycle. + slowpoke := ut.base // walks half as fast as ut.base + for { + pt, ok := ut.base.(*reflect.PtrType) + if !ok { + break + } + ut.base = pt.Elem() + if ut.base == slowpoke { // ut.base lapped slowpoke + // recursive pointer type. + return nil, os.ErrorString("can't represent recursive pointer type " + ut.base.String()) + } + if ut.indir%2 == 0 { + slowpoke = slowpoke.(*reflect.PtrType).Elem() + } + ut.indir++ + } + userTypeCache[rt] = ut + return +} +// userType returns, and saves, the information associated with user-provided type rt. +// If the user type is not valid, it calls error. +func userType(rt reflect.Type) *userTypeInfo { + ut, err := validUserType(rt) + if err != nil { + error(err) + } + return ut +} // A typeId represents a gob Type as an integer that can be passed on the wire. // Internally, typeIds are used as keys to a map to recover the underlying type info. type typeId int32 @@ -93,7 +156,7 @@ var ( tBool = bootstrapType("bool", false, 1) tInt = bootstrapType("int", int(0), 2) tUint = bootstrapType("uint", uint(0), 3) - tFloat = bootstrapType("float", 0.0, 4) + tFloat = bootstrapType("float", float64(0), 4) tBytes = bootstrapType("bytes", make([]byte, 0), 5) tString = bootstrapType("string", "", 6) tComplex = bootstrapType("complex", 0+0i, 7) @@ -110,6 +173,7 @@ var ( // Predefined because it's needed by the Decoder var tWireType = mustGetTypeInfo(reflect.Typeof(wireType{})).id +var wireTypeUserInfo *userTypeInfo // userTypeInfo of (*wireType) func init() { // Some magic numbers to make sure there are no surprises. @@ -133,6 +197,7 @@ func init() { } nextId = firstUserId registerBasics() + wireTypeUserInfo = userType(reflect.Typeof((*wireType)(nil))) } // Array type @@ -142,12 +207,18 @@ type arrayType struct { Len int } -func newArrayType(name string, elem gobType, length int) *arrayType { - a := &arrayType{CommonType{Name: name}, elem.id(), length} - setTypeId(a) +func newArrayType(name string) *arrayType { + a := &arrayType{CommonType{Name: name}, 0, 0} return a } +func (a *arrayType) init(elem gobType, len int) { + // Set our type id before evaluating the element's, in case it's our own. + setTypeId(a) + a.Elem = elem.id() + a.Len = len +} + func (a *arrayType) safeString(seen map[typeId]bool) string { if seen[a.Id] { return a.Name @@ -165,12 +236,18 @@ type mapType struct { Elem typeId } -func newMapType(name string, key, elem gobType) *mapType { - m := &mapType{CommonType{Name: name}, key.id(), elem.id()} - setTypeId(m) +func newMapType(name string) *mapType { + m := &mapType{CommonType{Name: name}, 0, 0} return m } +func (m *mapType) init(key, elem gobType) { + // Set our type id before evaluating the element's, in case it's our own. + setTypeId(m) + m.Key = key.id() + m.Elem = elem.id() +} + func (m *mapType) safeString(seen map[typeId]bool) string { if seen[m.Id] { return m.Name @@ -189,12 +266,17 @@ type sliceType struct { Elem typeId } -func newSliceType(name string, elem gobType) *sliceType { - s := &sliceType{CommonType{Name: name}, elem.id()} - setTypeId(s) +func newSliceType(name string) *sliceType { + s := &sliceType{CommonType{Name: name}, 0} return s } +func (s *sliceType) init(elem gobType) { + // Set our type id before evaluating the element's, in case it's our own. + setTypeId(s) + s.Elem = elem.id() +} + func (s *sliceType) safeString(seen map[typeId]bool) string { if seen[s.Id] { return s.Name @@ -236,26 +318,26 @@ func (s *structType) string() string { return s.safeString(make(map[typeId]bool) func newStructType(name string) *structType { s := &structType{CommonType{Name: name}, nil} + // For historical reasons we set the id here rather than init. + // Se the comment in newTypeObject for details. setTypeId(s) return s } -// Step through the indirections on a type to discover the base type. -// Return the base type and the number of indirections. -func indirect(t reflect.Type) (rt reflect.Type, count int) { - rt = t - for { - pt, ok := rt.(*reflect.PtrType) - if !ok { - break - } - rt = pt.Elem() - count++ - } - return +func (s *structType) init(field []*fieldType) { + s.Field = field } func newTypeObject(name string, rt reflect.Type) (gobType, os.Error) { + var err os.Error + var type0, type1 gobType + defer func() { + if err != nil { + types[rt] = nil, false + } + }() + // Install the top-level type before the subtypes (e.g. struct before + // fields) so recursive types can be constructed safely. switch t := rt.(type) { // All basic types are easy: they are predefined. case *reflect.BoolType: @@ -280,47 +362,62 @@ func newTypeObject(name string, rt reflect.Type) (gobType, os.Error) { return tInterface.gobType(), nil case *reflect.ArrayType: - gt, err := getType("", t.Elem()) + at := newArrayType(name) + types[rt] = at + type0, err = getType("", t.Elem()) if err != nil { return nil, err } - return newArrayType(name, gt, t.Len()), nil + // Historical aside: + // For arrays, maps, and slices, we set the type id after the elements + // are constructed. This is to retain the order of type id allocation after + // a fix made to handle recursive types, which changed the order in + // which types are built. Delaying the setting in this way preserves + // type ids while allowing recursive types to be described. Structs, + // done below, were already handling recursion correctly so they + // assign the top-level id before those of the field. + at.init(type0, t.Len()) + return at, nil case *reflect.MapType: - kt, err := getType("", t.Key()) + mt := newMapType(name) + types[rt] = mt + type0, err = getType("", t.Key()) if err != nil { return nil, err } - vt, err := getType("", t.Elem()) + type1, err = getType("", t.Elem()) if err != nil { return nil, err } - return newMapType(name, kt, vt), nil + mt.init(type0, type1) + return mt, nil case *reflect.SliceType: // []byte == []uint8 is a special case if t.Elem().Kind() == reflect.Uint8 { return tBytes.gobType(), nil } - gt, err := getType(t.Elem().Name(), t.Elem()) + st := newSliceType(name) + types[rt] = st + type0, err = getType(t.Elem().Name(), t.Elem()) if err != nil { return nil, err } - return newSliceType(name, gt), nil + st.init(type0) + return st, nil case *reflect.StructType: - // Install the struct type itself before the fields so recursive - // structures can be constructed safely. - strType := newStructType(name) - types[rt] = strType - idToType[strType.id()] = strType + st := newStructType(name) + types[rt] = st + idToType[st.id()] = st field := make([]*fieldType, t.NumField()) for i := 0; i < t.NumField(); i++ { f := t.Field(i) - typ, _ := indirect(f.Type) + typ := userType(f.Type).base tname := typ.Name() if tname == "" { - t, _ := indirect(f.Type) + t := userType(f.Type).base tname = t.String() } gt, err := getType(tname, f.Type) @@ -329,8 +426,8 @@ func newTypeObject(name string, rt reflect.Type) (gobType, os.Error) { } field[i] = &fieldType{f.Name, gt.id()} } - strType.Field = field - return strType, nil + st.init(field) + return st, nil default: return nil, os.ErrorString("gob NewTypeObject can't handle type: " + rt.String()) @@ -341,7 +438,7 @@ func newTypeObject(name string, rt reflect.Type) (gobType, os.Error) { // getType returns the Gob type describing the given reflect.Type. // typeLock must be held. func getType(name string, rt reflect.Type) (gobType, os.Error) { - rt, _ = indirect(rt) + rt = userType(rt).base typ, present := types[rt] if present { return typ, nil @@ -371,6 +468,7 @@ func bootstrapType(name string, e interface{}, expect typeId) typeId { types[rt] = typ setTypeId(typ) checkId(expect, nextId) + userType(rt) // might as well cache it now return nextId } @@ -381,7 +479,7 @@ func bootstrapType(name string, e interface{}, expect typeId) typeId { // For bootstrapping purposes, we assume that the recipient knows how // to decode a wireType; it is exactly the wireType struct here, interpreted // using the gob rules for sending a structure, except that we assume the -// ids for wireType and structType are known. The relevant pieces +// ids for wireType and structType etc. are known. The relevant pieces // are built in encode.go's init() function. // To maintain binary compatibility, if you extend this type, always put // the new fields last. @@ -473,18 +571,18 @@ func RegisterName(name string, value interface{}) { // reserved for nil panic("attempt to register empty name") } - rt, _ := indirect(reflect.Typeof(value)) + base := userType(reflect.Typeof(value)).base // Check for incompatible duplicates. - if t, ok := nameToConcreteType[name]; ok && t != rt { + if t, ok := nameToConcreteType[name]; ok && t != base { panic("gob: registering duplicate types for " + name) } - if n, ok := concreteTypeToName[rt]; ok && n != name { - panic("gob: registering duplicate names for " + rt.String()) + if n, ok := concreteTypeToName[base]; ok && n != name { + panic("gob: registering duplicate names for " + base.String()) } // Store the name and type provided by the user.... nameToConcreteType[name] = reflect.Typeof(value) // but the flattened type in the type table, since that's what decode needs. - concreteTypeToName[rt] = name + concreteTypeToName[base] = name } // Register records a type, identified by a value for that type, under its @@ -530,7 +628,7 @@ func registerBasics() { Register(uint32(0)) Register(uint64(0)) Register(float32(0)) - Register(0.0) + Register(float64(0)) Register(complex64(0i)) Register(complex128(0i)) Register(false) diff --git a/libgo/go/html/doc.go b/libgo/go/html/doc.go index c5338d0781d..4f5dee72da3 100644 --- a/libgo/go/html/doc.go +++ b/libgo/go/html/doc.go @@ -69,6 +69,9 @@ call to Next. For example, to extract an HTML page's anchor text: } } +A Tokenizer typically skips over HTML comments. To return comment tokens, set +Tokenizer.ReturnComments to true before looping over calls to Next. + Parsing is done by calling Parse with an io.Reader, which returns the root of the parse tree (the document element) as a *Node. It is the caller's responsibility to ensure that the Reader provides UTF-8 encoded HTML. For diff --git a/libgo/go/html/token.go b/libgo/go/html/token.go index d6388385051..ad03241ed92 100644 --- a/libgo/go/html/token.go +++ b/libgo/go/html/token.go @@ -25,6 +25,8 @@ const ( EndTagToken // A SelfClosingTagToken tag looks like <br/>. SelfClosingTagToken + // A CommentToken looks like <!--x-->. + CommentToken ) // String returns a string representation of the TokenType. @@ -40,6 +42,8 @@ func (t TokenType) String() string { return "EndTag" case SelfClosingTagToken: return "SelfClosingTag" + case CommentToken: + return "Comment" } return "Invalid(" + strconv.Itoa(int(t)) + ")" } @@ -52,8 +56,8 @@ type Attribute struct { } // A Token consists of a TokenType and some Data (tag name for start and end -// tags, content for text). A tag Token may also contain a slice of Attributes. -// Data is unescaped for both tag and text Tokens (it looks like "a<b" rather +// tags, content for text and comments). A tag Token may also contain a slice +// of Attributes. Data is unescaped for all Tokens (it looks like "a<b" rather // than "a<b"). type Token struct { Type TokenType @@ -91,12 +95,18 @@ func (t Token) String() string { return "</" + t.tagString() + ">" case SelfClosingTagToken: return "<" + t.tagString() + "/>" + case CommentToken: + return "<!--" + EscapeString(t.Data) + "-->" } return "Invalid(" + strconv.Itoa(int(t.Type)) + ")" } // A Tokenizer returns a stream of HTML Tokens. type Tokenizer struct { + // If ReturnComments is set, Next returns comment tokens; + // otherwise it skips over comments (default). + ReturnComments bool + // r is the source of the HTML text. r io.Reader // tt is the TokenType of the most recently read token. If tt == Error @@ -176,6 +186,39 @@ func (z *Tokenizer) readTo(x uint8) os.Error { panic("unreachable") } +// nextMarkupDeclaration returns the next TokenType starting with "<!". +func (z *Tokenizer) nextMarkupDeclaration() (TokenType, os.Error) { + // TODO: check for <!DOCTYPE ... >, don't just assume that it's a comment. + for i := 0; i < 2; i++ { + c, err := z.readByte() + if err != nil { + return TextToken, err + } + if c != '-' { + return z.nextText(), nil + } + } + // <!--> is a valid comment. + for dashCount := 2; ; { + c, err := z.readByte() + if err != nil { + return TextToken, err + } + switch c { + case '-': + dashCount++ + case '>': + if dashCount >= 2 { + return CommentToken, nil + } + fallthrough + default: + dashCount = 0 + } + } + panic("unreachable") +} + // nextTag returns the next TokenType starting from the tag open state. func (z *Tokenizer) nextTag() (tt TokenType, err os.Error) { c, err := z.readByte() @@ -189,7 +232,7 @@ func (z *Tokenizer) nextTag() (tt TokenType, err os.Error) { case 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z': tt = StartTagToken case c == '!': - return ErrorToken, os.NewError("html: TODO(nigeltao): implement comments") + return z.nextMarkupDeclaration() case c == '?': return ErrorToken, os.NewError("html: TODO(nigeltao): implement XML processing instructions") default: @@ -221,22 +264,8 @@ func (z *Tokenizer) nextTag() (tt TokenType, err os.Error) { panic("unreachable") } -// Next scans the next token and returns its type. -func (z *Tokenizer) Next() TokenType { - if z.err != nil { - z.tt = ErrorToken - return z.tt - } - z.p0 = z.p1 - c, err := z.readByte() - if err != nil { - z.tt, z.err = ErrorToken, err - return z.tt - } - if c == '<' { - z.tt, z.err = z.nextTag() - return z.tt - } +// nextText reads all text up until an '<'. +func (z *Tokenizer) nextText() TokenType { for { c, err := z.readByte() if err != nil { @@ -255,6 +284,31 @@ func (z *Tokenizer) Next() TokenType { panic("unreachable") } +// Next scans the next token and returns its type. +func (z *Tokenizer) Next() TokenType { + for { + if z.err != nil { + z.tt = ErrorToken + return z.tt + } + z.p0 = z.p1 + c, err := z.readByte() + if err != nil { + z.tt, z.err = ErrorToken, err + return z.tt + } + if c == '<' { + z.tt, z.err = z.nextTag() + if z.tt == CommentToken && !z.ReturnComments { + continue + } + return z.tt + } + return z.nextText() + } + panic("unreachable") +} + // trim returns the largest j such that z.buf[i:j] contains only white space, // or only white space plus the final ">" or "/>" of the raw data. func (z *Tokenizer) trim(i int) int { @@ -299,18 +353,33 @@ loop: return z.buf[i0:i], z.trim(i) } -// Text returns the raw data after unescaping. +// Text returns the unescaped text of a TextToken or a CommentToken. // The contents of the returned slice may change on the next call to Next. func (z *Tokenizer) Text() []byte { - s := unescape(z.Raw()) - z.p0 = z.p1 - return s + switch z.tt { + case TextToken: + s := unescape(z.Raw()) + z.p0 = z.p1 + return s + case CommentToken: + // We trim the "<!--" from the left and the "-->" from the right. + // "<!-->" is a valid comment, so the adjusted endpoints might overlap. + i0 := z.p0 + 4 + i1 := z.p1 - 3 + z.p0 = z.p1 + var s []byte + if i0 < i1 { + s = unescape(z.buf[i0:i1]) + } + return s + } + return nil } // TagName returns the lower-cased name of a tag token (the `img` out of -// `<IMG SRC="foo">`), and whether the tag has attributes. +// `<IMG SRC="foo">`) and whether the tag has attributes. // The contents of the returned slice may change on the next call to Next. -func (z *Tokenizer) TagName() (name []byte, remaining bool) { +func (z *Tokenizer) TagName() (name []byte, hasAttr bool) { i := z.p0 + 1 if i >= z.p1 { z.p0 = z.p1 @@ -320,14 +389,14 @@ func (z *Tokenizer) TagName() (name []byte, remaining bool) { i++ } name, z.p0 = z.lower(i) - remaining = z.p0 != z.p1 + hasAttr = z.p0 != z.p1 return } // TagAttr returns the lower-cased key and unescaped value of the next unparsed -// attribute for the current tag token, and whether there are more attributes. +// attribute for the current tag token and whether there are more attributes. // The contents of the returned slices may change on the next call to Next. -func (z *Tokenizer) TagAttr() (key, val []byte, remaining bool) { +func (z *Tokenizer) TagAttr() (key, val []byte, moreAttr bool) { key, i := z.lower(z.p0) // Get past the "=\"". if i == z.p1 || z.buf[i] != '=' { @@ -363,7 +432,7 @@ loop: } } val, z.p0 = z.buf[i:dst], z.trim(src) - remaining = z.p0 != z.p1 + moreAttr = z.p0 != z.p1 return } @@ -372,14 +441,14 @@ loop: func (z *Tokenizer) Token() Token { t := Token{Type: z.tt} switch z.tt { - case TextToken: + case TextToken, CommentToken: t.Data = string(z.Text()) case StartTagToken, EndTagToken, SelfClosingTagToken: var attr []Attribute - name, remaining := z.TagName() - for remaining { + name, moreAttr := z.TagName() + for moreAttr { var key, val []byte - key, val, remaining = z.TagAttr() + key, val, moreAttr = z.TagAttr() attr = append(attr, Attribute{string(key), string(val)}) } t.Data = string(name) diff --git a/libgo/go/html/token_test.go b/libgo/go/html/token_test.go index e07999ca5ad..5cf1f6dac30 100644 --- a/libgo/go/html/token_test.go +++ b/libgo/go/html/token_test.go @@ -7,6 +7,7 @@ package html import ( "bytes" "os" + "strings" "testing" ) @@ -15,8 +16,8 @@ type tokenTest struct { desc string // The HTML to parse. html string - // The string representations of the expected tokens. - tokens []string + // The string representations of the expected tokens, joined by '$'. + golden string } var tokenTests = []tokenTest{ @@ -25,61 +26,86 @@ var tokenTests = []tokenTest{ { "text", "foo bar", - []string{ - "foo bar", - }, + "foo bar", }, // An entity. { "entity", "one < two", - []string{ - "one < two", - }, + "one < two", }, // A start, self-closing and end tag. The tokenizer does not care if the start // and end tokens don't match; that is the job of the parser. { "tags", "<a>b<c/>d</e>", - []string{ - "<a>", - "b", - "<c/>", - "d", - "</e>", - }, + "<a>$b$<c/>$d$</e>", + }, + // Comments. + { + "comment0", + "abc<b><!-- skipme --></b>def", + "abc$<b>$</b>$def", + }, + { + "comment1", + "a<!-->z", + "a$z", + }, + { + "comment2", + "a<!--->z", + "a$z", + }, + { + "comment3", + "a<!--x>-->z", + "a$z", + }, + { + "comment4", + "a<!--x->-->z", + "a$z", + }, + { + "comment5", + "a<!>z", + "a$<!>z", + }, + { + "comment6", + "a<!->z", + "a$<!->z", + }, + { + "comment7", + "a<!---<>z", + "a$<!---<>z", + }, + { + "comment8", + "a<!--z", + "a$<!--z", }, // An attribute with a backslash. { "backslash", `<p id="a\"b">`, - []string{ - `<p id="a"b">`, - }, + `<p id="a"b">`, }, // Entities, tag name and attribute key lower-casing, and whitespace // normalization within a tag. { "tricky", "<p \t\n iD=\"a"B\" foo=\"bar\"><EM>te<&;xt</em></p>", - []string{ - `<p id="a"B" foo="bar">`, - "<em>", - "te<&;xt", - "</em>", - "</p>", - }, + `<p id="a"B" foo="bar">$<em>$te<&;xt$</em>$</p>`, }, // A non-existant entity. Tokenizing and converting back to a string should // escape the "&" to become "&". { "noSuchEntity", `<a b="c&noSuchEntity;d"><&alsoDoesntExist;&`, - []string{ - `<a b="c&noSuchEntity;d">`, - "<&alsoDoesntExist;&", - }, + `<a b="c&noSuchEntity;d">$<&alsoDoesntExist;&`, }, } @@ -87,7 +113,7 @@ func TestTokenizer(t *testing.T) { loop: for _, tt := range tokenTests { z := NewTokenizer(bytes.NewBuffer([]byte(tt.html))) - for i, s := range tt.tokens { + for i, s := range strings.Split(tt.golden, "$", -1) { if z.Next() == ErrorToken { t.Errorf("%s token %d: want %q got error %v", tt.desc, i, s, z.Error()) continue loop diff --git a/libgo/go/http/client.go b/libgo/go/http/client.go index 022f4f124a8..b1fe5ec6780 100644 --- a/libgo/go/http/client.go +++ b/libgo/go/http/client.go @@ -7,18 +7,41 @@ package http import ( - "bufio" "bytes" - "crypto/tls" "encoding/base64" "fmt" "io" - "net" "os" "strconv" "strings" ) +// A Client is an HTTP client. Its zero value (DefaultClient) is a usable client +// that uses DefaultTransport. +// Client is not yet very configurable. +type Client struct { + Transport ClientTransport // if nil, DefaultTransport is used +} + +// DefaultClient is the default Client and is used by Get, Head, and Post. +var DefaultClient = &Client{} + +// ClientTransport is an interface representing the ability to execute a +// single HTTP transaction, obtaining the Response for a given Request. +type ClientTransport interface { + // Do executes a single HTTP transaction, returning the Response for the + // request req. Do should not attempt to interpret the response. + // In particular, Do must return err == nil if it obtained a response, + // regardless of the response's HTTP status code. A non-nil err should + // be reserved for failure to obtain a response. Similarly, Do should + // not attempt to handle higher-level protocol details such as redirects, + // authentication, or cookies. + // + // Transports may modify the request. The request Headers field is + // guaranteed to be initalized. + Do(req *Request) (resp *Response, err os.Error) +} + // Given a string of the form "host", "host:port", or "[ipv6::address]:port", // return true if the string includes a port. func hasPort(s string) bool { return strings.LastIndex(s, ":") > strings.LastIndex(s, "]") } @@ -31,67 +54,83 @@ type readClose struct { io.Closer } -// Send issues an HTTP request. Caller should close resp.Body when done reading it. +// matchNoProxy returns true if requests to addr should not use a proxy, +// according to the NO_PROXY or no_proxy environment variable. +func matchNoProxy(addr string) bool { + if len(addr) == 0 { + return false + } + no_proxy := os.Getenv("NO_PROXY") + if len(no_proxy) == 0 { + no_proxy = os.Getenv("no_proxy") + } + if no_proxy == "*" { + return true + } + + addr = strings.ToLower(strings.TrimSpace(addr)) + if hasPort(addr) { + addr = addr[:strings.LastIndex(addr, ":")] + } + + for _, p := range strings.Split(no_proxy, ",", -1) { + p = strings.ToLower(strings.TrimSpace(p)) + if len(p) == 0 { + continue + } + if hasPort(p) { + p = p[:strings.LastIndex(p, ":")] + } + if addr == p || (p[0] == '.' && (strings.HasSuffix(addr, p) || addr == p[1:])) { + return true + } + } + return false +} + +// Do sends an HTTP request and returns an HTTP response, following +// policy (e.g. redirects, cookies, auth) as configured on the client. +// +// Callers should close resp.Body when done reading from it. +// +// Generally Get, Post, or PostForm will be used instead of Do. +func (c *Client) Do(req *Request) (resp *Response, err os.Error) { + return send(req, c.Transport) +} + + +// send issues an HTTP request. Caller should close resp.Body when done reading from it. // // TODO: support persistent connections (multiple requests on a single connection). // send() method is nonpublic because, when we refactor the code for persistent // connections, it may no longer make sense to have a method with this signature. -func send(req *Request) (resp *Response, err os.Error) { - if req.URL.Scheme != "http" && req.URL.Scheme != "https" { - return nil, &badStringError{"unsupported protocol scheme", req.URL.Scheme} +func send(req *Request, t ClientTransport) (resp *Response, err os.Error) { + if t == nil { + t = DefaultTransport + if t == nil { + err = os.NewError("no http.Client.Transport or http.DefaultTransport") + return + } } - addr := req.URL.Host - if !hasPort(addr) { - addr += ":" + req.URL.Scheme + // Most the callers of send (Get, Post, et al) don't need + // Headers, leaving it uninitialized. We guarantee to the + // ClientTransport that this has been initialized, though. + if req.Header == nil { + req.Header = Header(make(map[string][]string)) } + info := req.URL.RawUserinfo if len(info) > 0 { enc := base64.URLEncoding encoded := make([]byte, enc.EncodedLen(len(info))) enc.Encode(encoded, []byte(info)) if req.Header == nil { - req.Header = make(map[string]string) + req.Header = make(Header) } - req.Header["Authorization"] = "Basic " + string(encoded) - } - - var conn io.ReadWriteCloser - if req.URL.Scheme == "http" { - conn, err = net.Dial("tcp", "", addr) - if err != nil { - return nil, err - } - } else { // https - conn, err = tls.Dial("tcp", "", addr, nil) - if err != nil { - return nil, err - } - h := req.URL.Host - if hasPort(h) { - h = h[0:strings.LastIndex(h, ":")] - } - if err := conn.(*tls.Conn).VerifyHostname(h); err != nil { - return nil, err - } - } - - err = req.Write(conn) - if err != nil { - conn.Close() - return nil, err + req.Header.Set("Authorization", "Basic "+string(encoded)) } - - reader := bufio.NewReader(conn) - resp, err = ReadResponse(reader, req.Method) - if err != nil { - conn.Close() - return nil, err - } - - resp.Body = readClose{resp.Body, conn} - - return + return t.Do(req) } // True if the specified HTTP status code is one for which the Get utility should @@ -115,12 +154,32 @@ func shouldRedirect(statusCode int) bool { // finalURL is the URL from which the response was fetched -- identical to the // input URL unless redirects were followed. // -// Caller should close r.Body when done reading it. +// Caller should close r.Body when done reading from it. +// +// Get is a convenience wrapper around DefaultClient.Get. func Get(url string) (r *Response, finalURL string, err os.Error) { + return DefaultClient.Get(url) +} + +// Get issues a GET to the specified URL. If the response is one of the following +// redirect codes, it follows the redirect, up to a maximum of 10 redirects: +// +// 301 (Moved Permanently) +// 302 (Found) +// 303 (See Other) +// 307 (Temporary Redirect) +// +// finalURL is the URL from which the response was fetched -- identical to the +// input URL unless redirects were followed. +// +// Caller should close r.Body when done reading from it. +func (c *Client) Get(url string) (r *Response, finalURL string, err os.Error) { // TODO: if/when we add cookie support, the redirected request shouldn't // necessarily supply the same cookies as the original. // TODO: set referrer header on redirects. var base *URL + // TODO: remove this hard-coded 10 and use the Client's policy + // (ClientConfig) instead. for redirect := 0; ; redirect++ { if redirect >= 10 { err = os.ErrorString("stopped after 10 redirects") @@ -128,6 +187,9 @@ func Get(url string) (r *Response, finalURL string, err os.Error) { } var req Request + req.Method = "GET" + req.ProtoMajor = 1 + req.ProtoMinor = 1 if base == nil { req.URL, err = ParseURL(url) } else { @@ -137,12 +199,12 @@ func Get(url string) (r *Response, finalURL string, err os.Error) { break } url = req.URL.String() - if r, err = send(&req); err != nil { + if r, err = send(&req, c.Transport); err != nil { break } if shouldRedirect(r.StatusCode) { r.Body.Close() - if url = r.GetHeader("Location"); url == "" { + if url = r.Header.Get("Location"); url == "" { err = os.ErrorString(fmt.Sprintf("%d response missing Location header", r.StatusCode)) break } @@ -159,16 +221,25 @@ func Get(url string) (r *Response, finalURL string, err os.Error) { // Post issues a POST to the specified URL. // -// Caller should close r.Body when done reading it. +// Caller should close r.Body when done reading from it. +// +// Post is a wrapper around DefaultClient.Post func Post(url string, bodyType string, body io.Reader) (r *Response, err os.Error) { + return DefaultClient.Post(url, bodyType, body) +} + +// Post issues a POST to the specified URL. +// +// Caller should close r.Body when done reading from it. +func (c *Client) Post(url string, bodyType string, body io.Reader) (r *Response, err os.Error) { var req Request req.Method = "POST" req.ProtoMajor = 1 req.ProtoMinor = 1 req.Close = true req.Body = nopCloser{body} - req.Header = map[string]string{ - "Content-Type": bodyType, + req.Header = Header{ + "Content-Type": {bodyType}, } req.TransferEncoding = []string{"chunked"} @@ -177,14 +248,24 @@ func Post(url string, bodyType string, body io.Reader) (r *Response, err os.Erro return nil, err } - return send(&req) + return send(&req, c.Transport) } // PostForm issues a POST to the specified URL, // with data's keys and values urlencoded as the request body. // -// Caller should close r.Body when done reading it. +// Caller should close r.Body when done reading from it. +// +// PostForm is a wrapper around DefaultClient.PostForm func PostForm(url string, data map[string]string) (r *Response, err os.Error) { + return DefaultClient.PostForm(url, data) +} + +// PostForm issues a POST to the specified URL, +// with data's keys and values urlencoded as the request body. +// +// Caller should close r.Body when done reading from it. +func (c *Client) PostForm(url string, data map[string]string) (r *Response, err os.Error) { var req Request req.Method = "POST" req.ProtoMajor = 1 @@ -192,9 +273,9 @@ func PostForm(url string, data map[string]string) (r *Response, err os.Error) { req.Close = true body := urlencode(data) req.Body = nopCloser{body} - req.Header = map[string]string{ - "Content-Type": "application/x-www-form-urlencoded", - "Content-Length": strconv.Itoa(body.Len()), + req.Header = Header{ + "Content-Type": {"application/x-www-form-urlencoded"}, + "Content-Length": {strconv.Itoa(body.Len())}, } req.ContentLength = int64(body.Len()) @@ -203,7 +284,7 @@ func PostForm(url string, data map[string]string) (r *Response, err os.Error) { return nil, err } - return send(&req) + return send(&req, c.Transport) } // TODO: remove this function when PostForm takes a multimap. @@ -216,17 +297,20 @@ func urlencode(data map[string]string) (b *bytes.Buffer) { } // Head issues a HEAD to the specified URL. +// +// Head is a wrapper around DefaultClient.Head func Head(url string) (r *Response, err os.Error) { + return DefaultClient.Head(url) +} + +// Head issues a HEAD to the specified URL. +func (c *Client) Head(url string) (r *Response, err os.Error) { var req Request req.Method = "HEAD" if req.URL, err = ParseURL(url); err != nil { return } - url = req.URL.String() - if r, err = send(&req); err != nil { - return - } - return + return send(&req, c.Transport) } type nopCloser struct { diff --git a/libgo/go/http/client_test.go b/libgo/go/http/client_test.go index 013653a8296..c89ecbce2d0 100644 --- a/libgo/go/http/client_test.go +++ b/libgo/go/http/client_test.go @@ -8,6 +8,7 @@ package http import ( "io/ioutil" + "os" "strings" "testing" ) @@ -38,3 +39,28 @@ func TestClientHead(t *testing.T) { t.Error("Last-Modified header not found.") } } + +type recordingTransport struct { + req *Request +} + +func (t *recordingTransport) Do(req *Request) (resp *Response, err os.Error) { + t.req = req + return nil, os.NewError("dummy impl") +} + +func TestGetRequestFormat(t *testing.T) { + tr := &recordingTransport{} + client := &Client{Transport: tr} + url := "http://dummy.faketld/" + client.Get(url) // Note: doesn't hit network + if tr.req.Method != "GET" { + t.Errorf("expected method %q; got %q", "GET", tr.req.Method) + } + if tr.req.URL.String() != url { + t.Errorf("expected URL %q; got %q", url, tr.req.URL.String()) + } + if tr.req.Header == nil { + t.Errorf("expected non-nil request Header") + } +} diff --git a/libgo/go/http/fs.go b/libgo/go/http/fs.go index bbfa58d264d..8e16992e0f0 100644 --- a/libgo/go/http/fs.go +++ b/libgo/go/http/fs.go @@ -104,7 +104,7 @@ func serveFile(w ResponseWriter, r *Request, name string, redirect bool) { } } - if t, _ := time.Parse(TimeFormat, r.Header["If-Modified-Since"]); t != nil && d.Mtime_ns/1e9 <= t.Seconds() { + if t, _ := time.Parse(TimeFormat, r.Header.Get("If-Modified-Since")); t != nil && d.Mtime_ns/1e9 <= t.Seconds() { w.WriteHeader(StatusNotModified) return } @@ -153,7 +153,7 @@ func serveFile(w ResponseWriter, r *Request, name string, redirect bool) { // handle Content-Range header. // TODO(adg): handle multiple ranges - ranges, err := parseRange(r.Header["Range"], size) + ranges, err := parseRange(r.Header.Get("Range"), size) if err != nil || len(ranges) > 1 { Error(w, err.String(), StatusRequestedRangeNotSatisfiable) return diff --git a/libgo/go/http/fs_test.go b/libgo/go/http/fs_test.go index 0a5636b88d1..a8b67e3f08c 100644 --- a/libgo/go/http/fs_test.go +++ b/libgo/go/http/fs_test.go @@ -109,7 +109,7 @@ func TestServeFile(t *testing.T) { // set up the Request (re-used for all tests) var req Request - req.Header = make(map[string]string) + req.Header = make(Header) if req.URL, err = ParseURL("http://" + serverAddr + "/ServeFile"); err != nil { t.Fatal("ParseURL:", err) } @@ -123,9 +123,9 @@ func TestServeFile(t *testing.T) { // Range tests for _, rt := range ServeFileRangeTests { - req.Header["Range"] = "bytes=" + rt.r + req.Header.Set("Range", "bytes="+rt.r) if rt.r == "" { - req.Header["Range"] = "" + req.Header["Range"] = nil } r, body := getBody(t, req) if r.StatusCode != rt.code { @@ -138,8 +138,9 @@ func TestServeFile(t *testing.T) { if rt.r == "" { h = "" } - if r.Header["Content-Range"] != h { - t.Errorf("header mismatch: range=%q: got %q, want %q", rt.r, r.Header["Content-Range"], h) + cr := r.Header.Get("Content-Range") + if cr != h { + t.Errorf("header mismatch: range=%q: got %q, want %q", rt.r, cr, h) } if !equal(body, file[rt.start:rt.end]) { t.Errorf("body mismatch: range=%q: got %q, want %q", rt.r, body, file[rt.start:rt.end]) @@ -148,7 +149,7 @@ func TestServeFile(t *testing.T) { } func getBody(t *testing.T, req Request) (*Response, []byte) { - r, err := send(&req) + r, err := send(&req, DefaultTransport) if err != nil { t.Fatal(req.URL.String(), "send:", err) } diff --git a/libgo/go/http/header.go b/libgo/go/http/header.go new file mode 100644 index 00000000000..95b0f3db6bb --- /dev/null +++ b/libgo/go/http/header.go @@ -0,0 +1,43 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +import "net/textproto" + +// A Header represents the key-value pairs in an HTTP header. +type Header map[string][]string + +// Add adds the key, value pair to the header. +// It appends to any existing values associated with key. +func (h Header) Add(key, value string) { + textproto.MIMEHeader(h).Add(key, value) +} + +// Set sets the header entries associated with key to +// the single element value. It replaces any existing +// values associated with key. +func (h Header) Set(key, value string) { + textproto.MIMEHeader(h).Set(key, value) +} + +// Get gets the first value associated with the given key. +// If there are no values associated with the key, Get returns "". +// Get is a convenience method. For more complex queries, +// access the map directly. +func (h Header) Get(key string) string { + return textproto.MIMEHeader(h).Get(key) +} + +// Del deletes the values associated with key. +func (h Header) Del(key string) { + textproto.MIMEHeader(h).Del(key) +} + +// CanonicalHeaderKey returns the canonical format of the +// header key s. The canonicalization converts the first +// letter and any letter following a hyphen to upper case; +// the rest are converted to lowercase. For example, the +// canonical key for "accept-encoding" is "Accept-Encoding". +func CanonicalHeaderKey(s string) string { return textproto.CanonicalMIMEHeaderKey(s) } diff --git a/libgo/go/http/persist.go b/libgo/go/http/persist.go index 8bfc0975589..000a4200e59 100644 --- a/libgo/go/http/persist.go +++ b/libgo/go/http/persist.go @@ -6,14 +6,17 @@ package http import ( "bufio" - "container/list" "io" "net" + "net/textproto" "os" "sync" ) -var ErrPersistEOF = &ProtocolError{"persistent connection closed"} +var ( + ErrPersistEOF = &ProtocolError{"persistent connection closed"} + ErrPipeline = &ProtocolError{"pipeline error"} +) // A ServerConn reads requests and sends responses over an underlying // connection, until the HTTP keepalive logic commands an end. ServerConn @@ -26,8 +29,10 @@ type ServerConn struct { r *bufio.Reader clsd bool // indicates a graceful close re, we os.Error // read/write errors - lastBody io.ReadCloser + lastbody io.ReadCloser nread, nwritten int + pipe textproto.Pipeline + pipereq map[*Request]uint lk sync.Mutex // protected read/write to re,we } @@ -37,7 +42,7 @@ func NewServerConn(c net.Conn, r *bufio.Reader) *ServerConn { if r == nil { r = bufio.NewReader(c) } - return &ServerConn{c: c, r: r} + return &ServerConn{c: c, r: r, pipereq: make(map[*Request]uint)} } // Close detaches the ServerConn and returns the underlying connection as well @@ -57,10 +62,25 @@ func (sc *ServerConn) Close() (c net.Conn, r *bufio.Reader) { // Read returns the next request on the wire. An ErrPersistEOF is returned if // it is gracefully determined that there are no more requests (e.g. after the // first request on an HTTP/1.0 connection, or after a Connection:close on a -// HTTP/1.1 connection). Read can be called concurrently with Write, but not -// with another Read. +// HTTP/1.1 connection). func (sc *ServerConn) Read() (req *Request, err os.Error) { + // Ensure ordered execution of Reads and Writes + id := sc.pipe.Next() + sc.pipe.StartRequest(id) + defer func() { + sc.pipe.EndRequest(id) + if req == nil { + sc.pipe.StartResponse(id) + sc.pipe.EndResponse(id) + } else { + // Remember the pipeline id of this request + sc.lk.Lock() + sc.pipereq[req] = id + sc.lk.Unlock() + } + }() + sc.lk.Lock() if sc.we != nil { // no point receiving if write-side broken or closed defer sc.lk.Unlock() @@ -73,12 +93,12 @@ func (sc *ServerConn) Read() (req *Request, err os.Error) { sc.lk.Unlock() // Make sure body is fully consumed, even if user does not call body.Close - if sc.lastBody != nil { + if sc.lastbody != nil { // body.Close is assumed to be idempotent and multiple calls to // it should return the error that its first invokation // returned. - err = sc.lastBody.Close() - sc.lastBody = nil + err = sc.lastbody.Close() + sc.lastbody = nil if err != nil { sc.lk.Lock() defer sc.lk.Unlock() @@ -102,7 +122,7 @@ func (sc *ServerConn) Read() (req *Request, err os.Error) { return } } - sc.lastBody = req.Body + sc.lastbody = req.Body sc.nread++ if req.Close { sc.lk.Lock() @@ -121,11 +141,24 @@ func (sc *ServerConn) Pending() int { return sc.nread - sc.nwritten } -// Write writes a repsonse. To close the connection gracefully, set the +// Write writes resp in response to req. To close the connection gracefully, set the // Response.Close field to true. Write should be considered operational until // it returns an error, regardless of any errors returned on the Read side. -// Write can be called concurrently with Read, but not with another Write. -func (sc *ServerConn) Write(resp *Response) os.Error { +func (sc *ServerConn) Write(req *Request, resp *Response) os.Error { + + // Retrieve the pipeline ID of this request/response pair + sc.lk.Lock() + id, ok := sc.pipereq[req] + sc.pipereq[req] = 0, false + if !ok { + sc.lk.Unlock() + return ErrPipeline + } + sc.lk.Unlock() + + // Ensure pipeline order + sc.pipe.StartResponse(id) + defer sc.pipe.EndResponse(id) sc.lk.Lock() if sc.we != nil { @@ -166,10 +199,11 @@ type ClientConn struct { c net.Conn r *bufio.Reader re, we os.Error // read/write errors - lastBody io.ReadCloser + lastbody io.ReadCloser nread, nwritten int - reqm list.List // request methods in order of execution - lk sync.Mutex // protects read/write to reqm,re,we + pipe textproto.Pipeline + pipereq map[*Request]uint + lk sync.Mutex // protects read/write to re,we,pipereq,etc. } // NewClientConn returns a new ClientConn reading and writing c. If r is not @@ -178,7 +212,7 @@ func NewClientConn(c net.Conn, r *bufio.Reader) *ClientConn { if r == nil { r = bufio.NewReader(c) } - return &ClientConn{c: c, r: r} + return &ClientConn{c: c, r: r, pipereq: make(map[*Request]uint)} } // Close detaches the ClientConn and returns the underlying connection as well @@ -191,7 +225,6 @@ func (cc *ClientConn) Close() (c net.Conn, r *bufio.Reader) { r = cc.r cc.c = nil cc.r = nil - cc.reqm.Init() cc.lk.Unlock() return } @@ -201,8 +234,23 @@ func (cc *ClientConn) Close() (c net.Conn, r *bufio.Reader) { // keepalive connection is logically closed after this request and the opposing // server is informed. An ErrUnexpectedEOF indicates the remote closed the // underlying TCP connection, which is usually considered as graceful close. -// Write can be called concurrently with Read, but not with another Write. -func (cc *ClientConn) Write(req *Request) os.Error { +func (cc *ClientConn) Write(req *Request) (err os.Error) { + + // Ensure ordered execution of Writes + id := cc.pipe.Next() + cc.pipe.StartRequest(id) + defer func() { + cc.pipe.EndRequest(id) + if err != nil { + cc.pipe.StartResponse(id) + cc.pipe.EndResponse(id) + } else { + // Remember the pipeline id of this request + cc.lk.Lock() + cc.pipereq[req] = id + cc.lk.Unlock() + } + }() cc.lk.Lock() if cc.re != nil { // no point sending if read-side closed or broken @@ -223,7 +271,7 @@ func (cc *ClientConn) Write(req *Request) os.Error { cc.lk.Unlock() } - err := req.Write(cc.c) + err = req.Write(cc.c) if err != nil { cc.lk.Lock() defer cc.lk.Unlock() @@ -231,9 +279,6 @@ func (cc *ClientConn) Write(req *Request) os.Error { return err } cc.nwritten++ - cc.lk.Lock() - cc.reqm.PushBack(req.Method) - cc.lk.Unlock() return nil } @@ -250,7 +295,21 @@ func (cc *ClientConn) Pending() int { // returned together with an ErrPersistEOF, which means that the remote // requested that this be the last request serviced. Read can be called // concurrently with Write, but not with another Read. -func (cc *ClientConn) Read() (resp *Response, err os.Error) { +func (cc *ClientConn) Read(req *Request) (resp *Response, err os.Error) { + + // Retrieve the pipeline ID of this request/response pair + cc.lk.Lock() + id, ok := cc.pipereq[req] + cc.pipereq[req] = 0, false + if !ok { + cc.lk.Unlock() + return nil, ErrPipeline + } + cc.lk.Unlock() + + // Ensure pipeline order + cc.pipe.StartResponse(id) + defer cc.pipe.EndResponse(id) cc.lk.Lock() if cc.re != nil { @@ -259,17 +318,13 @@ func (cc *ClientConn) Read() (resp *Response, err os.Error) { } cc.lk.Unlock() - if cc.nread >= cc.nwritten { - return nil, os.NewError("persist client pipe count") - } - // Make sure body is fully consumed, even if user does not call body.Close - if cc.lastBody != nil { + if cc.lastbody != nil { // body.Close is assumed to be idempotent and multiple calls to // it should return the error that its first invokation // returned. - err = cc.lastBody.Close() - cc.lastBody = nil + err = cc.lastbody.Close() + cc.lastbody = nil if err != nil { cc.lk.Lock() defer cc.lk.Unlock() @@ -278,18 +333,14 @@ func (cc *ClientConn) Read() (resp *Response, err os.Error) { } } - cc.lk.Lock() - m := cc.reqm.Front() - cc.reqm.Remove(m) - cc.lk.Unlock() - resp, err = ReadResponse(cc.r, m.Value.(string)) + resp, err = ReadResponse(cc.r, req.Method) if err != nil { cc.lk.Lock() defer cc.lk.Unlock() cc.re = err return } - cc.lastBody = resp.Body + cc.lastbody = resp.Body cc.nread++ @@ -301,3 +352,12 @@ func (cc *ClientConn) Read() (resp *Response, err os.Error) { } return } + +// Do is convenience method that writes a request and reads a response. +func (cc *ClientConn) Do(req *Request) (resp *Response, err os.Error) { + err = cc.Write(req) + if err != nil { + return + } + return cc.Read(req) +} diff --git a/libgo/go/http/proxy_test.go b/libgo/go/http/proxy_test.go new file mode 100644 index 00000000000..0f2ca458fed --- /dev/null +++ b/libgo/go/http/proxy_test.go @@ -0,0 +1,45 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +import ( + "os" + "testing" +) + +// TODO(mattn): +// test ProxyAuth + +var MatchNoProxyTests = []struct { + host string + match bool +}{ + {"localhost", true}, // match completely + {"barbaz.net", true}, // match as .barbaz.net + {"foobar.com:443", true}, // have a port but match + {"foofoobar.com", false}, // not match as a part of foobar.com + {"baz.com", false}, // not match as a part of barbaz.com + {"localhost.net", false}, // not match as suffix of address + {"local.localhost", false}, // not match as prefix as address + {"barbarbaz.net", false}, // not match because NO_PROXY have a '.' + {"www.foobar.com", false}, // not match because NO_PROXY is not .foobar.com +} + +func TestMatchNoProxy(t *testing.T) { + oldenv := os.Getenv("NO_PROXY") + no_proxy := "foobar.com, .barbaz.net , localhost" + os.Setenv("NO_PROXY", no_proxy) + defer os.Setenv("NO_PROXY", oldenv) + + for _, test := range MatchNoProxyTests { + if matchNoProxy(test.host) != test.match { + if test.match { + t.Errorf("matchNoProxy(%v) = %v, want %v", test.host, !test.match, test.match) + } else { + t.Errorf("not expected: '%s' shouldn't match as '%s'", test.host, no_proxy) + } + } + } +} diff --git a/libgo/go/http/readrequest_test.go b/libgo/go/http/readrequest_test.go index 5e1cbcbcbdc..6ee07bc9148 100644 --- a/libgo/go/http/readrequest_test.go +++ b/libgo/go/http/readrequest_test.go @@ -50,14 +50,14 @@ var reqTests = []reqTest{ Proto: "HTTP/1.1", ProtoMajor: 1, ProtoMinor: 1, - Header: map[string]string{ - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "Accept-Language": "en-us,en;q=0.5", - "Accept-Encoding": "gzip,deflate", - "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", - "Keep-Alive": "300", - "Proxy-Connection": "keep-alive", - "Content-Length": "7", + Header: Header{ + "Accept": {"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"}, + "Accept-Language": {"en-us,en;q=0.5"}, + "Accept-Encoding": {"gzip,deflate"}, + "Accept-Charset": {"ISO-8859-1,utf-8;q=0.7,*;q=0.7"}, + "Keep-Alive": {"300"}, + "Proxy-Connection": {"keep-alive"}, + "Content-Length": {"7"}, }, Close: false, ContentLength: 7, @@ -93,7 +93,7 @@ var reqTests = []reqTest{ Proto: "HTTP/1.1", ProtoMajor: 1, ProtoMinor: 1, - Header: map[string]string{}, + Header: map[string][]string{}, Close: false, ContentLength: -1, Host: "test", diff --git a/libgo/go/http/request.go b/libgo/go/http/request.go index 04bebaaf55b..a7dc328a007 100644 --- a/libgo/go/http/request.go +++ b/libgo/go/http/request.go @@ -11,13 +11,13 @@ package http import ( "bufio" - "bytes" "container/vector" "fmt" "io" "io/ioutil" "mime" "mime/multipart" + "net/textproto" "os" "strconv" "strings" @@ -90,7 +90,7 @@ type Request struct { // The request parser implements this by canonicalizing the // name, making the first character and any characters // following a hyphen uppercase and the rest lowercase. - Header map[string]string + Header Header // The message body. Body io.ReadCloser @@ -133,7 +133,7 @@ type Request struct { // Trailer maps trailer keys to values. Like for Header, if the // response has multiple trailer lines with the same key, they will be // concatenated, delimited by commas. - Trailer map[string]string + Trailer Header } // ProtoAtLeast returns whether the HTTP protocol used @@ -146,8 +146,8 @@ func (r *Request) ProtoAtLeast(major, minor int) bool { // MultipartReader returns a MIME multipart reader if this is a // multipart/form-data POST request, else returns nil and an error. func (r *Request) MultipartReader() (multipart.Reader, os.Error) { - v, ok := r.Header["Content-Type"] - if !ok { + v := r.Header.Get("Content-Type") + if v == "" { return nil, ErrNotMultipart } d, params := mime.ParseMediaType(v) @@ -184,6 +184,17 @@ const defaultUserAgent = "Go http package" // If Body is present, Write forces "Transfer-Encoding: chunked" as a header // and then closes Body when finished sending it. func (req *Request) Write(w io.Writer) os.Error { + return req.write(w, false) +} + +// WriteProxy is like Write but writes the request in the form +// expected by an HTTP proxy. It includes the scheme and host +// name in the URI instead of using a separate Host: header line. +func (req *Request) WriteProxy(w io.Writer) os.Error { + return req.write(w, true) +} + +func (req *Request) write(w io.Writer, usingProxy bool) os.Error { host := req.Host if host == "" { host = req.URL.Host @@ -197,10 +208,19 @@ func (req *Request) Write(w io.Writer) os.Error { } } + if usingProxy { + if uri == "" || uri[0] != '/' { + uri = "/" + uri + } + uri = req.URL.Scheme + "://" + host + uri + } + fmt.Fprintf(w, "%s %s HTTP/1.1\r\n", valueOrDefault(req.Method, "GET"), uri) // Header lines - fmt.Fprintf(w, "Host: %s\r\n", host) + if !usingProxy { + fmt.Fprintf(w, "Host: %s\r\n", host) + } fmt.Fprintf(w, "User-Agent: %s\r\n", valueOrDefault(req.UserAgent, defaultUserAgent)) if req.Referer != "" { fmt.Fprintf(w, "Referer: %s\r\n", req.Referer) @@ -277,78 +297,6 @@ func readLine(b *bufio.Reader) (s string, err os.Error) { return string(p), nil } -var colon = []byte{':'} - -// Read a key/value pair from b. -// A key/value has the form Key: Value\r\n -// and the Value can continue on multiple lines if each continuation line -// starts with a space. -func readKeyValue(b *bufio.Reader) (key, value string, err os.Error) { - line, e := readLineBytes(b) - if e != nil { - return "", "", e - } - if len(line) == 0 { - return "", "", nil - } - - // Scan first line for colon. - i := bytes.Index(line, colon) - if i < 0 { - goto Malformed - } - - key = string(line[0:i]) - if strings.Contains(key, " ") { - // Key field has space - no good. - goto Malformed - } - - // Skip initial space before value. - for i++; i < len(line); i++ { - if line[i] != ' ' { - break - } - } - value = string(line[i:]) - - // Look for extension lines, which must begin with space. - for { - c, e := b.ReadByte() - if c != ' ' { - if e != os.EOF { - b.UnreadByte() - } - break - } - - // Eat leading space. - for c == ' ' { - if c, e = b.ReadByte(); e != nil { - if e == os.EOF { - e = io.ErrUnexpectedEOF - } - return "", "", e - } - } - b.UnreadByte() - - // Read the rest of the line and add to value. - if line, e = readLineBytes(b); e != nil { - return "", "", e - } - value += " " + string(line) - - if len(value) >= maxValueLength { - return "", "", &badStringError{"value too long for key", key} - } - } - return key, value, nil - -Malformed: - return "", "", &badStringError{"malformed header line", string(line)} -} - // Convert decimal at s[i:len(s)] to integer, // returning value, string position where the digits stopped, // and whether there was a valid number (digits, not too big). @@ -367,8 +315,9 @@ func atoi(s string, i int) (n, i1 int, ok bool) { return n, i, true } -// Parse HTTP version: "HTTP/1.2" -> (1, 2, true). -func parseHTTPVersion(vers string) (int, int, bool) { +// ParseHTTPVersion parses a HTTP version string. +// "HTTP/1.0" returns (1, 0, true). +func ParseHTTPVersion(vers string) (major, minor int, ok bool) { if len(vers) < 5 || vers[0:5] != "HTTP/" { return 0, 0, false } @@ -376,7 +325,6 @@ func parseHTTPVersion(vers string) (int, int, bool) { if !ok || i >= len(vers) || vers[i] != '.' { return 0, 0, false } - var minor int minor, i, ok = atoi(vers, i+1) if !ok || i != len(vers) { return 0, 0, false @@ -384,43 +332,6 @@ func parseHTTPVersion(vers string) (int, int, bool) { return major, minor, true } -// CanonicalHeaderKey returns the canonical format of the -// HTTP header key s. The canonicalization converts the first -// letter and any letter following a hyphen to upper case; -// the rest are converted to lowercase. For example, the -// canonical key for "accept-encoding" is "Accept-Encoding". -func CanonicalHeaderKey(s string) string { - // canonicalize: first letter upper case - // and upper case after each dash. - // (Host, User-Agent, If-Modified-Since). - // HTTP headers are ASCII only, so no Unicode issues. - var a []byte - upper := true - for i := 0; i < len(s); i++ { - v := s[i] - if upper && 'a' <= v && v <= 'z' { - if a == nil { - a = []byte(s) - } - a[i] = v + 'A' - 'a' - } - if !upper && 'A' <= v && v <= 'Z' { - if a == nil { - a = []byte(s) - } - a[i] = v + 'a' - 'A' - } - upper = false - if v == '-' { - upper = true - } - } - if a != nil { - return string(a) - } - return s -} - type chunkedReader struct { r *bufio.Reader n uint64 // unread bytes in chunk @@ -486,11 +397,16 @@ func (cr *chunkedReader) Read(b []uint8) (n int, err os.Error) { // ReadRequest reads and parses a request from b. func ReadRequest(b *bufio.Reader) (req *Request, err os.Error) { + + tp := textproto.NewReader(b) req = new(Request) // First line: GET /index.html HTTP/1.0 var s string - if s, err = readLine(b); err != nil { + if s, err = tp.ReadLine(); err != nil { + if err == os.EOF { + err = io.ErrUnexpectedEOF + } return nil, err } @@ -500,7 +416,7 @@ func ReadRequest(b *bufio.Reader) (req *Request, err os.Error) { } req.Method, req.RawURL, req.Proto = f[0], f[1], f[2] var ok bool - if req.ProtoMajor, req.ProtoMinor, ok = parseHTTPVersion(req.Proto); !ok { + if req.ProtoMajor, req.ProtoMinor, ok = ParseHTTPVersion(req.Proto); !ok { return nil, &badStringError{"malformed HTTP version", req.Proto} } @@ -509,32 +425,11 @@ func ReadRequest(b *bufio.Reader) (req *Request, err os.Error) { } // Subsequent lines: Key: value. - nheader := 0 - req.Header = make(map[string]string) - for { - var key, value string - if key, value, err = readKeyValue(b); err != nil { - return nil, err - } - if key == "" { - break - } - if nheader++; nheader >= maxHeaderLines { - return nil, ErrHeaderTooLong - } - - key = CanonicalHeaderKey(key) - - // RFC 2616 says that if you send the same header key - // multiple times, it has to be semantically equivalent - // to concatenating the values separated by commas. - oldvalue, present := req.Header[key] - if present { - req.Header[key] = oldvalue + "," + value - } else { - req.Header[key] = value - } + mimeHeader, err := tp.ReadMIMEHeader() + if err != nil { + return nil, err } + req.Header = Header(mimeHeader) // RFC2616: Must treat // GET /index.html HTTP/1.1 @@ -545,18 +440,18 @@ func ReadRequest(b *bufio.Reader) (req *Request, err os.Error) { // the same. In the second case, any Host line is ignored. req.Host = req.URL.Host if req.Host == "" { - req.Host = req.Header["Host"] + req.Host = req.Header.Get("Host") } - req.Header["Host"] = "", false + req.Header.Del("Host") fixPragmaCacheControl(req.Header) // Pull out useful fields as a convenience to clients. - req.Referer = req.Header["Referer"] - req.Header["Referer"] = "", false + req.Referer = req.Header.Get("Referer") + req.Header.Del("Referer") - req.UserAgent = req.Header["User-Agent"] - req.Header["User-Agent"] = "", false + req.UserAgent = req.Header.Get("User-Agent") + req.Header.Del("User-Agent") // TODO: Parse specific header values: // Accept @@ -642,7 +537,7 @@ func (r *Request) ParseForm() (err os.Error) { if r.Body == nil { return os.ErrorString("missing form body") } - ct := r.Header["Content-Type"] + ct := r.Header.Get("Content-Type") switch strings.Split(ct, ";", 2)[0] { case "text/plain", "application/x-www-form-urlencoded", "": b, e := ioutil.ReadAll(r.Body) @@ -677,17 +572,12 @@ func (r *Request) FormValue(key string) string { } func (r *Request) expectsContinue() bool { - expectation, ok := r.Header["Expect"] - return ok && strings.ToLower(expectation) == "100-continue" + return strings.ToLower(r.Header.Get("Expect")) == "100-continue" } func (r *Request) wantsHttp10KeepAlive() bool { if r.ProtoMajor != 1 || r.ProtoMinor != 0 { return false } - value, exists := r.Header["Connection"] - if !exists { - return false - } - return strings.Contains(strings.ToLower(value), "keep-alive") + return strings.Contains(strings.ToLower(r.Header.Get("Connection")), "keep-alive") } diff --git a/libgo/go/http/request_test.go b/libgo/go/http/request_test.go index d25e5e5e7e1..ae1c4e98245 100644 --- a/libgo/go/http/request_test.go +++ b/libgo/go/http/request_test.go @@ -74,7 +74,9 @@ func TestQuery(t *testing.T) { func TestPostQuery(t *testing.T) { req := &Request{Method: "POST"} req.URL, _ = ParseURL("http://www.google.com/search?q=foo&q=bar&both=x") - req.Header = map[string]string{"Content-Type": "application/x-www-form-urlencoded; boo!"} + req.Header = Header{ + "Content-Type": {"application/x-www-form-urlencoded; boo!"}, + } req.Body = nopCloser{strings.NewReader("z=post&both=y")} if q := req.FormValue("q"); q != "foo" { t.Errorf(`req.FormValue("q") = %q, want "foo"`, q) @@ -87,18 +89,18 @@ func TestPostQuery(t *testing.T) { } } -type stringMap map[string]string +type stringMap map[string][]string type parseContentTypeTest struct { contentType stringMap error bool } var parseContentTypeTests = []parseContentTypeTest{ - {contentType: stringMap{"Content-Type": "text/plain"}}, - {contentType: stringMap{"Content-Type": ""}}, - {contentType: stringMap{"Content-Type": "text/plain; boundary="}}, + {contentType: stringMap{"Content-Type": {"text/plain"}}}, + {contentType: stringMap{}}, // Non-existent keys are not placed. The value nil is illegal. + {contentType: stringMap{"Content-Type": {"text/plain; boundary="}}}, { - contentType: stringMap{"Content-Type": "application/unknown"}, + contentType: stringMap{"Content-Type": {"application/unknown"}}, error: true, }, } @@ -107,7 +109,7 @@ func TestPostContentTypeParsing(t *testing.T) { for i, test := range parseContentTypeTests { req := &Request{ Method: "POST", - Header: test.contentType, + Header: Header(test.contentType), Body: nopCloser{bytes.NewBufferString("body")}, } err := req.ParseForm() @@ -123,7 +125,7 @@ func TestPostContentTypeParsing(t *testing.T) { func TestMultipartReader(t *testing.T) { req := &Request{ Method: "POST", - Header: stringMap{"Content-Type": `multipart/form-data; boundary="foo123"`}, + Header: Header{"Content-Type": {`multipart/form-data; boundary="foo123"`}}, Body: nopCloser{new(bytes.Buffer)}, } multipart, err := req.MultipartReader() @@ -131,7 +133,7 @@ func TestMultipartReader(t *testing.T) { t.Errorf("expected multipart; error: %v", err) } - req.Header = stringMap{"Content-Type": "text/plain"} + req.Header = Header{"Content-Type": {"text/plain"}} multipart, err = req.MultipartReader() if multipart != nil { t.Errorf("unexpected multipart for text/plain") diff --git a/libgo/go/http/requestwrite_test.go b/libgo/go/http/requestwrite_test.go index 3ceabe4ee7a..55ca745d58c 100644 --- a/libgo/go/http/requestwrite_test.go +++ b/libgo/go/http/requestwrite_test.go @@ -34,13 +34,13 @@ var reqWriteTests = []reqWriteTest{ Proto: "HTTP/1.1", ProtoMajor: 1, ProtoMinor: 1, - Header: map[string]string{ - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", - "Accept-Encoding": "gzip,deflate", - "Accept-Language": "en-us,en;q=0.5", - "Keep-Alive": "300", - "Proxy-Connection": "keep-alive", + Header: Header{ + "Accept": {"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"}, + "Accept-Charset": {"ISO-8859-1,utf-8;q=0.7,*;q=0.7"}, + "Accept-Encoding": {"gzip,deflate"}, + "Accept-Language": {"en-us,en;q=0.5"}, + "Keep-Alive": {"300"}, + "Proxy-Connection": {"keep-alive"}, }, Body: nil, Close: false, @@ -53,10 +53,10 @@ var reqWriteTests = []reqWriteTest{ "GET http://www.techcrunch.com/ HTTP/1.1\r\n" + "Host: www.techcrunch.com\r\n" + "User-Agent: Fake\r\n" + + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n" + "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7\r\n" + "Accept-Encoding: gzip,deflate\r\n" + "Accept-Language: en-us,en;q=0.5\r\n" + - "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n" + "Keep-Alive: 300\r\n" + "Proxy-Connection: keep-alive\r\n\r\n", }, @@ -71,7 +71,7 @@ var reqWriteTests = []reqWriteTest{ }, ProtoMajor: 1, ProtoMinor: 1, - Header: map[string]string{}, + Header: map[string][]string{}, Body: nopCloser{bytes.NewBufferString("abcdef")}, TransferEncoding: []string{"chunked"}, }, @@ -93,7 +93,7 @@ var reqWriteTests = []reqWriteTest{ }, ProtoMajor: 1, ProtoMinor: 1, - Header: map[string]string{}, + Header: map[string][]string{}, Close: true, Body: nopCloser{bytes.NewBufferString("abcdef")}, TransferEncoding: []string{"chunked"}, diff --git a/libgo/go/http/response.go b/libgo/go/http/response.go index a24726110c8..3f919c86a3c 100644 --- a/libgo/go/http/response.go +++ b/libgo/go/http/response.go @@ -10,6 +10,7 @@ import ( "bufio" "fmt" "io" + "net/textproto" "os" "sort" "strconv" @@ -43,7 +44,7 @@ type Response struct { // omitted from Header. // // Keys in the map are canonicalized (see CanonicalHeaderKey). - Header map[string]string + Header Header // Body represents the response body. Body io.ReadCloser @@ -66,7 +67,7 @@ type Response struct { // Trailer maps trailer keys to values. Like for Header, if the // response has multiple trailer lines with the same key, they will be // concatenated, delimited by commas. - Trailer map[string]string + Trailer map[string][]string } // ReadResponse reads and returns an HTTP response from r. The RequestMethod @@ -76,13 +77,17 @@ type Response struct { // key/value pairs included in the response trailer. func ReadResponse(r *bufio.Reader, requestMethod string) (resp *Response, err os.Error) { + tp := textproto.NewReader(r) resp = new(Response) resp.RequestMethod = strings.ToUpper(requestMethod) // Parse the first line of the response. - line, err := readLine(r) + line, err := tp.ReadLine() if err != nil { + if err == os.EOF { + err = io.ErrUnexpectedEOF + } return nil, err } f := strings.Split(line, " ", 3) @@ -101,26 +106,16 @@ func ReadResponse(r *bufio.Reader, requestMethod string) (resp *Response, err os resp.Proto = f[0] var ok bool - if resp.ProtoMajor, resp.ProtoMinor, ok = parseHTTPVersion(resp.Proto); !ok { + if resp.ProtoMajor, resp.ProtoMinor, ok = ParseHTTPVersion(resp.Proto); !ok { return nil, &badStringError{"malformed HTTP version", resp.Proto} } // Parse the response headers. - nheader := 0 - resp.Header = make(map[string]string) - for { - key, value, err := readKeyValue(r) - if err != nil { - return nil, err - } - if key == "" { - break // end of response header - } - if nheader++; nheader >= maxHeaderLines { - return nil, ErrHeaderTooLong - } - resp.AddHeader(key, value) + mimeHeader, err := tp.ReadMIMEHeader() + if err != nil { + return nil, err } + resp.Header = Header(mimeHeader) fixPragmaCacheControl(resp.Header) @@ -136,34 +131,14 @@ func ReadResponse(r *bufio.Reader, requestMethod string) (resp *Response, err os // Pragma: no-cache // like // Cache-Control: no-cache -func fixPragmaCacheControl(header map[string]string) { - if header["Pragma"] == "no-cache" { +func fixPragmaCacheControl(header Header) { + if hp, ok := header["Pragma"]; ok && len(hp) > 0 && hp[0] == "no-cache" { if _, presentcc := header["Cache-Control"]; !presentcc { - header["Cache-Control"] = "no-cache" + header["Cache-Control"] = []string{"no-cache"} } } } -// AddHeader adds a value under the given key. Keys are not case sensitive. -func (r *Response) AddHeader(key, value string) { - key = CanonicalHeaderKey(key) - - oldValues, oldValuesPresent := r.Header[key] - if oldValuesPresent { - r.Header[key] = oldValues + "," + value - } else { - r.Header[key] = value - } -} - -// GetHeader returns the value of the response header with the given key. -// If there were multiple headers with this key, their values are concatenated, -// with a comma delimiter. If there were no response headers with the given -// key, GetHeader returns an empty string. Keys are not case sensitive. -func (r *Response) GetHeader(key string) (value string) { - return r.Header[CanonicalHeaderKey(key)] -} - // ProtoAtLeast returns whether the HTTP protocol used // in the response is at least major.minor. func (r *Response) ProtoAtLeast(major, minor int) bool { @@ -231,20 +206,19 @@ func (resp *Response) Write(w io.Writer) os.Error { return nil } -func writeSortedKeyValue(w io.Writer, kvm map[string]string, exclude map[string]bool) os.Error { - kva := make([]string, len(kvm)) - i := 0 - for k, v := range kvm { +func writeSortedKeyValue(w io.Writer, kvm map[string][]string, exclude map[string]bool) os.Error { + keys := make([]string, 0, len(kvm)) + for k := range kvm { if !exclude[k] { - kva[i] = fmt.Sprint(k + ": " + v + "\r\n") - i++ + keys = append(keys, k) } } - kva = kva[0:i] - sort.SortStrings(kva) - for _, l := range kva { - if _, err := io.WriteString(w, l); err != nil { - return err + sort.SortStrings(keys) + for _, k := range keys { + for _, v := range kvm[k] { + if _, err := fmt.Fprintf(w, "%s: %s\r\n", k, v); err != nil { + return err + } } } return nil diff --git a/libgo/go/http/response_test.go b/libgo/go/http/response_test.go index 89a8c3b44d2..bf63ccb9e96 100644 --- a/libgo/go/http/response_test.go +++ b/libgo/go/http/response_test.go @@ -34,8 +34,8 @@ var respTests = []respTest{ ProtoMajor: 1, ProtoMinor: 0, RequestMethod: "GET", - Header: map[string]string{ - "Connection": "close", // TODO(rsc): Delete? + Header: Header{ + "Connection": {"close"}, // TODO(rsc): Delete? }, Close: true, ContentLength: -1, @@ -44,6 +44,47 @@ var respTests = []respTest{ "Body here\n", }, + // Unchunked HTTP/1.1 response without Content-Length or + // Connection headers. + { + "HTTP/1.1 200 OK\r\n" + + "\r\n" + + "Body here\n", + + Response{ + Status: "200 OK", + StatusCode: 200, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + RequestMethod: "GET", + Close: true, + ContentLength: -1, + }, + + "Body here\n", + }, + + // Unchunked HTTP/1.1 204 response without Content-Length. + { + "HTTP/1.1 204 No Content\r\n" + + "\r\n" + + "Body should not be read!\n", + + Response{ + Status: "204 No Content", + StatusCode: 204, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + RequestMethod: "GET", + Close: false, + ContentLength: 0, + }, + + "", + }, + // Unchunked response with Content-Length. { "HTTP/1.0 200 OK\r\n" + @@ -59,9 +100,9 @@ var respTests = []respTest{ ProtoMajor: 1, ProtoMinor: 0, RequestMethod: "GET", - Header: map[string]string{ - "Connection": "close", // TODO(rsc): Delete? - "Content-Length": "10", // TODO(rsc): Delete? + Header: Header{ + "Connection": {"close"}, // TODO(rsc): Delete? + "Content-Length": {"10"}, // TODO(rsc): Delete? }, Close: true, ContentLength: 10, @@ -87,7 +128,7 @@ var respTests = []respTest{ ProtoMajor: 1, ProtoMinor: 0, RequestMethod: "GET", - Header: map[string]string{}, + Header: Header{}, Close: true, ContentLength: -1, TransferEncoding: []string{"chunked"}, @@ -114,7 +155,7 @@ var respTests = []respTest{ ProtoMajor: 1, ProtoMinor: 0, RequestMethod: "GET", - Header: map[string]string{}, + Header: Header{}, Close: true, ContentLength: -1, // TODO(rsc): Fix? TransferEncoding: []string{"chunked"}, @@ -134,7 +175,7 @@ var respTests = []respTest{ ProtoMajor: 1, ProtoMinor: 0, RequestMethod: "GET", - Header: map[string]string{}, + Header: Header{}, Close: true, ContentLength: -1, }, @@ -153,7 +194,7 @@ var respTests = []respTest{ ProtoMajor: 1, ProtoMinor: 0, RequestMethod: "GET", - Header: map[string]string{}, + Header: Header{}, Close: true, ContentLength: -1, }, diff --git a/libgo/go/http/responsewrite_test.go b/libgo/go/http/responsewrite_test.go index 9f10be5626d..aabb833f9c8 100644 --- a/libgo/go/http/responsewrite_test.go +++ b/libgo/go/http/responsewrite_test.go @@ -22,7 +22,7 @@ var respWriteTests = []respWriteTest{ ProtoMajor: 1, ProtoMinor: 0, RequestMethod: "GET", - Header: map[string]string{}, + Header: map[string][]string{}, Body: nopCloser{bytes.NewBufferString("abcdef")}, ContentLength: 6, }, @@ -38,7 +38,7 @@ var respWriteTests = []respWriteTest{ ProtoMajor: 1, ProtoMinor: 0, RequestMethod: "GET", - Header: map[string]string{}, + Header: map[string][]string{}, Body: nopCloser{bytes.NewBufferString("abcdef")}, ContentLength: -1, }, @@ -53,7 +53,7 @@ var respWriteTests = []respWriteTest{ ProtoMajor: 1, ProtoMinor: 1, RequestMethod: "GET", - Header: map[string]string{}, + Header: map[string][]string{}, Body: nopCloser{bytes.NewBufferString("abcdef")}, ContentLength: 6, TransferEncoding: []string{"chunked"}, diff --git a/libgo/go/http/serve_test.go b/libgo/go/http/serve_test.go index 053d6dca448..42fe3e5e4d2 100644 --- a/libgo/go/http/serve_test.go +++ b/libgo/go/http/serve_test.go @@ -9,10 +9,14 @@ package http import ( "bufio" "bytes" + "fmt" "io" + "io/ioutil" "os" "net" + "strings" "testing" + "time" ) type dummyAddr string @@ -136,6 +140,71 @@ func TestConsumingBodyOnNextConn(t *testing.T) { } } +type stringHandler string + +func (s stringHandler) ServeHTTP(w ResponseWriter, r *Request) { + w.SetHeader("Result", string(s)) +} + +var handlers = []struct { + pattern string + msg string +}{ + {"/", "Default"}, + {"/someDir/", "someDir"}, + {"someHost.com/someDir/", "someHost.com/someDir"}, +} + +var vtests = []struct { + url string + expected string +}{ + {"http://localhost/someDir/apage", "someDir"}, + {"http://localhost/otherDir/apage", "Default"}, + {"http://someHost.com/someDir/apage", "someHost.com/someDir"}, + {"http://otherHost.com/someDir/apage", "someDir"}, + {"http://otherHost.com/aDir/apage", "Default"}, +} + +func TestHostHandlers(t *testing.T) { + for _, h := range handlers { + Handle(h.pattern, stringHandler(h.msg)) + } + l, err := net.Listen("tcp", "127.0.0.1:0") // any port + if err != nil { + t.Fatal(err) + } + defer l.Close() + go Serve(l, nil) + conn, err := net.Dial("tcp", "", l.Addr().String()) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + cc := NewClientConn(conn, nil) + for _, vt := range vtests { + var r *Response + var req Request + if req.URL, err = ParseURL(vt.url); err != nil { + t.Errorf("cannot parse url: %v", err) + continue + } + if err := cc.Write(&req); err != nil { + t.Errorf("writing request: %v", err) + continue + } + r, err := cc.Read(&req) + if err != nil { + t.Errorf("reading response: %v", err) + continue + } + s := r.Header.Get("Result") + if s != vt.expected { + t.Errorf("Get(%q) = %q, want %q", vt.url, s, vt.expected) + } + } +} + type responseWriterMethodCall struct { method string headerKey, headerValue string // if method == "SetHeader" @@ -218,3 +287,148 @@ func TestMuxRedirectLeadingSlashes(t *testing.T) { } } } + +func TestServerTimeouts(t *testing.T) { + l, err := net.ListenTCP("tcp", &net.TCPAddr{Port: 0}) + if err != nil { + t.Fatalf("listen error: %v", err) + } + addr, _ := l.Addr().(*net.TCPAddr) + + reqNum := 0 + handler := HandlerFunc(func(res ResponseWriter, req *Request) { + reqNum++ + fmt.Fprintf(res, "req=%d", reqNum) + }) + + const second = 1000000000 /* nanos */ + server := &Server{Handler: handler, ReadTimeout: 0.25 * second, WriteTimeout: 0.25 * second} + go server.Serve(l) + + url := fmt.Sprintf("http://localhost:%d/", addr.Port) + + // Hit the HTTP server successfully. + r, _, err := Get(url) + if err != nil { + t.Fatalf("http Get #1: %v", err) + } + got, _ := ioutil.ReadAll(r.Body) + expected := "req=1" + if string(got) != expected { + t.Errorf("Unexpected response for request #1; got %q; expected %q", + string(got), expected) + } + + // Slow client that should timeout. + t1 := time.Nanoseconds() + conn, err := net.Dial("tcp", "", fmt.Sprintf("localhost:%d", addr.Port)) + if err != nil { + t.Fatalf("Dial: %v", err) + } + buf := make([]byte, 1) + n, err := conn.Read(buf) + latency := time.Nanoseconds() - t1 + if n != 0 || err != os.EOF { + t.Errorf("Read = %v, %v, wanted %v, %v", n, err, 0, os.EOF) + } + if latency < second*0.20 /* fudge from 0.25 above */ { + t.Errorf("got EOF after %d ns, want >= %d", latency, second*0.20) + } + + // Hit the HTTP server successfully again, verifying that the + // previous slow connection didn't run our handler. (that we + // get "req=2", not "req=3") + r, _, err = Get(url) + if err != nil { + t.Fatalf("http Get #2: %v", err) + } + got, _ = ioutil.ReadAll(r.Body) + expected = "req=2" + if string(got) != expected { + t.Errorf("Get #2 got %q, want %q", string(got), expected) + } + + l.Close() +} + +// TestIdentityResponse verifies that a handler can unset +func TestIdentityResponse(t *testing.T) { + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("failed to listen on a port: %v", err) + } + defer l.Close() + urlBase := "http://" + l.Addr().String() + "/" + + handler := HandlerFunc(func(rw ResponseWriter, req *Request) { + rw.SetHeader("Content-Length", "3") + rw.SetHeader("Transfer-Encoding", req.FormValue("te")) + switch { + case req.FormValue("overwrite") == "1": + _, err := rw.Write([]byte("foo TOO LONG")) + if err != ErrContentLength { + t.Errorf("expected ErrContentLength; got %v", err) + } + case req.FormValue("underwrite") == "1": + rw.SetHeader("Content-Length", "500") + rw.Write([]byte("too short")) + default: + rw.Write([]byte("foo")) + } + }) + + server := &Server{Handler: handler} + go server.Serve(l) + + // Note: this relies on the assumption (which is true) that + // Get sends HTTP/1.1 or greater requests. Otherwise the + // server wouldn't have the choice to send back chunked + // responses. + for _, te := range []string{"", "identity"} { + url := urlBase + "?te=" + te + res, _, err := Get(url) + if err != nil { + t.Fatalf("error with Get of %s: %v", url, err) + } + if cl, expected := res.ContentLength, int64(3); cl != expected { + t.Errorf("for %s expected res.ContentLength of %d; got %d", url, expected, cl) + } + if cl, expected := res.Header.Get("Content-Length"), "3"; cl != expected { + t.Errorf("for %s expected Content-Length header of %q; got %q", url, expected, cl) + } + if tl, expected := len(res.TransferEncoding), 0; tl != expected { + t.Errorf("for %s expected len(res.TransferEncoding) of %d; got %d (%v)", + url, expected, tl, res.TransferEncoding) + } + } + + // Verify that ErrContentLength is returned + url := urlBase + "?overwrite=1" + _, _, err = Get(url) + if err != nil { + t.Fatalf("error with Get of %s: %v", url, err) + } + + // Verify that the connection is closed when the declared Content-Length + // is larger than what the handler wrote. + conn, err := net.Dial("tcp", "", l.Addr().String()) + if err != nil { + t.Fatalf("error dialing: %v", err) + } + _, err = conn.Write([]byte("GET /?underwrite=1 HTTP/1.1\r\nHost: foo\r\n\r\n")) + if err != nil { + t.Fatalf("error writing: %v", err) + } + // The next ReadAll will hang for a failing test, so use a Timer instead + // to fail more traditionally + timer := time.AfterFunc(2e9, func() { + t.Fatalf("Timeout expired in ReadAll.") + }) + defer timer.Stop() + got, _ := ioutil.ReadAll(conn) + expectedSuffix := "\r\n\r\ntoo short" + if !strings.HasSuffix(string(got), expectedSuffix) { + t.Fatalf("Expected output to end with %q; got response body %q", + expectedSuffix, string(got)) + } +} diff --git a/libgo/go/http/server.go b/libgo/go/http/server.go index 644724f58e6..977c8c2297a 100644 --- a/libgo/go/http/server.go +++ b/libgo/go/http/server.go @@ -31,6 +31,7 @@ var ( ErrWriteAfterFlush = os.NewError("Conn.Write called after Flush") ErrBodyNotAllowed = os.NewError("http: response status code does not allow body") ErrHijacked = os.NewError("Conn has been hijacked") + ErrContentLength = os.NewError("Conn.Write wrote more than the declared Content-Length") ) // Objects implementing the Handler interface can be @@ -60,10 +61,10 @@ type ResponseWriter interface { // // Content-Type: text/html; charset=utf-8 // - // being sent. UTF-8 encoded HTML is the default setting for + // being sent. UTF-8 encoded HTML is the default setting for // Content-Type in this library, so users need not make that - // particular call. Calls to SetHeader after WriteHeader (or Write) - // are ignored. + // particular call. Calls to SetHeader after WriteHeader (or Write) + // are ignored. An empty value removes the header if previously set. SetHeader(string, string) // Write writes the data to the connection as part of an HTTP reply. @@ -108,6 +109,7 @@ type response struct { wroteContinue bool // 100 Continue response was written header map[string]string // reply header parameters written int64 // number of bytes written in body + contentLength int64 // explicitly-declared Content-Length; or -1 status int // status code passed to WriteHeader // close connection after this reply. set on request and @@ -170,33 +172,13 @@ func (c *conn) readRequest() (w *response, err os.Error) { w.conn = c w.req = req w.header = make(map[string]string) + w.contentLength = -1 // Expect 100 Continue support if req.expectsContinue() && req.ProtoAtLeast(1, 1) { // Wrap the Body reader with one that replies on the connection req.Body = &expectContinueReader{readCloser: req.Body, resp: w} } - - // Default output is HTML encoded in UTF-8. - w.SetHeader("Content-Type", "text/html; charset=utf-8") - w.SetHeader("Date", time.UTC().Format(TimeFormat)) - - if req.Method == "HEAD" { - // do nothing - } else if req.ProtoAtLeast(1, 1) { - // HTTP/1.1 or greater: use chunked transfer encoding - // to avoid closing the connection at EOF. - w.chunking = true - w.SetHeader("Transfer-Encoding", "chunked") - } else { - // HTTP version < 1.1: cannot do chunked transfer - // encoding, so signal EOF by closing connection. - // Will be overridden if the HTTP handler ends up - // writing a Content-Length and the client requested - // "Connection: keep-alive" - w.closeAfterReply = true - } - return w, nil } @@ -209,7 +191,10 @@ func (w *response) UsingTLS() bool { func (w *response) RemoteAddr() string { return w.conn.remoteAddr } // SetHeader implements the ResponseWriter.SetHeader method -func (w *response) SetHeader(hdr, val string) { w.header[CanonicalHeaderKey(hdr)] = val } +// An empty value removes the header from the map. +func (w *response) SetHeader(hdr, val string) { + w.header[CanonicalHeaderKey(hdr)] = val, val != "" +} // WriteHeader implements the ResponseWriter.WriteHeader method func (w *response) WriteHeader(code int) { @@ -225,13 +210,83 @@ func (w *response) WriteHeader(code int) { w.status = code if code == StatusNotModified { // Must not have body. - w.header["Content-Type"] = "", false - w.header["Transfer-Encoding"] = "", false + for _, header := range []string{"Content-Type", "Content-Length", "Transfer-Encoding"} { + if w.header[header] != "" { + // TODO: return an error if WriteHeader gets a return parameter + // or set a flag on w to make future Writes() write an error page? + // for now just log and drop the header. + log.Printf("http: StatusNotModified response with header %q defined", header) + w.header[header] = "", false + } + } + } else { + // Default output is HTML encoded in UTF-8. + if w.header["Content-Type"] == "" { + w.SetHeader("Content-Type", "text/html; charset=utf-8") + } + } + + if w.header["Date"] == "" { + w.SetHeader("Date", time.UTC().Format(TimeFormat)) + } + + // Check for a explicit (and valid) Content-Length header. + var hasCL bool + var contentLength int64 + if clenStr, ok := w.header["Content-Length"]; ok { + var err os.Error + contentLength, err = strconv.Atoi64(clenStr) + if err == nil { + hasCL = true + } else { + log.Printf("http: invalid Content-Length of %q sent", clenStr) + w.SetHeader("Content-Length", "") + } + } + + te, hasTE := w.header["Transfer-Encoding"] + if hasCL && hasTE && te != "identity" { + // TODO: return an error if WriteHeader gets a return parameter + // For now just ignore the Content-Length. + log.Printf("http: WriteHeader called with both Transfer-Encoding of %q and a Content-Length of %d", + te, contentLength) + w.SetHeader("Content-Length", "") + hasCL = false + } + + if w.req.Method == "HEAD" { + // do nothing + } else if hasCL { w.chunking = false + w.contentLength = contentLength + w.SetHeader("Transfer-Encoding", "") + } else if w.req.ProtoAtLeast(1, 1) { + // HTTP/1.1 or greater: use chunked transfer encoding + // to avoid closing the connection at EOF. + // TODO: this blows away any custom or stacked Transfer-Encoding they + // might have set. Deal with that as need arises once we have a valid + // use case. + w.chunking = true + w.SetHeader("Transfer-Encoding", "chunked") + } else { + // HTTP version < 1.1: cannot do chunked transfer + // encoding and we don't know the Content-Length so + // signal EOF by closing connection. + w.closeAfterReply = true + w.chunking = false // redundant + w.SetHeader("Transfer-Encoding", "") // in case already set + } + + if w.req.wantsHttp10KeepAlive() && (w.req.Method == "HEAD" || hasCL) { + _, connectionHeaderSet := w.header["Connection"] + if !connectionHeaderSet { + w.SetHeader("Connection", "keep-alive") + } } + // Cannot use Content-Length with non-identity Transfer-Encoding. if w.chunking { - w.header["Content-Length"] = "", false + w.SetHeader("Content-Length", "") } if !w.req.ProtoAtLeast(1, 0) { return @@ -259,15 +314,6 @@ func (w *response) Write(data []byte) (n int, err os.Error) { return 0, ErrHijacked } if !w.wroteHeader { - if w.req.wantsHttp10KeepAlive() { - _, hasLength := w.header["Content-Length"] - if hasLength { - _, connectionHeaderSet := w.header["Connection"] - if !connectionHeaderSet { - w.header["Connection"] = "keep-alive" - } - } - } w.WriteHeader(StatusOK) } if len(data) == 0 { @@ -280,6 +326,9 @@ func (w *response) Write(data []byte) (n int, err os.Error) { } w.written += int64(len(data)) // ignoring errors, for errorKludge + if w.contentLength != -1 && w.written > w.contentLength { + return 0, ErrContentLength + } // TODO(rsc): if chunking happened after the buffering, // then there would be fewer chunk headers. @@ -369,6 +418,11 @@ func (w *response) finishRequest() { } w.conn.buf.Flush() w.req.Body.Close() + + if w.contentLength != -1 && w.contentLength != w.written { + // Did not write enough. Avoid getting out of sync. + w.closeAfterReply = true + } } // Flush implements the ResponseWriter.Flush method. @@ -539,9 +593,8 @@ func RedirectHandler(url string, code int) Handler { // patterns and calls the handler for the pattern that // most closely matches the URL. // -// Patterns named fixed paths, like "/favicon.ico", -// or subtrees, like "/images/" (note the trailing slash). -// Patterns must begin with /. +// Patterns named fixed, rooted paths, like "/favicon.ico", +// or rooted subtrees, like "/images/" (note the trailing slash). // Longer patterns take precedence over shorter ones, so that // if there are handlers registered for both "/images/" // and "/images/thumbnails/", the latter handler will be @@ -549,11 +602,11 @@ func RedirectHandler(url string, code int) Handler { // former will receiver requests for any other paths in the // "/images/" subtree. // -// In the future, the pattern syntax may be relaxed to allow -// an optional host-name at the beginning of the pattern, -// so that a handler might register for the two patterns -// "/codesearch" and "codesearch.google.com/" -// without taking over requests for http://www.google.com/. +// Patterns may optionally begin with a host name, restricting matches to +// URLs on that host only. Host-specific patterns take precedence over +// general patterns, so that a handler might register for the two patterns +// "/codesearch" and "codesearch.google.com/" without also taking over +// requests for "http://www.google.com/". // // ServeMux also takes care of sanitizing the URL request path, // redirecting any request containing . or .. elements to an @@ -598,21 +651,13 @@ func cleanPath(p string) string { return np } -// ServeHTTP dispatches the request to the handler whose -// pattern most closely matches the request URL. -func (mux *ServeMux) ServeHTTP(w ResponseWriter, r *Request) { - // Clean path to canonical form and redirect. - if p := cleanPath(r.URL.Path); p != r.URL.Path { - w.SetHeader("Location", p) - w.WriteHeader(StatusMovedPermanently) - return - } - - // Most-specific (longest) pattern wins. +// Find a handler on a handler map given a path string +// Most-specific (longest) pattern wins +func (mux *ServeMux) match(path string) Handler { var h Handler var n = 0 for k, v := range mux.m { - if !pathMatch(k, r.URL.Path) { + if !pathMatch(k, path) { continue } if h == nil || len(k) > n { @@ -620,6 +665,23 @@ func (mux *ServeMux) ServeHTTP(w ResponseWriter, r *Request) { h = v } } + return h +} + +// ServeHTTP dispatches the request to the handler whose +// pattern most closely matches the request URL. +func (mux *ServeMux) ServeHTTP(w ResponseWriter, r *Request) { + // Clean path to canonical form and redirect. + if p := cleanPath(r.URL.Path); p != r.URL.Path { + w.SetHeader("Location", p) + w.WriteHeader(StatusMovedPermanently) + return + } + // Host-specific pattern takes precedence over generic ones + h := mux.match(r.Host + r.URL.Path) + if h == nil { + h = mux.match(r.URL.Path) + } if h == nil { h = NotFoundHandler() } @@ -628,7 +690,7 @@ func (mux *ServeMux) ServeHTTP(w ResponseWriter, r *Request) { // Handle registers the handler for the given pattern. func (mux *ServeMux) Handle(pattern string, handler Handler) { - if pattern == "" || pattern[0] != '/' { + if pattern == "" { panic("http: invalid pattern " + pattern) } @@ -649,10 +711,12 @@ func (mux *ServeMux) HandleFunc(pattern string, handler func(ResponseWriter, *Re // Handle registers the handler for the given pattern // in the DefaultServeMux. +// The documentation for ServeMux explains how patterns are matched. func Handle(pattern string, handler Handler) { DefaultServeMux.Handle(pattern, handler) } // HandleFunc registers the handler function for the given pattern // in the DefaultServeMux. +// The documentation for ServeMux explains how patterns are matched. func HandleFunc(pattern string, handler func(ResponseWriter, *Request)) { DefaultServeMux.HandleFunc(pattern, handler) } @@ -662,6 +726,39 @@ func HandleFunc(pattern string, handler func(ResponseWriter, *Request)) { // read requests and then call handler to reply to them. // Handler is typically nil, in which case the DefaultServeMux is used. func Serve(l net.Listener, handler Handler) os.Error { + srv := &Server{Handler: handler} + return srv.Serve(l) +} + +// A Server defines parameters for running an HTTP server. +type Server struct { + Addr string // TCP address to listen on, ":http" if empty + Handler Handler // handler to invoke, http.DefaultServeMux if nil + ReadTimeout int64 // the net.Conn.SetReadTimeout value for new connections + WriteTimeout int64 // the net.Conn.SetWriteTimeout value for new connections +} + +// ListenAndServe listens on the TCP network address srv.Addr and then +// calls Serve to handle requests on incoming connections. If +// srv.Addr is blank, ":http" is used. +func (srv *Server) ListenAndServe() os.Error { + addr := srv.Addr + if addr == "" { + addr = ":http" + } + l, e := net.Listen("tcp", addr) + if e != nil { + return e + } + return srv.Serve(l) +} + +// Serve accepts incoming connections on the Listener l, creating a +// new service thread for each. The service threads read requests and +// then call srv.Handler to reply to them. +func (srv *Server) Serve(l net.Listener) os.Error { + defer l.Close() + handler := srv.Handler if handler == nil { handler = DefaultServeMux } @@ -670,6 +767,12 @@ func Serve(l net.Listener, handler Handler) os.Error { if e != nil { return e } + if srv.ReadTimeout != 0 { + rw.SetReadTimeout(srv.ReadTimeout) + } + if srv.WriteTimeout != 0 { + rw.SetWriteTimeout(srv.WriteTimeout) + } c, err := newConn(rw, handler) if err != nil { continue @@ -703,17 +806,12 @@ func Serve(l net.Listener, handler Handler) os.Error { // http.HandleFunc("/hello", HelloServer) // err := http.ListenAndServe(":12345", nil) // if err != nil { -// log.Exit("ListenAndServe: ", err.String()) +// log.Fatal("ListenAndServe: ", err.String()) // } // } func ListenAndServe(addr string, handler Handler) os.Error { - l, e := net.Listen("tcp", addr) - if e != nil { - return e - } - e = Serve(l, handler) - l.Close() - return e + server := &Server{Addr: addr, Handler: handler} + return server.ListenAndServe() } // ListenAndServeTLS acts identically to ListenAndServe, except that it @@ -737,7 +835,7 @@ func ListenAndServe(addr string, handler Handler) os.Error { // log.Printf("About to listen on 10443. Go to https://127.0.0.1:10443/") // err := http.ListenAndServeTLS(":10443", "cert.pem", "key.pem", nil) // if err != nil { -// log.Exit(err) +// log.Fatal(err) // } // } // diff --git a/libgo/go/http/transfer.go b/libgo/go/http/transfer.go index e62885d62fd..996e2897325 100644 --- a/libgo/go/http/transfer.go +++ b/libgo/go/http/transfer.go @@ -21,7 +21,7 @@ type transferWriter struct { ContentLength int64 Close bool TransferEncoding []string - Trailer map[string]string + Trailer Header } func newTransferWriter(r interface{}) (t *transferWriter, err os.Error) { @@ -159,7 +159,7 @@ func (t *transferWriter) WriteBody(w io.Writer) (err os.Error) { type transferReader struct { // Input - Header map[string]string + Header Header StatusCode int RequestMethod string ProtoMajor int @@ -169,7 +169,21 @@ type transferReader struct { ContentLength int64 TransferEncoding []string Close bool - Trailer map[string]string + Trailer Header +} + +// bodyAllowedForStatus returns whether a given response status code +// permits a body. See RFC2616, section 4.4. +func bodyAllowedForStatus(status int) bool { + switch { + case status >= 100 && status <= 199: + return false + case status == 204: + return false + case status == 304: + return false + } + return true } // msg is *Request or *Response. @@ -217,6 +231,19 @@ func readTransfer(msg interface{}, r *bufio.Reader) (err os.Error) { return err } + // If there is no Content-Length or chunked Transfer-Encoding on a *Response + // and the status is not 1xx, 204 or 304, then the body is unbounded. + // See RFC2616, section 4.4. + switch msg.(type) { + case *Response: + if t.ContentLength == -1 && + !chunked(t.TransferEncoding) && + bodyAllowedForStatus(t.StatusCode) { + // Unbounded body. + t.Close = true + } + } + // Prepare body reader. ContentLength < 0 means chunked encoding // or close connection when finished, since multipart is not supported yet switch { @@ -262,14 +289,14 @@ func readTransfer(msg interface{}, r *bufio.Reader) (err os.Error) { func chunked(te []string) bool { return len(te) > 0 && te[0] == "chunked" } // Sanitize transfer encoding -func fixTransferEncoding(header map[string]string) ([]string, os.Error) { +func fixTransferEncoding(header Header) ([]string, os.Error) { raw, present := header["Transfer-Encoding"] if !present { return nil, nil } - header["Transfer-Encoding"] = "", false - encodings := strings.Split(raw, ",", -1) + header["Transfer-Encoding"] = nil, false + encodings := strings.Split(raw[0], ",", -1) te := make([]string, 0, len(encodings)) // TODO: Even though we only support "identity" and "chunked" // encodings, the loop below is designed with foresight. One @@ -294,7 +321,7 @@ func fixTransferEncoding(header map[string]string) ([]string, os.Error) { // Chunked encoding trumps Content-Length. See RFC 2616 // Section 4.4. Currently len(te) > 0 implies chunked // encoding. - header["Content-Length"] = "", false + header["Content-Length"] = nil, false return te, nil } @@ -304,7 +331,7 @@ func fixTransferEncoding(header map[string]string) ([]string, os.Error) { // Determine the expected body length, using RFC 2616 Section 4.4. This // function is not a method, because ultimately it should be shared by // ReadResponse and ReadRequest. -func fixLength(status int, requestMethod string, header map[string]string, te []string) (int64, os.Error) { +func fixLength(status int, requestMethod string, header Header, te []string) (int64, os.Error) { // Logic based on response type or status if noBodyExpected(requestMethod) { @@ -324,23 +351,21 @@ func fixLength(status int, requestMethod string, header map[string]string, te [] } // Logic based on Content-Length - if cl, present := header["Content-Length"]; present { - cl = strings.TrimSpace(cl) - if cl != "" { - n, err := strconv.Atoi64(cl) - if err != nil || n < 0 { - return -1, &badStringError{"bad Content-Length", cl} - } - return n, nil - } else { - header["Content-Length"] = "", false + cl := strings.TrimSpace(header.Get("Content-Length")) + if cl != "" { + n, err := strconv.Atoi64(cl) + if err != nil || n < 0 { + return -1, &badStringError{"bad Content-Length", cl} } + return n, nil + } else { + header.Del("Content-Length") } // Logic based on media type. The purpose of the following code is just // to detect whether the unsupported "multipart/byteranges" is being // used. A proper Content-Type parser is needed in the future. - if strings.Contains(strings.ToLower(header["Content-Type"]), "multipart/byteranges") { + if strings.Contains(strings.ToLower(header.Get("Content-Type")), "multipart/byteranges") { return -1, ErrNotSupported } @@ -351,24 +376,19 @@ func fixLength(status int, requestMethod string, header map[string]string, te [] // Determine whether to hang up after sending a request and body, or // receiving a response and body // 'header' is the request headers -func shouldClose(major, minor int, header map[string]string) bool { +func shouldClose(major, minor int, header Header) bool { if major < 1 { return true } else if major == 1 && minor == 0 { - v, present := header["Connection"] - if !present { - return true - } - v = strings.ToLower(v) - if !strings.Contains(v, "keep-alive") { + if !strings.Contains(strings.ToLower(header.Get("Connection")), "keep-alive") { return true } return false - } else if v, present := header["Connection"]; present { + } else { // TODO: Should split on commas, toss surrounding white space, // and check each field. - if v == "close" { - header["Connection"] = "", false + if strings.ToLower(header.Get("Connection")) == "close" { + header.Del("Connection") return true } } @@ -376,14 +396,14 @@ func shouldClose(major, minor int, header map[string]string) bool { } // Parse the trailer header -func fixTrailer(header map[string]string, te []string) (map[string]string, os.Error) { - raw, present := header["Trailer"] - if !present { +func fixTrailer(header Header, te []string) (Header, os.Error) { + raw := header.Get("Trailer") + if raw == "" { return nil, nil } - header["Trailer"] = "", false - trailer := make(map[string]string) + header.Del("Trailer") + trailer := make(Header) keys := strings.Split(raw, ",", -1) for _, key := range keys { key = CanonicalHeaderKey(strings.TrimSpace(key)) @@ -391,7 +411,7 @@ func fixTrailer(header map[string]string, te []string) (map[string]string, os.Er case "Transfer-Encoding", "Trailer", "Content-Length": return nil, &badStringError{"bad trailer key", key} } - trailer[key] = "" + trailer.Del(key) } if len(trailer) == 0 { return nil, nil diff --git a/libgo/go/http/transport.go b/libgo/go/http/transport.go new file mode 100644 index 00000000000..41d639c7e2f --- /dev/null +++ b/libgo/go/http/transport.go @@ -0,0 +1,147 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +import ( + "bufio" + "crypto/tls" + "encoding/base64" + "fmt" + "net" + "os" + "strings" + "sync" +) + +// DefaultTransport is the default implementation of ClientTransport +// and is used by DefaultClient. It establishes a new network connection for +// each call to Do and uses HTTP proxies as directed by the $HTTP_PROXY and +// $NO_PROXY (or $http_proxy and $no_proxy) environment variables. +var DefaultTransport ClientTransport = &transport{} + +// transport implements http.ClientTranport for the default case, +// using TCP connections to either the host or a proxy, serving +// http or https schemes. In the future this may become public +// and support options on keep-alive connection duration, pipelining +// controls, etc. For now this is simply a port of the old Go code +// client code to the http.ClientTransport interface. +type transport struct { + // TODO: keep-alives, pipelining, etc using a map from + // scheme/host to a connection. Something like: + l sync.Mutex + hostConn map[string]*ClientConn +} + +func (ct *transport) Do(req *Request) (resp *Response, err os.Error) { + if req.URL.Scheme != "http" && req.URL.Scheme != "https" { + return nil, &badStringError{"unsupported protocol scheme", req.URL.Scheme} + } + + addr := req.URL.Host + if !hasPort(addr) { + addr += ":" + req.URL.Scheme + } + + var proxyURL *URL + proxyAuth := "" + proxy := "" + if !matchNoProxy(addr) { + proxy = os.Getenv("HTTP_PROXY") + if proxy == "" { + proxy = os.Getenv("http_proxy") + } + } + + if proxy != "" { + proxyURL, err = ParseRequestURL(proxy) + if err != nil { + return nil, os.ErrorString("invalid proxy address") + } + if proxyURL.Host == "" { + proxyURL, err = ParseRequestURL("http://" + proxy) + if err != nil { + return nil, os.ErrorString("invalid proxy address") + } + } + addr = proxyURL.Host + proxyInfo := proxyURL.RawUserinfo + if proxyInfo != "" { + enc := base64.URLEncoding + encoded := make([]byte, enc.EncodedLen(len(proxyInfo))) + enc.Encode(encoded, []byte(proxyInfo)) + proxyAuth = "Basic " + string(encoded) + } + } + + // Connect to server or proxy + conn, err := net.Dial("tcp", "", addr) + if err != nil { + return nil, err + } + + if req.URL.Scheme == "http" { + // Include proxy http header if needed. + if proxyAuth != "" { + req.Header.Set("Proxy-Authorization", proxyAuth) + } + } else { // https + if proxyURL != nil { + // Ask proxy for direct connection to server. + // addr defaults above to ":https" but we need to use numbers + addr = req.URL.Host + if !hasPort(addr) { + addr += ":443" + } + fmt.Fprintf(conn, "CONNECT %s HTTP/1.1\r\n", addr) + fmt.Fprintf(conn, "Host: %s\r\n", addr) + if proxyAuth != "" { + fmt.Fprintf(conn, "Proxy-Authorization: %s\r\n", proxyAuth) + } + fmt.Fprintf(conn, "\r\n") + + // Read response. + // Okay to use and discard buffered reader here, because + // TLS server will not speak until spoken to. + br := bufio.NewReader(conn) + resp, err := ReadResponse(br, "CONNECT") + if err != nil { + return nil, err + } + if resp.StatusCode != 200 { + f := strings.Split(resp.Status, " ", 2) + return nil, os.ErrorString(f[1]) + } + } + + // Initiate TLS and check remote host name against certificate. + conn = tls.Client(conn, nil) + if err = conn.(*tls.Conn).Handshake(); err != nil { + return nil, err + } + h := req.URL.Host + if hasPort(h) { + h = h[:strings.LastIndex(h, ":")] + } + if err = conn.(*tls.Conn).VerifyHostname(h); err != nil { + return nil, err + } + } + + err = req.Write(conn) + if err != nil { + conn.Close() + return nil, err + } + + reader := bufio.NewReader(conn) + resp, err = ReadResponse(reader, req.Method) + if err != nil { + conn.Close() + return nil, err + } + + resp.Body = readClose{resp.Body, conn} + return +} diff --git a/libgo/go/image/decode_test.go b/libgo/go/image/decode_test.go new file mode 100644 index 00000000000..5b87344c50a --- /dev/null +++ b/libgo/go/image/decode_test.go @@ -0,0 +1,89 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package image_test + +import ( + "bufio" + "image" + "os" + "testing" + + // TODO(nigeltao): implement bmp, gif and tiff decoders. + _ "image/jpeg" + _ "image/png" +) + +const goldenFile = "testdata/video-001.png" + +type imageTest struct { + filename string + tolerance int +} + +var imageTests = []imageTest{ + //{"testdata/video-001.bmp", 0}, + // GIF images are restricted to a 256-color palette and the conversion + // to GIF loses significant image quality. + //{"testdata/video-001.gif", 64<<8}, + // JPEG is a lossy format and hence needs a non-zero tolerance. + {"testdata/video-001.jpeg", 8 << 8}, + {"testdata/video-001.png", 0}, + //{"testdata/video-001.tiff", 0}, +} + +func decode(filename string) (image.Image, string, os.Error) { + f, err := os.Open(filename, os.O_RDONLY, 0400) + if err != nil { + return nil, "", err + } + defer f.Close() + return image.Decode(bufio.NewReader(f)) +} + +func delta(u0, u1 uint32) int { + d := int(u0) - int(u1) + if d < 0 { + return -d + } + return d +} + +func withinTolerance(c0, c1 image.Color, tolerance int) bool { + r0, g0, b0, a0 := c0.RGBA() + r1, g1, b1, a1 := c1.RGBA() + r := delta(r0, r1) + g := delta(g0, g1) + b := delta(b0, b1) + a := delta(a0, a1) + return r <= tolerance && g <= tolerance && b <= tolerance && a <= tolerance +} + +func TestDecode(t *testing.T) { + golden, _, err := decode(goldenFile) + if err != nil { + t.Errorf("%s: %v", goldenFile, err) + } +loop: + for _, it := range imageTests { + m, _, err := decode(it.filename) + if err != nil { + t.Errorf("%s: %v", it.filename, err) + continue loop + } + b := golden.Bounds() + if !b.Eq(m.Bounds()) { + t.Errorf("%s: want bounds %v got %v", it.filename, b, m.Bounds()) + continue loop + } + for y := b.Min.Y; y < b.Max.Y; y++ { + for x := b.Min.X; x < b.Max.X; x++ { + if !withinTolerance(golden.At(x, y), m.At(x, y), it.tolerance) { + t.Errorf("%s: at (%d, %d), want %v got %v", it.filename, x, y, golden.At(x, y), m.At(x, y)) + continue loop + } + } + } + } +} diff --git a/libgo/go/image/png/reader.go b/libgo/go/image/png/reader.go index e2d679bb425..eee4eac2e15 100644 --- a/libgo/go/image/png/reader.go +++ b/libgo/go/image/png/reader.go @@ -29,11 +29,19 @@ const ( // A cb is a combination of color type and bit depth. const ( cbInvalid = iota + cbG1 + cbG2 + cbG4 cbG8 + cbGA8 cbTC8 + cbP1 + cbP2 + cbP4 cbP8 cbTCA8 cbG16 + cbGA16 cbTC16 cbTCA16 ) @@ -70,6 +78,7 @@ type imgOrErr struct { type decoder struct { width, height int + depth int palette image.PalettedColorModel cb int stage int @@ -138,7 +147,29 @@ func (d *decoder) parseIHDR(r io.Reader, crc hash.Hash32, length uint32) os.Erro return UnsupportedError("dimension overflow") } d.cb = cbInvalid - switch d.tmp[8] { + d.depth = int(d.tmp[8]) + switch d.depth { + case 1: + switch d.tmp[9] { + case ctGrayscale: + d.cb = cbG1 + case ctPaletted: + d.cb = cbP1 + } + case 2: + switch d.tmp[9] { + case ctGrayscale: + d.cb = cbG2 + case ctPaletted: + d.cb = cbP2 + } + case 4: + switch d.tmp[9] { + case ctGrayscale: + d.cb = cbG4 + case ctPaletted: + d.cb = cbP4 + } case 8: switch d.tmp[9] { case ctGrayscale: @@ -147,6 +178,8 @@ func (d *decoder) parseIHDR(r io.Reader, crc hash.Hash32, length uint32) os.Erro d.cb = cbTC8 case ctPaletted: d.cb = cbP8 + case ctGrayscaleAlpha: + d.cb = cbGA8 case ctTrueColorAlpha: d.cb = cbTCA8 } @@ -156,6 +189,8 @@ func (d *decoder) parseIHDR(r io.Reader, crc hash.Hash32, length uint32) os.Erro d.cb = cbG16 case ctTrueColor: d.cb = cbTC16 + case ctGrayscaleAlpha: + d.cb = cbGA16 case ctTrueColorAlpha: d.cb = cbTCA16 } @@ -169,7 +204,7 @@ func (d *decoder) parseIHDR(r io.Reader, crc hash.Hash32, length uint32) os.Erro func (d *decoder) parsePLTE(r io.Reader, crc hash.Hash32, length uint32) os.Error { np := int(length / 3) // The number of palette entries. - if length%3 != 0 || np <= 0 || np > 256 { + if length%3 != 0 || np <= 0 || np > 256 || np > 1<<uint(d.depth) { return FormatError("bad PLTE length") } n, err := io.ReadFull(r, d.tmp[0:3*np]) @@ -178,7 +213,7 @@ func (d *decoder) parsePLTE(r io.Reader, crc hash.Hash32, length uint32) os.Erro } crc.Write(d.tmp[0:n]) switch d.cb { - case cbP8: + case cbP1, cbP2, cbP4, cbP8: d.palette = image.PalettedColorModel(make([]image.Color, np)) for i := 0; i < np; i++ { d.palette[i] = image.RGBAColor{d.tmp[3*i+0], d.tmp[3*i+1], d.tmp[3*i+2], 0xff} @@ -206,7 +241,7 @@ func (d *decoder) parsetRNS(r io.Reader, crc hash.Hash32, length uint32) os.Erro return UnsupportedError("grayscale transparency") case cbTC8, cbTC16: return UnsupportedError("truecolor transparency") - case cbP8: + case cbP1, cbP2, cbP4, cbP8: if n > len(d.palette) { return FormatError("bad tRNS length") } @@ -214,7 +249,7 @@ func (d *decoder) parsetRNS(r io.Reader, crc hash.Hash32, length uint32) os.Erro rgba := d.palette[i].(image.RGBAColor) d.palette[i] = image.RGBAColor{rgba.R, rgba.G, rgba.B, d.tmp[i]} } - case cbTCA8, cbTCA16: + case cbGA8, cbGA16, cbTCA8, cbTCA16: return FormatError("tRNS, color type mismatch") } return nil @@ -240,7 +275,7 @@ func (d *decoder) idatReader(idat io.Reader) (image.Image, os.Error) { return nil, err } defer r.Close() - bpp := 0 // Bytes per pixel. + bitsPerPixel := 0 maxPalette := uint8(0) var ( gray *image.Gray @@ -253,40 +288,50 @@ func (d *decoder) idatReader(idat io.Reader) (image.Image, os.Error) { img image.Image ) switch d.cb { - case cbG8: - bpp = 1 + case cbG1, cbG2, cbG4, cbG8: + bitsPerPixel = d.depth gray = image.NewGray(d.width, d.height) img = gray + case cbGA8: + bitsPerPixel = 16 + nrgba = image.NewNRGBA(d.width, d.height) + img = nrgba case cbTC8: - bpp = 3 + bitsPerPixel = 24 rgba = image.NewRGBA(d.width, d.height) img = rgba - case cbP8: - bpp = 1 + case cbP1, cbP2, cbP4, cbP8: + bitsPerPixel = d.depth paletted = image.NewPaletted(d.width, d.height, d.palette) img = paletted maxPalette = uint8(len(d.palette) - 1) case cbTCA8: - bpp = 4 + bitsPerPixel = 32 nrgba = image.NewNRGBA(d.width, d.height) img = nrgba case cbG16: - bpp = 2 + bitsPerPixel = 16 gray16 = image.NewGray16(d.width, d.height) img = gray16 + case cbGA16: + bitsPerPixel = 32 + nrgba64 = image.NewNRGBA64(d.width, d.height) + img = nrgba64 case cbTC16: - bpp = 6 + bitsPerPixel = 48 rgba64 = image.NewRGBA64(d.width, d.height) img = rgba64 case cbTCA16: - bpp = 8 + bitsPerPixel = 64 nrgba64 = image.NewNRGBA64(d.width, d.height) img = nrgba64 } + bytesPerPixel := (bitsPerPixel + 7) / 8 + // cr and pr are the bytes for the current and previous row. // The +1 is for the per-row filter type, which is at cr[0]. - cr := make([]uint8, 1+bpp*d.width) - pr := make([]uint8, 1+bpp*d.width) + cr := make([]uint8, 1+(bitsPerPixel*d.width+7)/8) + pr := make([]uint8, 1+(bitsPerPixel*d.width+7)/8) for y := 0; y < d.height; y++ { // Read the decompressed bytes. @@ -302,26 +347,26 @@ func (d *decoder) idatReader(idat io.Reader) (image.Image, os.Error) { case ftNone: // No-op. case ftSub: - for i := bpp; i < len(cdat); i++ { - cdat[i] += cdat[i-bpp] + for i := bytesPerPixel; i < len(cdat); i++ { + cdat[i] += cdat[i-bytesPerPixel] } case ftUp: for i := 0; i < len(cdat); i++ { cdat[i] += pdat[i] } case ftAverage: - for i := 0; i < bpp; i++ { + for i := 0; i < bytesPerPixel; i++ { cdat[i] += pdat[i] / 2 } - for i := bpp; i < len(cdat); i++ { - cdat[i] += uint8((int(cdat[i-bpp]) + int(pdat[i])) / 2) + for i := bytesPerPixel; i < len(cdat); i++ { + cdat[i] += uint8((int(cdat[i-bytesPerPixel]) + int(pdat[i])) / 2) } case ftPaeth: - for i := 0; i < bpp; i++ { + for i := 0; i < bytesPerPixel; i++ { cdat[i] += paeth(0, pdat[i], 0) } - for i := bpp; i < len(cdat); i++ { - cdat[i] += paeth(cdat[i-bpp], pdat[i], pdat[i-bpp]) + for i := bytesPerPixel; i < len(cdat); i++ { + cdat[i] += paeth(cdat[i-bytesPerPixel], pdat[i], pdat[i-bytesPerPixel]) } default: return nil, FormatError("bad filter type") @@ -329,14 +374,79 @@ func (d *decoder) idatReader(idat io.Reader) (image.Image, os.Error) { // Convert from bytes to colors. switch d.cb { + case cbG1: + for x := 0; x < d.width; x += 8 { + b := cdat[x/8] + for x2 := 0; x2 < 8 && x+x2 < d.width; x2++ { + gray.Set(x+x2, y, image.GrayColor{(b >> 7) * 0xff}) + b <<= 1 + } + } + case cbG2: + for x := 0; x < d.width; x += 4 { + b := cdat[x/4] + for x2 := 0; x2 < 4 && x+x2 < d.width; x2++ { + gray.Set(x+x2, y, image.GrayColor{(b >> 6) * 0x55}) + b <<= 2 + } + } + case cbG4: + for x := 0; x < d.width; x += 2 { + b := cdat[x/2] + for x2 := 0; x2 < 2 && x+x2 < d.width; x2++ { + gray.Set(x+x2, y, image.GrayColor{(b >> 4) * 0x11}) + b <<= 4 + } + } case cbG8: for x := 0; x < d.width; x++ { gray.Set(x, y, image.GrayColor{cdat[x]}) } + case cbGA8: + for x := 0; x < d.width; x++ { + ycol := cdat[2*x+0] + nrgba.Set(x, y, image.NRGBAColor{ycol, ycol, ycol, cdat[2*x+1]}) + } case cbTC8: for x := 0; x < d.width; x++ { rgba.Set(x, y, image.RGBAColor{cdat[3*x+0], cdat[3*x+1], cdat[3*x+2], 0xff}) } + case cbP1: + for x := 0; x < d.width; x += 8 { + b := cdat[x/8] + for x2 := 0; x2 < 8 && x+x2 < d.width; x2++ { + idx := b >> 7 + if idx > maxPalette { + return nil, FormatError("palette index out of range") + } + paletted.SetColorIndex(x+x2, y, idx) + b <<= 1 + } + } + case cbP2: + for x := 0; x < d.width; x += 4 { + b := cdat[x/4] + for x2 := 0; x2 < 4 && x+x2 < d.width; x2++ { + idx := b >> 6 + if idx > maxPalette { + return nil, FormatError("palette index out of range") + } + paletted.SetColorIndex(x+x2, y, idx) + b <<= 2 + } + } + case cbP4: + for x := 0; x < d.width; x += 2 { + b := cdat[x/2] + for x2 := 0; x2 < 2 && x+x2 < d.width; x2++ { + idx := b >> 4 + if idx > maxPalette { + return nil, FormatError("palette index out of range") + } + paletted.SetColorIndex(x+x2, y, idx) + b <<= 4 + } + } case cbP8: for x := 0; x < d.width; x++ { if cdat[x] > maxPalette { @@ -353,6 +463,12 @@ func (d *decoder) idatReader(idat io.Reader) (image.Image, os.Error) { ycol := uint16(cdat[2*x+0])<<8 | uint16(cdat[2*x+1]) gray16.Set(x, y, image.Gray16Color{ycol}) } + case cbGA16: + for x := 0; x < d.width; x++ { + ycol := uint16(cdat[4*x+0])<<8 | uint16(cdat[4*x+1]) + acol := uint16(cdat[4*x+2])<<8 | uint16(cdat[4*x+3]) + nrgba64.Set(x, y, image.NRGBA64Color{ycol, ycol, ycol, acol}) + } case cbTC16: for x := 0; x < d.width; x++ { rcol := uint16(cdat[6*x+0])<<8 | uint16(cdat[6*x+1]) @@ -565,16 +681,20 @@ func DecodeConfig(r io.Reader) (image.Config, os.Error) { } var cm image.ColorModel switch d.cb { - case cbG8: + case cbG1, cbG2, cbG4, cbG8: cm = image.GrayColorModel + case cbGA8: + cm = image.NRGBAColorModel case cbTC8: cm = image.RGBAColorModel - case cbP8: + case cbP1, cbP2, cbP4, cbP8: cm = d.palette case cbTCA8: cm = image.NRGBAColorModel case cbG16: cm = image.Gray16ColorModel + case cbGA16: + cm = image.NRGBA64ColorModel case cbTC16: cm = image.RGBA64ColorModel case cbTCA16: diff --git a/libgo/go/image/png/reader_test.go b/libgo/go/image/png/reader_test.go index fefceee3a5d..8314a833877 100644 --- a/libgo/go/image/png/reader_test.go +++ b/libgo/go/image/png/reader_test.go @@ -13,23 +13,23 @@ import ( "testing" ) -// The go PNG library currently supports only a subset of the full PNG specification. -// In particular, bit depths other than 8 or 16 are not supported, nor are grayscale- -// alpha images. var filenames = []string{ - //"basn0g01", // bit depth is not 8 or 16 - //"basn0g02", // bit depth is not 8 or 16 - //"basn0g04", // bit depth is not 8 or 16 + "basn0g01", + "basn0g01-30", + "basn0g02", + "basn0g02-29", + "basn0g04", + "basn0g04-31", "basn0g08", "basn0g16", "basn2c08", "basn2c16", - //"basn3p01", // bit depth is not 8 or 16 - //"basn3p02", // bit depth is not 8 or 16 - //"basn3p04", // bit depth is not 8 or 16 + "basn3p01", + "basn3p02", + "basn3p04", "basn3p08", - //"basn4a08", // grayscale-alpha color model - //"basn4a16", // grayscale-alpha color model + "basn4a08", + "basn4a16", "basn6a08", "basn6a16", } @@ -58,7 +58,16 @@ func sng(w io.WriteCloser, filename string, png image.Image) { cpm, _ := cm.(image.PalettedColorModel) var paletted *image.Paletted if cpm != nil { - bitdepth = 8 + switch { + case len(cpm) <= 2: + bitdepth = 1 + case len(cpm) <= 4: + bitdepth = 2 + case len(cpm) <= 16: + bitdepth = 4 + default: + bitdepth = 8 + } paletted = png.(*image.Paletted) } @@ -131,8 +140,15 @@ func sng(w io.WriteCloser, filename string, png image.Image) { fmt.Fprintf(w, "%04x%04x%04x%04x ", nrgba64.R, nrgba64.G, nrgba64.B, nrgba64.A) } case cpm != nil: + var b, c int for x := bounds.Min.X; x < bounds.Max.X; x++ { - fmt.Fprintf(w, "%02x", paletted.ColorIndexAt(x, y)) + b = b<<uint(bitdepth) | int(paletted.ColorIndexAt(x, y)) + c++ + if c == 8/bitdepth { + fmt.Fprintf(w, "%02x", b) + b = 0 + c = 0 + } } } io.WriteString(w, "\n") @@ -143,14 +159,25 @@ func sng(w io.WriteCloser, filename string, png image.Image) { func TestReader(t *testing.T) { for _, fn := range filenames { // Read the .png file. - image, err := readPng("testdata/pngsuite/" + fn + ".png") + img, err := readPng("testdata/pngsuite/" + fn + ".png") if err != nil { t.Error(fn, err) continue } + + if fn == "basn4a16" { + // basn4a16.sng is gray + alpha but sng() will produce true color + alpha + // so we just check a single random pixel. + c := img.At(2, 1).(image.NRGBA64Color) + if c.R != 0x11a7 || c.G != 0x11a7 || c.B != 0x11a7 || c.A != 0x1085 { + t.Error(fn, fmt.Errorf("wrong pixel value at (2, 1): %x", c)) + } + continue + } + piper, pipew := io.Pipe() pb := bufio.NewReader(piper) - go sng(pipew, fn, image) + go sng(pipew, fn, img) defer piper.Close() // Read the .sng file. diff --git a/libgo/go/image/png/testdata/pngsuite/README b/libgo/go/image/png/testdata/pngsuite/README index 27e7c65586b..abe3ecb201d 100644 --- a/libgo/go/image/png/testdata/pngsuite/README +++ b/libgo/go/image/png/testdata/pngsuite/README @@ -5,5 +5,14 @@ README.original gives the following license for those files: Permission to use, copy, and distribute these images for any purpose and without fee is hereby granted. + +The files basn0g01-30.png, basn0g02-29.png and basn0g04-31.png are in fact +not part of pngsuite but were created from files in pngsuite. Their non-power- +of-two sizes makes them useful for testing bit-depths smaller than a byte. + The *.sng files in this directory were generated from the *.png files -by the sng command-line tool. +by the sng command-line tool and some hand editing. The files +basn0g0{1,2,4}.sng were actually generated by first converting the PNG +to a bitdepth of 8 and then running sng on them. basn4a08.sng was generated +by from a 16-bit rgba version of basn4a08.png rather than the original +gray + alpha. diff --git a/libgo/go/image/png/testdata/pngsuite/basn0g01-30.sng b/libgo/go/image/png/testdata/pngsuite/basn0g01-30.sng new file mode 100644 index 00000000000..7fa35710ca5 --- /dev/null +++ b/libgo/go/image/png/testdata/pngsuite/basn0g01-30.sng @@ -0,0 +1,39 @@ +#SNG: from basn0g01-30.png +IHDR { + width: 30; height: 30; bitdepth: 8; + using grayscale; +} +gAMA {1.0000} +IMAGE { + pixels hex +ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff +ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff +ffffffffffffffffffffffffffffffffffffffffffffffffffffffffff00 +ffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000 +ffffffff0000ffffffffffff0000ffffffffffffffffffffffffff000000 +ffffffff0000ffffffffffff0000ffffffffffffffffffffffff00000000 +ffffffff0000ffffffffffff0000ffffffffffffffffffffff0000000000 +ffffffff0000ffff0000ffff0000ffffffffffffffffffff000000000000 +ffffffff0000ffff0000ffff0000ffffffffffffffffff00000000000000 +ffffffff0000ffff0000ffff0000ffffffffffffffff0000000000000000 +ffffffffff0000000000000000ffffffffffffffff000000000000000000 +ffffffffff0000000000000000ffffffffffffff00000000000000000000 +ffffffffffff0000ffff0000ffffffffffffff0000000000000000000000 +ffffffffffff0000ffff0000ffffffffffff000000000000000000000000 +ffffffffffffffffffffffffffffffffff00000000000000000000000000 +ffffffffffffffffffffffffffffffff0000000000000000000000000000 +ffffffffffffffffffffffffffffff000000000000000000000000000000 +ffffffffffffffffffffffffffff00000000000000000000000000000000 +ffffffffffffffffffffffffff00000000000000ffffffffffffff000000 +ffffffffffffffffffffffff0000000000000000ffffffffffffff000000 +ffffffffffffffffffffff000000000000000000ffff00000000ffff0000 +ffffffffffffffffffff00000000000000000000ffff00000000ffff0000 +ffffffffffffffffff0000000000000000000000ffffffffffffff000000 +ffffffffffffffff000000000000000000000000ffffffffffffff000000 +ffffffffffffff00000000000000000000000000ffff00000000ffff0000 +ffffffffffff0000000000000000000000000000ffff00000000ffff0000 +ffffffffff000000000000000000000000000000ffffffffffffff000000 +ffffffff00000000000000000000000000000000ffffffffffffff000000 +ffffff000000000000000000000000000000000000000000000000000000 +ffff00000000000000000000000000000000000000000000000000000000 +} diff --git a/libgo/go/image/png/testdata/pngsuite/basn0g01.sng b/libgo/go/image/png/testdata/pngsuite/basn0g01.sng index e712d8ec678..2ce069de264 100644 --- a/libgo/go/image/png/testdata/pngsuite/basn0g01.sng +++ b/libgo/go/image/png/testdata/pngsuite/basn0g01.sng @@ -1,41 +1,41 @@ #SNG: from basn0g01.png IHDR { - width: 32; height: 32; bitdepth: 1; + width: 32; height: 32; bitdepth: 8; using grayscale; } gAMA {1.0000} IMAGE { pixels hex -fffffffe -fffffffc -fffffff8 -fffffff0 -f3f3ffe0 -f3f3ffc0 -f3f3ff80 -f333ff00 -f333fe00 -f333fc00 -f807f800 -f807f000 -fccfe000 -fccfc000 -ffff8000 -ffff0000 -fffe0000 -fffc0000 -fff80fe0 -fff00fe0 -ffe00c30 -ffc00c30 -ff800fe0 -ff000fe0 -fe000c30 -fc000c30 -f8000fe0 -f0000fe0 -e0000000 -c0000000 -80000000 -00000000 +ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff00 +ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000 +ffffffffffffffffffffffffffffffffffffffffffffffffffffffffff000000 +ffffffffffffffffffffffffffffffffffffffffffffffffffffffff00000000 +ffffffff0000ffffffffffff0000ffffffffffffffffffffffffff0000000000 +ffffffff0000ffffffffffff0000ffffffffffffffffffffffff000000000000 +ffffffff0000ffffffffffff0000ffffffffffffffffffffff00000000000000 +ffffffff0000ffff0000ffff0000ffffffffffffffffffff0000000000000000 +ffffffff0000ffff0000ffff0000ffffffffffffffffff000000000000000000 +ffffffff0000ffff0000ffff0000ffffffffffffffff00000000000000000000 +ffffffffff0000000000000000ffffffffffffffff0000000000000000000000 +ffffffffff0000000000000000ffffffffffffff000000000000000000000000 +ffffffffffff0000ffff0000ffffffffffffff00000000000000000000000000 +ffffffffffff0000ffff0000ffffffffffff0000000000000000000000000000 +ffffffffffffffffffffffffffffffffff000000000000000000000000000000 +ffffffffffffffffffffffffffffffff00000000000000000000000000000000 +ffffffffffffffffffffffffffffff0000000000000000000000000000000000 +ffffffffffffffffffffffffffff000000000000000000000000000000000000 +ffffffffffffffffffffffffff00000000000000ffffffffffffff0000000000 +ffffffffffffffffffffffff0000000000000000ffffffffffffff0000000000 +ffffffffffffffffffffff000000000000000000ffff00000000ffff00000000 +ffffffffffffffffffff00000000000000000000ffff00000000ffff00000000 +ffffffffffffffffff0000000000000000000000ffffffffffffff0000000000 +ffffffffffffffff000000000000000000000000ffffffffffffff0000000000 +ffffffffffffff00000000000000000000000000ffff00000000ffff00000000 +ffffffffffff0000000000000000000000000000ffff00000000ffff00000000 +ffffffffff000000000000000000000000000000ffffffffffffff0000000000 +ffffffff00000000000000000000000000000000ffffffffffffff0000000000 +ffffff0000000000000000000000000000000000000000000000000000000000 +ffff000000000000000000000000000000000000000000000000000000000000 +ff00000000000000000000000000000000000000000000000000000000000000 +0000000000000000000000000000000000000000000000000000000000000000 } diff --git a/libgo/go/image/png/testdata/pngsuite/basn0g02-29.sng b/libgo/go/image/png/testdata/pngsuite/basn0g02-29.sng new file mode 100644 index 00000000000..afb5dba4800 --- /dev/null +++ b/libgo/go/image/png/testdata/pngsuite/basn0g02-29.sng @@ -0,0 +1,38 @@ +#SNG: from basn0g02-29.png +IHDR { + width: 29; height: 29; bitdepth: 8; + using grayscale; +} +gAMA {1.0000} +IMAGE { + pixels hex +0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaaff +0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaaff +0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaaff +0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaaff +55555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff00 +55555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff00 +55555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff00 +55555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff00 +aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff0000000055 +aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff0000000055 +aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff0000000055 +aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff0000000055 +ffffffff0000000055555555aaaaaaaaffffffff0000000055555555aa +ffffffff0000000055555555aaaaaaaaffffffff0000000055555555aa +ffffffff0000000055555555aaaaaaaaffffffff0000000055555555aa +ffffffff0000000055555555aaaaaaaaffffffff0000000055555555aa +0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaaff +0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaaff +0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaaff +0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaaff +55555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff00 +55555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff00 +55555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff00 +55555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff00 +aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff0000000055 +aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff0000000055 +aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff0000000055 +aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff0000000055 +ffffffff0000000055555555aaaaaaaaffffffff0000000055555555aa +} diff --git a/libgo/go/image/png/testdata/pngsuite/basn0g02.sng b/libgo/go/image/png/testdata/pngsuite/basn0g02.sng index e7f2d7ea918..bb53d750df9 100644 --- a/libgo/go/image/png/testdata/pngsuite/basn0g02.sng +++ b/libgo/go/image/png/testdata/pngsuite/basn0g02.sng @@ -1,41 +1,41 @@ #SNG: from basn0g02.png IHDR { - width: 32; height: 32; bitdepth: 2; + width: 32; height: 32; bitdepth: 8; using grayscale; } gAMA {1.0000} IMAGE { pixels hex -0055aaff0055aaff -0055aaff0055aaff -0055aaff0055aaff -0055aaff0055aaff -55aaff0055aaff00 -55aaff0055aaff00 -55aaff0055aaff00 -55aaff0055aaff00 -aaff0055aaff0055 -aaff0055aaff0055 -aaff0055aaff0055 -aaff0055aaff0055 -ff0055aaff0055aa -ff0055aaff0055aa -ff0055aaff0055aa -ff0055aaff0055aa -0055aaff0055aaff -0055aaff0055aaff -0055aaff0055aaff -0055aaff0055aaff -55aaff0055aaff00 -55aaff0055aaff00 -55aaff0055aaff00 -55aaff0055aaff00 -aaff0055aaff0055 -aaff0055aaff0055 -aaff0055aaff0055 -aaff0055aaff0055 -ff0055aaff0055aa -ff0055aaff0055aa -ff0055aaff0055aa -ff0055aaff0055aa +0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff +0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff +0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff +0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff +55555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff00000000 +55555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff00000000 +55555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff00000000 +55555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff00000000 +aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff0000000055555555 +aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff0000000055555555 +aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff0000000055555555 +aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff0000000055555555 +ffffffff0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaa +ffffffff0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaa +ffffffff0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaa +ffffffff0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaa +0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff +0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff +0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff +0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff +55555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff00000000 +55555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff00000000 +55555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff00000000 +55555555aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff00000000 +aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff0000000055555555 +aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff0000000055555555 +aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff0000000055555555 +aaaaaaaaffffffff0000000055555555aaaaaaaaffffffff0000000055555555 +ffffffff0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaa +ffffffff0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaa +ffffffff0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaa +ffffffff0000000055555555aaaaaaaaffffffff0000000055555555aaaaaaaa } diff --git a/libgo/go/image/png/testdata/pngsuite/basn0g04-31.sng b/libgo/go/image/png/testdata/pngsuite/basn0g04-31.sng new file mode 100644 index 00000000000..7f7948e1f7b --- /dev/null +++ b/libgo/go/image/png/testdata/pngsuite/basn0g04-31.sng @@ -0,0 +1,40 @@ +#SNG: from basn0g04-31.png +IHDR { + width: 31; height: 31; bitdepth: 8; + using grayscale; +} +gAMA {1.0000} +IMAGE { + pixels hex +00000000111111112222222233333333444444445555555566666666777777 +00000000111111112222222233333333444444445555555566666666777777 +00000000111111112222222233333333444444445555555566666666777777 +00000000111111112222222233333333444444445555555566666666777777 +11111111222222223333333344444444555555556666666677777777888888 +11111111222222223333333344444444555555556666666677777777888888 +11111111222222223333333344444444555555556666666677777777888888 +11111111222222223333333344444444555555556666666677777777888888 +22222222333333334444444455555555666666667777777788888888999999 +22222222333333334444444455555555666666667777777788888888999999 +22222222333333334444444455555555666666667777777788888888999999 +22222222333333334444444455555555666666667777777788888888999999 +33333333444444445555555566666666777777778888888899999999aaaaaa +33333333444444445555555566666666777777778888888899999999aaaaaa +33333333444444445555555566666666777777778888888899999999aaaaaa +33333333444444445555555566666666777777778888888899999999aaaaaa +444444445555555566666666777777778888888899999999aaaaaaaabbbbbb +444444445555555566666666777777778888888899999999aaaaaaaabbbbbb +444444445555555566666666777777778888888899999999aaaaaaaabbbbbb +444444445555555566666666777777778888888899999999aaaaaaaabbbbbb +5555555566666666777777778888888899999999aaaaaaaabbbbbbbbcccccc +5555555566666666777777778888888899999999aaaaaaaabbbbbbbbcccccc +5555555566666666777777778888888899999999aaaaaaaabbbbbbbbcccccc +5555555566666666777777778888888899999999aaaaaaaabbbbbbbbcccccc +66666666777777778888888899999999aaaaaaaabbbbbbbbccccccccdddddd +66666666777777778888888899999999aaaaaaaabbbbbbbbccccccccdddddd +66666666777777778888888899999999aaaaaaaabbbbbbbbccccccccdddddd +66666666777777778888888899999999aaaaaaaabbbbbbbbccccccccdddddd +777777778888888899999999aaaaaaaabbbbbbbbccccccccddddddddeeeeee +777777778888888899999999aaaaaaaabbbbbbbbccccccccddddddddeeeeee +777777778888888899999999aaaaaaaabbbbbbbbccccccccddddddddeeeeee +} diff --git a/libgo/go/image/png/testdata/pngsuite/basn0g04.sng b/libgo/go/image/png/testdata/pngsuite/basn0g04.sng index 396c508b20d..a95ad01a355 100644 --- a/libgo/go/image/png/testdata/pngsuite/basn0g04.sng +++ b/libgo/go/image/png/testdata/pngsuite/basn0g04.sng @@ -1,41 +1,41 @@ #SNG: from basn0g04.png IHDR { - width: 32; height: 32; bitdepth: 4; + width: 32; height: 32; bitdepth: 8; using grayscale; } gAMA {1.0000} IMAGE { pixels hex -00001111222233334444555566667777 -00001111222233334444555566667777 -00001111222233334444555566667777 -00001111222233334444555566667777 -11112222333344445555666677778888 -11112222333344445555666677778888 -11112222333344445555666677778888 -11112222333344445555666677778888 -22223333444455556666777788889999 -22223333444455556666777788889999 -22223333444455556666777788889999 -22223333444455556666777788889999 -3333444455556666777788889999aaaa -3333444455556666777788889999aaaa -3333444455556666777788889999aaaa -3333444455556666777788889999aaaa -444455556666777788889999aaaabbbb -444455556666777788889999aaaabbbb -444455556666777788889999aaaabbbb -444455556666777788889999aaaabbbb -55556666777788889999aaaabbbbcccc -55556666777788889999aaaabbbbcccc -55556666777788889999aaaabbbbcccc -55556666777788889999aaaabbbbcccc -6666777788889999aaaabbbbccccdddd -6666777788889999aaaabbbbccccdddd -6666777788889999aaaabbbbccccdddd -6666777788889999aaaabbbbccccdddd -777788889999aaaabbbbccccddddeeee -777788889999aaaabbbbccccddddeeee -777788889999aaaabbbbccccddddeeee -777788889999aaaabbbbccccddddeeee +0000000011111111222222223333333344444444555555556666666677777777 +0000000011111111222222223333333344444444555555556666666677777777 +0000000011111111222222223333333344444444555555556666666677777777 +0000000011111111222222223333333344444444555555556666666677777777 +1111111122222222333333334444444455555555666666667777777788888888 +1111111122222222333333334444444455555555666666667777777788888888 +1111111122222222333333334444444455555555666666667777777788888888 +1111111122222222333333334444444455555555666666667777777788888888 +2222222233333333444444445555555566666666777777778888888899999999 +2222222233333333444444445555555566666666777777778888888899999999 +2222222233333333444444445555555566666666777777778888888899999999 +2222222233333333444444445555555566666666777777778888888899999999 +33333333444444445555555566666666777777778888888899999999aaaaaaaa +33333333444444445555555566666666777777778888888899999999aaaaaaaa +33333333444444445555555566666666777777778888888899999999aaaaaaaa +33333333444444445555555566666666777777778888888899999999aaaaaaaa +444444445555555566666666777777778888888899999999aaaaaaaabbbbbbbb +444444445555555566666666777777778888888899999999aaaaaaaabbbbbbbb +444444445555555566666666777777778888888899999999aaaaaaaabbbbbbbb +444444445555555566666666777777778888888899999999aaaaaaaabbbbbbbb +5555555566666666777777778888888899999999aaaaaaaabbbbbbbbcccccccc +5555555566666666777777778888888899999999aaaaaaaabbbbbbbbcccccccc +5555555566666666777777778888888899999999aaaaaaaabbbbbbbbcccccccc +5555555566666666777777778888888899999999aaaaaaaabbbbbbbbcccccccc +66666666777777778888888899999999aaaaaaaabbbbbbbbccccccccdddddddd +66666666777777778888888899999999aaaaaaaabbbbbbbbccccccccdddddddd +66666666777777778888888899999999aaaaaaaabbbbbbbbccccccccdddddddd +66666666777777778888888899999999aaaaaaaabbbbbbbbccccccccdddddddd +777777778888888899999999aaaaaaaabbbbbbbbccccccccddddddddeeeeeeee +777777778888888899999999aaaaaaaabbbbbbbbccccccccddddddddeeeeeeee +777777778888888899999999aaaaaaaabbbbbbbbccccccccddddddddeeeeeeee +777777778888888899999999aaaaaaaabbbbbbbbccccccccddddddddeeeeeeee } diff --git a/libgo/go/image/png/testdata/pngsuite/basn3p02.sng b/libgo/go/image/png/testdata/pngsuite/basn3p02.sng index f2321aa3d0e..ab3fb375fc4 100644 --- a/libgo/go/image/png/testdata/pngsuite/basn3p02.sng +++ b/libgo/go/image/png/testdata/pngsuite/basn3p02.sng @@ -4,14 +4,11 @@ IHDR { using color palette; } gAMA {1.0000} -sBIT { - red: 1; green: 1; blue: 1; -} PLTE { - ( 0,255, 0) # rgb = (0x00,0xff,0x00) green1 - (255, 0, 0) # rgb = (0xff,0x00,0x00) red1 - (255,255, 0) # rgb = (0xff,0xff,0x00) yellow1 - ( 0, 0,255) # rgb = (0x00,0x00,0xff) blue1 + ( 0,255, 0) # rgb = (0x00,0xff,0x00) + (255, 0, 0) # rgb = (0xff,0x00,0x00) + (255,255, 0) # rgb = (0xff,0xff,0x00) + ( 0, 0,255) # rgb = (0x00,0x00,0xff) } IMAGE { pixels hex diff --git a/libgo/go/image/png/testdata/pngsuite/basn3p04.sng b/libgo/go/image/png/testdata/pngsuite/basn3p04.sng index e52885dba0c..a2b2fb53c28 100644 --- a/libgo/go/image/png/testdata/pngsuite/basn3p04.sng +++ b/libgo/go/image/png/testdata/pngsuite/basn3p04.sng @@ -4,19 +4,16 @@ IHDR { using color palette; } gAMA {1.0000} -sBIT { - red: 4; green: 4; blue: 4; -} PLTE { ( 34, 0,255) # rgb = (0x22,0x00,0xff) - ( 0,255,255) # rgb = (0x00,0xff,0xff) cyan1 + ( 0,255,255) # rgb = (0x00,0xff,0xff) (136, 0,255) # rgb = (0x88,0x00,0xff) ( 34,255, 0) # rgb = (0x22,0xff,0x00) ( 0,153,255) # rgb = (0x00,0x99,0xff) (255,102, 0) # rgb = (0xff,0x66,0x00) (221, 0,255) # rgb = (0xdd,0x00,0xff) (119,255, 0) # rgb = (0x77,0xff,0x00) - (255, 0, 0) # rgb = (0xff,0x00,0x00) red1 + (255, 0, 0) # rgb = (0xff,0x00,0x00) ( 0,255,153) # rgb = (0x00,0xff,0x99) (221,255, 0) # rgb = (0xdd,0xff,0x00) (255, 0,187) # rgb = (0xff,0x00,0xbb) diff --git a/libgo/go/image/png/testdata/pngsuite/basn4a08.sng b/libgo/go/image/png/testdata/pngsuite/basn4a08.sng index b760382a022..cc4096fac5d 100644 --- a/libgo/go/image/png/testdata/pngsuite/basn4a08.sng +++ b/libgo/go/image/png/testdata/pngsuite/basn4a08.sng @@ -1,41 +1,41 @@ #SNG: from basn4a08.png IHDR { width: 32; height: 32; bitdepth: 8; - using grayscale alpha; + using color alpha; } gAMA {1.0000} IMAGE { pixels hex -ff00 ff08 ff10 ff18 ff20 ff29 ff31 ff39 ff41 ff4a ff52 ff5a ff62 ff6a ff73 ff7b ff83 ff8b ff94 ff9c ffa4 ffac ffb4 ffbd ffc5 ffcd ffd5 ffde ffe6 ffee fff6 ffff -f600 f608 f610 f618 f620 f629 f631 f639 f641 f64a f652 f65a f662 f66a f673 f67b f683 f68b f694 f69c f6a4 f6ac f6b4 f6bd f6c5 f6cd f6d5 f6de f6e6 f6ee f6f6 f6ff -ee00 ee08 ee10 ee18 ee20 ee29 ee31 ee39 ee41 ee4a ee52 ee5a ee62 ee6a ee73 ee7b ee83 ee8b ee94 ee9c eea4 eeac eeb4 eebd eec5 eecd eed5 eede eee6 eeee eef6 eeff -e600 e608 e610 e618 e620 e629 e631 e639 e641 e64a e652 e65a e662 e66a e673 e67b e683 e68b e694 e69c e6a4 e6ac e6b4 e6bd e6c5 e6cd e6d5 e6de e6e6 e6ee e6f6 e6ff -de00 de08 de10 de18 de20 de29 de31 de39 de41 de4a de52 de5a de62 de6a de73 de7b de83 de8b de94 de9c dea4 deac deb4 debd dec5 decd ded5 dede dee6 deee def6 deff -d500 d508 d510 d518 d520 d529 d531 d539 d541 d54a d552 d55a d562 d56a d573 d57b d583 d58b d594 d59c d5a4 d5ac d5b4 d5bd d5c5 d5cd d5d5 d5de d5e6 d5ee d5f6 d5ff -cd00 cd08 cd10 cd18 cd20 cd29 cd31 cd39 cd41 cd4a cd52 cd5a cd62 cd6a cd73 cd7b cd83 cd8b cd94 cd9c cda4 cdac cdb4 cdbd cdc5 cdcd cdd5 cdde cde6 cdee cdf6 cdff -c500 c508 c510 c518 c520 c529 c531 c539 c541 c54a c552 c55a c562 c56a c573 c57b c583 c58b c594 c59c c5a4 c5ac c5b4 c5bd c5c5 c5cd c5d5 c5de c5e6 c5ee c5f6 c5ff -bd00 bd08 bd10 bd18 bd20 bd29 bd31 bd39 bd41 bd4a bd52 bd5a bd62 bd6a bd73 bd7b bd83 bd8b bd94 bd9c bda4 bdac bdb4 bdbd bdc5 bdcd bdd5 bdde bde6 bdee bdf6 bdff -b400 b408 b410 b418 b420 b429 b431 b439 b441 b44a b452 b45a b462 b46a b473 b47b b483 b48b b494 b49c b4a4 b4ac b4b4 b4bd b4c5 b4cd b4d5 b4de b4e6 b4ee b4f6 b4ff -ac00 ac08 ac10 ac18 ac20 ac29 ac31 ac39 ac41 ac4a ac52 ac5a ac62 ac6a ac73 ac7b ac83 ac8b ac94 ac9c aca4 acac acb4 acbd acc5 accd acd5 acde ace6 acee acf6 acff -a400 a408 a410 a418 a420 a429 a431 a439 a441 a44a a452 a45a a462 a46a a473 a47b a483 a48b a494 a49c a4a4 a4ac a4b4 a4bd a4c5 a4cd a4d5 a4de a4e6 a4ee a4f6 a4ff -9c00 9c08 9c10 9c18 9c20 9c29 9c31 9c39 9c41 9c4a 9c52 9c5a 9c62 9c6a 9c73 9c7b 9c83 9c8b 9c94 9c9c 9ca4 9cac 9cb4 9cbd 9cc5 9ccd 9cd5 9cde 9ce6 9cee 9cf6 9cff -9400 9408 9410 9418 9420 9429 9431 9439 9441 944a 9452 945a 9462 946a 9473 947b 9483 948b 9494 949c 94a4 94ac 94b4 94bd 94c5 94cd 94d5 94de 94e6 94ee 94f6 94ff -8b00 8b08 8b10 8b18 8b20 8b29 8b31 8b39 8b41 8b4a 8b52 8b5a 8b62 8b6a 8b73 8b7b 8b83 8b8b 8b94 8b9c 8ba4 8bac 8bb4 8bbd 8bc5 8bcd 8bd5 8bde 8be6 8bee 8bf6 8bff -8300 8308 8310 8318 8320 8329 8331 8339 8341 834a 8352 835a 8362 836a 8373 837b 8383 838b 8394 839c 83a4 83ac 83b4 83bd 83c5 83cd 83d5 83de 83e6 83ee 83f6 83ff -7b00 7b08 7b10 7b18 7b20 7b29 7b31 7b39 7b41 7b4a 7b52 7b5a 7b62 7b6a 7b73 7b7b 7b83 7b8b 7b94 7b9c 7ba4 7bac 7bb4 7bbd 7bc5 7bcd 7bd5 7bde 7be6 7bee 7bf6 7bff -7300 7308 7310 7318 7320 7329 7331 7339 7341 734a 7352 735a 7362 736a 7373 737b 7383 738b 7394 739c 73a4 73ac 73b4 73bd 73c5 73cd 73d5 73de 73e6 73ee 73f6 73ff -6a00 6a08 6a10 6a18 6a20 6a29 6a31 6a39 6a41 6a4a 6a52 6a5a 6a62 6a6a 6a73 6a7b 6a83 6a8b 6a94 6a9c 6aa4 6aac 6ab4 6abd 6ac5 6acd 6ad5 6ade 6ae6 6aee 6af6 6aff -6200 6208 6210 6218 6220 6229 6231 6239 6241 624a 6252 625a 6262 626a 6273 627b 6283 628b 6294 629c 62a4 62ac 62b4 62bd 62c5 62cd 62d5 62de 62e6 62ee 62f6 62ff -5a00 5a08 5a10 5a18 5a20 5a29 5a31 5a39 5a41 5a4a 5a52 5a5a 5a62 5a6a 5a73 5a7b 5a83 5a8b 5a94 5a9c 5aa4 5aac 5ab4 5abd 5ac5 5acd 5ad5 5ade 5ae6 5aee 5af6 5aff -5200 5208 5210 5218 5220 5229 5231 5239 5241 524a 5252 525a 5262 526a 5273 527b 5283 528b 5294 529c 52a4 52ac 52b4 52bd 52c5 52cd 52d5 52de 52e6 52ee 52f6 52ff -4a00 4a08 4a10 4a18 4a20 4a29 4a31 4a39 4a41 4a4a 4a52 4a5a 4a62 4a6a 4a73 4a7b 4a83 4a8b 4a94 4a9c 4aa4 4aac 4ab4 4abd 4ac5 4acd 4ad5 4ade 4ae6 4aee 4af6 4aff -4100 4108 4110 4118 4120 4129 4131 4139 4141 414a 4152 415a 4162 416a 4173 417b 4183 418b 4194 419c 41a4 41ac 41b4 41bd 41c5 41cd 41d5 41de 41e6 41ee 41f6 41ff -3900 3908 3910 3918 3920 3929 3931 3939 3941 394a 3952 395a 3962 396a 3973 397b 3983 398b 3994 399c 39a4 39ac 39b4 39bd 39c5 39cd 39d5 39de 39e6 39ee 39f6 39ff -3100 3108 3110 3118 3120 3129 3131 3139 3141 314a 3152 315a 3162 316a 3173 317b 3183 318b 3194 319c 31a4 31ac 31b4 31bd 31c5 31cd 31d5 31de 31e6 31ee 31f6 31ff -2900 2908 2910 2918 2920 2929 2931 2939 2941 294a 2952 295a 2962 296a 2973 297b 2983 298b 2994 299c 29a4 29ac 29b4 29bd 29c5 29cd 29d5 29de 29e6 29ee 29f6 29ff -2000 2008 2010 2018 2020 2029 2031 2039 2041 204a 2052 205a 2062 206a 2073 207b 2083 208b 2094 209c 20a4 20ac 20b4 20bd 20c5 20cd 20d5 20de 20e6 20ee 20f6 20ff -1800 1808 1810 1818 1820 1829 1831 1839 1841 184a 1852 185a 1862 186a 1873 187b 1883 188b 1894 189c 18a4 18ac 18b4 18bd 18c5 18cd 18d5 18de 18e6 18ee 18f6 18ff -1000 1008 1010 1018 1020 1029 1031 1039 1041 104a 1052 105a 1062 106a 1073 107b 1083 108b 1094 109c 10a4 10ac 10b4 10bd 10c5 10cd 10d5 10de 10e6 10ee 10f6 10ff -0800 0808 0810 0818 0820 0829 0831 0839 0841 084a 0852 085a 0862 086a 0873 087b 0883 088b 0894 089c 08a4 08ac 08b4 08bd 08c5 08cd 08d5 08de 08e6 08ee 08f6 08ff -0000 0008 0010 0018 0020 0029 0031 0039 0041 004a 0052 005a 0062 006a 0073 007b 0083 008b 0094 009c 00a4 00ac 00b4 00bd 00c5 00cd 00d5 00de 00e6 00ee 00f6 00ff +ffffff00 ffffff08 ffffff10 ffffff18 ffffff20 ffffff29 ffffff31 ffffff39 ffffff41 ffffff4a ffffff52 ffffff5a ffffff62 ffffff6a ffffff73 ffffff7b ffffff83 ffffff8b ffffff94 ffffff9c ffffffa4 ffffffac ffffffb4 ffffffbd ffffffc5 ffffffcd ffffffd5 ffffffde ffffffe6 ffffffee fffffff6 ffffffff +f6f6f600 f6f6f608 f6f6f610 f6f6f618 f6f6f620 f6f6f629 f6f6f631 f6f6f639 f6f6f641 f6f6f64a f6f6f652 f6f6f65a f6f6f662 f6f6f66a f6f6f673 f6f6f67b f6f6f683 f6f6f68b f6f6f694 f6f6f69c f6f6f6a4 f6f6f6ac f6f6f6b4 f6f6f6bd f6f6f6c5 f6f6f6cd f6f6f6d5 f6f6f6de f6f6f6e6 f6f6f6ee f6f6f6f6 f6f6f6ff +eeeeee00 eeeeee08 eeeeee10 eeeeee18 eeeeee20 eeeeee29 eeeeee31 eeeeee39 eeeeee41 eeeeee4a eeeeee52 eeeeee5a eeeeee62 eeeeee6a eeeeee73 eeeeee7b eeeeee83 eeeeee8b eeeeee94 eeeeee9c eeeeeea4 eeeeeeac eeeeeeb4 eeeeeebd eeeeeec5 eeeeeecd eeeeeed5 eeeeeede eeeeeee6 eeeeeeee eeeeeef6 eeeeeeff +e6e6e600 e6e6e608 e6e6e610 e6e6e618 e6e6e620 e6e6e629 e6e6e631 e6e6e639 e6e6e641 e6e6e64a e6e6e652 e6e6e65a e6e6e662 e6e6e66a e6e6e673 e6e6e67b e6e6e683 e6e6e68b e6e6e694 e6e6e69c e6e6e6a4 e6e6e6ac e6e6e6b4 e6e6e6bd e6e6e6c5 e6e6e6cd e6e6e6d5 e6e6e6de e6e6e6e6 e6e6e6ee e6e6e6f6 e6e6e6ff +dedede00 dedede08 dedede10 dedede18 dedede20 dedede29 dedede31 dedede39 dedede41 dedede4a dedede52 dedede5a dedede62 dedede6a dedede73 dedede7b dedede83 dedede8b dedede94 dedede9c dededea4 dededeac dededeb4 dededebd dededec5 dededecd dededed5 dededede dededee6 dededeee dededef6 dededeff +d5d5d500 d5d5d508 d5d5d510 d5d5d518 d5d5d520 d5d5d529 d5d5d531 d5d5d539 d5d5d541 d5d5d54a d5d5d552 d5d5d55a d5d5d562 d5d5d56a d5d5d573 d5d5d57b d5d5d583 d5d5d58b d5d5d594 d5d5d59c d5d5d5a4 d5d5d5ac d5d5d5b4 d5d5d5bd d5d5d5c5 d5d5d5cd d5d5d5d5 d5d5d5de d5d5d5e6 d5d5d5ee d5d5d5f6 d5d5d5ff +cdcdcd00 cdcdcd08 cdcdcd10 cdcdcd18 cdcdcd20 cdcdcd29 cdcdcd31 cdcdcd39 cdcdcd41 cdcdcd4a cdcdcd52 cdcdcd5a cdcdcd62 cdcdcd6a cdcdcd73 cdcdcd7b cdcdcd83 cdcdcd8b cdcdcd94 cdcdcd9c cdcdcda4 cdcdcdac cdcdcdb4 cdcdcdbd cdcdcdc5 cdcdcdcd cdcdcdd5 cdcdcdde cdcdcde6 cdcdcdee cdcdcdf6 cdcdcdff +c5c5c500 c5c5c508 c5c5c510 c5c5c518 c5c5c520 c5c5c529 c5c5c531 c5c5c539 c5c5c541 c5c5c54a c5c5c552 c5c5c55a c5c5c562 c5c5c56a c5c5c573 c5c5c57b c5c5c583 c5c5c58b c5c5c594 c5c5c59c c5c5c5a4 c5c5c5ac c5c5c5b4 c5c5c5bd c5c5c5c5 c5c5c5cd c5c5c5d5 c5c5c5de c5c5c5e6 c5c5c5ee c5c5c5f6 c5c5c5ff +bdbdbd00 bdbdbd08 bdbdbd10 bdbdbd18 bdbdbd20 bdbdbd29 bdbdbd31 bdbdbd39 bdbdbd41 bdbdbd4a bdbdbd52 bdbdbd5a bdbdbd62 bdbdbd6a bdbdbd73 bdbdbd7b bdbdbd83 bdbdbd8b bdbdbd94 bdbdbd9c bdbdbda4 bdbdbdac bdbdbdb4 bdbdbdbd bdbdbdc5 bdbdbdcd bdbdbdd5 bdbdbdde bdbdbde6 bdbdbdee bdbdbdf6 bdbdbdff +b4b4b400 b4b4b408 b4b4b410 b4b4b418 b4b4b420 b4b4b429 b4b4b431 b4b4b439 b4b4b441 b4b4b44a b4b4b452 b4b4b45a b4b4b462 b4b4b46a b4b4b473 b4b4b47b b4b4b483 b4b4b48b b4b4b494 b4b4b49c b4b4b4a4 b4b4b4ac b4b4b4b4 b4b4b4bd b4b4b4c5 b4b4b4cd b4b4b4d5 b4b4b4de b4b4b4e6 b4b4b4ee b4b4b4f6 b4b4b4ff +acacac00 acacac08 acacac10 acacac18 acacac20 acacac29 acacac31 acacac39 acacac41 acacac4a acacac52 acacac5a acacac62 acacac6a acacac73 acacac7b acacac83 acacac8b acacac94 acacac9c acacaca4 acacacac acacacb4 acacacbd acacacc5 acacaccd acacacd5 acacacde acacace6 acacacee acacacf6 acacacff +a4a4a400 a4a4a408 a4a4a410 a4a4a418 a4a4a420 a4a4a429 a4a4a431 a4a4a439 a4a4a441 a4a4a44a a4a4a452 a4a4a45a a4a4a462 a4a4a46a a4a4a473 a4a4a47b a4a4a483 a4a4a48b a4a4a494 a4a4a49c a4a4a4a4 a4a4a4ac a4a4a4b4 a4a4a4bd a4a4a4c5 a4a4a4cd a4a4a4d5 a4a4a4de a4a4a4e6 a4a4a4ee a4a4a4f6 a4a4a4ff +9c9c9c00 9c9c9c08 9c9c9c10 9c9c9c18 9c9c9c20 9c9c9c29 9c9c9c31 9c9c9c39 9c9c9c41 9c9c9c4a 9c9c9c52 9c9c9c5a 9c9c9c62 9c9c9c6a 9c9c9c73 9c9c9c7b 9c9c9c83 9c9c9c8b 9c9c9c94 9c9c9c9c 9c9c9ca4 9c9c9cac 9c9c9cb4 9c9c9cbd 9c9c9cc5 9c9c9ccd 9c9c9cd5 9c9c9cde 9c9c9ce6 9c9c9cee 9c9c9cf6 9c9c9cff +94949400 94949408 94949410 94949418 94949420 94949429 94949431 94949439 94949441 9494944a 94949452 9494945a 94949462 9494946a 94949473 9494947b 94949483 9494948b 94949494 9494949c 949494a4 949494ac 949494b4 949494bd 949494c5 949494cd 949494d5 949494de 949494e6 949494ee 949494f6 949494ff +8b8b8b00 8b8b8b08 8b8b8b10 8b8b8b18 8b8b8b20 8b8b8b29 8b8b8b31 8b8b8b39 8b8b8b41 8b8b8b4a 8b8b8b52 8b8b8b5a 8b8b8b62 8b8b8b6a 8b8b8b73 8b8b8b7b 8b8b8b83 8b8b8b8b 8b8b8b94 8b8b8b9c 8b8b8ba4 8b8b8bac 8b8b8bb4 8b8b8bbd 8b8b8bc5 8b8b8bcd 8b8b8bd5 8b8b8bde 8b8b8be6 8b8b8bee 8b8b8bf6 8b8b8bff +83838300 83838308 83838310 83838318 83838320 83838329 83838331 83838339 83838341 8383834a 83838352 8383835a 83838362 8383836a 83838373 8383837b 83838383 8383838b 83838394 8383839c 838383a4 838383ac 838383b4 838383bd 838383c5 838383cd 838383d5 838383de 838383e6 838383ee 838383f6 838383ff +7b7b7b00 7b7b7b08 7b7b7b10 7b7b7b18 7b7b7b20 7b7b7b29 7b7b7b31 7b7b7b39 7b7b7b41 7b7b7b4a 7b7b7b52 7b7b7b5a 7b7b7b62 7b7b7b6a 7b7b7b73 7b7b7b7b 7b7b7b83 7b7b7b8b 7b7b7b94 7b7b7b9c 7b7b7ba4 7b7b7bac 7b7b7bb4 7b7b7bbd 7b7b7bc5 7b7b7bcd 7b7b7bd5 7b7b7bde 7b7b7be6 7b7b7bee 7b7b7bf6 7b7b7bff +73737300 73737308 73737310 73737318 73737320 73737329 73737331 73737339 73737341 7373734a 73737352 7373735a 73737362 7373736a 73737373 7373737b 73737383 7373738b 73737394 7373739c 737373a4 737373ac 737373b4 737373bd 737373c5 737373cd 737373d5 737373de 737373e6 737373ee 737373f6 737373ff +6a6a6a00 6a6a6a08 6a6a6a10 6a6a6a18 6a6a6a20 6a6a6a29 6a6a6a31 6a6a6a39 6a6a6a41 6a6a6a4a 6a6a6a52 6a6a6a5a 6a6a6a62 6a6a6a6a 6a6a6a73 6a6a6a7b 6a6a6a83 6a6a6a8b 6a6a6a94 6a6a6a9c 6a6a6aa4 6a6a6aac 6a6a6ab4 6a6a6abd 6a6a6ac5 6a6a6acd 6a6a6ad5 6a6a6ade 6a6a6ae6 6a6a6aee 6a6a6af6 6a6a6aff +62626200 62626208 62626210 62626218 62626220 62626229 62626231 62626239 62626241 6262624a 62626252 6262625a 62626262 6262626a 62626273 6262627b 62626283 6262628b 62626294 6262629c 626262a4 626262ac 626262b4 626262bd 626262c5 626262cd 626262d5 626262de 626262e6 626262ee 626262f6 626262ff +5a5a5a00 5a5a5a08 5a5a5a10 5a5a5a18 5a5a5a20 5a5a5a29 5a5a5a31 5a5a5a39 5a5a5a41 5a5a5a4a 5a5a5a52 5a5a5a5a 5a5a5a62 5a5a5a6a 5a5a5a73 5a5a5a7b 5a5a5a83 5a5a5a8b 5a5a5a94 5a5a5a9c 5a5a5aa4 5a5a5aac 5a5a5ab4 5a5a5abd 5a5a5ac5 5a5a5acd 5a5a5ad5 5a5a5ade 5a5a5ae6 5a5a5aee 5a5a5af6 5a5a5aff +52525200 52525208 52525210 52525218 52525220 52525229 52525231 52525239 52525241 5252524a 52525252 5252525a 52525262 5252526a 52525273 5252527b 52525283 5252528b 52525294 5252529c 525252a4 525252ac 525252b4 525252bd 525252c5 525252cd 525252d5 525252de 525252e6 525252ee 525252f6 525252ff +4a4a4a00 4a4a4a08 4a4a4a10 4a4a4a18 4a4a4a20 4a4a4a29 4a4a4a31 4a4a4a39 4a4a4a41 4a4a4a4a 4a4a4a52 4a4a4a5a 4a4a4a62 4a4a4a6a 4a4a4a73 4a4a4a7b 4a4a4a83 4a4a4a8b 4a4a4a94 4a4a4a9c 4a4a4aa4 4a4a4aac 4a4a4ab4 4a4a4abd 4a4a4ac5 4a4a4acd 4a4a4ad5 4a4a4ade 4a4a4ae6 4a4a4aee 4a4a4af6 4a4a4aff +41414100 41414108 41414110 41414118 41414120 41414129 41414131 41414139 41414141 4141414a 41414152 4141415a 41414162 4141416a 41414173 4141417b 41414183 4141418b 41414194 4141419c 414141a4 414141ac 414141b4 414141bd 414141c5 414141cd 414141d5 414141de 414141e6 414141ee 414141f6 414141ff +39393900 39393908 39393910 39393918 39393920 39393929 39393931 39393939 39393941 3939394a 39393952 3939395a 39393962 3939396a 39393973 3939397b 39393983 3939398b 39393994 3939399c 393939a4 393939ac 393939b4 393939bd 393939c5 393939cd 393939d5 393939de 393939e6 393939ee 393939f6 393939ff +31313100 31313108 31313110 31313118 31313120 31313129 31313131 31313139 31313141 3131314a 31313152 3131315a 31313162 3131316a 31313173 3131317b 31313183 3131318b 31313194 3131319c 313131a4 313131ac 313131b4 313131bd 313131c5 313131cd 313131d5 313131de 313131e6 313131ee 313131f6 313131ff +29292900 29292908 29292910 29292918 29292920 29292929 29292931 29292939 29292941 2929294a 29292952 2929295a 29292962 2929296a 29292973 2929297b 29292983 2929298b 29292994 2929299c 292929a4 292929ac 292929b4 292929bd 292929c5 292929cd 292929d5 292929de 292929e6 292929ee 292929f6 292929ff +20202000 20202008 20202010 20202018 20202020 20202029 20202031 20202039 20202041 2020204a 20202052 2020205a 20202062 2020206a 20202073 2020207b 20202083 2020208b 20202094 2020209c 202020a4 202020ac 202020b4 202020bd 202020c5 202020cd 202020d5 202020de 202020e6 202020ee 202020f6 202020ff +18181800 18181808 18181810 18181818 18181820 18181829 18181831 18181839 18181841 1818184a 18181852 1818185a 18181862 1818186a 18181873 1818187b 18181883 1818188b 18181894 1818189c 181818a4 181818ac 181818b4 181818bd 181818c5 181818cd 181818d5 181818de 181818e6 181818ee 181818f6 181818ff +10101000 10101008 10101010 10101018 10101020 10101029 10101031 10101039 10101041 1010104a 10101052 1010105a 10101062 1010106a 10101073 1010107b 10101083 1010108b 10101094 1010109c 101010a4 101010ac 101010b4 101010bd 101010c5 101010cd 101010d5 101010de 101010e6 101010ee 101010f6 101010ff +08080800 08080808 08080810 08080818 08080820 08080829 08080831 08080839 08080841 0808084a 08080852 0808085a 08080862 0808086a 08080873 0808087b 08080883 0808088b 08080894 0808089c 080808a4 080808ac 080808b4 080808bd 080808c5 080808cd 080808d5 080808de 080808e6 080808ee 080808f6 080808ff +00000000 00000008 00000010 00000018 00000020 00000029 00000031 00000039 00000041 0000004a 00000052 0000005a 00000062 0000006a 00000073 0000007b 00000083 0000008b 00000094 0000009c 000000a4 000000ac 000000b4 000000bd 000000c5 000000cd 000000d5 000000de 000000e6 000000ee 000000f6 000000ff } diff --git a/libgo/go/index/suffixarray/qsufsort.go b/libgo/go/index/suffixarray/qsufsort.go index 0e6894a8b57..9751b5c7663 100644 --- a/libgo/go/index/suffixarray/qsufsort.go +++ b/libgo/go/index/suffixarray/qsufsort.go @@ -146,19 +146,25 @@ func (x *suffixSortable) Swap(i, j int) { x.sa[i], x.sa[j] = x.sa[j], x.sa[ func (x *suffixSortable) updateGroups(offset int) { - prev := len(x.sa) - 1 - group := x.inv[x.sa[prev]+x.h] - for i := prev; i >= 0; i-- { - if g := x.inv[x.sa[i]+x.h]; g < group { - if prev == i+1 { // previous group had size 1 and is thus sorted - x.sa[i+1] = -1 - } + bounds := make([]int, 0, 4) + group := x.inv[x.sa[0]+x.h] + for i := 1; i < len(x.sa); i++ { + if g := x.inv[x.sa[i]+x.h]; g > group { + bounds = append(bounds, i) group = g - prev = i } - x.inv[x.sa[i]] = prev + offset - if prev == 0 { // first group has size 1 and is thus sorted - x.sa[0] = -1 + } + bounds = append(bounds, len(x.sa)) + + // update the group numberings after all new groups are determined + prev := 0 + for _, b := range bounds { + for i := prev; i < b; i++ { + x.inv[x.sa[i]] = offset + b - 1 + } + if b-prev == 1 { + x.sa[prev] = -1 } + prev = b } } diff --git a/libgo/go/index/suffixarray/suffixarray.go b/libgo/go/index/suffixarray/suffixarray.go index 628e000e1d3..d8c6fc91b48 100644 --- a/libgo/go/index/suffixarray/suffixarray.go +++ b/libgo/go/index/suffixarray/suffixarray.go @@ -50,27 +50,33 @@ func (x *Index) at(i int) []byte { } -func (x *Index) search(s []byte) int { - return sort.Search(len(x.sa), func(i int) bool { return bytes.Compare(x.at(i), s) >= 0 }) +// lookupAll returns a slice into the matching region of the index. +// The runtime is O(log(N)*len(s)). +func (x *Index) lookupAll(s []byte) []int { + // find matching suffix index range [i:j] + // find the first index where s would be the prefix + i := sort.Search(len(x.sa), func(i int) bool { return bytes.Compare(x.at(i), s) >= 0 }) + // starting at i, find the first index at which s is not a prefix + j := i + sort.Search(len(x.sa)-i, func(j int) bool { return !bytes.HasPrefix(x.at(j+i), s) }) + return x.sa[i:j] } // Lookup returns an unsorted list of at most n indices where the byte string s // occurs in the indexed data. If n < 0, all occurrences are returned. // The result is nil if s is empty, s is not found, or n == 0. -// Lookup time is O((log(N) + len(result))*len(s)) where N is the +// Lookup time is O(log(N)*len(s) + len(result)) where N is the // size of the indexed data. // func (x *Index) Lookup(s []byte, n int) (result []int) { if len(s) > 0 && n != 0 { - // find matching suffix index i - i := x.search(s) - // x.at(i-1) < s <= x.at(i) - - // collect the following suffixes with matching prefixes - for (n < 0 || len(result) < n) && i < len(x.sa) && bytes.HasPrefix(x.at(i), s) { - result = append(result, x.sa[i]) - i++ + matches := x.lookupAll(s) + if len(matches) < n || n < 0 { + n = len(matches) + } + if n > 0 { + result = make([]int, n) + copy(result, matches) } } return diff --git a/libgo/go/index/suffixarray/suffixarray_test.go b/libgo/go/index/suffixarray/suffixarray_test.go index b3486a96d08..e85267f17f5 100644 --- a/libgo/go/index/suffixarray/suffixarray_test.go +++ b/libgo/go/index/suffixarray/suffixarray_test.go @@ -99,6 +99,12 @@ var testCases = []testCase{ "to (come|the)?", }, }, + + { + "godoc simulation", + "package main\n\nimport(\n \"rand\"\n ", + []string{}, + }, } diff --git a/libgo/go/io/io.go b/libgo/go/io/io.go index 1a6eca95a0d..3b879189798 100644 --- a/libgo/go/io/io.go +++ b/libgo/go/io/io.go @@ -150,14 +150,23 @@ type WriterAt interface { WriteAt(p []byte, off int64) (n int, err os.Error) } -// ReadByter is the interface that wraps the ReadByte method. +// ByteReader is the interface that wraps the ReadByte method. // // ReadByte reads and returns the next byte from the input. // If no byte is available, err will be set. -type ReadByter interface { +type ByteReader interface { ReadByte() (c byte, err os.Error) } +// RuneReader is the interface that wraps the ReadRune method. +// +// ReadRune reads a single UTF-8 encoded Unicode character +// and returns the rune and its size in bytes. If no character is +// available, err will be set. +type RuneReader interface { + ReadRune() (rune int, size int, err os.Error) +} + // WriteString writes the contents of the string s to w, which accepts an array of bytes. func WriteString(w Writer, s string) (n int, err os.Error) { return w.Write([]byte(s)) diff --git a/libgo/go/io/ioutil/tempfile.go b/libgo/go/io/ioutil/tempfile.go index 114eca2b50e..c7cc67b1b74 100644 --- a/libgo/go/io/ioutil/tempfile.go +++ b/libgo/go/io/ioutil/tempfile.go @@ -46,6 +46,7 @@ func TempFile(dir, prefix string) (f *os.File, err os.Error) { nconflict := 0 for i := 0; i < 10000; i++ { + // TODO(rsc): use filepath.Join name := dir + "/" + prefix + nextSuffix() f, err = os.Open(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600) if pe, ok := err.(*os.PathError); ok && pe.Error == os.EEXIST { @@ -58,3 +59,34 @@ func TempFile(dir, prefix string) (f *os.File, err os.Error) { } return } + +// TempDir creates a new temporary directory in the directory dir +// with a name beginning with prefix and returns the path of the +// new directory. If dir is the empty string, TempDir uses the +// default directory for temporary files (see os.TempDir). +// Multiple programs calling TempDir simultaneously +// will not choose the same directory. It is the caller's responsibility +// to remove the directory when no longer needed. +func TempDir(dir, prefix string) (name string, err os.Error) { + if dir == "" { + dir = os.TempDir() + } + + nconflict := 0 + for i := 0; i < 10000; i++ { + // TODO(rsc): use filepath.Join + try := dir + "/" + prefix + nextSuffix() + err = os.Mkdir(try, 0700) + if pe, ok := err.(*os.PathError); ok && pe.Error == os.EEXIST { + if nconflict++; nconflict > 10 { + rand = reseed() + } + continue + } + if err == nil { + name = try + } + break + } + return +} diff --git a/libgo/go/io/ioutil/tempfile_test.go b/libgo/go/io/ioutil/tempfile_test.go index d949a86cf09..6013ec1d4a4 100644 --- a/libgo/go/io/ioutil/tempfile_test.go +++ b/libgo/go/io/ioutil/tempfile_test.go @@ -23,11 +23,31 @@ func TestTempFile(t *testing.T) { t.Errorf("TempFile(dir, `ioutil_test`) = %v, %v", f, err) } if f != nil { + f.Close() + os.Remove(f.Name()) re := regexp.MustCompile("^" + regexp.QuoteMeta(dir) + "/ioutil_test[0-9]+$") if !re.MatchString(f.Name()) { t.Errorf("TempFile(`"+dir+"`, `ioutil_test`) created bad name %s", f.Name()) } - os.Remove(f.Name()) } - f.Close() +} + +func TestTempDir(t *testing.T) { + name, err := TempDir("/_not_exists_", "foo") + if name != "" || err == nil { + t.Errorf("TempDir(`/_not_exists_`, `foo`) = %v, %v", name, err) + } + + dir := os.TempDir() + name, err = TempDir(dir, "ioutil_test") + if name == "" || err != nil { + t.Errorf("TempDir(dir, `ioutil_test`) = %v, %v", name, err) + } + if name != "" { + os.Remove(name) + re := regexp.MustCompile("^" + regexp.QuoteMeta(dir) + "/ioutil_test[0-9]+$") + if !re.MatchString(name) { + t.Errorf("TempDir(`"+dir+"`, `ioutil_test`) created bad name %s", name) + } + } } diff --git a/libgo/go/json/decode.go b/libgo/go/json/decode.go index ff91dd83c33..501230c0c05 100644 --- a/libgo/go/json/decode.go +++ b/libgo/go/json/decode.go @@ -9,6 +9,7 @@ package json import ( "container/vector" + "encoding/base64" "os" "reflect" "runtime" @@ -466,13 +467,15 @@ func (d *decodeState) object(v reflect.Value) { } else { var f reflect.StructField var ok bool - // First try for field with that tag. st := sv.Type().(*reflect.StructType) - for i := 0; i < sv.NumField(); i++ { - f = st.Field(i) - if f.Tag == key { - ok = true - break + // First try for field with that tag. + if isValidTag(key) { + for i := 0; i < sv.NumField(); i++ { + f = st.Field(i) + if f.Tag == key { + ok = true + break + } } } if !ok { @@ -568,17 +571,29 @@ func (d *decodeState) literal(v reflect.Value) { } case '"': // string - s, ok := unquote(item) + s, ok := unquoteBytes(item) if !ok { d.error(errPhase) } switch v := v.(type) { default: d.saveError(&UnmarshalTypeError{"string", v.Type()}) + case *reflect.SliceValue: + if v.Type() != byteSliceType { + d.saveError(&UnmarshalTypeError{"string", v.Type()}) + break + } + b := make([]byte, base64.StdEncoding.DecodedLen(len(s))) + n, err := base64.StdEncoding.Decode(b, s) + if err != nil { + d.saveError(err) + break + } + v.Set(reflect.NewValue(b[0:n]).(*reflect.SliceValue)) case *reflect.StringValue: - v.Set(s) + v.Set(string(s)) case *reflect.InterfaceValue: - v.Set(reflect.NewValue(s)) + v.Set(reflect.NewValue(string(s))) } default: // number @@ -772,12 +787,43 @@ func getu4(s []byte) int { // unquote converts a quoted JSON string literal s into an actual string t. // The rules are different than for Go, so cannot use strconv.Unquote. func unquote(s []byte) (t string, ok bool) { + s, ok = unquoteBytes(s) + t = string(s) + return +} + +func unquoteBytes(s []byte) (t []byte, ok bool) { if len(s) < 2 || s[0] != '"' || s[len(s)-1] != '"' { return } + s = s[1 : len(s)-1] + + // Check for unusual characters. If there are none, + // then no unquoting is needed, so return a slice of the + // original bytes. + r := 0 + for r < len(s) { + c := s[r] + if c == '\\' || c == '"' || c < ' ' { + break + } + if c < utf8.RuneSelf { + r++ + continue + } + rune, size := utf8.DecodeRune(s[r:]) + if rune == utf8.RuneError && size == 1 { + break + } + r += size + } + if r == len(s) { + return s, true + } + b := make([]byte, len(s)+2*utf8.UTFMax) - w := 0 - for r := 1; r < len(s)-1; { + w := copy(b, s[0:r]) + for r < len(s) { // Out of room? Can only happen if s is full of // malformed UTF-8 and we're replacing each // byte with RuneError. @@ -789,7 +835,7 @@ func unquote(s []byte) (t string, ok bool) { switch c := s[r]; { case c == '\\': r++ - if r >= len(s)-1 { + if r >= len(s) { return } switch s[r] { @@ -857,5 +903,5 @@ func unquote(s []byte) (t string, ok bool) { w += utf8.EncodeRune(b[w:], rune) } } - return string(b[0:w]), true + return b[0:w], true } diff --git a/libgo/go/json/decode_test.go b/libgo/go/json/decode_test.go index 9cb27af412a..ad6026363b5 100644 --- a/libgo/go/json/decode_test.go +++ b/libgo/go/json/decode_test.go @@ -40,6 +40,11 @@ var ( umtrue = unmarshaler{true} ) +type badTag struct { + X string + Y string "y" + Z string "@#*%(#@" +} type unmarshalTest struct { in string @@ -62,6 +67,9 @@ var unmarshalTests = []unmarshalTest{ {`{"X": [1,2,3], "Y": 4}`, new(T), T{Y: 4}, &UnmarshalTypeError{"array", reflect.Typeof("")}}, {`{"x": 1}`, new(tx), tx{}, &UnmarshalFieldError{"x", txType, txType.Field(0)}}, + // skip invalid tags + {`{"X":"a", "y":"b", "Z":"c"}`, new(badTag), badTag{"a", "b", "c"}, nil}, + // syntax errors {`{"X": "foo", "Y"}`, nil, nil, SyntaxError("invalid character '}' after object key")}, @@ -164,6 +172,25 @@ func TestUnmarshalMarshal(t *testing.T) { } } +func TestLargeByteSlice(t *testing.T) { + s0 := make([]byte, 2000) + for i := range s0 { + s0[i] = byte(i) + } + b, err := Marshal(s0) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + var s1 []byte + if err := Unmarshal(b, &s1); err != nil { + t.Fatalf("Unmarshal: %v", err) + } + if bytes.Compare(s0, s1) != 0 { + t.Errorf("Marshal large byte slice") + diff(t, s0, s1) + } +} + type Xint struct { X int } @@ -412,11 +439,7 @@ var allValueIndent = `{ "str25", "str26" ], - "ByteSlice": [ - 27, - 28, - 29 - ], + "ByteSlice": "Gxwd", "Small": { "Tag": "tag30" }, @@ -502,7 +525,7 @@ var pallValueIndent = `{ "EmptySlice": [], "NilSlice": [], "StringSlice": [], - "ByteSlice": [], + "ByteSlice": "", "Small": { "Tag": "" }, diff --git a/libgo/go/json/encode.go b/libgo/go/json/encode.go index 759b49dbeb4..26ce47039f6 100644 --- a/libgo/go/json/encode.go +++ b/libgo/go/json/encode.go @@ -7,12 +7,14 @@ package json import ( - "os" "bytes" + "encoding/base64" + "os" "reflect" "runtime" "sort" "strconv" + "unicode" "utf8" ) @@ -31,13 +33,14 @@ import ( // String values encode as JSON strings, with each invalid UTF-8 sequence // replaced by the encoding of the Unicode replacement character U+FFFD. // -// Array and slice values encode as JSON arrays. +// Array and slice values encode as JSON arrays, except that +// []byte encodes as a base64-encoded string. // // Struct values encode as JSON objects. Each struct field becomes // a member of the object. By default the object's key name is the -// struct field name converted to lower case. If the struct field -// has a tag, that tag will be used as the name instead. -// Only exported fields will be encoded. +// struct field name. If the struct field has a non-empty tag consisting +// of only Unicode letters, digits, and underscores, that tag will be used +// as the name instead. Only exported fields will be encoded. // // Map values encode as JSON objects. // The map's key type must be string; the object keys are used directly @@ -177,6 +180,8 @@ func (e *encodeState) error(err os.Error) { panic(err) } +var byteSliceType = reflect.Typeof([]byte(nil)) + func (e *encodeState) reflectValue(v reflect.Value) { if v == nil { e.WriteString("null") @@ -231,7 +236,7 @@ func (e *encodeState) reflectValue(v reflect.Value) { } else { e.WriteByte(',') } - if f.Tag != "" { + if isValidTag(f.Tag) { e.string(f.Tag) } else { e.string(f.Name) @@ -263,6 +268,24 @@ func (e *encodeState) reflectValue(v reflect.Value) { e.WriteByte('}') case reflect.ArrayOrSliceValue: + if v.Type() == byteSliceType { + e.WriteByte('"') + s := v.Interface().([]byte) + if len(s) < 1024 { + // for small buffers, using Encode directly is much faster. + dst := make([]byte, base64.StdEncoding.EncodedLen(len(s))) + base64.StdEncoding.Encode(dst, s) + e.Write(dst) + } else { + // for large buffers, avoid unnecessary extra temporary + // buffer space. + enc := base64.NewEncoder(base64.StdEncoding, e) + enc.Write(s) + enc.Close() + } + e.WriteByte('"') + break + } e.WriteByte('[') n := v.Len() for i := 0; i < n; i++ { @@ -286,6 +309,18 @@ func (e *encodeState) reflectValue(v reflect.Value) { return } +func isValidTag(s string) bool { + if s == "" { + return false + } + for _, c := range s { + if c != '_' && !unicode.IsLetter(c) && !unicode.IsDigit(c) { + return false + } + } + return true +} + // stringValues is a slice of reflect.Value holding *reflect.StringValue. // It implements the methods to sort by string. type stringValues []reflect.Value diff --git a/libgo/go/json/scanner.go b/libgo/go/json/scanner.go index 112c8f9c350..e98ddef5cc1 100644 --- a/libgo/go/json/scanner.go +++ b/libgo/go/json/scanner.go @@ -416,7 +416,7 @@ func state0(s *scanner, c int) int { s.step = stateDot return scanContinue } - if c == 'e' { + if c == 'e' || c == 'E' { s.step = stateE return scanContinue } @@ -440,7 +440,7 @@ func stateDot0(s *scanner, c int) int { s.step = stateDot0 return scanContinue } - if c == 'e' { + if c == 'e' || c == 'E' { s.step = stateE return scanContinue } diff --git a/libgo/go/log/log.go b/libgo/go/log/log.go index d34af9e5e45..658e3bd94b1 100644 --- a/libgo/go/log/log.go +++ b/libgo/go/log/log.go @@ -4,11 +4,11 @@ // Simple logging package. It defines a type, Logger, with methods // for formatting output. It also has a predefined 'standard' Logger -// accessible through helper functions Print[f|ln], Exit[f|ln], and +// accessible through helper functions Print[f|ln], Fatal[f|ln], and // Panic[f|ln], which are easier to use than creating a Logger manually. // That logger writes to standard error and prints the date and time // of each logged message. -// The Exit functions call os.Exit(1) after writing the log message. +// The Fatal functions call os.Exit(1) after writing the log message. // The Panic functions call panic after writing the log message. package log @@ -164,20 +164,20 @@ func (l *Logger) Print(v ...interface{}) { l.Output(2, fmt.Sprint(v...)) } // Arguments are handled in the manner of fmt.Println. func (l *Logger) Println(v ...interface{}) { l.Output(2, fmt.Sprintln(v...)) } -// Exit is equivalent to l.Print() followed by a call to os.Exit(1). -func (l *Logger) Exit(v ...interface{}) { +// Fatal is equivalent to l.Print() followed by a call to os.Exit(1). +func (l *Logger) Fatal(v ...interface{}) { l.Output(2, fmt.Sprint(v...)) os.Exit(1) } -// Exitf is equivalent to l.Printf() followed by a call to os.Exit(1). -func (l *Logger) Exitf(format string, v ...interface{}) { +// Fatalf is equivalent to l.Printf() followed by a call to os.Exit(1). +func (l *Logger) Fatalf(format string, v ...interface{}) { l.Output(2, fmt.Sprintf(format, v...)) os.Exit(1) } -// Exitln is equivalent to l.Println() followed by a call to os.Exit(1). -func (l *Logger) Exitln(v ...interface{}) { +// Fatalln is equivalent to l.Println() followed by a call to os.Exit(1). +func (l *Logger) Fatalln(v ...interface{}) { l.Output(2, fmt.Sprintln(v...)) os.Exit(1) } @@ -238,20 +238,20 @@ func Println(v ...interface{}) { std.Output(2, fmt.Sprintln(v...)) } -// Exit is equivalent to Print() followed by a call to os.Exit(1). -func Exit(v ...interface{}) { +// Fatal is equivalent to Print() followed by a call to os.Exit(1). +func Fatal(v ...interface{}) { std.Output(2, fmt.Sprint(v...)) os.Exit(1) } -// Exitf is equivalent to Printf() followed by a call to os.Exit(1). -func Exitf(format string, v ...interface{}) { +// Fatalf is equivalent to Printf() followed by a call to os.Exit(1). +func Fatalf(format string, v ...interface{}) { std.Output(2, fmt.Sprintf(format, v...)) os.Exit(1) } -// Exitln is equivalent to Println() followed by a call to os.Exit(1). -func Exitln(v ...interface{}) { +// Fatalln is equivalent to Println() followed by a call to os.Exit(1). +func Fatalln(v ...interface{}) { std.Output(2, fmt.Sprintln(v...)) os.Exit(1) } diff --git a/libgo/go/net/dial.go b/libgo/go/net/dial.go index 03b9d87be33..1cf8e79159e 100644 --- a/libgo/go/net/dial.go +++ b/libgo/go/net/dial.go @@ -24,7 +24,7 @@ import "os" // Dial("tcp", "127.0.0.1:123", "127.0.0.1:88") // func Dial(net, laddr, raddr string) (c Conn, err os.Error) { - switch prefixBefore(net, ':') { + switch net { case "tcp", "tcp4", "tcp6": var la, ra *TCPAddr if laddr != "" { @@ -137,7 +137,7 @@ func Listen(net, laddr string) (l Listener, err os.Error) { // The network string net must be a packet-oriented network: // "udp", "udp4", "udp6", or "unixgram". func ListenPacket(net, laddr string) (c PacketConn, err os.Error) { - switch prefixBefore(net, ':') { + switch net { case "udp", "udp4", "udp6": var la *UDPAddr if laddr != "" { @@ -162,18 +162,24 @@ func ListenPacket(net, laddr string) (c PacketConn, err os.Error) { return nil, err } return c, nil - case "ip", "ip4", "ip6": - var la *IPAddr - if laddr != "" { - if la, err = ResolveIPAddr(laddr); err != nil { + } + + if i := last(net, ':'); i > 0 { + switch net[0:i] { + case "ip", "ip4", "ip6": + var la *IPAddr + if laddr != "" { + if la, err = ResolveIPAddr(laddr); err != nil { + return nil, err + } + } + c, err := ListenIP(net, la) + if err != nil { return nil, err } + return c, nil } - c, err := ListenIP(net, la) - if err != nil { - return nil, err - } - return c, nil } + return nil, UnknownNetworkError(net) } diff --git a/libgo/go/net/dnsclient.go b/libgo/go/net/dnsclient.go index 87d76261f8e..3252dd45406 100644 --- a/libgo/go/net/dnsclient.go +++ b/libgo/go/net/dnsclient.go @@ -98,18 +98,18 @@ func exchange(cfg *dnsConfig, c Conn, name string, qtype uint16) (*dnsMsg, os.Er // Find answer for name in dns message. // On return, if err == nil, addrs != nil. -func answer(name, server string, dns *dnsMsg, qtype uint16) (addrs []dnsRR, err os.Error) { +func answer(name, server string, dns *dnsMsg, qtype uint16) (cname string, addrs []dnsRR, err os.Error) { addrs = make([]dnsRR, 0, len(dns.answer)) if dns.rcode == dnsRcodeNameError && dns.recursion_available { - return nil, &DNSError{Error: noSuchHost, Name: name} + return "", nil, &DNSError{Error: noSuchHost, Name: name} } if dns.rcode != dnsRcodeSuccess { // None of the error codes make sense // for the query we sent. If we didn't get // a name error and we didn't get success, // the server is behaving incorrectly. - return nil, &DNSError{Error: "server misbehaving", Name: name, Server: server} + return "", nil, &DNSError{Error: "server misbehaving", Name: name, Server: server} } // Look for the name. @@ -137,19 +137,19 @@ Cname: } } if len(addrs) == 0 { - return nil, &DNSError{Error: noSuchHost, Name: name, Server: server} + return "", nil, &DNSError{Error: noSuchHost, Name: name, Server: server} } - return addrs, nil + return name, addrs, nil } - return nil, &DNSError{Error: "too many redirects", Name: name, Server: server} + return "", nil, &DNSError{Error: "too many redirects", Name: name, Server: server} } // Do a lookup for a single name, which must be rooted // (otherwise answer will not find the answers). -func tryOneName(cfg *dnsConfig, name string, qtype uint16) (addrs []dnsRR, err os.Error) { +func tryOneName(cfg *dnsConfig, name string, qtype uint16) (cname string, addrs []dnsRR, err os.Error) { if len(cfg.servers) == 0 { - return nil, &DNSError{Error: "no DNS servers", Name: name} + return "", nil, &DNSError{Error: "no DNS servers", Name: name} } for i := 0; i < len(cfg.servers); i++ { // Calling Dial here is scary -- we have to be sure @@ -170,7 +170,7 @@ func tryOneName(cfg *dnsConfig, name string, qtype uint16) (addrs []dnsRR, err o err = merr continue } - addrs, err = answer(name, server, msg, qtype) + cname, addrs, err = answer(name, server, msg, qtype) if err == nil || err.(*DNSError).Error == noSuchHost { break } @@ -261,9 +261,8 @@ func lookup(name string, qtype uint16) (cname string, addrs []dnsRR, err os.Erro rname += "." } // Can try as ordinary name. - addrs, err = tryOneName(cfg, rname, qtype) + cname, addrs, err = tryOneName(cfg, rname, qtype) if err == nil { - cname = rname return } } @@ -277,9 +276,8 @@ func lookup(name string, qtype uint16) (cname string, addrs []dnsRR, err os.Erro if rname[len(rname)-1] != '.' { rname += "." } - addrs, err = tryOneName(cfg, rname, qtype) + cname, addrs, err = tryOneName(cfg, rname, qtype) if err == nil { - cname = rname return } } @@ -289,9 +287,8 @@ func lookup(name string, qtype uint16) (cname string, addrs []dnsRR, err os.Erro if !rooted { rname += "." } - addrs, err = tryOneName(cfg, rname, qtype) + cname, addrs, err = tryOneName(cfg, rname, qtype) if err == nil { - cname = rname return } return diff --git a/libgo/go/net/fd.go b/libgo/go/net/fd.go index 26d17d4e0e7..d48aefe2cdd 100644 --- a/libgo/go/net/fd.go +++ b/libgo/go/net/fd.go @@ -220,11 +220,16 @@ func (s *pollServer) Run() { nn, _ = s.pr.Read(scratch[0:]) } // Read from channels - for fd, ok := <-s.cr; ok; fd, ok = <-s.cr { - s.AddFD(fd, 'r') - } - for fd, ok := <-s.cw; ok; fd, ok = <-s.cw { - s.AddFD(fd, 'w') + Update: + for { + select { + case fd := <-s.cr: + s.AddFD(fd, 'r') + case fd := <-s.cw: + s.AddFD(fd, 'w') + default: + break Update + } } } else { netfd := s.LookupFD(fd, mode) diff --git a/libgo/go/net/fd_windows.go b/libgo/go/net/fd_windows.go index 9b91eb398cb..63a8fbc4489 100644 --- a/libgo/go/net/fd_windows.go +++ b/libgo/go/net/fd_windows.go @@ -13,147 +13,241 @@ import ( "unsafe" ) +type InvalidConnError struct{} + +func (e *InvalidConnError) String() string { return "invalid net.Conn" } +func (e *InvalidConnError) Temporary() bool { return false } +func (e *InvalidConnError) Timeout() bool { return false } + +var initErr os.Error + +func init() { + var d syscall.WSAData + e := syscall.WSAStartup(uint32(0x101), &d) + if e != 0 { + initErr = os.NewSyscallError("WSAStartup", e) + } +} + +func closesocket(s int) (errno int) { + return syscall.Closesocket(int32(s)) +} + +// Interface for all io operations. +type anOpIface interface { + Op() *anOp + Name() string + Submit() (errno int) +} + // IO completion result parameters. type ioResult struct { - key uint32 - qty uint32 - errno int + qty uint32 + err int } -// Network file descriptor. -type netFD struct { - // locking/lifetime of sysfd - sysmu sync.Mutex - sysref int - closing bool +// anOp implements functionality common to all io operations. +type anOp struct { + // Used by IOCP interface, it must be first field + // of the struct, as our code rely on it. + o syscall.Overlapped - // immutable until Close - sysfd int - family int - proto int - cr chan *ioResult - cw chan *ioResult - net string - laddr Addr - raddr Addr + resultc chan ioResult // io completion results + errnoc chan int // io submit / cancel operation errors + fd *netFD +} - // owned by client - rdeadline_delta int64 - rdeadline int64 - rio sync.Mutex - wdeadline_delta int64 - wdeadline int64 - wio sync.Mutex +func (o *anOp) Init(fd *netFD) { + o.fd = fd + o.resultc = make(chan ioResult, 1) + o.errnoc = make(chan int) } -type InvalidConnError struct{} +func (o *anOp) Op() *anOp { + return o +} -func (e *InvalidConnError) String() string { return "invalid net.Conn" } -func (e *InvalidConnError) Temporary() bool { return false } -func (e *InvalidConnError) Timeout() bool { return false } +// bufOp is used by io operations that read / write +// data from / to client buffer. +type bufOp struct { + anOp + buf syscall.WSABuf +} -// pollServer will run around waiting for io completion request -// to arrive. Every request received will contain channel to signal -// io owner about the completion. +func (o *bufOp) Init(fd *netFD, buf []byte) { + o.anOp.Init(fd) + o.buf.Len = uint32(len(buf)) + if len(buf) == 0 { + o.buf.Buf = nil + } else { + o.buf.Buf = (*byte)(unsafe.Pointer(&buf[0])) + } +} -type pollServer struct { +// resultSrv will retreive all io completion results from +// iocp and send them to the correspondent waiting client +// goroutine via channel supplied in the request. +type resultSrv struct { iocp int32 } -func newPollServer() (s *pollServer, err os.Error) { - s = new(pollServer) - var e int - if s.iocp, e = syscall.CreateIoCompletionPort(-1, 0, 0, 1); e != 0 { - return nil, os.NewSyscallError("CreateIoCompletionPort", e) +func (s *resultSrv) Run() { + var o *syscall.Overlapped + var key uint32 + var r ioResult + for { + r.err = syscall.GetQueuedCompletionStatus(s.iocp, &(r.qty), &key, &o, syscall.INFINITE) + switch { + case r.err == 0: + // Dequeued successfully completed io packet. + case r.err == syscall.WAIT_TIMEOUT && o == nil: + // Wait has timed out (should not happen now, but might be used in the future). + panic("GetQueuedCompletionStatus timed out") + case o == nil: + // Failed to dequeue anything -> report the error. + panic("GetQueuedCompletionStatus failed " + syscall.Errstr(r.err)) + default: + // Dequeued failed io packet. + } + (*anOp)(unsafe.Pointer(o)).resultc <- r } - go s.Run() - return s, nil } -type ioPacket struct { - // Used by IOCP interface, - // it must be first field of the struct, - // as our code rely on it. - o syscall.Overlapped - // Link to the io owner. - c chan *ioResult - - w *syscall.WSABuf +// ioSrv executes net io requests. +type ioSrv struct { + submchan chan anOpIface // submit io requests + canchan chan anOpIface // cancel io requests } -func (s *pollServer) getCompletedIO() (ov *syscall.Overlapped, result *ioResult, err os.Error) { - var r ioResult - var o *syscall.Overlapped - _, e := syscall.GetQueuedCompletionStatus(s.iocp, &r.qty, &r.key, &o, syscall.INFINITE) - switch { - case e == 0: - // Dequeued successfully completed io packet. - return o, &r, nil - case e == syscall.WAIT_TIMEOUT && o == nil: - // Wait has timed out (should not happen now, but might be used in the future). - return nil, &r, os.NewSyscallError("GetQueuedCompletionStatus", e) - case o == nil: - // Failed to dequeue anything -> report the error. - return nil, &r, os.NewSyscallError("GetQueuedCompletionStatus", e) - default: - // Dequeued failed io packet. - r.errno = e - return o, &r, nil +// ProcessRemoteIO will execute submit io requests on behalf +// of other goroutines, all on a single os thread, so it can +// cancel them later. Results of all operations will be sent +// back to their requesters via channel supplied in request. +func (s *ioSrv) ProcessRemoteIO() { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + for { + select { + case o := <-s.submchan: + o.Op().errnoc <- o.Submit() + case o := <-s.canchan: + o.Op().errnoc <- syscall.CancelIo(uint32(o.Op().fd.sysfd)) + } } - return } -func (s *pollServer) Run() { - for { - o, r, err := s.getCompletedIO() - if err != nil { - panic("Run pollServer: " + err.String() + "\n") +// ExecIO executes a single io operation. It either executes it +// inline, or, if timeouts are employed, passes the request onto +// a special goroutine and waits for completion or cancels request. +func (s *ioSrv) ExecIO(oi anOpIface, deadline_delta int64) (n int, err os.Error) { + var e int + o := oi.Op() + if deadline_delta > 0 { + // Send request to a special dedicated thread, + // so it can stop the io with CancelIO later. + s.submchan <- oi + e = <-o.errnoc + } else { + e = oi.Submit() + } + switch e { + case 0: + // IO completed immediately, but we need to get our completion message anyway. + case syscall.ERROR_IO_PENDING: + // IO started, and we have to wait for it's completion. + default: + return 0, &OpError{oi.Name(), o.fd.net, o.fd.laddr, os.Errno(e)} + } + // Wait for our request to complete. + var r ioResult + if deadline_delta > 0 { + select { + case r = <-o.resultc: + case <-time.After(deadline_delta): + s.canchan <- oi + <-o.errnoc + r = <-o.resultc + if r.err == syscall.ERROR_OPERATION_ABORTED { // IO Canceled + r.err = syscall.EWOULDBLOCK + } } - p := (*ioPacket)(unsafe.Pointer(o)) - p.c <- r + } else { + r = <-o.resultc } + if r.err != 0 { + err = &OpError{oi.Name(), o.fd.net, o.fd.laddr, os.Errno(r.err)} + } + return int(r.qty), err } -// Network FD methods. -// All the network FDs use a single pollServer. - -var pollserver *pollServer +// Start helper goroutines. +var resultsrv *resultSrv +var iosrv *ioSrv var onceStartServer sync.Once func startServer() { - p, err := newPollServer() - if err != nil { - panic("Start pollServer: " + err.String() + "\n") - } - pollserver = p - - go timeoutIO() + resultsrv = new(resultSrv) + var errno int + resultsrv.iocp, errno = syscall.CreateIoCompletionPort(-1, 0, 0, 1) + if errno != 0 { + panic("CreateIoCompletionPort failed " + syscall.Errstr(errno)) + } + go resultsrv.Run() + + iosrv = new(ioSrv) + iosrv.submchan = make(chan anOpIface) + iosrv.canchan = make(chan anOpIface) + go iosrv.ProcessRemoteIO() } -var initErr os.Error +// Network file descriptor. +type netFD struct { + // locking/lifetime of sysfd + sysmu sync.Mutex + sysref int + closing bool -func newFD(fd, family, proto int, net string, laddr, raddr Addr) (f *netFD, err os.Error) { - if initErr != nil { - return nil, initErr - } - onceStartServer.Do(startServer) - // Associate our socket with pollserver.iocp. - if _, e := syscall.CreateIoCompletionPort(int32(fd), pollserver.iocp, 0, 0); e != 0 { - return nil, &OpError{"CreateIoCompletionPort", net, laddr, os.Errno(e)} - } + // immutable until Close + sysfd int + family int + proto int + net string + laddr Addr + raddr Addr + + // owned by client + rdeadline_delta int64 + rdeadline int64 + rio sync.Mutex + wdeadline_delta int64 + wdeadline int64 + wio sync.Mutex +} + +func allocFD(fd, family, proto int, net string, laddr, raddr Addr) (f *netFD) { f = &netFD{ sysfd: fd, family: family, proto: proto, - cr: make(chan *ioResult, 1), - cw: make(chan *ioResult, 1), net: net, laddr: laddr, raddr: raddr, } runtime.SetFinalizer(f, (*netFD).Close) - return f, nil + return f +} + +func newFD(fd, family, proto int, net string, laddr, raddr Addr) (f *netFD, err os.Error) { + if initErr != nil { + return nil, initErr + } + onceStartServer.Do(startServer) + // Associate our socket with resultsrv.iocp. + if _, e := syscall.CreateIoCompletionPort(int32(fd), resultsrv.iocp, 0, 0); e != 0 { + return nil, &OpError{"CreateIoCompletionPort", net, laddr, os.Errno(e)} + } + return allocFD(fd, family, proto, net, laddr, raddr), nil } // Add a reference to this fd. @@ -172,7 +266,7 @@ func (fd *netFD) decref() { // In case the user has set linger, switch to blocking mode so // the close blocks. As long as this doesn't happen often, we // can handle the extra OS processes. Otherwise we'll need to - // use the pollserver for Close too. Sigh. + // use the resultsrv for Close too. Sigh. syscall.SetNonblock(fd.sysfd, false) closesocket(fd.sysfd) fd.sysfd = -1 @@ -194,89 +288,22 @@ func (fd *netFD) Close() os.Error { return nil } -func newWSABuf(p []byte) *syscall.WSABuf { - var p0 *byte - if len(p) > 0 { - p0 = (*byte)(unsafe.Pointer(&p[0])) - } - return &syscall.WSABuf{uint32(len(p)), p0} -} - -func waitPacket(fd *netFD, pckt *ioPacket, mode int) (r *ioResult) { - var delta int64 - if mode == 'r' { - delta = fd.rdeadline_delta - } - if mode == 'w' { - delta = fd.wdeadline_delta - } - if delta <= 0 { - return <-pckt.c - } +// Read from network. - select { - case r = <-pckt.c: - case <-time.After(delta): - a := &arg{f: cancel, fd: fd, pckt: pckt, c: make(chan int)} - ioChan <- a - <-a.c - r = <-pckt.c - if r.errno == 995 { // IO Canceled - r.errno = syscall.EWOULDBLOCK - } - } - return r +type readOp struct { + bufOp } -const ( - read = iota - readfrom - write - writeto - cancel -) +func (o *readOp) Submit() (errno int) { + var d, f uint32 + return syscall.WSARecv(uint32(o.fd.sysfd), &o.buf, 1, &d, &f, &o.o, nil) +} -type arg struct { - f int - fd *netFD - pckt *ioPacket - done *uint32 - flags *uint32 - rsa *syscall.RawSockaddrAny - size *int32 - sa *syscall.Sockaddr - c chan int -} - -var ioChan chan *arg = make(chan *arg) - -func timeoutIO() { - // CancelIO only cancels all pending input and output (I/O) operations that are - // issued by the calling thread for the specified file, does not cancel I/O - // operations that other threads issue for a file handle. So we need do all timeout - // I/O in single OS thread. - runtime.LockOSThread() - defer runtime.UnlockOSThread() - for { - o := <-ioChan - var e int - switch o.f { - case read: - e = syscall.WSARecv(uint32(o.fd.sysfd), o.pckt.w, 1, o.done, o.flags, &o.pckt.o, nil) - case readfrom: - e = syscall.WSARecvFrom(uint32(o.fd.sysfd), o.pckt.w, 1, o.done, o.flags, o.rsa, o.size, &o.pckt.o, nil) - case write: - e = syscall.WSASend(uint32(o.fd.sysfd), o.pckt.w, 1, o.done, uint32(0), &o.pckt.o, nil) - case writeto: - e = syscall.WSASendto(uint32(o.fd.sysfd), o.pckt.w, 1, o.done, 0, *o.sa, &o.pckt.o, nil) - case cancel: - _, e = syscall.CancelIo(uint32(o.fd.sysfd)) - } - o.c <- e - } +func (o *readOp) Name() string { + return "WSARecv" } -func (fd *netFD) Read(p []byte) (n int, err os.Error) { +func (fd *netFD) Read(buf []byte) (n int, err os.Error) { if fd == nil { return 0, os.EINVAL } @@ -287,45 +314,37 @@ func (fd *netFD) Read(p []byte) (n int, err os.Error) { if fd.sysfd == -1 { return 0, os.EINVAL } - // Submit receive request. - var pckt ioPacket - pckt.c = fd.cr - pckt.w = newWSABuf(p) - var done uint32 - flags := uint32(0) - var e int - if fd.rdeadline_delta > 0 { - a := &arg{f: read, fd: fd, pckt: &pckt, done: &done, flags: &flags, c: make(chan int)} - ioChan <- a - e = <-a.c - } else { - e = syscall.WSARecv(uint32(fd.sysfd), pckt.w, 1, &done, &flags, &pckt.o, nil) - } - switch e { - case 0: - // IO completed immediately, but we need to get our completion message anyway. - case syscall.ERROR_IO_PENDING: - // IO started, and we have to wait for it's completion. - default: - return 0, &OpError{"WSARecv", fd.net, fd.laddr, os.Errno(e)} - } - // Wait for our request to complete. - r := waitPacket(fd, &pckt, 'r') - if r.errno != 0 { - err = &OpError{"WSARecv", fd.net, fd.laddr, os.Errno(r.errno)} - } - n = int(r.qty) + var o readOp + o.Init(fd, buf) + n, err = iosrv.ExecIO(&o, fd.rdeadline_delta) if err == nil && n == 0 { err = os.EOF } return } -func (fd *netFD) ReadFrom(p []byte) (n int, sa syscall.Sockaddr, err os.Error) { +// ReadFrom from network. + +type readFromOp struct { + bufOp + rsa syscall.RawSockaddrAny +} + +func (o *readFromOp) Submit() (errno int) { + var d, f uint32 + l := int32(unsafe.Sizeof(o.rsa)) + return syscall.WSARecvFrom(uint32(o.fd.sysfd), &o.buf, 1, &d, &f, &o.rsa, &l, &o.o, nil) +} + +func (o *readFromOp) Name() string { + return "WSARecvFrom" +} + +func (fd *netFD) ReadFrom(buf []byte) (n int, sa syscall.Sockaddr, err os.Error) { if fd == nil { return 0, nil, os.EINVAL } - if len(p) == 0 { + if len(buf) == 0 { return 0, nil, nil } fd.rio.Lock() @@ -335,41 +354,29 @@ func (fd *netFD) ReadFrom(p []byte) (n int, sa syscall.Sockaddr, err os.Error) { if fd.sysfd == -1 { return 0, nil, os.EINVAL } - // Submit receive request. - var pckt ioPacket - pckt.c = fd.cr - pckt.w = newWSABuf(p) - var done uint32 - flags := uint32(0) - var rsa syscall.RawSockaddrAny - l := int32(unsafe.Sizeof(rsa)) - var e int - if fd.rdeadline_delta > 0 { - a := &arg{f: readfrom, fd: fd, pckt: &pckt, done: &done, flags: &flags, rsa: &rsa, size: &l, c: make(chan int)} - ioChan <- a - e = <-a.c - } else { - e = syscall.WSARecvFrom(uint32(fd.sysfd), pckt.w, 1, &done, &flags, &rsa, &l, &pckt.o, nil) - } - switch e { - case 0: - // IO completed immediately, but we need to get our completion message anyway. - case syscall.ERROR_IO_PENDING: - // IO started, and we have to wait for it's completion. - default: - return 0, nil, &OpError{"WSARecvFrom", fd.net, fd.laddr, os.Errno(e)} - } - // Wait for our request to complete. - r := waitPacket(fd, &pckt, 'r') - if r.errno != 0 { - err = &OpError{"WSARecvFrom", fd.net, fd.laddr, os.Errno(r.errno)} - } - n = int(r.qty) - sa, _ = rsa.Sockaddr() + var o readFromOp + o.Init(fd, buf) + n, err = iosrv.ExecIO(&o, fd.rdeadline_delta) + sa, _ = o.rsa.Sockaddr() return } -func (fd *netFD) Write(p []byte) (n int, err os.Error) { +// Write to network. + +type writeOp struct { + bufOp +} + +func (o *writeOp) Submit() (errno int) { + var d uint32 + return syscall.WSASend(uint32(o.fd.sysfd), &o.buf, 1, &d, 0, &o.o, nil) +} + +func (o *writeOp) Name() string { + return "WSASend" +} + +func (fd *netFD) Write(buf []byte) (n int, err os.Error) { if fd == nil { return 0, os.EINVAL } @@ -380,41 +387,32 @@ func (fd *netFD) Write(p []byte) (n int, err os.Error) { if fd.sysfd == -1 { return 0, os.EINVAL } - // Submit send request. - var pckt ioPacket - pckt.c = fd.cw - pckt.w = newWSABuf(p) - var done uint32 - var e int - if fd.wdeadline_delta > 0 { - a := &arg{f: write, fd: fd, pckt: &pckt, done: &done, c: make(chan int)} - ioChan <- a - e = <-a.c - } else { - e = syscall.WSASend(uint32(fd.sysfd), pckt.w, 1, &done, uint32(0), &pckt.o, nil) - } - switch e { - case 0: - // IO completed immediately, but we need to get our completion message anyway. - case syscall.ERROR_IO_PENDING: - // IO started, and we have to wait for it's completion. - default: - return 0, &OpError{"WSASend", fd.net, fd.laddr, os.Errno(e)} - } - // Wait for our request to complete. - r := waitPacket(fd, &pckt, 'w') - if r.errno != 0 { - err = &OpError{"WSASend", fd.net, fd.laddr, os.Errno(r.errno)} - } - n = int(r.qty) - return + var o writeOp + o.Init(fd, buf) + return iosrv.ExecIO(&o, fd.wdeadline_delta) +} + +// WriteTo to network. + +type writeToOp struct { + bufOp + sa syscall.Sockaddr +} + +func (o *writeToOp) Submit() (errno int) { + var d uint32 + return syscall.WSASendto(uint32(o.fd.sysfd), &o.buf, 1, &d, 0, o.sa, &o.o, nil) +} + +func (o *writeToOp) Name() string { + return "WSASendto" } -func (fd *netFD) WriteTo(p []byte, sa syscall.Sockaddr) (n int, err os.Error) { +func (fd *netFD) WriteTo(buf []byte, sa syscall.Sockaddr) (n int, err os.Error) { if fd == nil { return 0, os.EINVAL } - if len(p) == 0 { + if len(buf) == 0 { return 0, nil } fd.wio.Lock() @@ -424,34 +422,29 @@ func (fd *netFD) WriteTo(p []byte, sa syscall.Sockaddr) (n int, err os.Error) { if fd.sysfd == -1 { return 0, os.EINVAL } - // Submit send request. - var pckt ioPacket - pckt.c = fd.cw - pckt.w = newWSABuf(p) - var done uint32 - var e int - if fd.wdeadline_delta > 0 { - a := &arg{f: writeto, fd: fd, pckt: &pckt, done: &done, sa: &sa, c: make(chan int)} - ioChan <- a - e = <-a.c - } else { - e = syscall.WSASendto(uint32(fd.sysfd), pckt.w, 1, &done, 0, sa, &pckt.o, nil) - } - switch e { - case 0: - // IO completed immediately, but we need to get our completion message anyway. - case syscall.ERROR_IO_PENDING: - // IO started, and we have to wait for it's completion. - default: - return 0, &OpError{"WSASendTo", fd.net, fd.laddr, os.Errno(e)} - } - // Wait for our request to complete. - r := waitPacket(fd, &pckt, 'w') - if r.errno != 0 { - err = &OpError{"WSASendTo", fd.net, fd.laddr, os.Errno(r.errno)} - } - n = int(r.qty) - return + var o writeToOp + o.Init(fd, buf) + o.sa = sa + return iosrv.ExecIO(&o, fd.wdeadline_delta) +} + +// Accept new network connections. + +type acceptOp struct { + anOp + newsock int + attrs [2]syscall.RawSockaddrAny // space for local and remote address only +} + +func (o *acceptOp) Submit() (errno int) { + var d uint32 + l := uint32(unsafe.Sizeof(o.attrs[0])) + return syscall.AcceptEx(uint32(o.fd.sysfd), uint32(o.newsock), + (*byte)(unsafe.Pointer(&o.attrs[0])), 0, l, l, &d, &o.o) +} + +func (o *acceptOp) Name() string { + return "AcceptEx" } func (fd *netFD) accept(toAddr func(syscall.Sockaddr) Addr) (nfd *netFD, err os.Error) { @@ -474,72 +467,40 @@ func (fd *netFD) accept(toAddr func(syscall.Sockaddr) Addr) (nfd *netFD, err os. // Associate our new socket with IOCP. onceStartServer.Do(startServer) - if _, e = syscall.CreateIoCompletionPort(int32(s), pollserver.iocp, 0, 0); e != 0 { + if _, e = syscall.CreateIoCompletionPort(int32(s), resultsrv.iocp, 0, 0); e != 0 { return nil, &OpError{"CreateIoCompletionPort", fd.net, fd.laddr, os.Errno(e)} } // Submit accept request. - // Will use new unique channel here, because, unlike Read or Write, - // Accept is expected to be executed by many goroutines simultaniously. - var pckt ioPacket - pckt.c = make(chan *ioResult) - attrs, e := syscall.AcceptIOCP(fd.sysfd, s, &pckt.o) - switch e { - case 0: - // IO completed immediately, but we need to get our completion message anyway. - case syscall.ERROR_IO_PENDING: - // IO started, and we have to wait for it's completion. - default: - closesocket(s) - return nil, &OpError{"AcceptEx", fd.net, fd.laddr, os.Errno(e)} - } - - // Wait for peer connection. - r := <-pckt.c - if r.errno != 0 { + var o acceptOp + o.Init(fd) + o.newsock = s + _, err = iosrv.ExecIO(&o, 0) + if err != nil { closesocket(s) - return nil, &OpError{"AcceptEx", fd.net, fd.laddr, os.Errno(r.errno)} + return nil, err } // Inherit properties of the listening socket. e = syscall.SetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_UPDATE_ACCEPT_CONTEXT, fd.sysfd) if e != 0 { closesocket(s) - return nil, &OpError{"Setsockopt", fd.net, fd.laddr, os.Errno(r.errno)} + return nil, err } // Get local and peer addr out of AcceptEx buffer. - lsa, rsa := syscall.GetAcceptIOCPSockaddrs(attrs) - - // Create our netFD and return it for further use. - laddr := toAddr(lsa) - raddr := toAddr(rsa) - - f := &netFD{ - sysfd: s, - family: fd.family, - proto: fd.proto, - cr: make(chan *ioResult, 1), - cw: make(chan *ioResult, 1), - net: fd.net, - laddr: laddr, - raddr: raddr, - } - runtime.SetFinalizer(f, (*netFD).Close) - return f, nil -} - -func closesocket(s int) (errno int) { - return syscall.Closesocket(int32(s)) + var lrsa, rrsa *syscall.RawSockaddrAny + var llen, rlen int32 + l := uint32(unsafe.Sizeof(*lrsa)) + syscall.GetAcceptExSockaddrs((*byte)(unsafe.Pointer(&o.attrs[0])), + 0, l, l, &lrsa, &llen, &rrsa, &rlen) + lsa, _ := lrsa.Sockaddr() + rsa, _ := rrsa.Sockaddr() + + return allocFD(s, fd.family, fd.proto, fd.net, toAddr(lsa), toAddr(rsa)), nil } -func init() { - var d syscall.WSAData - e := syscall.WSAStartup(uint32(0x101), &d) - if e != 0 { - initErr = os.NewSyscallError("WSAStartup", e) - } -} +// Not implemeted functions. func (fd *netFD) dup() (f *os.File, err os.Error) { // TODO: Implement this diff --git a/libgo/go/net/iprawsock.go b/libgo/go/net/iprawsock.go index 241be150956..81a918ce5cb 100644 --- a/libgo/go/net/iprawsock.go +++ b/libgo/go/net/iprawsock.go @@ -245,7 +245,7 @@ func hostToIP(host string) (ip IP, err os.Error) { err = err1 goto Error } - addr = ParseIP(addrs[0]) + addr = firstSupportedAddr(addrs) if addr == nil { // should not happen err = &AddrError{"LookupHost returned invalid address", addrs[0]} @@ -311,7 +311,7 @@ func DialIP(netProto string, laddr, raddr *IPAddr) (c *IPConn, err os.Error) { if err != nil { return } - switch prefixBefore(net, ':') { + switch net { case "ip", "ip4", "ip6": default: return nil, UnknownNetworkError(net) @@ -335,7 +335,7 @@ func ListenIP(netProto string, laddr *IPAddr) (c *IPConn, err os.Error) { if err != nil { return } - switch prefixBefore(net, ':') { + switch net { case "ip", "ip4", "ip6": default: return nil, UnknownNetworkError(net) diff --git a/libgo/go/net/ipsock.go b/libgo/go/net/ipsock.go index 4ba6a55b96f..ae4204b48aa 100644 --- a/libgo/go/net/ipsock.go +++ b/libgo/go/net/ipsock.go @@ -18,19 +18,34 @@ import ( // Unfortunately, we need to run on kernels built without IPv6 support too. // So probe the kernel to figure it out. func kernelSupportsIPv6() bool { - // FreeBSD does not support this sort of interface. - if syscall.OS == "freebsd" { + s, err := syscall.Socket(syscall.AF_INET6, syscall.SOCK_STREAM, syscall.IPPROTO_TCP) + if err != 0 { return false } - fd, e := syscall.Socket(syscall.AF_INET6, syscall.SOCK_STREAM, syscall.IPPROTO_TCP) - if fd >= 0 { - closesocket(fd) + defer closesocket(s) + + la := &TCPAddr{IP: IPv4(127, 0, 0, 1)} + sa, oserr := la.toAddr().sockaddr(syscall.AF_INET6) + if oserr != nil { + return false } - return e == 0 + + return syscall.Bind(s, sa) == 0 } var preferIPv4 = !kernelSupportsIPv6() +func firstSupportedAddr(addrs []string) (addr IP) { + for _, s := range addrs { + addr = ParseIP(s) + if !preferIPv4 || addr.To4() != nil { + break + } + addr = nil + } + return addr +} + // TODO(rsc): if syscall.OS == "linux", we're supposd to read // /proc/sys/net/core/somaxconn, // to take advantage of kernels that have raised the limit. @@ -208,7 +223,7 @@ func hostPortToIP(net, hostport string) (ip IP, iport int, err os.Error) { err = err1 goto Error } - addr = ParseIP(addrs[0]) + addr = firstSupportedAddr(addrs) if addr == nil { // should not happen err = &AddrError{"LookupHost returned invalid address", addrs[0]} diff --git a/libgo/go/net/multicast_test.go b/libgo/go/net/multicast_test.go new file mode 100644 index 00000000000..32fdec85bde --- /dev/null +++ b/libgo/go/net/multicast_test.go @@ -0,0 +1,62 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "runtime" + "testing" +) + +func TestMulticastJoinAndLeave(t *testing.T) { + if runtime.GOOS == "windows" { + return + } + + addr := &UDPAddr{ + IP: IPv4zero, + Port: 0, + } + // open a UDPConn + conn, err := ListenUDP("udp4", addr) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + + // try to join group + mcast := IPv4(224, 0, 0, 254) + err = conn.JoinGroup(mcast) + if err != nil { + t.Fatal(err) + } + + // try to leave group + err = conn.LeaveGroup(mcast) + if err != nil { + t.Fatal(err) + } +} + +func TestJoinFailureWithIPv6Address(t *testing.T) { + addr := &UDPAddr{ + IP: IPv4zero, + Port: 0, + } + + // open a UDPConn + conn, err := ListenUDP("udp4", addr) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + + // try to join group + mcast := ParseIP("ff02::1") + err = conn.JoinGroup(mcast) + if err == nil { + t.Fatal("JoinGroup succeeded, should fail") + } + t.Logf("%s", err) +} diff --git a/libgo/go/net/net.go b/libgo/go/net/net.go index c0c1c3b8ab1..04a898a9aac 100644 --- a/libgo/go/net/net.go +++ b/libgo/go/net/net.go @@ -31,7 +31,6 @@ type Conn interface { Write(b []byte) (n int, err os.Error) // Close closes the connection. - // The error returned is an os.Error to satisfy io.Closer; Close() os.Error // LocalAddr returns the local network address. @@ -83,7 +82,6 @@ type PacketConn interface { WriteTo(b []byte, addr Addr) (n int, err os.Error) // Close closes the connection. - // The error returned is an os.Error to satisfy io.Closer; Close() os.Error // LocalAddr returns the local network address. @@ -112,7 +110,6 @@ type Listener interface { Accept() (c Conn, err os.Error) // Close closes the listener. - // The error returned is an os.Error to satisfy io.Closer; Close() os.Error // Addr returns the listener's network address. diff --git a/libgo/go/net/parse.go b/libgo/go/net/parse.go index 605f3110b7b..2bc0db4655a 100644 --- a/libgo/go/net/parse.go +++ b/libgo/go/net/parse.go @@ -192,16 +192,6 @@ func count(s string, b byte) int { return n } -// Returns the prefix of s up to but not including the character c -func prefixBefore(s string, c byte) string { - for i, v := range s { - if v == int(c) { - return s[0:i] - } - } - return s -} - // Index of rightmost occurrence of b in s. func last(s string, b byte) int { i := len(s) diff --git a/libgo/go/net/server_test.go b/libgo/go/net/server_test.go index 3f2442a462d..3dda500e585 100644 --- a/libgo/go/net/server_test.go +++ b/libgo/go/net/server_test.go @@ -25,7 +25,7 @@ func runEcho(fd io.ReadWriter, done chan<- int) { for { n, err := fd.Read(buf[0:]) - if err != nil || n == 0 { + if err != nil || n == 0 || string(buf[:n]) == "END" { break } fd.Write(buf[0:n]) @@ -79,6 +79,13 @@ func connect(t *testing.T, network, addr string, isEmpty bool) { if n != len(b) || err1 != nil { t.Fatalf("fd.Read() = %d, %v (want %d, nil)", n, err1, len(b)) } + + // Send explicit ending for unixpacket. + // Older Linux kernels do stop reads on close. + if network == "unixpacket" { + fd.Write([]byte("END")) + } + fd.Close() } @@ -133,13 +140,16 @@ func runPacket(t *testing.T, network, addr string, listening chan<- string, done listening <- c.LocalAddr().String() c.SetReadTimeout(10e6) // 10ms var buf [1000]byte +Run: for { n, addr, err := c.ReadFrom(buf[0:]) if e, ok := err.(Error); ok && e.Timeout() { - if done <- 1 { - break + select { + case done <- 1: + break Run + default: + continue Run } - continue } if err != nil { break diff --git a/libgo/go/net/textproto/header.go b/libgo/go/net/textproto/header.go new file mode 100644 index 00000000000..288deb2ceb4 --- /dev/null +++ b/libgo/go/net/textproto/header.go @@ -0,0 +1,43 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package textproto + +// A MIMEHeader represents a MIME-style header mapping +// keys to sets of values. +type MIMEHeader map[string][]string + +// Add adds the key, value pair to the header. +// It appends to any existing values associated with key. +func (h MIMEHeader) Add(key, value string) { + key = CanonicalMIMEHeaderKey(key) + h[key] = append(h[key], value) +} + +// Set sets the header entries associated with key to +// the single element value. It replaces any existing +// values associated with key. +func (h MIMEHeader) Set(key, value string) { + h[CanonicalMIMEHeaderKey(key)] = []string{value} +} + +// Get gets the first value associated with the given key. +// If there are no values associated with the key, Get returns "". +// Get is a convenience method. For more complex queries, +// access the map directly. +func (h MIMEHeader) Get(key string) string { + if h == nil { + return "" + } + v := h[CanonicalMIMEHeaderKey(key)] + if len(v) == 0 { + return "" + } + return v[0] +} + +// Del deletes the values associated with key. +func (h MIMEHeader) Del(key string) { + h[CanonicalMIMEHeaderKey(key)] = nil, false +} diff --git a/libgo/go/net/textproto/reader.go b/libgo/go/net/textproto/reader.go index c8e34b7589d..ac1278689a4 100644 --- a/libgo/go/net/textproto/reader.go +++ b/libgo/go/net/textproto/reader.go @@ -402,7 +402,7 @@ func (r *Reader) ReadDotLines() ([]string, os.Error) { // ReadMIMEHeader reads a MIME-style header from r. // The header is a sequence of possibly continued Key: Value lines // ending in a blank line. -// The returned map m maps CanonicalHeaderKey(key) to a +// The returned map m maps CanonicalMIMEHeaderKey(key) to a // sequence of values in the same order encountered in the input. // // For example, consider this input: @@ -415,12 +415,12 @@ func (r *Reader) ReadDotLines() ([]string, os.Error) { // Given that input, ReadMIMEHeader returns the map: // // map[string][]string{ -// "My-Key": []string{"Value 1", "Value 2"}, -// "Long-Key": []string{"Even Longer Value"}, +// "My-Key": {"Value 1", "Value 2"}, +// "Long-Key": {"Even Longer Value"}, // } // -func (r *Reader) ReadMIMEHeader() (map[string][]string, os.Error) { - m := make(map[string][]string) +func (r *Reader) ReadMIMEHeader() (MIMEHeader, os.Error) { + m := make(MIMEHeader) for { kv, err := r.ReadContinuedLineBytes() if len(kv) == 0 { @@ -432,7 +432,7 @@ func (r *Reader) ReadMIMEHeader() (map[string][]string, os.Error) { if i < 0 || bytes.IndexByte(kv[0:i], ' ') >= 0 { return m, ProtocolError("malformed MIME header line: " + string(kv)) } - key := CanonicalHeaderKey(string(kv[0:i])) + key := CanonicalMIMEHeaderKey(string(kv[0:i])) // Skip initial spaces in value. i++ // skip colon @@ -452,12 +452,12 @@ func (r *Reader) ReadMIMEHeader() (map[string][]string, os.Error) { panic("unreachable") } -// CanonicalHeaderKey returns the canonical format of the +// CanonicalMIMEHeaderKey returns the canonical format of the // MIME header key s. The canonicalization converts the first // letter and any letter following a hyphen to upper case; // the rest are converted to lowercase. For example, the // canonical key for "accept-encoding" is "Accept-Encoding". -func CanonicalHeaderKey(s string) string { +func CanonicalMIMEHeaderKey(s string) string { // Quick check for canonical encoding. needUpper := true for i := 0; i < len(s); i++ { diff --git a/libgo/go/net/textproto/reader_test.go b/libgo/go/net/textproto/reader_test.go index 2cecbc75f25..0658e58b82d 100644 --- a/libgo/go/net/textproto/reader_test.go +++ b/libgo/go/net/textproto/reader_test.go @@ -26,10 +26,10 @@ var canonicalHeaderKeyTests = []canonicalHeaderKeyTest{ {"USER-AGENT", "User-Agent"}, } -func TestCanonicalHeaderKey(t *testing.T) { +func TestCanonicalMIMEHeaderKey(t *testing.T) { for _, tt := range canonicalHeaderKeyTests { - if s := CanonicalHeaderKey(tt.in); s != tt.out { - t.Errorf("CanonicalHeaderKey(%q) = %q, want %q", tt.in, s, tt.out) + if s := CanonicalMIMEHeaderKey(tt.in); s != tt.out { + t.Errorf("CanonicalMIMEHeaderKey(%q) = %q, want %q", tt.in, s, tt.out) } } } @@ -130,7 +130,7 @@ func TestReadDotBytes(t *testing.T) { func TestReadMIMEHeader(t *testing.T) { r := reader("my-key: Value 1 \r\nLong-key: Even \n Longer Value\r\nmy-Key: Value 2\r\n\n") m, err := r.ReadMIMEHeader() - want := map[string][]string{ + want := MIMEHeader{ "My-Key": {"Value 1", "Value 2"}, "Long-Key": {"Even Longer Value"}, } diff --git a/libgo/go/net/udpsock.go b/libgo/go/net/udpsock.go index 0270954c176..f9274493e60 100644 --- a/libgo/go/net/udpsock.go +++ b/libgo/go/net/udpsock.go @@ -279,3 +279,44 @@ func (c *UDPConn) BindToDevice(device string) os.Error { // It is the caller's responsibility to close f when finished. // Closing c does not affect f, and closing f does not affect c. func (c *UDPConn) File() (f *os.File, err os.Error) { return c.fd.dup() } + +var errInvalidMulticast = os.ErrorString("invalid IPv4 multicast address") + +// JoinGroup joins the IPv4 multicast group named by addr. +// The UDPConn must use the "udp4" network. +func (c *UDPConn) JoinGroup(addr IP) os.Error { + if !c.ok() { + return os.EINVAL + } + ip := addr.To4() + if ip == nil { + return &OpError{"joingroup", "udp", &IPAddr{ip}, errInvalidMulticast} + } + mreq := &syscall.IpMreq{ + Multiaddr: [4]byte{ip[0], ip[1], ip[2], ip[3]}, + } + err := os.NewSyscallError("setsockopt", syscall.SetsockoptIpMreq(c.fd.sysfd, syscall.IPPROTO_IP, syscall.IP_ADD_MEMBERSHIP, mreq)) + if err != nil { + return &OpError{"joingroup", "udp", &IPAddr{ip}, err} + } + return nil +} + +// LeaveGroup exits the IPv4 multicast group named by addr. +func (c *UDPConn) LeaveGroup(addr IP) os.Error { + if !c.ok() { + return os.EINVAL + } + ip := addr.To4() + if ip == nil { + return &OpError{"leavegroup", "udp", &IPAddr{ip}, errInvalidMulticast} + } + mreq := &syscall.IpMreq{ + Multiaddr: [4]byte{ip[0], ip[1], ip[2], ip[3]}, + } + err := os.NewSyscallError("setsockopt", syscall.SetsockoptIpMreq(c.fd.sysfd, syscall.IPPROTO_IP, syscall.IP_DROP_MEMBERSHIP, mreq)) + if err != nil { + return &OpError{"leavegroup", "udp", &IPAddr{ip}, err} + } + return nil +} diff --git a/libgo/go/netchan/common.go b/libgo/go/netchan/common.go index 56c0b25199f..dd06050ee50 100644 --- a/libgo/go/netchan/common.go +++ b/libgo/go/netchan/common.go @@ -6,7 +6,7 @@ package netchan import ( "gob" - "net" + "io" "os" "reflect" "sync" @@ -93,7 +93,7 @@ type encDec struct { enc *gob.Encoder } -func newEncDec(conn net.Conn) *encDec { +func newEncDec(conn io.ReadWriter) *encDec { return &encDec{ dec: gob.NewDecoder(conn), enc: gob.NewEncoder(conn), @@ -199,9 +199,10 @@ func (cs *clientSet) sync(timeout int64) os.Error { // are delivered into the local channel. type netChan struct { *chanDir - name string - id int - size int // buffer size of channel. + name string + id int + size int // buffer size of channel. + closed bool // sender-specific state ackCh chan bool // buffered with space for all the acks we need @@ -227,6 +228,9 @@ func newNetChan(name string, id int, ch *chanDir, ed *encDec, size int, count in // Close the channel. func (nch *netChan) close() { + if nch.closed { + return + } if nch.dir == Recv { if nch.sendCh != nil { // If the sender goroutine is active, close the channel to it. @@ -239,6 +243,7 @@ func (nch *netChan) close() { nch.ch.Close() close(nch.ackCh) } + nch.closed = true } // Send message from remote side to local receiver. @@ -256,7 +261,10 @@ func (nch *netChan) send(val reflect.Value) { nch.sendCh = make(chan reflect.Value, nch.size) go nch.sender() } - if ok := nch.sendCh <- val; !ok { + select { + case nch.sendCh <- val: + // ok + default: // TODO: should this be more resilient? panic("netchan: remote sender sent more values than allowed") } @@ -318,8 +326,11 @@ func (nch *netChan) acked() { if nch.dir != Send { panic("recv on wrong direction of channel") } - if ok := nch.ackCh <- true; !ok { - panic("netchan: remote receiver sent too many acks") + select { + case nch.ackCh <- true: + // ok + default: // TODO: should this be more resilient? + panic("netchan: remote receiver sent too many acks") } } diff --git a/libgo/go/netchan/export.go b/libgo/go/netchan/export.go index 0f72ca7a940..55eba0e2e0f 100644 --- a/libgo/go/netchan/export.go +++ b/libgo/go/netchan/export.go @@ -23,6 +23,7 @@ package netchan import ( "log" + "io" "net" "os" "reflect" @@ -43,7 +44,6 @@ func expLog(args ...interface{}) { // but they must use different ports. type Exporter struct { *clientSet - listener net.Listener } type expClient struct { @@ -57,7 +57,7 @@ type expClient struct { seqLock sync.Mutex // guarantees messages are in sequence, only locked under mu } -func newClient(exp *Exporter, conn net.Conn) *expClient { +func newClient(exp *Exporter, conn io.ReadWriter) *expClient { client := new(expClient) client.exp = exp client.encDec = newEncDec(conn) @@ -118,7 +118,9 @@ func (client *expClient) run() { for { *hdr = header{} if err := client.decode(hdrValue); err != nil { - expLog("error decoding client header:", err) + if err != os.EOF { + expLog("error decoding client header:", err) + } break } switch hdr.PayloadType { @@ -169,7 +171,7 @@ func (client *expClient) run() { nch.acked() } default: - log.Exit("netchan export: unknown payload type", hdr.PayloadType) + log.Fatal("netchan export: unknown payload type", hdr.PayloadType) } } client.exp.delClient(client) @@ -258,39 +260,50 @@ func (client *expClient) ack() int64 { return n } -// Wait for incoming connections, start a new runner for each -func (exp *Exporter) listen() { +// Serve waits for incoming connections on the listener +// and serves the Exporter's channels on each. +// It blocks until the listener is closed. +func (exp *Exporter) Serve(listener net.Listener) { for { - conn, err := exp.listener.Accept() + conn, err := listener.Accept() if err != nil { expLog("listen:", err) break } - client := exp.addClient(conn) - go client.run() + go exp.ServeConn(conn) } } -// NewExporter creates a new Exporter to export channels -// on the network and local address defined as in net.Listen. -func NewExporter(network, localaddr string) (*Exporter, os.Error) { - listener, err := net.Listen(network, localaddr) - if err != nil { - return nil, err - } +// ServeConn exports the Exporter's channels on conn. +// It blocks until the connection is terminated. +func (exp *Exporter) ServeConn(conn io.ReadWriter) { + exp.addClient(conn).run() +} + +// NewExporter creates a new Exporter that exports a set of channels. +func NewExporter() *Exporter { e := &Exporter{ - listener: listener, clientSet: &clientSet{ names: make(map[string]*chanDir), clients: make(map[unackedCounter]bool), }, } - go e.listen() - return e, nil + return e +} + +// ListenAndServe exports the exporter's channels through the +// given network and local address defined as in net.Listen. +func (exp *Exporter) ListenAndServe(network, localaddr string) os.Error { + listener, err := net.Listen(network, localaddr) + if err != nil { + return err + } + go exp.Serve(listener) + return nil } // addClient creates a new expClient and records its existence -func (exp *Exporter) addClient(conn net.Conn) *expClient { +func (exp *Exporter) addClient(conn io.ReadWriter) *expClient { client := newClient(exp, conn) exp.mu.Lock() exp.clients[client] = true @@ -327,9 +340,6 @@ func (exp *Exporter) Sync(timeout int64) os.Error { return exp.clientSet.sync(timeout) } -// Addr returns the Exporter's local network address. -func (exp *Exporter) Addr() net.Addr { return exp.listener.Addr() } - func checkChan(chT interface{}, dir Dir) (*reflect.ChanValue, os.Error) { chanType, ok := reflect.Typeof(chT).(*reflect.ChanType) if !ok { diff --git a/libgo/go/netchan/import.go b/libgo/go/netchan/import.go index 22b0f69ba38..30edcd8123b 100644 --- a/libgo/go/netchan/import.go +++ b/libgo/go/netchan/import.go @@ -5,6 +5,7 @@ package netchan import ( + "io" "log" "net" "os" @@ -25,7 +26,6 @@ func impLog(args ...interface{}) { // importers, even from the same machine/network port. type Importer struct { *encDec - conn net.Conn chanLock sync.Mutex // protects access to channel map names map[string]*netChan chans map[int]*netChan @@ -33,23 +33,26 @@ type Importer struct { maxId int } -// NewImporter creates a new Importer object to import channels -// from an Exporter at the network and remote address as defined in net.Dial. -// The Exporter must be available and serving when the Importer is -// created. -func NewImporter(network, remoteaddr string) (*Importer, os.Error) { - conn, err := net.Dial(network, "", remoteaddr) - if err != nil { - return nil, err - } +// NewImporter creates a new Importer object to import a set of channels +// from the given connection. The Exporter must be available and serving when +// the Importer is created. +func NewImporter(conn io.ReadWriter) *Importer { imp := new(Importer) imp.encDec = newEncDec(conn) - imp.conn = conn imp.chans = make(map[int]*netChan) imp.names = make(map[string]*netChan) imp.errors = make(chan os.Error, 10) go imp.run() - return imp, nil + return imp +} + +// Import imports a set of channels from the given network and address. +func Import(network, remoteaddr string) (*Importer, os.Error) { + conn, err := net.Dial(network, "", remoteaddr) + if err != nil { + return nil, err + } + return NewImporter(conn), nil } // shutdown closes all channels for which we are receiving data from the remote side. @@ -91,11 +94,13 @@ func (imp *Importer) run() { } if err.Error != "" { impLog("response error:", err.Error) - if sent := imp.errors <- os.ErrorString(err.Error); !sent { + select { + case imp.errors <- os.ErrorString(err.Error): + continue // errors are not acknowledged + default: imp.shutdown() return } - continue // errors are not acknowledged. } case payClosed: nch := imp.getChan(hdr.Id, false) @@ -171,13 +176,13 @@ func (imp *Importer) Import(name string, chT interface{}, dir Dir, size int) os. // The channel to be bound to the remote site's channel is provided // in the call and may be of arbitrary channel type. // Despite the literal signature, the effective signature is -// ImportNValues(name string, chT chan T, dir Dir, n int) os.Error +// ImportNValues(name string, chT chan T, dir Dir, size, n int) os.Error // Example usage: // imp, err := NewImporter("tcp", "netchanserver.mydomain.com:1234") -// if err != nil { log.Exit(err) } +// if err != nil { log.Fatal(err) } // ch := make(chan myType) -// err = imp.ImportNValues("name", ch, Recv, 1) -// if err != nil { log.Exit(err) } +// err = imp.ImportNValues("name", ch, Recv, 1, 1) +// if err != nil { log.Fatal(err) } // fmt.Printf("%+v\n", <-ch) func (imp *Importer) ImportNValues(name string, chT interface{}, dir Dir, size, n int) os.Error { ch, err := checkChan(chT, dir) @@ -229,15 +234,13 @@ func (imp *Importer) ImportNValues(name string, chT interface{}, dir Dir, size, // the channel. Messages in flight for the channel may be dropped. func (imp *Importer) Hangup(name string) os.Error { imp.chanLock.Lock() - nc, ok := imp.names[name] - if ok { - imp.names[name] = nil, false - imp.chans[nc.id] = nil, false - } - imp.chanLock.Unlock() - if !ok { + defer imp.chanLock.Unlock() + nc := imp.names[name] + if nc == nil { return os.ErrorString("netchan import: hangup: no such channel: " + name) } + imp.names[name] = nil, false + imp.chans[nc.id] = nil, false nc.close() return nil } diff --git a/libgo/go/netchan/netchan_test.go b/libgo/go/netchan/netchan_test.go index 6d7d63f9882..1c84a9d14de 100644 --- a/libgo/go/netchan/netchan_test.go +++ b/libgo/go/netchan/netchan_test.go @@ -5,6 +5,7 @@ package netchan import ( + "net" "strings" "testing" "time" @@ -23,7 +24,7 @@ func exportSend(exp *Exporter, n int, t *testing.T, done chan bool) { } go func() { for i := 0; i < n; i++ { - ch <- base+i + ch <- base + i } close(ch) if done != nil { @@ -61,7 +62,7 @@ func importSend(imp *Importer, n int, t *testing.T, done chan bool) { } go func() { for i := 0; i < n; i++ { - ch <- base+i + ch <- base + i } close(ch) if done != nil { @@ -94,27 +95,13 @@ func importReceive(imp *Importer, t *testing.T, done chan bool) { } func TestExportSendImportReceive(t *testing.T) { - exp, err := NewExporter("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal("new exporter:", err) - } - imp, err := NewImporter("tcp", exp.Addr().String()) - if err != nil { - t.Fatal("new importer:", err) - } + exp, imp := pair(t) exportSend(exp, count, t, nil) importReceive(imp, t, nil) } func TestExportReceiveImportSend(t *testing.T) { - exp, err := NewExporter("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal("new exporter:", err) - } - imp, err := NewImporter("tcp", exp.Addr().String()) - if err != nil { - t.Fatal("new importer:", err) - } + exp, imp := pair(t) expDone := make(chan bool) done := make(chan bool) go func() { @@ -127,27 +114,13 @@ func TestExportReceiveImportSend(t *testing.T) { } func TestClosingExportSendImportReceive(t *testing.T) { - exp, err := NewExporter("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal("new exporter:", err) - } - imp, err := NewImporter("tcp", exp.Addr().String()) - if err != nil { - t.Fatal("new importer:", err) - } + exp, imp := pair(t) exportSend(exp, closeCount, t, nil) importReceive(imp, t, nil) } func TestClosingImportSendExportReceive(t *testing.T) { - exp, err := NewExporter("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal("new exporter:", err) - } - imp, err := NewImporter("tcp", exp.Addr().String()) - if err != nil { - t.Fatal("new importer:", err) - } + exp, imp := pair(t) expDone := make(chan bool) done := make(chan bool) go func() { @@ -160,17 +133,10 @@ func TestClosingImportSendExportReceive(t *testing.T) { } func TestErrorForIllegalChannel(t *testing.T) { - exp, err := NewExporter("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal("new exporter:", err) - } - imp, err := NewImporter("tcp", exp.Addr().String()) - if err != nil { - t.Fatal("new importer:", err) - } + exp, imp := pair(t) // Now export a channel. ch := make(chan int, 1) - err = exp.Export("aChannel", ch, Send) + err := exp.Export("aChannel", ch, Send) if err != nil { t.Fatal("export:", err) } @@ -200,14 +166,7 @@ func TestErrorForIllegalChannel(t *testing.T) { // Not a great test but it does at least invoke Drain. func TestExportDrain(t *testing.T) { - exp, err := NewExporter("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal("new exporter:", err) - } - imp, err := NewImporter("tcp", exp.Addr().String()) - if err != nil { - t.Fatal("new importer:", err) - } + exp, imp := pair(t) done := make(chan bool) go func() { exportSend(exp, closeCount, t, nil) @@ -221,14 +180,7 @@ func TestExportDrain(t *testing.T) { // Not a great test but it does at least invoke Sync. func TestExportSync(t *testing.T) { - exp, err := NewExporter("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal("new exporter:", err) - } - imp, err := NewImporter("tcp", exp.Addr().String()) - if err != nil { - t.Fatal("new importer:", err) - } + exp, imp := pair(t) done := make(chan bool) exportSend(exp, closeCount, t, nil) go importReceive(imp, t, done) @@ -239,16 +191,9 @@ func TestExportSync(t *testing.T) { // Test hanging up the send side of an export. // TODO: test hanging up the receive side of an export. func TestExportHangup(t *testing.T) { - exp, err := NewExporter("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal("new exporter:", err) - } - imp, err := NewImporter("tcp", exp.Addr().String()) - if err != nil { - t.Fatal("new importer:", err) - } + exp, imp := pair(t) ech := make(chan int) - err = exp.Export("exportedSend", ech, Send) + err := exp.Export("exportedSend", ech, Send) if err != nil { t.Fatal("export:", err) } @@ -276,16 +221,9 @@ func TestExportHangup(t *testing.T) { // Test hanging up the send side of an import. // TODO: test hanging up the receive side of an import. func TestImportHangup(t *testing.T) { - exp, err := NewExporter("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal("new exporter:", err) - } - imp, err := NewImporter("tcp", exp.Addr().String()) - if err != nil { - t.Fatal("new importer:", err) - } + exp, imp := pair(t) ech := make(chan int) - err = exp.Export("exportedRecv", ech, Recv) + err := exp.Export("exportedRecv", ech, Recv) if err != nil { t.Fatal("export:", err) } @@ -343,14 +281,7 @@ func exportLoopback(exp *Exporter, t *testing.T) { // This test checks that channel operations can proceed // even when other concurrent operations are blocked. func TestIndependentSends(t *testing.T) { - exp, err := NewExporter("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal("new exporter:", err) - } - imp, err := NewImporter("tcp", exp.Addr().String()) - if err != nil { - t.Fatal("new importer:", err) - } + exp, imp := pair(t) exportLoopback(exp, t) @@ -377,23 +308,8 @@ type value struct { } func TestCrossConnect(t *testing.T) { - e1, err := NewExporter("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal("new exporter:", err) - } - i1, err := NewImporter("tcp", e1.Addr().String()) - if err != nil { - t.Fatal("new importer:", err) - } - - e2, err := NewExporter("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal("new exporter:", err) - } - i2, err := NewImporter("tcp", e2.Addr().String()) - if err != nil { - t.Fatal("new importer:", err) - } + e1, i1 := pair(t) + e2, i2 := pair(t) crossExport(e1, e2, t) crossImport(i1, i2, t) @@ -452,20 +368,13 @@ const flowCount = 100 // test flow control from exporter to importer. func TestExportFlowControl(t *testing.T) { - exp, err := NewExporter("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal("new exporter:", err) - } - imp, err := NewImporter("tcp", exp.Addr().String()) - if err != nil { - t.Fatal("new importer:", err) - } + exp, imp := pair(t) sendDone := make(chan bool, 1) exportSend(exp, flowCount, t, sendDone) ch := make(chan int) - err = imp.ImportNValues("exportedSend", ch, Recv, 20, -1) + err := imp.ImportNValues("exportedSend", ch, Recv, 20, -1) if err != nil { t.Fatal("importReceive:", err) } @@ -475,17 +384,10 @@ func TestExportFlowControl(t *testing.T) { // test flow control from importer to exporter. func TestImportFlowControl(t *testing.T) { - exp, err := NewExporter("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal("new exporter:", err) - } - imp, err := NewImporter("tcp", exp.Addr().String()) - if err != nil { - t.Fatal("new importer:", err) - } + exp, imp := pair(t) ch := make(chan int) - err = exp.Export("exportedRecv", ch, Recv) + err := exp.Export("exportedRecv", ch, Recv) if err != nil { t.Fatal("importReceive:", err) } @@ -513,3 +415,11 @@ func testFlow(sendDone chan bool, ch <-chan int, N int, t *testing.T) { t.Fatalf("expected %d values; got %d", N, n) } } + +func pair(t *testing.T) (*Exporter, *Importer) { + c0, c1 := net.Pipe() + exp := NewExporter() + go exp.ServeConn(c0) + imp := NewImporter(c1) + return exp, imp +} diff --git a/libgo/go/os/env_windows.go b/libgo/go/os/env_windows.go index d2b159dfba7..a45d79be32a 100644 --- a/libgo/go/os/env_windows.go +++ b/libgo/go/os/env_windows.go @@ -50,8 +50,8 @@ func Setenv(key, value string) Error { if len(value) > 0 { v = syscall.StringToUTF16Ptr(value) } - ok, e := syscall.SetEnvironmentVariable(syscall.StringToUTF16Ptr(key), v) - if !ok { + e := syscall.SetEnvironmentVariable(syscall.StringToUTF16Ptr(key), v) + if e != 0 { return NewSyscallError("SetEnvironmentVariable", e) } return nil diff --git a/libgo/go/os/error.go b/libgo/go/os/error.go index 8cdf532548c..635a3fe50aa 100644 --- a/libgo/go/os/error.go +++ b/libgo/go/os/error.go @@ -37,7 +37,7 @@ func (e Errno) Temporary() bool { } func (e Errno) Timeout() bool { - return e == Errno(syscall.EAGAIN) || e == Errno(syscall.EWOULDBLOCK) + return e == Errno(syscall.EAGAIN) || e == Errno(syscall.EWOULDBLOCK) || e == Errno(syscall.ETIMEDOUT) } // Commonly known Unix errors. @@ -79,6 +79,8 @@ var ( ECONNREFUSED Error = Errno(syscall.ECONNREFUSED) ENAMETOOLONG Error = Errno(syscall.ENAMETOOLONG) EAFNOSUPPORT Error = Errno(syscall.EAFNOSUPPORT) + ETIMEDOUT Error = Errno(syscall.ETIMEDOUT) + ENOTCONN Error = Errno(syscall.ENOTCONN) ) // PathError records an error and the operation and file path that caused it. diff --git a/libgo/go/os/exec.go b/libgo/go/os/exec.go index 100d984d170..dbdfacc5857 100644 --- a/libgo/go/os/exec.go +++ b/libgo/go/os/exec.go @@ -5,17 +5,29 @@ package os import ( + "runtime" "syscall" ) -// ForkExec forks the current process and invokes Exec with the program, arguments, -// and environment specified by name, argv, and envv. It returns the process -// id of the forked process and an Error, if any. The fd array specifies the +// Process stores the information about a process created by StartProcess. +type Process struct { + Pid int + handle int +} + +func newProcess(pid, handle int) *Process { + p := &Process{pid, handle} + runtime.SetFinalizer(p, (*Process).Release) + return p +} + +// StartProcess starts a new process with the program, arguments, +// and environment specified by name, argv, and envv. The fd array specifies the // file descriptors to be set up in the new process: fd[0] will be Unix file // descriptor 0 (standard input), fd[1] descriptor 1, and so on. A nil entry // will cause the child to have no open file descriptor with that index. // If dir is not empty, the child chdirs into the directory before execing the program. -func ForkExec(name string, argv []string, envv []string, dir string, fd []*File) (pid int, err Error) { +func StartProcess(name string, argv []string, envv []string, dir string, fd []*File) (p *Process, err Error) { if envv == nil { envv = Environ() } @@ -29,17 +41,17 @@ func ForkExec(name string, argv []string, envv []string, dir string, fd []*File) } } - p, e := syscall.ForkExec(name, argv, envv, dir, intfd) + pid, h, e := syscall.StartProcess(name, argv, envv, dir, intfd) if e != 0 { - return 0, &PathError{"fork/exec", name, Errno(e)} + return nil, &PathError{"fork/exec", name, Errno(e)} } - return p, nil + return newProcess(pid, h), nil } // Exec replaces the current process with an execution of the // named binary, with arguments argv and environment envv. // If successful, Exec never returns. If it fails, it returns an Error. -// ForkExec is almost always a better way to execute a program. +// StartProcess is almost always a better way to execute a program. func Exec(name string, argv []string, envv []string) Error { if envv == nil { envv = Environ() @@ -65,37 +77,18 @@ type Waitmsg struct { Rusage *syscall.Rusage // System-dependent resource usage info. } -// Options for Wait. -const ( - WNOHANG = syscall.WNOHANG // Don't wait if no process has exited. - WSTOPPED = syscall.WSTOPPED // If set, status of stopped subprocesses is also reported. - WUNTRACED = syscall.WUNTRACED // Usually an alias for WSTOPPED. - WRUSAGE = 1 << 20 // Record resource usage. -) - -// WRUSAGE must not be too high a bit, to avoid clashing with Linux's -// WCLONE, WALL, and WNOTHREAD flags, which sit in the top few bits of -// the options - // Wait waits for process pid to exit or stop, and then returns a // Waitmsg describing its status and an Error, if any. The options // (WNOHANG etc.) affect the behavior of the Wait call. +// Wait is equivalent to calling FindProcess and then Wait +// and Release on the result. func Wait(pid int, options int) (w *Waitmsg, err Error) { - var status syscall.WaitStatus - var rusage *syscall.Rusage - if options&WRUSAGE != 0 { - rusage = new(syscall.Rusage) - options ^= WRUSAGE - } - pid1, e := syscall.Wait4(pid, &status, options, rusage) - if e != 0 { - return nil, NewSyscallError("wait", e) + p, e := FindProcess(pid) + if e != nil { + return nil, e } - w = new(Waitmsg) - w.Pid = pid1 - w.WaitStatus = status - w.Rusage = rusage - return w, nil + defer p.Release() + return p.Wait(options) } // Convert i to decimal string. diff --git a/libgo/go/os/exec_unix.go b/libgo/go/os/exec_unix.go new file mode 100644 index 00000000000..8990d6a97ec --- /dev/null +++ b/libgo/go/os/exec_unix.go @@ -0,0 +1,63 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package os + +import ( + "runtime" + "syscall" +) + +// Options for Wait. +const ( + WNOHANG = syscall.WNOHANG // Don't wait if no process has exited. + WSTOPPED = syscall.WSTOPPED // If set, status of stopped subprocesses is also reported. + WUNTRACED = syscall.WUNTRACED // Usually an alias for WSTOPPED. + WRUSAGE = 1 << 20 // Record resource usage. +) + +// WRUSAGE must not be too high a bit, to avoid clashing with Linux's +// WCLONE, WALL, and WNOTHREAD flags, which sit in the top few bits of +// the options + +// Wait waits for the Process to exit or stop, and then returns a +// Waitmsg describing its status and an Error, if any. The options +// (WNOHANG etc.) affect the behavior of the Wait call. +func (p *Process) Wait(options int) (w *Waitmsg, err Error) { + if p.Pid == -1 { + return nil, EINVAL + } + var status syscall.WaitStatus + var rusage *syscall.Rusage + if options&WRUSAGE != 0 { + rusage = new(syscall.Rusage) + options ^= WRUSAGE + } + pid1, e := syscall.Wait4(p.Pid, &status, options, rusage) + if e != 0 { + return nil, NewSyscallError("wait", e) + } + w = new(Waitmsg) + w.Pid = pid1 + w.WaitStatus = status + w.Rusage = rusage + return w, nil +} + +// Release releases any resources associated with the Process. +func (p *Process) Release() Error { + // NOOP for unix. + p.Pid = -1 + // no need for a finalizer anymore + runtime.SetFinalizer(p, nil) + return nil +} + +// FindProcess looks for a running process by its pid. +// The Process it returns can be used to obtain information +// about the underlying operating system process. +func FindProcess(pid int) (p *Process, err Error) { + // NOOP for unix. + return newProcess(pid, 0), nil +} diff --git a/libgo/go/os/exec_windows.go b/libgo/go/os/exec_windows.go new file mode 100644 index 00000000000..ae8ffeab2e9 --- /dev/null +++ b/libgo/go/os/exec_windows.go @@ -0,0 +1,52 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package os + +import ( + "runtime" + "syscall" +) + +func (p *Process) Wait(options int) (w *Waitmsg, err Error) { + s, e := syscall.WaitForSingleObject(int32(p.handle), syscall.INFINITE) + switch s { + case syscall.WAIT_OBJECT_0: + break + case syscall.WAIT_FAILED: + return nil, NewSyscallError("WaitForSingleObject", e) + default: + return nil, ErrorString("os: unexpected result from WaitForSingleObject") + } + var ec uint32 + e = syscall.GetExitCodeProcess(uint32(p.handle), &ec) + if e != 0 { + return nil, NewSyscallError("GetExitCodeProcess", e) + } + return &Waitmsg{p.Pid, syscall.WaitStatus{s, ec}, new(syscall.Rusage)}, nil +} + +func (p *Process) Release() Error { + if p.handle == -1 { + return EINVAL + } + e := syscall.CloseHandle(int32(p.handle)) + if e != 0 { + return NewSyscallError("CloseHandle", e) + } + p.handle = -1 + // no need for a finalizer anymore + runtime.SetFinalizer(p, nil) + return nil +} + +func FindProcess(pid int) (p *Process, err Error) { + const da = syscall.STANDARD_RIGHTS_READ | + syscall.PROCESS_QUERY_INFORMATION | syscall.SYNCHRONIZE + h, e := syscall.OpenProcess(da, false, uint32(pid)) + if e != 0 { + return nil, NewSyscallError("OpenProcess", e) + } + return newProcess(pid, int(h)), nil +} diff --git a/libgo/go/os/inotify/inotify_linux.go b/libgo/go/os/inotify/inotify_linux.go index 1e74c7fbc53..96c229e7b74 100644 --- a/libgo/go/os/inotify/inotify_linux.go +++ b/libgo/go/os/inotify/inotify_linux.go @@ -8,11 +8,11 @@ This package implements a wrapper for the Linux inotify system. Example: watcher, err := inotify.NewWatcher() if err != nil { - log.Exit(err) + log.Fatal(err) } err = watcher.Watch("/tmp") if err != nil { - log.Exit(err) + log.Fatal(err) } for { select { @@ -153,7 +153,11 @@ func (w *Watcher) readEvents() { for { n, errno = syscall.Read(w.fd, buf[0:]) // See if there is a message on the "done" channel - _, done := <-w.done + var done bool + select { + case done = <-w.done: + default: + } // If EOF or a "done" message is received if n == 0 || done { diff --git a/libgo/go/os/os_test.go b/libgo/go/os/os_test.go index 5b577065ad8..be5f4824e4f 100644 --- a/libgo/go/os/os_test.go +++ b/libgo/go/os/os_test.go @@ -423,10 +423,11 @@ func TestForkExec(t *testing.T) { adir = "/" expect = "/\n" } - pid, err := ForkExec(cmd, args, nil, adir, []*File{nil, w, Stderr}) + p, err := StartProcess(cmd, args, nil, adir, []*File{nil, w, Stderr}) if err != nil { - t.Fatalf("ForkExec: %v", err) + t.Fatalf("StartProcess: %v", err) } + defer p.Release() w.Close() var b bytes.Buffer @@ -436,7 +437,7 @@ func TestForkExec(t *testing.T) { args[0] = cmd t.Errorf("exec %q returned %q wanted %q", strings.Join(args, " "), output, expect) } - Wait(pid, 0) + p.Wait(0) } func checkMode(t *testing.T, path string, mode uint32) { @@ -746,15 +747,16 @@ func run(t *testing.T, cmd []string) string { if err != nil { t.Fatal(err) } - pid, err := ForkExec("/bin/hostname", []string{"hostname"}, nil, "/", []*File{nil, w, Stderr}) + p, err := StartProcess("/bin/hostname", []string{"hostname"}, nil, "/", []*File{nil, w, Stderr}) if err != nil { t.Fatal(err) } + defer p.Release() w.Close() var b bytes.Buffer io.Copy(&b, r) - Wait(pid, 0) + p.Wait(0) output := b.String() if n := len(output); n > 0 && output[n-1] == '\n' { output = output[0 : n-1] diff --git a/libgo/go/path/path_test.go b/libgo/go/path/path_test.go index 6b4be07a958..ab0b48ad6ad 100644 --- a/libgo/go/path/path_test.go +++ b/libgo/go/path/path_test.go @@ -256,8 +256,11 @@ func TestWalk(t *testing.T) { // 2) handle errors, expect none errors := make(chan os.Error, 64) Walk(tree.name, v, errors) - if err, ok := <-errors; ok { + select { + case err := <-errors: t.Errorf("no error expected, found: %s", err) + default: + // ok } checkMarks(t) @@ -276,14 +279,21 @@ func TestWalk(t *testing.T) { errors = make(chan os.Error, 64) os.Chmod(Join(tree.name, tree.entries[1].name), 0) Walk(tree.name, v, errors) + Loop: for i := 1; i <= 2; i++ { - if _, ok := <-errors; !ok { + select { + case <-errors: + // ok + default: t.Errorf("%d. error expected, none found", i) - break + break Loop } } - if err, ok := <-errors; ok { + select { + case err := <-errors: t.Errorf("only two errors expected, found 3rd: %v", err) + default: + // ok } // the inaccessible subtrees were marked manually checkMarks(t) diff --git a/libgo/go/reflect/all_test.go b/libgo/go/reflect/all_test.go index 320f4420302..42afadd85c0 100644 --- a/libgo/go/reflect/all_test.go +++ b/libgo/go/reflect/all_test.go @@ -1093,6 +1093,18 @@ func TestMethod(t *testing.T) { t.Errorf("Value Method returned %d; want 250", i) } + // Curried method of pointer. + i = NewValue(&p).Method(0).Call([]Value{NewValue(10)})[0].(*IntValue).Get() + if i != 250 { + t.Errorf("Value Method returned %d; want 250", i) + } + + // Curried method of pointer to value. + i = NewValue(p).Addr().Method(0).Call([]Value{NewValue(10)})[0].(*IntValue).Get() + if i != 250 { + t.Errorf("Value Method returned %d; want 250", i) + } + // Curried method of interface value. // Have to wrap interface value in a struct to get at it. // Passing it to NewValue directly would @@ -1390,3 +1402,66 @@ func TestEmbeddedMethods(t *testing.T) { t.Errorf("f(o) = %d, want 2", v) } } + +func TestPtrTo(t *testing.T) { + var i int + + typ := Typeof(i) + for i = 0; i < 100; i++ { + typ = PtrTo(typ) + } + for i = 0; i < 100; i++ { + typ = typ.(*PtrType).Elem() + } + if typ != Typeof(i) { + t.Errorf("after 100 PtrTo and Elem, have %s, want %s", typ, Typeof(i)) + } +} + +func TestAddr(t *testing.T) { + var p struct { + X, Y int + } + + v := NewValue(&p) + v = v.(*PtrValue).Elem() + v = v.Addr() + v = v.(*PtrValue).Elem() + v = v.(*StructValue).Field(0) + v.(*IntValue).Set(2) + if p.X != 2 { + t.Errorf("Addr.Elem.Set failed to set value") + } + + // Again but take address of the NewValue value. + // Exercises generation of PtrTypes not present in the binary. + v = NewValue(&p) + v = v.Addr() + v = v.(*PtrValue).Elem() + v = v.(*PtrValue).Elem() + v = v.Addr() + v = v.(*PtrValue).Elem() + v = v.(*StructValue).Field(0) + v.(*IntValue).Set(3) + if p.X != 3 { + t.Errorf("Addr.Elem.Set failed to set value") + } + + // Starting without pointer we should get changed value + // in interface. + v = NewValue(p) + v0 := v + v = v.Addr() + v = v.(*PtrValue).Elem() + v = v.(*StructValue).Field(0) + v.(*IntValue).Set(4) + if p.X != 3 { // should be unchanged from last time + t.Errorf("somehow value Set changed original p") + } + p = v0.Interface().(struct { + X, Y int + }) + if p.X != 4 { + t.Errorf("Addr.Elem.Set valued to set value in top value") + } +} diff --git a/libgo/go/reflect/deepequal.go b/libgo/go/reflect/deepequal.go index a50925e51e6..c9beec50666 100644 --- a/libgo/go/reflect/deepequal.go +++ b/libgo/go/reflect/deepequal.go @@ -31,8 +31,8 @@ func deepValueEqual(v1, v2 Value, visited map[uintptr]*visit, depth int) bool { // if depth > 10 { panic("deepValueEqual") } // for debugging - addr1 := v1.Addr() - addr2 := v2.Addr() + addr1 := v1.UnsafeAddr() + addr2 := v2.UnsafeAddr() if addr1 > addr2 { // Canonicalize order to reduce number of entries in visited. addr1, addr2 = addr2, addr1 diff --git a/libgo/go/reflect/type.go b/libgo/go/reflect/type.go index 4ad4c5f2b4c..fb798802815 100644 --- a/libgo/go/reflect/type.go +++ b/libgo/go/reflect/type.go @@ -50,6 +50,7 @@ type commonType struct { equalfn func(unsafe.Pointer, unsafe.Pointer, uintptr) string *string *uncommonType + ptrToThis *runtime.Type } type method struct { @@ -164,6 +165,10 @@ type SliceType struct { elem *runtime.Type } +// arrayOrSliceType is an unexported method that guarantees only +// arrays and slices implement ArrayOrSliceType. +func (*SliceType) arrayOrSliceType() {} + // Struct field type structField struct { name *string @@ -236,11 +241,19 @@ type Type interface { // Kind returns the specific kind of this type. Kind() Kind - // For non-interface types, Method returns the i'th method with receiver T. - // For interface types, Method returns the i'th method in the interface. - // NumMethod returns the number of such methods. + // Method returns the i'th method in the type's method set. + // + // For a non-interface type T or *T, the returned Method's Type and Func + // fields describe a function whose first argument is the receiver. + // + // For an interface type, the returned Method's Type field gives the + // method signature, without a receiver, and the Func field is nil. Method(int) Method + + // NumMethods returns the number of methods in the type's method set. NumMethod() int + + common() *commonType uncommon() *uncommonType } @@ -351,6 +364,8 @@ func (t *commonType) FieldAlign() int { return int(t.fieldAlign) } func (t *commonType) Kind() Kind { return Kind(t.kind & kindMask) } +func (t *commonType) common() *commonType { return t } + func (t *uncommonType) Method(i int) (m Method) { if t == nil || i < 0 || i >= len(t.methods) { return @@ -364,7 +379,7 @@ func (t *uncommonType) Method(i int) (m Method) { } m.Type = runtimeToType(p.typ).(*FuncType) fn := p.tfn - m.Func = &FuncValue{value: value{m.Type, addr(&fn), true}} + m.Func = &FuncValue{value: value{m.Type, addr(&fn), canSet}} return } @@ -392,6 +407,10 @@ func (t *ArrayType) Len() int { return int(t.len) } // Elem returns the type of the array's elements. func (t *ArrayType) Elem() Type { return runtimeToType(t.elem) } +// arrayOrSliceType is an unexported method that guarantees only +// arrays and slices implement ArrayOrSliceType. +func (*ArrayType) arrayOrSliceType() {} + // Dir returns the channel direction. func (t *ChanType) Dir() ChanDir { return ChanDir(t.dir) } @@ -445,7 +464,7 @@ func (t *FuncType) Out(i int) Type { // NumOut returns the number of function output parameters. func (t *FuncType) NumOut() int { return len(t.out) } -// Method returns the i'th interface method. +// Method returns the i'th method in the type's method set. func (t *InterfaceType) Method(i int) (m Method) { if i < 0 || i >= len(t.methods) { return @@ -459,7 +478,7 @@ func (t *InterfaceType) Method(i int) (m Method) { return } -// NumMethod returns the number of interface methods. +// NumMethod returns the number of interface methods in the type's method set. func (t *InterfaceType) NumMethod() int { return len(t.methods) } // Key returns the map key type. @@ -737,7 +756,86 @@ func runtimeToType(v *runtime.Type) Type { type ArrayOrSliceType interface { Type Elem() Type + arrayOrSliceType() // Guarantees only Array and Slice implement this interface. } // Typeof returns the reflection Type of the value in the interface{}. func Typeof(i interface{}) Type { return canonicalize(toType(unsafe.Typeof(i))) } + +// ptrMap is the cache for PtrTo. +var ptrMap struct { + sync.RWMutex + m map[Type]*PtrType +} + +// runtimePtrType is the runtime layout for a *PtrType. +// The memory immediately before the *PtrType is always +// the canonical runtime.Type to be used for a *runtime.Type +// describing this PtrType. +type runtimePtrType struct { + runtime.Type + runtime.PtrType +} + +// PtrTo returns the pointer type with element t. +// For example, if t represents type Foo, PtrTo(t) represents *Foo. +func PtrTo(t Type) *PtrType { + // If t records its pointer-to type, use it. + ct := t.common() + if p := ct.ptrToThis; p != nil { + return runtimeToType(p).(*PtrType) + } + + // Otherwise, synthesize one. + // This only happens for pointers with no methods. + // We keep the mapping in a map on the side, because + // this operation is rare and a separate map lets us keep + // the type structures in read-only memory. + ptrMap.RLock() + if m := ptrMap.m; m != nil { + if p := m[t]; p != nil { + ptrMap.RUnlock() + return p + } + } + ptrMap.RUnlock() + ptrMap.Lock() + if ptrMap.m == nil { + ptrMap.m = make(map[Type]*PtrType) + } + p := ptrMap.m[t] + if p != nil { + // some other goroutine won the race and created it + ptrMap.Unlock() + return p + } + + rt := (*runtime.Type)(unsafe.Pointer(ct)) + + rp := new(runtime.PtrType) + + // initialize rp used *byte's PtrType as a prototype. + // have to do assignment as PtrType, not runtime.PtrType, + // in order to write to unexported fields. + p = (*PtrType)(unsafe.Pointer(rp)) + bp := (*PtrType)(unsafe.Pointer(unsafe.Typeof((*byte)(nil)).(*runtime.PtrType))) + *p = *bp + + s := "*" + *ct.string + p.string = &s + + // For the type structures linked into the binary, the + // compiler provides a good hash of the string. + // Create a good hash for the new string by using + // the FNV-1 hash's mixing function to combine the + // old hash and the new "*". + p.hash = ct.hash*16777619 ^ '*' + + p.uncommonType = nil + p.ptrToThis = nil + p.elem = rt + + ptrMap.m[t] = (*PtrType)(unsafe.Pointer(rp)) + ptrMap.Unlock() + return p +} diff --git a/libgo/go/reflect/value.go b/libgo/go/reflect/value.go index 8ef402bbc79..e7b68b3e720 100644 --- a/libgo/go/reflect/value.go +++ b/libgo/go/reflect/value.go @@ -11,7 +11,7 @@ import ( ) const ptrSize = uintptr(unsafe.Sizeof((*byte)(nil))) -const cannotSet = "cannot set value obtained via unexported struct field" +const cannotSet = "cannot set value obtained from unexported struct field" type addr unsafe.Pointer @@ -51,20 +51,32 @@ type Value interface { // Interface returns the value as an interface{}. Interface() interface{} - // CanSet returns whether the value can be changed. + // CanSet returns true if the value can be changed. // Values obtained by the use of non-exported struct fields // can be used in Get but not Set. - // If CanSet() returns false, calling the type-specific Set - // will cause a crash. + // If CanSet returns false, calling the type-specific Set will panic. CanSet() bool // SetValue assigns v to the value; v must have the same type as the value. SetValue(v Value) - // Addr returns a pointer to the underlying data. - // It is for advanced clients that also - // import the "unsafe" package. - Addr() uintptr + // CanAddr returns true if the value's address can be obtained with Addr. + // Such values are called addressable. A value is addressable if it is + // an element of a slice, an element of an addressable array, + // a field of an addressable struct, the result of dereferencing a pointer, + // or the result of a call to NewValue, MakeChan, MakeMap, or MakeZero. + // If CanAddr returns false, calling Addr will panic. + CanAddr() bool + + // Addr returns the address of the value. + // If the value is not addressable, Addr panics. + // Addr is typically used to obtain a pointer to a struct field or slice element + // in order to call a method that requires a pointer receiver. + Addr() *PtrValue + + // UnsafeAddr returns a pointer to the underlying data. + // It is for advanced clients that also import the "unsafe" package. + UnsafeAddr() uintptr // Method returns a FuncValue corresponding to the value's i'th method. // The arguments to a Call on the returned FuncValue @@ -75,19 +87,42 @@ type Value interface { getAddr() addr } +// flags for value +const ( + canSet uint32 = 1 << iota // can set value (write to *v.addr) + canAddr // can take address of value + canStore // can store through value (write to **v.addr) +) + // value is the common implementation of most values. // It is embedded in other, public struct types, but always // with a unique tag like "uint" or "float" so that the client cannot // convert from, say, *UintValue to *FloatValue. type value struct { - typ Type - addr addr - canSet bool + typ Type + addr addr + flag uint32 } func (v *value) Type() Type { return v.typ } -func (v *value) Addr() uintptr { return uintptr(v.addr) } +func (v *value) Addr() *PtrValue { + if !v.CanAddr() { + panic("reflect: cannot take address of value") + } + a := v.addr + flag := canSet + if v.CanSet() { + flag |= canStore + } + // We could safely set canAddr here too - + // the caller would get the address of a - + // but it doesn't match the Go model. + // The language doesn't let you say &&v. + return newValue(PtrTo(v.typ), addr(&a), flag).(*PtrValue) +} + +func (v *value) UnsafeAddr() uintptr { return uintptr(v.addr) } func (v *value) getAddr() addr { return v.addr } @@ -109,7 +144,10 @@ func (v *value) Interface() interface{} { return unsafe.Unreflect(v.typ, unsafe.Pointer(v.addr)) } -func (v *value) CanSet() bool { return v.canSet } +func (v *value) CanSet() bool { return v.flag&canSet != 0 } + +func (v *value) CanAddr() bool { return v.flag&canAddr != 0 } + /* * basic types @@ -125,7 +163,7 @@ func (v *BoolValue) Get() bool { return *(*bool)(v.addr) } // Set sets v to the value x. func (v *BoolValue) Set(x bool) { - if !v.canSet { + if !v.CanSet() { panic(cannotSet) } *(*bool)(v.addr) = x @@ -152,7 +190,7 @@ func (v *FloatValue) Get() float64 { // Set sets v to the value x. func (v *FloatValue) Set(x float64) { - if !v.canSet { + if !v.CanSet() { panic(cannotSet) } switch v.typ.Kind() { @@ -197,7 +235,7 @@ func (v *ComplexValue) Get() complex128 { // Set sets v to the value x. func (v *ComplexValue) Set(x complex128) { - if !v.canSet { + if !v.CanSet() { panic(cannotSet) } switch v.typ.Kind() { @@ -237,7 +275,7 @@ func (v *IntValue) Get() int64 { // Set sets v to the value x. func (v *IntValue) Set(x int64) { - if !v.canSet { + if !v.CanSet() { panic(cannotSet) } switch v.typ.Kind() { @@ -282,7 +320,7 @@ func (v *StringValue) Get() string { return *(*string)(v.addr) } // Set sets v to the value x. func (v *StringValue) Set(x string) { - if !v.canSet { + if !v.CanSet() { panic(cannotSet) } *(*string)(v.addr) = x @@ -317,7 +355,7 @@ func (v *UintValue) Get() uint64 { // Set sets v to the value x. func (v *UintValue) Set(x uint64) { - if !v.canSet { + if !v.CanSet() { panic(cannotSet) } switch v.typ.Kind() { @@ -361,7 +399,7 @@ func (v *UnsafePointerValue) Get() uintptr { return uintptr(*(*unsafe.Pointer)(v // Set sets v to the value x. func (v *UnsafePointerValue) Set(x unsafe.Pointer) { - if !v.canSet { + if !v.CanSet() { panic(cannotSet) } *(*unsafe.Pointer)(v.addr) = x @@ -473,7 +511,7 @@ func (v *ArrayValue) addr() addr { return v.value.addr } // Set assigns x to v. // The new value x must have the same type as v. func (v *ArrayValue) Set(x *ArrayValue) { - if !v.canSet { + if !v.CanSet() { panic(cannotSet) } typesMustMatch(v.typ, x.typ) @@ -491,7 +529,7 @@ func (v *ArrayValue) Elem(i int) Value { panic("array index out of bounds") } p := addr(uintptr(v.addr()) + uintptr(i)*typ.Size()) - return newValue(typ, p, v.canSet) + return newValue(typ, p, v.flag) } /* @@ -537,7 +575,7 @@ func (v *SliceValue) SetLen(n int) { // Set assigns x to v. // The new value x must have the same type as v. func (v *SliceValue) Set(x *SliceValue) { - if !v.canSet { + if !v.CanSet() { panic(cannotSet) } typesMustMatch(v.typ, x.typ) @@ -566,7 +604,14 @@ func (v *SliceValue) Slice(beg, end int) *SliceValue { s.Data = uintptr(v.addr()) + uintptr(beg)*typ.Elem().Size() s.Len = end - beg s.Cap = cap - beg - return newValue(typ, addr(s), v.canSet).(*SliceValue) + + // Like the result of Addr, we treat Slice as an + // unaddressable temporary, so don't set canAddr. + flag := canSet + if v.flag&canStore != 0 { + flag |= canStore + } + return newValue(typ, addr(s), flag).(*SliceValue) } // Elem returns the i'th element of v. @@ -577,7 +622,11 @@ func (v *SliceValue) Elem(i int) Value { panic("reflect: slice index out of range") } p := addr(uintptr(v.addr()) + uintptr(i)*typ.Size()) - return newValue(typ, p, v.canSet) + flag := canAddr + if v.flag&canStore != 0 { + flag |= canSet | canStore + } + return newValue(typ, p, flag) } // MakeSlice creates a new zero-initialized slice value @@ -588,7 +637,7 @@ func MakeSlice(typ *SliceType, len, cap int) *SliceValue { Len: len, Cap: cap, } - return newValue(typ, addr(s), true).(*SliceValue) + return newValue(typ, addr(s), canAddr|canSet|canStore).(*SliceValue) } /* @@ -606,7 +655,7 @@ func (v *ChanValue) IsNil() bool { return *(*uintptr)(v.addr) == 0 } // Set assigns x to v. // The new value x must have the same type as v. func (v *ChanValue) Set(x *ChanValue) { - if !v.canSet { + if !v.CanSet() { panic(cannotSet) } typesMustMatch(v.typ, x.typ) @@ -733,7 +782,7 @@ func (v *FuncValue) Get() uintptr { return *(*uintptr)(v.addr) } // Set assigns x to v. // The new value x must have the same type as v. func (v *FuncValue) Set(x *FuncValue) { - if !v.canSet { + if !v.CanSet() { panic(cannotSet) } typesMustMatch(v.typ, x.typ) @@ -754,13 +803,24 @@ func (v *value) Method(i int) *FuncValue { } p := &t.methods[i] fn := p.tfn - fv := &FuncValue{value: value{runtimeToType(p.typ), addr(&fn), true}, first: v, isInterface: false} + fv := &FuncValue{value: value{runtimeToType(p.typ), addr(&fn), 0}, first: v, isInterface: false} return fv } // implemented in ../pkg/runtime/*/asm.s func call(typ *FuncType, fnaddr *byte, isInterface bool, params *addr, results *addr) +// Interface returns the fv as an interface value. +// If fv is a method obtained by invoking Value.Method +// (as opposed to Type.Method), Interface cannot return an +// interface value, so it panics. +func (fv *FuncValue) Interface() interface{} { + if fv.first != nil { + panic("FuncValue: cannot create interface value for method with bound receiver") + } + return fv.value.Interface() +} + // Call calls the function fv with input parameters in. // It returns the function's output parameters as Values. func (fv *FuncValue) Call(in []Value) []Value { @@ -875,7 +935,7 @@ func (v *InterfaceValue) Set(x Value) { if x != nil { i = x.Interface() } - if !v.canSet { + if !v.CanSet() { panic(cannotSet) } // Two different representations; see comment in Get. @@ -906,10 +966,10 @@ func (v *InterfaceValue) Method(i int) *FuncValue { // Interface is two words: itable, data. tab := *(**[10000]addr)(v.addr) - data := &value{Typeof((*byte)(nil)), addr(uintptr(v.addr) + ptrSize), true} + data := &value{Typeof((*byte)(nil)), addr(uintptr(v.addr) + ptrSize), 0} fn := tab[i+1] - fv := &FuncValue{value: value{runtimeToType(p.typ), addr(&fn), true}, first: data, isInterface: true} + fv := &FuncValue{value: value{runtimeToType(p.typ), addr(&fn), 0}, first: data, isInterface: true} return fv } @@ -928,7 +988,7 @@ func (v *MapValue) IsNil() bool { return *(*uintptr)(v.addr) == 0 } // Set assigns x to v. // The new value x must have the same type as v. func (v *MapValue) Set(x *MapValue) { - if !v.canSet { + if !v.CanSet() { panic(cannotSet) } if x == nil { @@ -1047,15 +1107,18 @@ func (v *PtrValue) IsNil() bool { return *(*uintptr)(v.addr) == 0 } func (v *PtrValue) Get() uintptr { return *(*uintptr)(v.addr) } // Set assigns x to v. -// The new value x must have the same type as v. +// The new value x must have the same type as v, and x.Elem().CanSet() must be true. func (v *PtrValue) Set(x *PtrValue) { if x == nil { *(**uintptr)(v.addr) = nil return } - if !v.canSet { + if !v.CanSet() { panic(cannotSet) } + if x.flag&canStore == 0 { + panic("cannot copy pointer obtained from unexported struct field") + } typesMustMatch(v.typ, x.typ) // TODO: This will have to move into the runtime // once the new gc goes in @@ -1084,7 +1147,7 @@ func (v *PtrValue) PointTo(x Value) { typesMustMatch(v.typ.(*PtrType).Elem(), x.Type()) // TODO: This will have to move into the runtime // once the new gc goes in. - *(*uintptr)(v.addr) = x.Addr() + *(*uintptr)(v.addr) = x.UnsafeAddr() } // Elem returns the value that v points to. @@ -1093,7 +1156,11 @@ func (v *PtrValue) Elem() Value { if v.IsNil() { return nil } - return newValue(v.typ.(*PtrType).Elem(), *(*addr)(v.addr), v.canSet) + flag := canAddr + if v.flag&canStore != 0 { + flag |= canSet | canStore + } + return newValue(v.typ.(*PtrType).Elem(), *(*addr)(v.addr), flag) } // Indirect returns the value that v points to. @@ -1120,7 +1187,7 @@ type StructValue struct { func (v *StructValue) Set(x *StructValue) { // TODO: This will have to move into the runtime // once the gc goes in. - if !v.canSet { + if !v.CanSet() { panic(cannotSet) } typesMustMatch(v.typ, x.typ) @@ -1137,7 +1204,12 @@ func (v *StructValue) Field(i int) Value { return nil } f := t.Field(i) - return newValue(f.Type, addr(uintptr(v.addr)+f.Offset), v.canSet && f.PkgPath == "") + flag := v.flag + if f.PkgPath != "" { + // unexported field + flag &^= canSet | canStore + } + return newValue(f.Type, addr(uintptr(v.addr)+f.Offset), flag) } // FieldByIndex returns the nested field corresponding to index. @@ -1193,11 +1265,11 @@ func NewValue(i interface{}) Value { return nil } t, a := unsafe.Reflect(i) - return newValue(canonicalize(toType(t)), addr(a), true) + return newValue(canonicalize(toType(t)), addr(a), canSet|canAddr|canStore) } -func newValue(typ Type, addr addr, canSet bool) Value { - v := value{typ, addr, canSet} +func newValue(typ Type, addr addr, flag uint32) Value { + v := value{typ, addr, flag} switch typ.(type) { case *ArrayType: return &ArrayValue{v} @@ -1238,5 +1310,5 @@ func MakeZero(typ Type) Value { if typ == nil { return nil } - return newValue(typ, addr(unsafe.New(typ)), true) + return newValue(typ, addr(unsafe.New(typ)), canSet|canAddr|canStore) } diff --git a/libgo/go/regexp/all_test.go b/libgo/go/regexp/all_test.go index aed73306454..c7ee4c87970 100644 --- a/libgo/go/regexp/all_test.go +++ b/libgo/go/regexp/all_test.go @@ -316,9 +316,9 @@ func TestNumSubexp(t *testing.T) { } func BenchmarkLiteral(b *testing.B) { - x := strings.Repeat("x", 50) + x := strings.Repeat("x", 50) + "y" b.StopTimer() - re := MustCompile(x) + re := MustCompile("y") b.StartTimer() for i := 0; i < b.N; i++ { if !re.MatchString(x) { @@ -329,9 +329,9 @@ func BenchmarkLiteral(b *testing.B) { } func BenchmarkNotLiteral(b *testing.B) { - x := strings.Repeat("x", 49) + x := strings.Repeat("x", 50) + "y" b.StopTimer() - re := MustCompile("^" + x) + re := MustCompile(".y") b.StartTimer() for i := 0; i < b.N; i++ { if !re.MatchString(x) { diff --git a/libgo/go/regexp/find_test.go b/libgo/go/regexp/find_test.go index 1690711dd78..83b249e3cef 100644 --- a/libgo/go/regexp/find_test.go +++ b/libgo/go/regexp/find_test.go @@ -6,6 +6,7 @@ package regexp import ( "fmt" + "strings" "testing" ) @@ -191,6 +192,12 @@ func TestFindStringIndex(t *testing.T) { } } +func TestFindReaderIndex(t *testing.T) { + for _, test := range findTests { + testFindIndex(&test, MustCompile(test.pat).FindReaderIndex(strings.NewReader(test.text)), t) + } +} + // Now come the simple All cases. func TestFindAll(t *testing.T) { @@ -381,12 +388,18 @@ func TestFindSubmatchIndex(t *testing.T) { } } -func TestFindStringSubmatchndex(t *testing.T) { +func TestFindStringSubmatchIndex(t *testing.T) { for _, test := range findTests { testFindSubmatchIndex(&test, MustCompile(test.pat).FindStringSubmatchIndex(test.text), t) } } +func TestFindReaderSubmatchIndex(t *testing.T) { + for _, test := range findTests { + testFindSubmatchIndex(&test, MustCompile(test.pat).FindReaderSubmatchIndex(strings.NewReader(test.text)), t) + } +} + // Now come the monster AllSubmatch cases. func TestFindAllSubmatch(t *testing.T) { @@ -452,7 +465,7 @@ func TestFindAllSubmatchIndex(t *testing.T) { } } -func TestFindAllStringSubmatchndex(t *testing.T) { +func TestFindAllStringSubmatchIndex(t *testing.T) { for _, test := range findTests { testFindAllSubmatchIndex(&test, MustCompile(test.pat).FindAllStringSubmatchIndex(test.text, -1), t) } diff --git a/libgo/go/regexp/regexp.go b/libgo/go/regexp/regexp.go index d274ccdf5a5..e3221ac9d68 100644 --- a/libgo/go/regexp/regexp.go +++ b/libgo/go/regexp/regexp.go @@ -54,6 +54,16 @@ // text of the match/submatch. If an index is negative, it means that // subexpression did not match any string in the input. // +// There is also a subset of the methods that can be applied to text read +// from a RuneReader: +// +// MatchReader, FindReaderIndex, FindReaderSubmatchIndex +// +// This set may grow. Note that regular expression matches may need to +// examine text beyond the text returned by a match, so the methods that +// match text from a RuneReader may read arbitrarily far into the input +// before returning. +// // (There are a few other methods that do not match this pattern.) // package regexp @@ -231,13 +241,13 @@ func (p *parser) error(err Error) { panic(err) } -const endOfFile = -1 +const endOfText = -1 func (p *parser) c() int { return p.ch } func (p *parser) nextc() int { if p.pos >= len(p.re.expr) { - p.ch = endOfFile + p.ch = endOfText } else { c, w := utf8.DecodeRuneInString(p.re.expr[p.pos:]) p.ch = c @@ -288,7 +298,7 @@ func (p *parser) checkBackslash() int { if c == '\\' { c = p.nextc() switch { - case c == endOfFile: + case c == endOfText: p.error(ErrExtraneousBackslash) case ispunct(c): // c is as delivered @@ -311,7 +321,7 @@ func (p *parser) charClass() *instr { left := -1 for { switch c := p.c(); c { - case ']', endOfFile: + case ']', endOfText: if left >= 0 { p.error(ErrBadRange) } @@ -356,7 +366,7 @@ func (p *parser) charClass() *instr { func (p *parser) term() (start, end *instr) { switch c := p.c(); c { - case '|', endOfFile: + case '|', endOfText: return nil, nil case '*', '+', '?': p.error(ErrBareClosure) @@ -638,8 +648,11 @@ func (re *Regexp) NumSubexp() int { return re.nbra } // match vectors away as we execute. Matches are ref counted and returned // to a free list when no longer active. Increases a simple benchmark by 22X. type matchArena struct { - head *matchVec - len int // length of match vector + head *matchVec + len int // length of match vector + pos int + atBOT bool // whether we're at beginning of text + atEOT bool // whether we're at end of text } type matchVec struct { @@ -699,21 +712,21 @@ type state struct { // Append new state to to-do list. Leftmost-longest wins so avoid // adding a state that's already active. The matchVec will be inc-ref'ed // if it is assigned to a state. -func (a *matchArena) addState(s []state, inst *instr, prefixed bool, match *matchVec, pos, end int) []state { +func (a *matchArena) addState(s []state, inst *instr, prefixed bool, match *matchVec) []state { switch inst.kind { case iBOT: - if pos == 0 { - s = a.addState(s, inst.next, prefixed, match, pos, end) + if a.atBOT { + s = a.addState(s, inst.next, prefixed, match) } return s case iEOT: - if pos == end { - s = a.addState(s, inst.next, prefixed, match, pos, end) + if a.atEOT { + s = a.addState(s, inst.next, prefixed, match) } return s case iBra: - match.m[inst.braNum] = pos - s = a.addState(s, inst.next, prefixed, match, pos, end) + match.m[inst.braNum] = a.pos + s = a.addState(s, inst.next, prefixed, match) return s } l := len(s) @@ -727,62 +740,157 @@ func (a *matchArena) addState(s []state, inst *instr, prefixed bool, match *matc s = append(s, state{inst, prefixed, match}) match.ref++ if inst.kind == iAlt { - s = a.addState(s, inst.left, prefixed, a.copy(match), pos, end) + s = a.addState(s, inst.left, prefixed, a.copy(match)) // give other branch a copy of this match vector - s = a.addState(s, inst.next, prefixed, a.copy(match), pos, end) + s = a.addState(s, inst.next, prefixed, a.copy(match)) } return s } -// Accepts either string or bytes - the logic is identical either way. -// If bytes == nil, scan str. -func (re *Regexp) doExecute(str string, bytestr []byte, pos int) []int { +// input abstracts different representations of the input text. It provides +// one-character lookahead. +type input interface { + step(pos int) (rune int, width int) // advance one rune + canCheckPrefix() bool // can we look ahead without losing info? + hasPrefix(re *Regexp) bool + index(re *Regexp, pos int) int +} + +// inputString scans a string. +type inputString struct { + str string +} + +func newInputString(str string) *inputString { + return &inputString{str: str} +} + +func (i *inputString) step(pos int) (int, int) { + if pos < len(i.str) { + return utf8.DecodeRuneInString(i.str[pos:len(i.str)]) + } + return endOfText, 0 +} + +func (i *inputString) canCheckPrefix() bool { + return true +} + +func (i *inputString) hasPrefix(re *Regexp) bool { + return strings.HasPrefix(i.str, re.prefix) +} + +func (i *inputString) index(re *Regexp, pos int) int { + return strings.Index(i.str[pos:], re.prefix) +} + +// inputBytes scans a byte slice. +type inputBytes struct { + str []byte +} + +func newInputBytes(str []byte) *inputBytes { + return &inputBytes{str: str} +} + +func (i *inputBytes) step(pos int) (int, int) { + if pos < len(i.str) { + return utf8.DecodeRune(i.str[pos:len(i.str)]) + } + return endOfText, 0 +} + +func (i *inputBytes) canCheckPrefix() bool { + return true +} + +func (i *inputBytes) hasPrefix(re *Regexp) bool { + return bytes.HasPrefix(i.str, re.prefixBytes) +} + +func (i *inputBytes) index(re *Regexp, pos int) int { + return bytes.Index(i.str[pos:], re.prefixBytes) +} + +// inputReader scans a RuneReader. +type inputReader struct { + r io.RuneReader + atEOT bool + pos int +} + +func newInputReader(r io.RuneReader) *inputReader { + return &inputReader{r: r} +} + +func (i *inputReader) step(pos int) (int, int) { + if !i.atEOT && pos != i.pos { + return endOfText, 0 + + } + r, w, err := i.r.ReadRune() + if err != nil { + i.atEOT = true + return endOfText, 0 + } + i.pos += w + return r, w +} + +func (i *inputReader) canCheckPrefix() bool { + return false +} + +func (i *inputReader) hasPrefix(re *Regexp) bool { + return false +} + +func (i *inputReader) index(re *Regexp, pos int) int { + return -1 +} + +// Search match starting from pos bytes into the input. +func (re *Regexp) doExecute(i input, pos int) []int { var s [2][]state s[0] = make([]state, 0, 10) s[1] = make([]state, 0, 10) in, out := 0, 1 var final state found := false - end := len(str) - if bytestr != nil { - end = len(bytestr) - } anchored := re.inst[0].next.kind == iBOT if anchored && pos > 0 { return nil } // fast check for initial plain substring - if re.prefix != "" { + if i.canCheckPrefix() && re.prefix != "" { advance := 0 if anchored { - if bytestr == nil { - if !strings.HasPrefix(str, re.prefix) { - return nil - } - } else { - if !bytes.HasPrefix(bytestr, re.prefixBytes) { - return nil - } + if !i.hasPrefix(re) { + return nil } } else { - if bytestr == nil { - advance = strings.Index(str[pos:], re.prefix) - } else { - advance = bytes.Index(bytestr[pos:], re.prefixBytes) + advance = i.index(re, pos) + if advance == -1 { + return nil } } - if advance == -1 { - return nil - } pos += advance } - arena := &matchArena{nil, 2 * (re.nbra + 1)} - for startPos := pos; pos <= end; { + // We look one character ahead so we can match $, which checks whether + // we are at EOT. + nextChar, nextWidth := i.step(pos) + arena := &matchArena{ + len: 2 * (re.nbra + 1), + pos: pos, + atBOT: pos == 0, + atEOT: nextChar == endOfText, + } + for c, startPos := 0, pos; c != endOfText; { if !found && (pos == startPos || !anchored) { // prime the pump if we haven't seen a match yet match := arena.noMatch() match.m[0] = pos - s[out] = arena.addState(s[out], re.start.next, false, match, pos, end) + s[out] = arena.addState(s[out], re.start.next, false, match) arena.free(match) // if addState saved it, ref was incremented } else if len(s[out]) == 0 { // machine has completed @@ -795,35 +903,32 @@ func (re *Regexp) doExecute(str string, bytestr []byte, pos int) []int { arena.free(state.match) } s[out] = old[0:0] // truncate state vector - charwidth := 1 - c := endOfFile - if pos < end { - if bytestr == nil { - c, charwidth = utf8.DecodeRuneInString(str[pos:end]) - } else { - c, charwidth = utf8.DecodeRune(bytestr[pos:end]) - } - } - pos += charwidth + c = nextChar + thisPos := pos + pos += nextWidth + nextChar, nextWidth = i.step(pos) + arena.atEOT = nextChar == endOfText + arena.atBOT = false + arena.pos = pos for _, st := range s[in] { switch st.inst.kind { case iBOT: case iEOT: case iChar: if c == st.inst.char { - s[out] = arena.addState(s[out], st.inst.next, st.prefixed, st.match, pos, end) + s[out] = arena.addState(s[out], st.inst.next, st.prefixed, st.match) } case iCharClass: if st.inst.cclass.matches(c) { - s[out] = arena.addState(s[out], st.inst.next, st.prefixed, st.match, pos, end) + s[out] = arena.addState(s[out], st.inst.next, st.prefixed, st.match) } case iAny: - if c != endOfFile { - s[out] = arena.addState(s[out], st.inst.next, st.prefixed, st.match, pos, end) + if c != endOfText { + s[out] = arena.addState(s[out], st.inst.next, st.prefixed, st.match) } case iNotNL: - if c != endOfFile && c != '\n' { - s[out] = arena.addState(s[out], st.inst.next, st.prefixed, st.match, pos, end) + if c != endOfText && c != '\n' { + s[out] = arena.addState(s[out], st.inst.next, st.prefixed, st.match) } case iBra: case iAlt: @@ -831,13 +936,13 @@ func (re *Regexp) doExecute(str string, bytestr []byte, pos int) []int { // choose leftmost longest if !found || // first st.match.m[0] < final.match.m[0] || // leftmost - (st.match.m[0] == final.match.m[0] && pos-charwidth > final.match.m[1]) { // longest + (st.match.m[0] == final.match.m[0] && thisPos > final.match.m[1]) { // longest if final.match != nil { arena.free(final.match) } final = st final.match.ref++ - final.match.m[1] = pos - charwidth + final.match.m[1] = thisPos } found = true default: @@ -874,14 +979,31 @@ func (re *Regexp) LiteralPrefix() (prefix string, complete bool) { return string(c[:i]), true } +// MatchReader returns whether the Regexp matches the text read by the +// RuneReader. The return value is a boolean: true for match, false for no +// match. +func (re *Regexp) MatchReader(r io.RuneReader) bool { + return len(re.doExecute(newInputReader(r), 0)) > 0 +} + // MatchString returns whether the Regexp matches the string s. // The return value is a boolean: true for match, false for no match. -func (re *Regexp) MatchString(s string) bool { return len(re.doExecute(s, nil, 0)) > 0 } +func (re *Regexp) MatchString(s string) bool { return len(re.doExecute(newInputString(s), 0)) > 0 } // Match returns whether the Regexp matches the byte slice b. // The return value is a boolean: true for match, false for no match. -func (re *Regexp) Match(b []byte) bool { return len(re.doExecute("", b, 0)) > 0 } +func (re *Regexp) Match(b []byte) bool { return len(re.doExecute(newInputBytes(b), 0)) > 0 } +// MatchReader checks whether a textual regular expression matches the text +// read by the RuneReader. More complicated queries need to use Compile and +// the full Regexp interface. +func MatchReader(pattern string, r io.RuneReader) (matched bool, error os.Error) { + re, err := Compile(pattern) + if err != nil { + return false, err + } + return re.MatchReader(r), nil +} // MatchString checks whether a textual regular expression // matches a string. More complicated queries need @@ -921,7 +1043,7 @@ func (re *Regexp) ReplaceAllStringFunc(src string, repl func(string) string) str searchPos := 0 // position where we next look for a match buf := new(bytes.Buffer) for searchPos <= len(src) { - a := re.doExecute(src, nil, searchPos) + a := re.doExecute(newInputString(src), searchPos) if len(a) == 0 { break // no more matches } @@ -973,7 +1095,7 @@ func (re *Regexp) ReplaceAllFunc(src []byte, repl func([]byte) []byte) []byte { searchPos := 0 // position where we next look for a match buf := new(bytes.Buffer) for searchPos <= len(src) { - a := re.doExecute("", src, searchPos) + a := re.doExecute(newInputBytes(src), searchPos) if len(a) == 0 { break // no more matches } @@ -1038,7 +1160,13 @@ func (re *Regexp) allMatches(s string, b []byte, n int, deliver func([]int)) { } for pos, i, prevMatchEnd := 0, 0, -1; i < n && pos <= end; { - matches := re.doExecute(s, b, pos) + var in input + if b == nil { + in = newInputString(s) + } else { + in = newInputBytes(b) + } + matches := re.doExecute(in, pos) if len(matches) == 0 { break } @@ -1052,6 +1180,7 @@ func (re *Regexp) allMatches(s string, b []byte, n int, deliver func([]int)) { accept = false } var width int + // TODO: use step() if b == nil { _, width = utf8.DecodeRuneInString(s[pos:end]) } else { @@ -1077,7 +1206,7 @@ func (re *Regexp) allMatches(s string, b []byte, n int, deliver func([]int)) { // Find returns a slice holding the text of the leftmost match in b of the regular expression. // A return value of nil indicates no match. func (re *Regexp) Find(b []byte) []byte { - a := re.doExecute("", b, 0) + a := re.doExecute(newInputBytes(b), 0) if a == nil { return nil } @@ -1089,7 +1218,7 @@ func (re *Regexp) Find(b []byte) []byte { // b[loc[0]:loc[1]]. // A return value of nil indicates no match. func (re *Regexp) FindIndex(b []byte) (loc []int) { - a := re.doExecute("", b, 0) + a := re.doExecute(newInputBytes(b), 0) if a == nil { return nil } @@ -1102,7 +1231,7 @@ func (re *Regexp) FindIndex(b []byte) (loc []int) { // an empty string. Use FindStringIndex or FindStringSubmatch if it is // necessary to distinguish these cases. func (re *Regexp) FindString(s string) string { - a := re.doExecute(s, nil, 0) + a := re.doExecute(newInputString(s), 0) if a == nil { return "" } @@ -1114,7 +1243,19 @@ func (re *Regexp) FindString(s string) string { // itself is at s[loc[0]:loc[1]]. // A return value of nil indicates no match. func (re *Regexp) FindStringIndex(s string) []int { - a := re.doExecute(s, nil, 0) + a := re.doExecute(newInputString(s), 0) + if a == nil { + return nil + } + return a[0:2] +} + +// FindReaderIndex returns a two-element slice of integers defining the +// location of the leftmost match of the regular expression in text read from +// the RuneReader. The match itself is at s[loc[0]:loc[1]]. A return +// value of nil indicates no match. +func (re *Regexp) FindReaderIndex(r io.RuneReader) []int { + a := re.doExecute(newInputReader(r), 0) if a == nil { return nil } @@ -1127,7 +1268,7 @@ func (re *Regexp) FindStringIndex(s string) []int { // comment. // A return value of nil indicates no match. func (re *Regexp) FindSubmatch(b []byte) [][]byte { - a := re.doExecute("", b, 0) + a := re.doExecute(newInputBytes(b), 0) if a == nil { return nil } @@ -1146,7 +1287,7 @@ func (re *Regexp) FindSubmatch(b []byte) [][]byte { // in the package comment. // A return value of nil indicates no match. func (re *Regexp) FindSubmatchIndex(b []byte) []int { - return re.doExecute("", b, 0) + return re.doExecute(newInputBytes(b), 0) } // FindStringSubmatch returns a slice of strings holding the text of the @@ -1155,7 +1296,7 @@ func (re *Regexp) FindSubmatchIndex(b []byte) []int { // package comment. // A return value of nil indicates no match. func (re *Regexp) FindStringSubmatch(s string) []string { - a := re.doExecute(s, nil, 0) + a := re.doExecute(newInputString(s), 0) if a == nil { return nil } @@ -1174,7 +1315,16 @@ func (re *Regexp) FindStringSubmatch(s string) []string { // 'Index' descriptions in the package comment. // A return value of nil indicates no match. func (re *Regexp) FindStringSubmatchIndex(s string) []int { - return re.doExecute(s, nil, 0) + return re.doExecute(newInputString(s), 0) +} + +// FindReaderSubmatchIndex returns a slice holding the index pairs +// identifying the leftmost match of the regular expression of text read by +// the RuneReader, and the matches, if any, of its subexpressions, as defined +// by the 'Submatch' and 'Index' descriptions in the package comment. A +// return value of nil indicates no match. +func (re *Regexp) FindReaderSubmatchIndex(r io.RuneReader) []int { + return re.doExecute(newInputReader(r), 0) } const startSize = 10 // The size at which to start a slice in the 'All' routines. diff --git a/libgo/go/rpc/client.go b/libgo/go/rpc/client.go index 601c49715b3..6de6d1325b6 100644 --- a/libgo/go/rpc/client.go +++ b/libgo/go/rpc/client.go @@ -15,6 +15,16 @@ import ( "sync" ) +// ServerError represents an error that has been returned from +// the remote side of the RPC connection. +type ServerError string + +func (e ServerError) String() string { + return string(e) +} + +const ErrShutdown = os.ErrorString("connection is shut down") + // Call represents an active RPC. type Call struct { ServiceMethod string // The name of the service and method to call. @@ -30,12 +40,12 @@ type Call struct { // with a single Client. type Client struct { mutex sync.Mutex // protects pending, seq - shutdown os.Error // non-nil if the client is shut down sending sync.Mutex seq uint64 codec ClientCodec pending map[uint64]*Call closing bool + shutdown bool } // A ClientCodec implements writing of RPC requests and @@ -43,7 +53,9 @@ type Client struct { // The client calls WriteRequest to write a request to the connection // and calls ReadResponseHeader and ReadResponseBody in pairs // to read responses. The client calls Close when finished with the -// connection. +// connection. ReadResponseBody may be called with a nil +// argument to force the body of the response to be read and then +// discarded. type ClientCodec interface { WriteRequest(*Request, interface{}) os.Error ReadResponseHeader(*Response) os.Error @@ -55,10 +67,10 @@ type ClientCodec interface { func (client *Client) send(c *Call) { // Register this call. client.mutex.Lock() - if client.shutdown != nil { - c.Error = client.shutdown + if client.shutdown { + c.Error = ErrShutdown client.mutex.Unlock() - _ = c.Done <- c // do not block + c.done() return } c.seq = client.seq @@ -93,25 +105,30 @@ func (client *Client) input() { c := client.pending[seq] client.pending[seq] = c, false client.mutex.Unlock() - err = client.codec.ReadResponseBody(c.Reply) - if response.Error != "" { - c.Error = os.ErrorString(response.Error) - } else if err != nil { - c.Error = err + + if response.Error == "" { + err = client.codec.ReadResponseBody(c.Reply) + if err != nil { + c.Error = os.ErrorString("reading body " + err.String()) + } } else { - // Empty strings should turn into nil os.Errors - c.Error = nil + // We've got an error response. Give this to the request; + // any subsequent requests will get the ReadResponseBody + // error if there is one. + c.Error = ServerError(response.Error) + err = client.codec.ReadResponseBody(nil) + if err != nil { + err = os.ErrorString("reading error body: " + err.String()) + } } - // We don't want to block here. It is the caller's responsibility to make - // sure the channel has enough buffer space. See comment in Go(). - _ = c.Done <- c // do not block + c.done() } // Terminate pending calls. client.mutex.Lock() - client.shutdown = err + client.shutdown = true for _, call := range client.pending { call.Error = err - _ = call.Done <- call // do not block + call.done() } client.mutex.Unlock() if err != os.EOF || !client.closing { @@ -119,6 +136,16 @@ func (client *Client) input() { } } +func (call *Call) done() { + select { + case call.Done <- call: + // ok + default: + // We don't want to block here. It is the caller's responsibility to make + // sure the channel has enough buffer space. See comment in Go(). + } +} + // NewClient returns a new Client to handle requests to the // set of services at the other end of the connection. func NewClient(conn io.ReadWriteCloser) *Client { @@ -201,10 +228,11 @@ func Dial(network, address string) (*Client, os.Error) { } func (client *Client) Close() os.Error { - if client.shutdown != nil || client.closing { - return os.ErrorString("rpc: already closed") - } client.mutex.Lock() + if client.shutdown || client.closing { + client.mutex.Unlock() + return ErrShutdown + } client.closing = true client.mutex.Unlock() return client.codec.Close() @@ -231,9 +259,9 @@ func (client *Client) Go(serviceMethod string, args interface{}, reply interface } } c.Done = done - if client.shutdown != nil { - c.Error = client.shutdown - _ = c.Done <- c // do not block + if client.shutdown { + c.Error = ErrShutdown + c.done() return c } client.send(c) @@ -242,8 +270,8 @@ func (client *Client) Go(serviceMethod string, args interface{}, reply interface // Call invokes the named function, waits for it to complete, and returns its error status. func (client *Client) Call(serviceMethod string, args interface{}, reply interface{}) os.Error { - if client.shutdown != nil { - return client.shutdown + if client.shutdown { + return ErrShutdown } call := <-client.Go(serviceMethod, args, reply, nil).Done return call.Error diff --git a/libgo/go/rpc/debug.go b/libgo/go/rpc/debug.go index 44b32e04ba0..32dc8a18ba2 100644 --- a/libgo/go/rpc/debug.go +++ b/libgo/go/rpc/debug.go @@ -83,7 +83,7 @@ func (server debugHTTP) ServeHTTP(w http.ResponseWriter, req *http.Request) { } server.Unlock() sort.Sort(services) - err := debug.Execute(services, w) + err := debug.Execute(w, services) if err != nil { fmt.Fprintln(w, "rpc: error executing template:", err.String()) } diff --git a/libgo/go/rpc/jsonrpc/client.go b/libgo/go/rpc/jsonrpc/client.go index dcaa69f9dfc..5b806bd6e25 100644 --- a/libgo/go/rpc/jsonrpc/client.go +++ b/libgo/go/rpc/jsonrpc/client.go @@ -98,6 +98,9 @@ func (c *clientCodec) ReadResponseHeader(r *rpc.Response) os.Error { } func (c *clientCodec) ReadResponseBody(x interface{}) os.Error { + if x == nil { + return nil + } return json.Unmarshal(*c.resp.Result, x) } diff --git a/libgo/go/rpc/jsonrpc/server.go b/libgo/go/rpc/jsonrpc/server.go index bf53bda8daa..9c6b8b40d68 100644 --- a/libgo/go/rpc/jsonrpc/server.go +++ b/libgo/go/rpc/jsonrpc/server.go @@ -85,6 +85,9 @@ func (c *serverCodec) ReadRequestHeader(r *rpc.Request) os.Error { } func (c *serverCodec) ReadRequestBody(x interface{}) os.Error { + if x == nil { + return nil + } // JSON params is array value. // RPC params is struct. // Unmarshal into array containing struct for now. diff --git a/libgo/go/rpc/server.go b/libgo/go/rpc/server.go index 5c50bcc3a37..9dcda414807 100644 --- a/libgo/go/rpc/server.go +++ b/libgo/go/rpc/server.go @@ -73,7 +73,7 @@ rpc.HandleHTTP() l, e := net.Listen("tcp", ":1234") if e != nil { - log.Exit("listen error:", e) + log.Fatal("listen error:", e) } go http.Serve(l, nil) @@ -82,7 +82,7 @@ client, err := rpc.DialHTTP("tcp", serverAddress + ":1234") if err != nil { - log.Exit("dialing:", err) + log.Fatal("dialing:", err) } Then it can make a remote call: @@ -92,7 +92,7 @@ var reply int err = client.Call("Arith.Multiply", args, &reply) if err != nil { - log.Exit("arith error:", err) + log.Fatal("arith error:", err) } fmt.Printf("Arith: %d*%d=%d", args.A, args.B, *reply) @@ -225,7 +225,7 @@ func (server *Server) register(rcvr interface{}, name string, useName bool) os.E sname = name } if sname == "" { - log.Exit("rpc: no service name for type", s.typ.String()) + log.Fatal("rpc: no service name for type", s.typ.String()) } if s.typ.PkgPath() != "" && !isExported(sname) && !useName { s := "rpc Register: type " + sname + " is not exported" @@ -299,10 +299,10 @@ func (server *Server) register(rcvr interface{}, name string, useName bool) os.E // A value sent as a placeholder for the response when the server receives an invalid request. type InvalidRequest struct { - marker int + Marker int } -var invalidRequest = InvalidRequest{1} +var invalidRequest = InvalidRequest{} func _new(t *reflect.PtrType) *reflect.PtrValue { v := reflect.MakeZero(t).(*reflect.PtrValue) @@ -316,6 +316,7 @@ func sendResponse(sending *sync.Mutex, req *Request, reply interface{}, codec Se resp.ServiceMethod = req.ServiceMethod if errmsg != "" { resp.Error = errmsg + reply = invalidRequest } resp.Seq = req.Seq sending.Lock() @@ -389,54 +390,74 @@ func (server *Server) ServeConn(conn io.ReadWriteCloser) { func (server *Server) ServeCodec(codec ServerCodec) { sending := new(sync.Mutex) for { - // Grab the request header. - req := new(Request) - err := codec.ReadRequestHeader(req) + req, service, mtype, err := server.readRequest(codec) if err != nil { + if err != os.EOF { + log.Println("rpc:", err) + } if err == os.EOF || err == io.ErrUnexpectedEOF { - if err == io.ErrUnexpectedEOF { - log.Println("rpc:", err) - } break } - s := "rpc: server cannot decode request: " + err.String() - sendResponse(sending, req, invalidRequest, codec, s) - break - } - serviceMethod := strings.Split(req.ServiceMethod, ".", -1) - if len(serviceMethod) != 2 { - s := "rpc: service/method request ill-formed: " + req.ServiceMethod - sendResponse(sending, req, invalidRequest, codec, s) - continue - } - // Look up the request. - server.Lock() - service, ok := server.serviceMap[serviceMethod[0]] - server.Unlock() - if !ok { - s := "rpc: can't find service " + req.ServiceMethod - sendResponse(sending, req, invalidRequest, codec, s) - continue - } - mtype, ok := service.method[serviceMethod[1]] - if !ok { - s := "rpc: can't find method " + req.ServiceMethod - sendResponse(sending, req, invalidRequest, codec, s) + // discard body + codec.ReadRequestBody(nil) + + // send a response if we actually managed to read a header. + if req != nil { + sendResponse(sending, req, invalidRequest, codec, err.String()) + } continue } + // Decode the argument value. argv := _new(mtype.ArgType) replyv := _new(mtype.ReplyType) err = codec.ReadRequestBody(argv.Interface()) if err != nil { - log.Println("rpc: tearing down", serviceMethod[0], "connection:", err) + if err == os.EOF || err == io.ErrUnexpectedEOF { + if err == io.ErrUnexpectedEOF { + log.Println("rpc:", err) + } + break + } sendResponse(sending, req, replyv.Interface(), codec, err.String()) - break + continue } go service.call(sending, mtype, req, argv, replyv, codec) } codec.Close() } +func (server *Server) readRequest(codec ServerCodec) (req *Request, service *service, mtype *methodType, err os.Error) { + // Grab the request header. + req = new(Request) + err = codec.ReadRequestHeader(req) + if err != nil { + req = nil + if err == os.EOF || err == io.ErrUnexpectedEOF { + return + } + err = os.ErrorString("rpc: server cannot decode request: " + err.String()) + return + } + + serviceMethod := strings.Split(req.ServiceMethod, ".", -1) + if len(serviceMethod) != 2 { + err = os.ErrorString("rpc: service/method request ill-formed: " + req.ServiceMethod) + return + } + // Look up the request. + server.Lock() + service = server.serviceMap[serviceMethod[0]] + server.Unlock() + if service == nil { + err = os.ErrorString("rpc: can't find service " + req.ServiceMethod) + return + } + mtype = service.method[serviceMethod[1]] + if mtype == nil { + err = os.ErrorString("rpc: can't find method " + req.ServiceMethod) + } + return +} // Accept accepts connections on the listener and serves requests // for each incoming connection. Accept blocks; the caller typically @@ -445,7 +466,7 @@ func (server *Server) Accept(lis net.Listener) { for { conn, err := lis.Accept() if err != nil { - log.Exit("rpc.Serve: accept:", err.String()) // TODO(r): exit? + log.Fatal("rpc.Serve: accept:", err.String()) // TODO(r): exit? } go server.ServeConn(conn) } @@ -465,7 +486,8 @@ func RegisterName(name string, rcvr interface{}) os.Error { // The server calls ReadRequestHeader and ReadRequestBody in pairs // to read requests from the connection, and it calls WriteResponse to // write a response back. The server calls Close when finished with the -// connection. +// connection. ReadRequestBody may be called with a nil +// argument to force the body of the request to be read and discarded. type ServerCodec interface { ReadRequestHeader(*Request) os.Error ReadRequestBody(interface{}) os.Error diff --git a/libgo/go/rpc/server_test.go b/libgo/go/rpc/server_test.go index 355d51ce46b..05aaebceb48 100644 --- a/libgo/go/rpc/server_test.go +++ b/libgo/go/rpc/server_test.go @@ -72,7 +72,7 @@ func (t *Arith) Error(args *Args, reply *Reply) os.Error { func listenTCP() (net.Listener, string) { l, e := net.Listen("tcp", "127.0.0.1:0") // any available address if e != nil { - log.Exitf("net.Listen tcp :0: %v", e) + log.Fatalf("net.Listen tcp :0: %v", e) } return l, l.Addr().String() } @@ -134,14 +134,25 @@ func testRPC(t *testing.T, addr string) { t.Errorf("Add: expected %d got %d", reply.C, args.A+args.B) } - args = &Args{7, 8} + // Nonexistent method + args = &Args{7, 0} reply = new(Reply) - err = client.Call("Arith.Mul", args, reply) - if err != nil { - t.Errorf("Mul: expected no error but got string %q", err.String()) + err = client.Call("Arith.BadOperation", args, reply) + // expect an error + if err == nil { + t.Error("BadOperation: expected error") + } else if !strings.HasPrefix(err.String(), "rpc: can't find method ") { + t.Errorf("BadOperation: expected can't find method error; got %q", err) } - if reply.C != args.A*args.B { - t.Errorf("Mul: expected %d got %d", reply.C, args.A*args.B) + + // Unknown service + args = &Args{7, 8} + reply = new(Reply) + err = client.Call("Arith.Unknown", args, reply) + if err == nil { + t.Error("expected error calling unknown service") + } else if strings.Index(err.String(), "method") < 0 { + t.Error("expected error about method; got", err) } // Out of order. @@ -178,6 +189,15 @@ func testRPC(t *testing.T, addr string) { t.Error("Div: expected divide by zero error; got", err) } + // Bad type. + reply = new(Reply) + err = client.Call("Arith.Add", reply, reply) // args, reply would be the correct thing to use + if err == nil { + t.Error("expected error calling Arith.Add with wrong arg type") + } else if strings.Index(err.String(), "type") < 0 { + t.Error("expected error about type; got", err) + } + // Non-struct argument const Val = 12345 str := fmt.Sprint(Val) @@ -200,9 +220,19 @@ func testRPC(t *testing.T, addr string) { if str != expect { t.Errorf("String: expected %s got %s", expect, str) } + + args = &Args{7, 8} + reply = new(Reply) + err = client.Call("Arith.Mul", args, reply) + if err != nil { + t.Errorf("Mul: expected no error but got string %q", err.String()) + } + if reply.C != args.A*args.B { + t.Errorf("Mul: expected %d got %d", reply.C, args.A*args.B) + } } -func TestHTTPRPC(t *testing.T) { +func TestHTTP(t *testing.T) { once.Do(startServer) testHTTPRPC(t, "") newOnce.Do(startNewServer) @@ -233,65 +263,6 @@ func testHTTPRPC(t *testing.T, path string) { } } -func TestCheckUnknownService(t *testing.T) { - once.Do(startServer) - - conn, err := net.Dial("tcp", "", serverAddr) - if err != nil { - t.Fatal("dialing:", err) - } - - client := NewClient(conn) - - args := &Args{7, 8} - reply := new(Reply) - err = client.Call("Unknown.Add", args, reply) - if err == nil { - t.Error("expected error calling unknown service") - } else if strings.Index(err.String(), "service") < 0 { - t.Error("expected error about service; got", err) - } -} - -func TestCheckUnknownMethod(t *testing.T) { - once.Do(startServer) - - conn, err := net.Dial("tcp", "", serverAddr) - if err != nil { - t.Fatal("dialing:", err) - } - - client := NewClient(conn) - - args := &Args{7, 8} - reply := new(Reply) - err = client.Call("Arith.Unknown", args, reply) - if err == nil { - t.Error("expected error calling unknown service") - } else if strings.Index(err.String(), "method") < 0 { - t.Error("expected error about method; got", err) - } -} - -func TestCheckBadType(t *testing.T) { - once.Do(startServer) - - conn, err := net.Dial("tcp", "", serverAddr) - if err != nil { - t.Fatal("dialing:", err) - } - - client := NewClient(conn) - - reply := new(Reply) - err = client.Call("Arith.Add", reply, reply) // args, reply would be the correct thing to use - if err == nil { - t.Error("expected error calling Arith.Add with wrong arg type") - } else if strings.Index(err.String(), "type") < 0 { - t.Error("expected error about type; got", err) - } -} - type ArgNotPointer int type ReplyNotPointer int type ArgNotPublic int @@ -364,14 +335,12 @@ func TestSendDeadlock(t *testing.T) { testSendDeadlock(client) done <- true }() - for i := 0; i < 50; i++ { - time.Sleep(100 * 1e6) - _, ok := <-done - if ok { - return - } + select { + case <-done: + return + case <-time.After(5e9): + t.Fatal("deadlock") } - t.Fatal("deadlock") } func testSendDeadlock(client *Client) { diff --git a/libgo/go/runtime/chan_defs.go b/libgo/go/runtime/chan_defs.go deleted file mode 100644 index 5cfea6e15a8..00000000000 --- a/libgo/go/runtime/chan_defs.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Go definitions of internal structures. Master is chan.c - -package runtime - -type sudoG struct { - g *g_ - selgen uint32 - offset int16 - isfree int8 - link *sudoG - elem [8]byte -} - -type waitQ struct { - first *sudoG - last *sudoG -} - -type hChan struct { - qcount uint32 - dataqsiz uint32 - elemsize uint16 - closed uint16 - elemalign uint8 - elemalg *alg - senddataq *link - recvdataq *link - recvq waitQ - sendq waitQ - free sudoG - lock -} - -type link struct { - link *link - elem [8]byte -} - -type scase struct { - chan_ *hChan - pc *byte - send uint16 - so uint16 - elemp *byte // union elem [8]byte -} - -type select_ struct { - tcase uint16 - ncase uint16 - link *select_ - scase [1]*scase -} diff --git a/libgo/go/runtime/debug.go b/libgo/go/runtime/debug.go index 803ea4921c1..74010b394b0 100644 --- a/libgo/go/runtime/debug.go +++ b/libgo/go/runtime/debug.go @@ -57,7 +57,6 @@ type MemStatsType struct { MSpanSys uint64 MCacheInuse uint64 // mcache structures MCacheSys uint64 - MHeapMapSys uint64 // heap map BuckHashSys uint64 // profiling bucket hash table // Garbage collector statistics. @@ -70,7 +69,8 @@ type MemStatsType struct { // Per-size allocation statistics. // Not locked during update; approximate. - BySize [67]struct { + // 61 is NumSizeClasses in the C code. + BySize [61]struct { Size uint32 Mallocs uint64 Frees uint64 diff --git a/libgo/go/runtime/extern.go b/libgo/go/runtime/extern.go index 77c3e8e3a64..c6e664abbbb 100644 --- a/libgo/go/runtime/extern.go +++ b/libgo/go/runtime/extern.go @@ -31,6 +31,19 @@ func Caller(skip int) (pc uintptr, file string, line int, ok bool) // It returns the number of entries written to pc. func Callers(skip int, pc []uintptr) int +type Func struct { // Keep in sync with runtime.h:struct Func + name string + typ string // go type string + src string // src file name + pcln []byte // pc/ln tab for this func + entry uintptr // entry pc + pc0 uintptr // starting pc, ln for table + ln0 int32 + frame int32 // stack frame size + args int32 // number of 32-bit in/out args + locals int32 // number of 32-bit locals +} + // FuncForPC returns a *Func describing the function that contains the // given program counter address, or else nil. func FuncForPC(pc uintptr) *Func @@ -47,31 +60,47 @@ func (f *Func) Entry() uintptr { return f.entry } // counter within f. func (f *Func) FileLine(pc uintptr) (file string, line int) { // NOTE(rsc): If you edit this function, also edit - // symtab.c:/^funcline. + // symtab.c:/^funcline. That function also has the + // comments explaining the logic. + targetpc := pc + var pcQuant uintptr = 1 if GOARCH == "arm" { pcQuant = 4 } - targetpc := pc p := f.pcln pc = f.pc0 line = int(f.ln0) - file = f.src - for i := 0; i < len(p) && pc <= targetpc; i++ { - switch { - case p[i] == 0: + i := 0 + //print("FileLine start pc=", pc, " targetpc=", targetpc, " line=", line, + // " tab=", p, " ", p[0], " quant=", pcQuant, " GOARCH=", GOARCH, "\n") + for { + for i < len(p) && p[i] > 128 { + pc += pcQuant * uintptr(p[i]-128) + i++ + } + //print("pc<", pc, " targetpc=", targetpc, " line=", line, "\n") + if pc > targetpc || i >= len(p) { + break + } + if p[i] == 0 { + if i+5 > len(p) { + break + } line += int(p[i+1]<<24) | int(p[i+2]<<16) | int(p[i+3]<<8) | int(p[i+4]) - i += 4 - case p[i] <= 64: + i += 5 + } else if p[i] <= 64 { line += int(p[i]) - case p[i] <= 128: + i++ + } else { line -= int(p[i] - 64) - default: - pc += pcQuant * uintptr(p[i]-129) + i++ } + //print("pc=", pc, " targetpc=", targetpc, " line=", line, "\n") pc += pcQuant } + file = f.src return } diff --git a/libgo/go/runtime/hashmap_defs.go b/libgo/go/runtime/hashmap_defs.go deleted file mode 100644 index 57780df879b..00000000000 --- a/libgo/go/runtime/hashmap_defs.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Go definitions of internal structures. Master is hashmap.[c,h] - -package runtime - -type hash_hash uintptr - -type hash_entry struct { - hash hash_hash - key byte // dwarf.c substitutes the real type - val byte // for key and val -} - -type hash_subtable struct { - power uint8 - used uint8 - datasize uint8 - max_probes uint8 - limit_bytes int16 - end *hash_entry - entry hash_entry // TODO: [0]hash_entry -} - -type hash struct { - count uint32 - datasize uint8 - max_power uint8 - max_probes uint8 - indirectval uint8 - changes int32 - data_hash func(uint32, uintptr) hash_hash - data_eq func(uint32, uintptr, uintptr) uint32 - data_del func(uint32, uintptr, uintptr) - st *hash_subtable - keysize uint32 - valsize uint32 - datavo uint32 - ko0 uint32 - vo0 uint32 - ko1 uint32 - vo1 uint32 - po1 uint32 - ko2 uint32 - vo2 uint32 - po2 uint32 - keyalg *alg - valalg *alg -} diff --git a/libgo/go/runtime/iface_defs.go b/libgo/go/runtime/iface_defs.go deleted file mode 100644 index 69d52ef9a6b..00000000000 --- a/libgo/go/runtime/iface_defs.go +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package runtime - -/* - * Must match iface.c:/Itable and compilers. - * NOTE: type.go has an Itable, that is the version of Itab used by the reflection code. - */ -type itab struct { - Itype *Type - Type *Type - link *itab - bad int32 - unused int32 - Fn func() // TODO: [0]func() -} diff --git a/libgo/go/runtime/malloc_defs.go b/libgo/go/runtime/malloc_defs.go deleted file mode 100644 index 11d6627e110..00000000000 --- a/libgo/go/runtime/malloc_defs.go +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Go definitions of internal structures. Master is malloc.h - -package runtime - -import "unsafe" - -const ( - pageShift = 12 - pageSize = 1 << pageShift - pageMask = pageSize - 1 -) - -type pageID uintptr - -const ( - numSizeClasses = 67 - maxSmallSize = 32 << 10 - fixAllocChunk = 128 << 10 - maxMCacheListLen = 256 - maxMCacheSize = 2 << 20 - maxMHeapList = 1 << 8 // 1 << (20 - pageShift) - heapAllocChunk = 1 << 20 -) - -type mLink struct { - next *mLink -} - -type fixAlloc struct { - size uintptr - alloc func(uintptr) - first func(unsafe.Pointer, *byte) - arg unsafe.Pointer - list *mLink - chunk *byte - nchunk uint32 - inuse uintptr - sys uintptr -} - - -// MStats? used to be in extern.go - -type mCacheList struct { - list *mLink - nlist uint32 - nlistmin uint32 -} - -type mCache struct { - list [numSizeClasses]mCacheList - size uint64 - local_alloc int64 - local_objects int64 - next_sample int32 -} - -type mSpan struct { - next *mSpan - prev *mSpan - allnext *mSpan - start pageID - npages uintptr - freelist *mLink - ref uint32 - sizeclass uint32 - state uint32 - // union { - gcref *uint32 // sizeclass > 0 - // gcref0 uint32; // sizeclass == 0 - // } -} - -type mCentral struct { - // lock - sizeclass int32 - nonempty mSpan - empty mSpan - nfree int32 -} - -type mHeap struct { - // lock - free [maxMHeapList]mSpan - large mSpan - allspans *mSpan - // map_ mHeapMap - min *byte - max *byte - closure_min *byte - closure_max *byte - - central [numSizeClasses]struct { - pad [64]byte - // union: mCentral - } - - spanalloc fixAlloc - cachealloc fixAlloc -} - -const ( - refFree = iota - refStack - refNone - refSome - refcountOverhead = 4 - refNoPointers = 0x80000000 - refHasFinalizer = 0x40000000 - refProfiled = 0x20000000 - refNoProfiling = 0x10000000 - refFlags = 0xFFFF0000 -) - -const ( - mProf_None = iota - mProf_Sample - mProf_All -) - -type finalizer struct { - next *finalizer - fn func(unsafe.Pointer) - arg unsafe.Pointer - nret int32 -} diff --git a/libgo/go/runtime/mheapmap32_defs.go b/libgo/go/runtime/mheapmap32_defs.go deleted file mode 100644 index 755725b4606..00000000000 --- a/libgo/go/runtime/mheapmap32_defs.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package runtime - -const ( - mHeapMap_Level1Bits = 10 - mHeapMap_Level2Bits = 10 - mHeapMap_TotalBits = mHeapMap_Level1Bits + mHeapMap_Level2Bits - - mHeapMap_Level1Mask = (1 << mHeapMap_Level1Bits) - 1 - mHeapMap_Level2Mask = (1 << mHeapMap_Level2Bits) - 1 -) - -type mHeapMap struct { - allocator func(uintptr) - p [1 << mHeapMap_Level1Bits]*mHeapMapNode2 -} - -type mHeapMapNode2 struct { - s [1 << mHeapMap_Level2Bits]*mSpan -} diff --git a/libgo/go/runtime/mheapmap64_defs.go b/libgo/go/runtime/mheapmap64_defs.go deleted file mode 100644 index d7ba2b42001..00000000000 --- a/libgo/go/runtime/mheapmap64_defs.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package runtime - -const ( - mHeapMap_Level1Bits = 18 - mHeapMap_Level2Bits = 18 - mHeapMap_Level3Bits = 16 - mHeapMap_TotalBits = mHeapMap_Level1Bits + mHeapMap_Level2Bits + mHeapMap_Level3Bits - - mHeapMap_Level1Mask = (1 << mHeapMap_Level1Bits) - 1 - mHeapMap_Level2Mask = (1 << mHeapMap_Level2Bits) - 1 - mHeapMap_Level3Mask = (1 << mHeapMap_Level3Bits) - 1 -) - -type mHeapMap struct { - allocator func(uintptr) - p [1 << mHeapMap_Level1Bits]*mHeapMapNode2 -} - - -type mHeapMapNode2 struct { - p [1 << mHeapMap_Level2Bits]*mHeapMapNode3 -} - - -type mHeapMapNode3 struct { - s [1 << mHeapMap_Level3Bits]*mSpan -} diff --git a/libgo/go/runtime/pprof/pprof.go b/libgo/go/runtime/pprof/pprof.go index d0cc730899d..9bee5112819 100644 --- a/libgo/go/runtime/pprof/pprof.go +++ b/libgo/go/runtime/pprof/pprof.go @@ -88,7 +88,6 @@ func WriteHeapProfile(w io.Writer) os.Error { fmt.Fprintf(b, "# Stack = %d / %d\n", s.StackInuse, s.StackSys) fmt.Fprintf(b, "# MSpan = %d / %d\n", s.MSpanInuse, s.MSpanSys) fmt.Fprintf(b, "# MCache = %d / %d\n", s.MCacheInuse, s.MCacheSys) - fmt.Fprintf(b, "# MHeapMapSys = %d\n", s.MHeapMapSys) fmt.Fprintf(b, "# BuckHashSys = %d\n", s.BuckHashSys) fmt.Fprintf(b, "# NextGC = %d\n", s.NextGC) diff --git a/libgo/go/runtime/runtime_defs.go b/libgo/go/runtime/runtime_defs.go deleted file mode 100644 index deea320b5af..00000000000 --- a/libgo/go/runtime/runtime_defs.go +++ /dev/null @@ -1,200 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Go definitions of internal structures. Master is runtime.h - -// TODO(lvd): automate conversion to all the _defs.go files - -package runtime - -import "unsafe" - -const ( - gidle = iota - grunnable - grunning - gsyscall - gwaiting - gmoribund - gdead - grecovery -) - -// const ( Structrnd = sizeof(uintptr) ) - -type string_ struct { - str *byte - len int32 -} - -type iface struct { - // tab *itab - data unsafe.Pointer -} - -type eface struct { - type_ *Type - data unsafe.Pointer -} - -type complex64 struct { - real float32 - imag float32 -} - -type complex128 struct { - real float64 - imag float64 -} - -type slice struct { - array *byte - len uint32 - cap uint32 -} - -type gobuf struct { - sp unsafe.Pointer - pc unsafe.Pointer - g *g_ -} - -type g_ struct { - stackguard unsafe.Pointer - stackbase unsafe.Pointer - defer_ *defer_ - panic_ *panic_ - sched gobuf - stack0 unsafe.Pointer - entry unsafe.Pointer - alllink *g_ - param unsafe.Pointer - status int16 - goid int32 - selgen uint32 - schedlink *g_ - readyonstop bool - ispanic bool - m *m_ - lockedm *m_ - sig int32 - sigcode0 uintptr - sigcode1 uintptr -} - -type m_ struct { - g0 *g_ - morepc unsafe.Pointer - moreargp unsafe.Pointer - morebuf gobuf - moreframesize uint32 - moreargsize uint32 - cret uintptr - procid uint64 - gsignal *g_ - tls [8]uint32 - sched gobuf - curg *g_ - id int32 - mallocing int32 - gcing int32 - locks int32 - nomemprof int32 - waitnextg int32 - // havenextg note - nextg *g_ - alllink *m_ - schedlink *m_ - machport uint32 - mcache *mCache - lockedg *g_ - freg [8]uint64 - // gostack unsafe.Pointer // __WINDOWS__ -} - -type stktop struct { - stackguard *uint8 - stackbase *uint8 - gobuf gobuf - args uint32 - fp *uint8 - free bool - panic_ bool -} - -type alg struct { - hash func(uint32, unsafe.Pointer) uintptr - equal func(uint32, unsafe.Pointer, unsafe.Pointer) uint32 - print func(uint32, unsafe.Pointer) - copy func(uint32, unsafe.Pointer, unsafe.Pointer) -} - -type sigtab struct { - flags int32 - name *int8 -} - -const ( - sigCatch = (1 << iota) - sigIgnore - sigRestart - sigQueue - sigPanic -) - -type Func struct { - name string - typ string - src string - pcln []byte - entry uintptr - pc0 uintptr - ln0 int32 - frame int32 - args int32 - locals int32 -} - -const ( - aMEM = iota - aNOEQ - aSTRING - aINTER - aNILINTER - aMEMWORD - amax -) - -type defer_ struct { - siz int32 - sp unsafe.Pointer - pc unsafe.Pointer - fn unsafe.Pointer - link *defer_ - args [8]byte // padded to actual size -} - -type panic_ struct { - arg eface - stackbase unsafe.Pointer - link *panic_ - recovered bool -} - -/* - * External data. - */ - -var ( - algarray [amax]alg - emptystring string - allg *g_ - allm *m_ - goidgen int32 - gomaxprocs int32 - panicking int32 - fd int32 - gcwaiting int32 - goos *int8 -) diff --git a/libgo/go/runtime/type.go b/libgo/go/runtime/type.go index 645e3647e83..f5f3ef1baad 100644 --- a/libgo/go/runtime/type.go +++ b/libgo/go/runtime/type.go @@ -9,7 +9,7 @@ * data structures and must be kept in sync with this file: * * ../../cmd/gc/reflect.c - * ../../cmd/ld/dwarf.c + * ../../cmd/ld/dwarf.c decodetype_* * ../reflect/type.go * type.h */ @@ -32,6 +32,7 @@ type commonType struct { string *string // string form; unnecessary but undeniably useful *uncommonType // (relatively) uncommon fields + ptrToThis *Type // pointer to this type, if used in binary or has methods } // Values for commonType.kind. @@ -71,7 +72,7 @@ const ( type Type commonType // Method on non-interface type -type method struct { +type _method struct { // underscore is to avoid collision with C name *string // name of method pkgPath *string // nil for exported Names; otherwise import path mtyp *Type // method type (without receiver) @@ -84,9 +85,9 @@ type method struct { // Using a pointer to this struct reduces the overall size required // to describe an unnamed type with no methods. type uncommonType struct { - name *string // name of type - pkgPath *string // import path; nil for built-in types like int, string - methods []method // methods associated with type + name *string // name of type + pkgPath *string // import path; nil for built-in types like int, string + methods []_method // methods associated with type } // BoolType represents a boolean type. @@ -151,7 +152,7 @@ type FuncType struct { } // Method on interface type -type imethod struct { +type _imethod struct { // underscore is to avoid collision with C name *string // name of method pkgPath *string // nil for exported Names; otherwise import path typ *Type // .(*FuncType) underneath @@ -160,7 +161,7 @@ type imethod struct { // InterfaceType represents an interface type. type InterfaceType struct { commonType - methods []imethod // sorted by hash + methods []_imethod // sorted by hash } // MapType represents a map type. diff --git a/libgo/go/scanner/scanner.go b/libgo/go/scanner/scanner.go index 11aa9f43f33..2396cdd9a19 100644 --- a/libgo/go/scanner/scanner.go +++ b/libgo/go/scanner/scanner.go @@ -34,13 +34,15 @@ import ( ) +// TODO(gri): Consider changing this to use the new (token) Position package. + // A source position is represented by a Position value. // A position is valid if Line > 0. type Position struct { Filename string // filename, if any Offset int // byte offset, starting at 0 Line int // line number, starting at 1 - Column int // column number, starting at 0 (character count per line) + Column int // column number, starting at 1 (character count per line) } @@ -136,15 +138,17 @@ type Scanner struct { // Source position srcBufOffset int // byte offset of srcBuf[0] in source - line int // newline count + 1 - column int // character count on line + line int // line count + column int // character count + lastLineLen int // length of last line in characters (for correct column reporting) + lastCharLen int // length of last character in bytes // Token text buffer // Typically, token text is stored completely in srcBuf, but in general // the token text's head may be buffered in tokBuf while the token text's // tail is stored in srcBuf. tokBuf bytes.Buffer // token text head that is not in srcBuf anymore - tokPos int // token text tail position (srcBuf index) + tokPos int // token text tail position (srcBuf index); valid if >= 0 tokEnd int // token text tail end (srcBuf index) // One character look-ahead @@ -175,13 +179,14 @@ type Scanner struct { } -// Init initializes a Scanner with a new source and returns itself. +// Init initializes a Scanner with a new source and returns s. // Error is set to nil, ErrorCount is set to 0, Mode is set to GoTokens, // and Whitespace is set to GoWhitespace. func (s *Scanner) Init(src io.Reader) *Scanner { s.src = src // initialize source buffer + // (the first call to next() will fill it by calling src.Read) s.srcBuf[0] = utf8.RuneSelf // sentinel s.srcPos = 0 s.srcEnd = 0 @@ -190,12 +195,15 @@ func (s *Scanner) Init(src io.Reader) *Scanner { s.srcBufOffset = 0 s.line = 1 s.column = 0 + s.lastLineLen = 0 + s.lastCharLen = 0 // initialize token text buffer + // (required for first call to next()). s.tokPos = -1 // initialize one character look-ahead - s.ch = s.next() + s.ch = -1 // no char read yet // initialize public fields s.Error = nil @@ -207,12 +215,17 @@ func (s *Scanner) Init(src io.Reader) *Scanner { } +// TODO(gri): The code for next() and the internal scanner state could benefit +// from a rethink. While next() is optimized for the common ASCII +// case, the "corrections" needed for proper position tracking undo +// some of the attempts for fast-path optimization. + // next reads and returns the next Unicode character. It is designed such // that only a minimal amount of work needs to be done in the common ASCII // case (one test to check for both ASCII and end-of-buffer, and one test // to check for newlines). func (s *Scanner) next() int { - ch := int(s.srcBuf[s.srcPos]) + ch, width := int(s.srcBuf[s.srcPos]), 1 if ch >= utf8.RuneSelf { // uncommon case: not ASCII or not enough bytes @@ -222,47 +235,64 @@ func (s *Scanner) next() int { if s.tokPos >= 0 { s.tokBuf.Write(s.srcBuf[s.tokPos:s.srcPos]) s.tokPos = 0 + // s.tokEnd is set by Scan() } // move unread bytes to beginning of buffer copy(s.srcBuf[0:], s.srcBuf[s.srcPos:s.srcEnd]) s.srcBufOffset += s.srcPos // read more bytes + // (an io.Reader must return os.EOF when it reaches + // the end of what it is reading - simply returning + // n == 0 will make this loop retry forever; but the + // error is in the reader implementation in that case) i := s.srcEnd - s.srcPos n, err := s.src.Read(s.srcBuf[i:bufLen]) - s.srcEnd = i + n s.srcPos = 0 + s.srcEnd = i + n s.srcBuf[s.srcEnd] = utf8.RuneSelf // sentinel if err != nil { if s.srcEnd == 0 { + if s.lastCharLen > 0 { + // previous character was not EOF + s.column++ + } + s.lastCharLen = 0 return EOF } if err != os.EOF { s.error(err.String()) - break } + // If err == EOF, we won't be getting more + // bytes; break to avoid infinite loop. If + // err is something else, we don't know if + // we can get more bytes; thus also break. + break } } // at least one byte ch = int(s.srcBuf[s.srcPos]) if ch >= utf8.RuneSelf { // uncommon case: not ASCII - var width int ch, width = utf8.DecodeRune(s.srcBuf[s.srcPos:s.srcEnd]) if ch == utf8.RuneError && width == 1 { s.error("illegal UTF-8 encoding") } - s.srcPos += width - 1 } } - s.srcPos++ + // advance + s.srcPos += width + s.lastCharLen = width s.column++ + + // special situations switch ch { case 0: // implementation restriction for compatibility with other tools s.error("illegal character NUL") case '\n': s.line++ + s.lastLineLen = s.column s.column = 0 } @@ -272,13 +302,13 @@ func (s *Scanner) next() int { // Next reads and returns the next Unicode character. // It returns EOF at the end of the source. It reports -// a read error by calling s.Error, if set, or else -// prints an error message to os.Stderr. Next does not +// a read error by calling s.Error, if not nil; otherwise +// it prints an error message to os.Stderr. Next does not // update the Scanner's Position field; use Pos() to // get the current position. func (s *Scanner) Next() int { s.tokPos = -1 // don't collect token text - ch := s.ch + ch := s.Peek() s.ch = s.next() return ch } @@ -288,6 +318,9 @@ func (s *Scanner) Next() int { // the scanner. It returns EOF if the scanner's position is at the last // character of the source. func (s *Scanner) Peek() int { + if s.ch < 0 { + s.ch = s.next() + } return s.ch } @@ -511,10 +544,10 @@ func (s *Scanner) scanComment(ch int) { // Scan reads the next token or Unicode character from source and returns it. // It only recognizes tokens t for which the respective Mode bit (1<<-t) is set. // It returns EOF at the end of the source. It reports scanner errors (read and -// token errors) by calling s.Error, if set; otherwise it prints an error message -// to os.Stderr. +// token errors) by calling s.Error, if not nil; otherwise it prints an error +// message to os.Stderr. func (s *Scanner) Scan() int { - ch := s.ch + ch := s.Peek() // reset token text position s.tokPos = -1 @@ -527,12 +560,22 @@ redo: // start collecting token text s.tokBuf.Reset() - s.tokPos = s.srcPos - 1 + s.tokPos = s.srcPos - s.lastCharLen // set token position + // (this is a slightly optimized version of the code in Pos()) s.Offset = s.srcBufOffset + s.tokPos - s.Line = s.line - s.Column = s.column + if s.column > 0 { + // common case: last character was not a '\n' + s.Line = s.line + s.Column = s.column + } else { + // last character was a '\n' + // (we cannot be at the beginning of the source + // since we have called next() at least once) + s.Line = s.line - 1 + s.Column = s.lastLineLen + } // determine token value tok := ch @@ -596,25 +639,33 @@ redo: } // end of token text - s.tokEnd = s.srcPos - 1 + s.tokEnd = s.srcPos - s.lastCharLen s.ch = ch return tok } -// Position returns the current source position. If called before Next() -// or Scan(), it returns the position of the next Unicode character or token -// returned by these functions. If called afterwards, it returns the position -// immediately after the last character of the most recent token or character -// scanned. -func (s *Scanner) Pos() Position { - return Position{ - s.Filename, - s.srcBufOffset + s.srcPos - 1, - s.line, - s.column, +// Pos returns the position of the character immediately after +// the character or token returned by the last call to Next or Scan. +func (s *Scanner) Pos() (pos Position) { + pos.Filename = s.Filename + pos.Offset = s.srcBufOffset + s.srcPos - s.lastCharLen + switch { + case s.column > 0: + // common case: last character was not a '\n' + pos.Line = s.line + pos.Column = s.column + case s.lastLineLen > 0: + // last character was a '\n' + pos.Line = s.line - 1 + pos.Column = s.lastLineLen + default: + // at the beginning of the source + pos.Line = 1 + pos.Column = 1 } + return } diff --git a/libgo/go/scanner/scanner_test.go b/libgo/go/scanner/scanner_test.go index 506f434fe7b..002252de8a8 100644 --- a/libgo/go/scanner/scanner_test.go +++ b/libgo/go/scanner/scanner_test.go @@ -10,6 +10,7 @@ import ( "os" "strings" "testing" + "utf8" ) @@ -408,7 +409,7 @@ func TestScanWhitespace(t *testing.T) { func testError(t *testing.T, src, msg string, tok int) { s := new(Scanner).Init(bytes.NewBufferString(src)) errorCalled := false - s.Error = func(s *Scanner, m string) { + s.Error = func(_ *Scanner, m string) { if !errorCalled { // only look at first error if m != msg { @@ -431,6 +432,8 @@ func testError(t *testing.T, src, msg string, tok int) { func TestError(t *testing.T) { + testError(t, "\x00", "illegal character NUL", 0) + testError(t, "\xff", "illegal UTF-8 encoding", utf8.RuneError) testError(t, `01238`, "illegal octal number", Int) testError(t, `'\"'`, "illegal char escape", Char) testError(t, `'aa'`, "illegal char literal", Char) @@ -445,38 +448,99 @@ func TestError(t *testing.T) { } -func checkPos(t *testing.T, s *Scanner, offset, line, column, char int) { - pos := s.Pos() - if pos.Offset != offset { - t.Errorf("offset = %d, want %d", pos.Offset, offset) +func checkPos(t *testing.T, got, want Position) { + if got.Offset != want.Offset || got.Line != want.Line || got.Column != want.Column { + t.Errorf("got offset, line, column = %d, %d, %d; want %d, %d, %d", + got.Offset, got.Line, got.Column, want.Offset, want.Line, want.Column) } - if pos.Line != line { - t.Errorf("line = %d, want %d", pos.Line, line) - } - if pos.Column != column { - t.Errorf("column = %d, want %d", pos.Column, column) +} + + +func checkNextPos(t *testing.T, s *Scanner, offset, line, column, char int) { + if ch := s.Next(); ch != char { + t.Errorf("ch = %s, want %s", TokenString(ch), TokenString(char)) } - ch := s.Scan() - if ch != char { + want := Position{Offset: offset, Line: line, Column: column} + checkPos(t, s.Pos(), want) +} + + +func checkScanPos(t *testing.T, s *Scanner, offset, line, column, char int) { + want := Position{Offset: offset, Line: line, Column: column} + checkPos(t, s.Pos(), want) + if ch := s.Scan(); ch != char { t.Errorf("ch = %s, want %s", TokenString(ch), TokenString(char)) + if string(ch) != s.TokenText() { + t.Errorf("tok = %q, want %q", s.TokenText(), string(ch)) + } } + checkPos(t, s.Position, want) } func TestPos(t *testing.T) { - s := new(Scanner).Init(bytes.NewBufferString("abc\n012\n\nx")) + // corner case: empty source + s := new(Scanner).Init(bytes.NewBufferString("")) + checkPos(t, s.Pos(), Position{Offset: 0, Line: 1, Column: 1}) + s.Peek() // peek doesn't affect the position + checkPos(t, s.Pos(), Position{Offset: 0, Line: 1, Column: 1}) + + // corner case: source with only a newline + s = new(Scanner).Init(bytes.NewBufferString("\n")) + checkPos(t, s.Pos(), Position{Offset: 0, Line: 1, Column: 1}) + checkNextPos(t, s, 1, 2, 1, '\n') + // after EOF position doesn't change + for i := 10; i > 0; i-- { + checkScanPos(t, s, 1, 2, 1, EOF) + } + + // corner case: source with only a single character + s = new(Scanner).Init(bytes.NewBufferString("本")) + checkPos(t, s.Pos(), Position{Offset: 0, Line: 1, Column: 1}) + checkNextPos(t, s, 3, 1, 2, '本') + // after EOF position doesn't change + for i := 10; i > 0; i-- { + checkScanPos(t, s, 3, 1, 2, EOF) + } + + // positions after calling Next + s = new(Scanner).Init(bytes.NewBufferString(" foo६४ \n\n本語\n")) + checkNextPos(t, s, 1, 1, 2, ' ') + s.Peek() // peek doesn't affect the position + checkNextPos(t, s, 2, 1, 3, ' ') + checkNextPos(t, s, 3, 1, 4, 'f') + checkNextPos(t, s, 4, 1, 5, 'o') + checkNextPos(t, s, 5, 1, 6, 'o') + checkNextPos(t, s, 8, 1, 7, '६') + checkNextPos(t, s, 11, 1, 8, '४') + checkNextPos(t, s, 12, 1, 9, ' ') + checkNextPos(t, s, 13, 1, 10, ' ') + checkNextPos(t, s, 14, 2, 1, '\n') + checkNextPos(t, s, 15, 3, 1, '\n') + checkNextPos(t, s, 18, 3, 2, '本') + checkNextPos(t, s, 21, 3, 3, '語') + checkNextPos(t, s, 22, 4, 1, '\n') + // after EOF position doesn't change + for i := 10; i > 0; i-- { + checkScanPos(t, s, 22, 4, 1, EOF) + } + + // positions after calling Scan + s = new(Scanner).Init(bytes.NewBufferString("abc\n本語\n\nx")) s.Mode = 0 s.Whitespace = 0 - checkPos(t, s, 0, 1, 1, 'a') - checkPos(t, s, 1, 1, 2, 'b') - checkPos(t, s, 2, 1, 3, 'c') - checkPos(t, s, 3, 2, 0, '\n') - checkPos(t, s, 4, 2, 1, '0') - checkPos(t, s, 5, 2, 2, '1') - checkPos(t, s, 6, 2, 3, '2') - checkPos(t, s, 7, 3, 0, '\n') - checkPos(t, s, 8, 4, 0, '\n') - checkPos(t, s, 9, 4, 1, 'x') - checkPos(t, s, 9, 4, 1, EOF) - checkPos(t, s, 9, 4, 1, EOF) // after EOF, position doesn't change + checkScanPos(t, s, 0, 1, 1, 'a') + s.Peek() // peek doesn't affect the position + checkScanPos(t, s, 1, 1, 2, 'b') + checkScanPos(t, s, 2, 1, 3, 'c') + checkScanPos(t, s, 3, 1, 4, '\n') + checkScanPos(t, s, 4, 2, 1, '本') + checkScanPos(t, s, 7, 2, 2, '語') + checkScanPos(t, s, 10, 2, 3, '\n') + checkScanPos(t, s, 11, 3, 1, '\n') + checkScanPos(t, s, 12, 4, 1, 'x') + // after EOF position doesn't change + for i := 10; i > 0; i-- { + checkScanPos(t, s, 13, 4, 2, EOF) + } } diff --git a/libgo/go/strconv/atof_test.go b/libgo/go/strconv/atof_test.go index 6cc60e549da..6d8396ee73b 100644 --- a/libgo/go/strconv/atof_test.go +++ b/libgo/go/strconv/atof_test.go @@ -103,6 +103,11 @@ var atoftests = []atofTest{ {"1e", "0", os.EINVAL}, {"1e-", "0", os.EINVAL}, {".e-1", "0", os.EINVAL}, + + // http://www.exploringbinary.com/java-hangs-when-converting-2-2250738585072012e-308/ + {"2.2250738585072012e-308", "2.2250738585072014e-308", nil}, + // http://www.exploringbinary.com/php-hangs-on-numeric-value-2-2250738585072011e-308/ + {"2.2250738585072011e-308", "2.225073858507201e-308", nil}, } func init() { diff --git a/libgo/go/strconv/ftoa.go b/libgo/go/strconv/ftoa.go index 4ec3cdbb974..b6049c54589 100644 --- a/libgo/go/strconv/ftoa.go +++ b/libgo/go/strconv/ftoa.go @@ -64,7 +64,7 @@ func FtoaN(f float64, fmt byte, prec int, n int) string { } func genericFtoa(bits uint64, fmt byte, prec int, flt *floatInfo) string { - neg := bits>>flt.expbits>>flt.mantbits != 0 + neg := bits>>(flt.expbits+flt.mantbits) != 0 exp := int(bits>>flt.mantbits) & (1<<flt.expbits - 1) mant := bits & (uint64(1)<<flt.mantbits - 1) diff --git a/libgo/go/strconv/ftoa_test.go b/libgo/go/strconv/ftoa_test.go index 3a862a2beeb..6d361a138ef 100644 --- a/libgo/go/strconv/ftoa_test.go +++ b/libgo/go/strconv/ftoa_test.go @@ -118,6 +118,11 @@ var ftoatests = []ftoaTest{ {0.5, 'f', 1, "0.5"}, {0.5, 'f', 0, "0"}, {1.5, 'f', 0, "2"}, + + // http://www.exploringbinary.com/java-hangs-when-converting-2-2250738585072012e-308/ + {2.2250738585072012e-308, 'g', -1, "2.2250738585072014e-308"}, + // http://www.exploringbinary.com/php-hangs-on-numeric-value-2-2250738585072011e-308/ + {2.2250738585072011e-308, 'g', -1, "2.225073858507201e-308"}, } func TestFtoa(t *testing.T) { diff --git a/libgo/go/sync/atomic/atomic.c b/libgo/go/sync/atomic/atomic.c new file mode 100644 index 00000000000..e2d9b242fbd --- /dev/null +++ b/libgo/go/sync/atomic/atomic.c @@ -0,0 +1,97 @@ +/* atomic.c -- implement atomic routines for Go. + + Copyright 2011 The Go Authors. All rights reserved. + Use of this source code is governed by a BSD-style + license that can be found in the LICENSE file. */ + +#include <stdint.h> + +_Bool CompareAndSwapInt32 (int32_t *, int32_t, int32_t) + asm ("libgo_sync.atomic.CompareAndSwapInt32"); + +_Bool +CompareAndSwapInt32 (int32_t *val, int32_t old, int32_t new) +{ + return __sync_bool_compare_and_swap (val, old, new); +} + +_Bool CompareAndSwapInt64 (int64_t *, int64_t, int64_t) + asm ("libgo_sync.atomic.CompareAndSwapInt64"); + +_Bool +CompareAndSwapInt64 (int64_t *val, int64_t old, int64_t new) +{ + return __sync_bool_compare_and_swap (val, old, new); +} + +_Bool CompareAndSwapUint32 (uint32_t *, uint32_t, uint32_t) + asm ("libgo_sync.atomic.CompareAndSwapUint32"); + +_Bool +CompareAndSwapUint32 (uint32_t *val, uint32_t old, uint32_t new) +{ + return __sync_bool_compare_and_swap (val, old, new); +} + +_Bool CompareAndSwapUint64 (uint64_t *, uint64_t, uint64_t) + asm ("libgo_sync.atomic.CompareAndSwapUint64"); + +_Bool +CompareAndSwapUint64 (uint64_t *val, uint64_t old, uint64_t new) +{ + return __sync_bool_compare_and_swap (val, old, new); +} + +_Bool CompareAndSwapUintptr (uintptr_t *, uintptr_t, uintptr_t) + asm ("libgo_sync.atomic.CompareAndSwapUintptr"); + +_Bool +CompareAndSwapUintptr (uintptr_t *val, uintptr_t old, uintptr_t new) +{ + return __sync_bool_compare_and_swap (val, old, new); +} + +int32_t AddInt32 (int32_t *, int32_t) + asm ("libgo_sync.atomic.AddInt32"); + +int32_t +AddInt32 (int32_t *val, int32_t delta) +{ + return __sync_add_and_fetch (val, delta); +} + +uint32_t AddUint32 (uint32_t *, uint32_t) + asm ("libgo_sync.atomic.AddUint32"); + +uint32_t +AddUint32 (uint32_t *val, uint32_t delta) +{ + return __sync_add_and_fetch (val, delta); +} + +int64_t AddInt64 (int64_t *, int64_t) + asm ("libgo_sync.atomic.AddInt64"); + +int64_t +AddInt64 (int64_t *val, int64_t delta) +{ + return __sync_add_and_fetch (val, delta); +} + +uint64_t AddUint64 (uint64_t *, uint64_t) + asm ("libgo_sync.atomic.AddUint64"); + +uint64_t +AddUint64 (uint64_t *val, uint64_t delta) +{ + return __sync_add_and_fetch (val, delta); +} + +uintptr_t AddUintptr (uintptr_t *, uintptr_t) + asm ("libgo_sync.atomic.AddUintptr"); + +uintptr_t +AddUintptr (uintptr_t *val, uintptr_t delta) +{ + return __sync_add_and_fetch (val, delta); +} diff --git a/libgo/go/sync/atomic/atomic_test.go b/libgo/go/sync/atomic/atomic_test.go new file mode 100644 index 00000000000..7b204b1d9f0 --- /dev/null +++ b/libgo/go/sync/atomic/atomic_test.go @@ -0,0 +1,506 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package atomic + +import ( + "runtime" + "testing" + "unsafe" +) + +// Tests of correct behavior, without contention. +// (Does the function work as advertised?) +// +// Test that the Add functions add correctly. +// Test that the CompareAndSwap functions actually +// do the comparison and the swap correctly. +// +// The loop over power-of-two values is meant to +// ensure that the operations apply to the full word size. +// The struct fields x.before and x.after check that the +// operations do not extend past the full word size. + +const ( + magic32 = 0xdedbeef + magic64 = 0xdeddeadbeefbeef +) + +func TestAddInt32(t *testing.T) { + var x struct { + before int32 + i int32 + after int32 + } + x.before = magic32 + x.after = magic32 + var j int32 + for delta := int32(1); delta+delta > delta; delta += delta { + k := AddInt32(&x.i, delta) + j += delta + if x.i != j || k != j { + t.Fatalf("delta=%d i=%d j=%d k=%d", delta, x.i, j, k) + } + } + if x.before != magic32 || x.after != magic32 { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magic32, magic32) + } +} + +func TestAddUint32(t *testing.T) { + var x struct { + before uint32 + i uint32 + after uint32 + } + x.before = magic32 + x.after = magic32 + var j uint32 + for delta := uint32(1); delta+delta > delta; delta += delta { + k := AddUint32(&x.i, delta) + j += delta + if x.i != j || k != j { + t.Fatalf("delta=%d i=%d j=%d k=%d", delta, x.i, j, k) + } + } + if x.before != magic32 || x.after != magic32 { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magic32, magic32) + } +} + +func TestAddInt64(t *testing.T) { + var x struct { + before int64 + i int64 + after int64 + } + x.before = magic64 + x.after = magic64 + var j int64 + for delta := int64(1); delta+delta > delta; delta += delta { + k := AddInt64(&x.i, delta) + j += delta + if x.i != j || k != j { + t.Fatalf("delta=%d i=%d j=%d k=%d", delta, x.i, j, k) + } + } + if x.before != magic64 || x.after != magic64 { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, int64(magic64), int64(magic64)) + } +} + +func TestAddUint64(t *testing.T) { + var x struct { + before uint64 + i uint64 + after uint64 + } + x.before = magic64 + x.after = magic64 + var j uint64 + for delta := uint64(1); delta+delta > delta; delta += delta { + k := AddUint64(&x.i, delta) + j += delta + if x.i != j || k != j { + t.Fatalf("delta=%d i=%d j=%d k=%d", delta, x.i, j, k) + } + } + if x.before != magic64 || x.after != magic64 { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, uint64(magic64), uint64(magic64)) + } +} + +func TestAddUintptr(t *testing.T) { + var x struct { + before uintptr + i uintptr + after uintptr + } + var m uint64 = magic64 + magicptr := uintptr(m) + x.before = magicptr + x.after = magicptr + var j uintptr + for delta := uintptr(1); delta+delta > delta; delta += delta { + k := AddUintptr(&x.i, delta) + j += delta + if x.i != j || k != j { + t.Fatalf("delta=%d i=%d j=%d k=%d", delta, x.i, j, k) + } + } + if x.before != magicptr || x.after != magicptr { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magicptr, magicptr) + } +} + +func TestCompareAndSwapInt32(t *testing.T) { + var x struct { + before int32 + i int32 + after int32 + } + x.before = magic32 + x.after = magic32 + for val := int32(1); val+val > val; val += val { + x.i = val + if !CompareAndSwapInt32(&x.i, val, val+1) { + t.Errorf("should have swapped %#x %#x", val, val+1) + } + if x.i != val+1 { + t.Errorf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1) + } + x.i = val + 1 + if CompareAndSwapInt32(&x.i, val, val+2) { + t.Errorf("should not have swapped %#x %#x", val, val+2) + } + if x.i != val+1 { + t.Errorf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1) + } + } + if x.before != magic32 || x.after != magic32 { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magic32, magic32) + } +} + +func TestCompareAndSwapUint32(t *testing.T) { + var x struct { + before uint32 + i uint32 + after uint32 + } + x.before = magic32 + x.after = magic32 + for val := uint32(1); val+val > val; val += val { + x.i = val + if !CompareAndSwapUint32(&x.i, val, val+1) { + t.Errorf("should have swapped %#x %#x", val, val+1) + } + if x.i != val+1 { + t.Errorf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1) + } + x.i = val + 1 + if CompareAndSwapUint32(&x.i, val, val+2) { + t.Errorf("should not have swapped %#x %#x", val, val+2) + } + if x.i != val+1 { + t.Errorf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1) + } + } + if x.before != magic32 || x.after != magic32 { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magic32, magic32) + } +} + +func TestCompareAndSwapInt64(t *testing.T) { + var x struct { + before int64 + i int64 + after int64 + } + x.before = magic64 + x.after = magic64 + for val := int64(1); val+val > val; val += val { + x.i = val + if !CompareAndSwapInt64(&x.i, val, val+1) { + t.Errorf("should have swapped %#x %#x", val, val+1) + } + if x.i != val+1 { + t.Errorf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1) + } + x.i = val + 1 + if CompareAndSwapInt64(&x.i, val, val+2) { + t.Errorf("should not have swapped %#x %#x", val, val+2) + } + if x.i != val+1 { + t.Errorf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1) + } + } + if x.before != magic64 || x.after != magic64 { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, uint64(magic64), uint64(magic64)) + } +} + +func TestCompareAndSwapUint64(t *testing.T) { + var x struct { + before uint64 + i uint64 + after uint64 + } + x.before = magic64 + x.after = magic64 + for val := uint64(1); val+val > val; val += val { + x.i = val + if !CompareAndSwapUint64(&x.i, val, val+1) { + t.Errorf("should have swapped %#x %#x", val, val+1) + } + if x.i != val+1 { + t.Errorf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1) + } + x.i = val + 1 + if CompareAndSwapUint64(&x.i, val, val+2) { + t.Errorf("should not have swapped %#x %#x", val, val+2) + } + if x.i != val+1 { + t.Errorf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1) + } + } + if x.before != magic64 || x.after != magic64 { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, uint64(magic64), uint64(magic64)) + } +} + +func TestCompareAndSwapUintptr(t *testing.T) { + var x struct { + before uintptr + i uintptr + after uintptr + } + var m uint64 = magic64 + magicptr := uintptr(m) + x.before = magicptr + x.after = magicptr + for val := uintptr(1); val+val > val; val += val { + x.i = val + if !CompareAndSwapUintptr(&x.i, val, val+1) { + t.Errorf("should have swapped %#x %#x", val, val+1) + } + if x.i != val+1 { + t.Errorf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1) + } + x.i = val + 1 + if CompareAndSwapUintptr(&x.i, val, val+2) { + t.Errorf("should not have swapped %#x %#x", val, val+2) + } + if x.i != val+1 { + t.Errorf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i, val+1) + } + } + if x.before != magicptr || x.after != magicptr { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magicptr, magicptr) + } +} + +// Tests of correct behavior, with contention. +// (Is the function atomic?) +// +// For each function, we write a "hammer" function that repeatedly +// uses the atomic operation to add 1 to a value. After running +// multiple hammers in parallel, check that we end with the correct +// total. + +var hammer32 = []struct { + name string + f func(*uint32, int) +}{ + {"AddInt32", hammerAddInt32}, + {"AddUint32", hammerAddUint32}, + {"AddUintptr", hammerAddUintptr32}, + {"CompareAndSwapInt32", hammerCompareAndSwapInt32}, + {"CompareAndSwapUint32", hammerCompareAndSwapUint32}, + {"CompareAndSwapUintptr", hammerCompareAndSwapUintptr32}, +} + +func init() { + var v uint64 = 1 << 50 + if uintptr(v) != 0 { + // 64-bit system; clear uintptr tests + hammer32[2].f = nil + hammer32[5].f = nil + } +} + +func hammerAddInt32(uval *uint32, count int) { + val := (*int32)(unsafe.Pointer(uval)) + for i := 0; i < count; i++ { + AddInt32(val, 1) + } +} + +func hammerAddUint32(val *uint32, count int) { + for i := 0; i < count; i++ { + AddUint32(val, 1) + } +} + +func hammerAddUintptr32(uval *uint32, count int) { + // only safe when uintptr is 32-bit. + // not called on 64-bit systems. + val := (*uintptr)(unsafe.Pointer(uval)) + for i := 0; i < count; i++ { + AddUintptr(val, 1) + } +} + +func hammerCompareAndSwapInt32(uval *uint32, count int) { + val := (*int32)(unsafe.Pointer(uval)) + for i := 0; i < count; i++ { + for { + v := *val + if CompareAndSwapInt32(val, v, v+1) { + break + } + } + } +} + +func hammerCompareAndSwapUint32(val *uint32, count int) { + for i := 0; i < count; i++ { + for { + v := *val + if CompareAndSwapUint32(val, v, v+1) { + break + } + } + } +} + +func hammerCompareAndSwapUintptr32(uval *uint32, count int) { + // only safe when uintptr is 32-bit. + // not called on 64-bit systems. + val := (*uintptr)(unsafe.Pointer(uval)) + for i := 0; i < count; i++ { + for { + v := *val + if CompareAndSwapUintptr(val, v, v+1) { + break + } + } + } +} + +func TestHammer32(t *testing.T) { + const ( + n = 100000 + p = 4 + ) + defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(p)) + + for _, tt := range hammer32 { + if tt.f == nil { + continue + } + c := make(chan int) + var val uint32 + for i := 0; i < p; i++ { + go func() { + tt.f(&val, n) + c <- 1 + }() + } + for i := 0; i < p; i++ { + <-c + } + if val != n*p { + t.Errorf("%s: val=%d want %d", tt.name, val, n*p) + } + } +} + +var hammer64 = []struct { + name string + f func(*uint64, int) +}{ + {"AddInt64", hammerAddInt64}, + {"AddUint64", hammerAddUint64}, + {"AddUintptr", hammerAddUintptr64}, + {"CompareAndSwapInt64", hammerCompareAndSwapInt64}, + {"CompareAndSwapUint64", hammerCompareAndSwapUint64}, + {"CompareAndSwapUintptr", hammerCompareAndSwapUintptr64}, +} + +func init() { + var v uint64 = 1 << 50 + if uintptr(v) == 0 { + // 32-bit system; clear uintptr tests + hammer64[2].f = nil + hammer64[5].f = nil + } +} + +func hammerAddInt64(uval *uint64, count int) { + val := (*int64)(unsafe.Pointer(uval)) + for i := 0; i < count; i++ { + AddInt64(val, 1) + } +} + +func hammerAddUint64(val *uint64, count int) { + for i := 0; i < count; i++ { + AddUint64(val, 1) + } +} + +func hammerAddUintptr64(uval *uint64, count int) { + // only safe when uintptr is 64-bit. + // not called on 32-bit systems. + val := (*uintptr)(unsafe.Pointer(uval)) + for i := 0; i < count; i++ { + AddUintptr(val, 1) + } +} + +func hammerCompareAndSwapInt64(uval *uint64, count int) { + val := (*int64)(unsafe.Pointer(uval)) + for i := 0; i < count; i++ { + for { + v := *val + if CompareAndSwapInt64(val, v, v+1) { + break + } + } + } +} + +func hammerCompareAndSwapUint64(val *uint64, count int) { + for i := 0; i < count; i++ { + for { + v := *val + if CompareAndSwapUint64(val, v, v+1) { + break + } + } + } +} + +func hammerCompareAndSwapUintptr64(uval *uint64, count int) { + // only safe when uintptr is 64-bit. + // not called on 32-bit systems. + val := (*uintptr)(unsafe.Pointer(uval)) + for i := 0; i < count; i++ { + for { + v := *val + if CompareAndSwapUintptr(val, v, v+1) { + break + } + } + } +} + +func TestHammer64(t *testing.T) { + const ( + n = 100000 + p = 4 + ) + defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(p)) + + for _, tt := range hammer64 { + if tt.f == nil { + continue + } + c := make(chan int) + var val uint64 + for i := 0; i < p; i++ { + go func() { + tt.f(&val, n) + c <- 1 + }() + } + for i := 0; i < p; i++ { + <-c + } + if val != n*p { + t.Errorf("%s: val=%d want %d", tt.name, val, n*p) + } + } +} diff --git a/libgo/go/sync/atomic/doc.go b/libgo/go/sync/atomic/doc.go new file mode 100644 index 00000000000..1335def59f6 --- /dev/null +++ b/libgo/go/sync/atomic/doc.go @@ -0,0 +1,57 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package atomic provides low-level atomic memory primitives +// useful for implementing synchronization algorithms. +// +// These functions require great care to be used correctly. +// Except for special, low-level applications, synchronization is better +// done with channels or the facilities of the sync package. +// Share memory by communicating; +// don't communicate by sharing memory. +// +// The compare-and-swap operation, implemented by the CompareAndSwapT +// functions, is the atomic equivalent of: +// +// if *val == old { +// *val = new +// return true +// } +// return false +// +package atomic + +// BUG(rsc): On ARM, the 64-bit functions use instructions unavailable before ARM 11. +// +// On x86-32, the 64-bit functions use instructions unavailable before the Pentium. + +// CompareAndSwapInt32 executes the compare-and-swap operation for an int32 value. +func CompareAndSwapInt32(val *int32, old, new int32) (swapped bool) + +// CompareAndSwapInt64 executes the compare-and-swap operation for an int64 value. +func CompareAndSwapInt64(val *int64, old, new int64) (swapped bool) + +// CompareAndSwapUint32 executes the compare-and-swap operation for a uint32 value. +func CompareAndSwapUint32(val *uint32, old, new uint32) (swapped bool) + +// CompareAndSwapUint64 executes the compare-and-swap operation for a uint64 value. +func CompareAndSwapUint64(val *uint64, old, new uint64) (swapped bool) + +// CompareAndSwapUintptr executes the compare-and-swap operation for a uintptr value. +func CompareAndSwapUintptr(val *uintptr, old, new uintptr) (swapped bool) + +// AddInt32 atomically adds delta to *val and returns the new value. +func AddInt32(val *int32, delta int32) (new int32) + +// AddUint32 atomically adds delta to *val and returns the new value. +func AddUint32(val *uint32, delta uint32) (new uint32) + +// AddInt64 atomically adds delta to *val and returns the new value. +func AddInt64(val *int64, delta int64) (new int64) + +// AddUint64 atomically adds delta to *val and returns the new value. +func AddUint64(val *uint64, delta uint64) (new uint64) + +// AddUintptr atomically adds delta to *val and returns the new value. +func AddUintptr(val *uintptr, delta uintptr) (new uintptr) diff --git a/libgo/go/sync/cond.go b/libgo/go/sync/cond.go new file mode 100644 index 00000000000..ea48f2e7a98 --- /dev/null +++ b/libgo/go/sync/cond.go @@ -0,0 +1,90 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sync + +import "runtime" + +// Cond implements a condition variable, a rendezvous point +// for goroutines waiting for or announcing the occurrence +// of an event. +// +// Each Cond has an associated Locker L (often a *Mutex or *RWMutex), +// which must be held when changing the condition and +// when calling the Wait method. +type Cond struct { + L Locker // held while observing or changing the condition + m Mutex // held to avoid internal races + waiters int // number of goroutines blocked on Wait + sema *uint32 +} + +// NewCond returns a new Cond with Locker l. +func NewCond(l Locker) *Cond { + return &Cond{L: l} +} + +// Wait atomically unlocks c.L and suspends execution +// of the calling goroutine. After later resuming execution, +// Wait locks c.L before returning. +// +// Because L is not locked when Wait first resumes, the caller +// typically cannot assume that the condition is true when +// Wait returns. Instead, the caller should Wait in a loop: +// +// c.L.Lock() +// for !condition() { +// c.Wait() +// } +// ... make use of condition ... +// c.L.Unlock() +// +func (c *Cond) Wait() { + c.m.Lock() + if c.sema == nil { + c.sema = new(uint32) + } + s := c.sema + c.waiters++ + c.m.Unlock() + c.L.Unlock() + runtime.Semacquire(s) + c.L.Lock() +} + +// Signal wakes one goroutine waiting on c, if there is any. +// +// It is allowed but not required for the caller to hold c.L +// during the call. +func (c *Cond) Signal() { + c.m.Lock() + if c.waiters > 0 { + c.waiters-- + runtime.Semrelease(c.sema) + } + c.m.Unlock() +} + +// Broadcast wakes all goroutines waiting on c. +// +// It is allowed but not required for the caller to hold c.L +// during the call. +func (c *Cond) Broadcast() { + c.m.Lock() + if c.waiters > 0 { + s := c.sema + n := c.waiters + for i := 0; i < n; i++ { + runtime.Semrelease(s) + } + // We just issued n wakeups via the semaphore s. + // To ensure that they wake up the existing waiters + // and not waiters that arrive after Broadcast returns, + // clear c.sema. The next operation will allocate + // a new one. + c.sema = nil + c.waiters = 0 + } + c.m.Unlock() +} diff --git a/libgo/go/sync/cond_test.go b/libgo/go/sync/cond_test.go new file mode 100644 index 00000000000..846f98bf39d --- /dev/null +++ b/libgo/go/sync/cond_test.go @@ -0,0 +1,99 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +package sync_test + +import ( + . "sync" + "testing" +) + +func TestCondSignal(t *testing.T) { + var m Mutex + c := NewCond(&m) + n := 2 + running := make(chan bool, n) + awake := make(chan bool, n) + for i := 0; i < n; i++ { + go func() { + m.Lock() + running <- true + c.Wait() + awake <- true + m.Unlock() + }() + } + for i := 0; i < n; i++ { + <-running // Wait for everyone to run. + } + for n > 0 { + select { + case <-awake: + t.Fatal("goroutine not asleep") + default: + } + m.Lock() + c.Signal() + m.Unlock() + <-awake // Will deadlock if no goroutine wakes up + select { + case <-awake: + t.Fatal("too many goroutines awake") + default: + } + n-- + } + c.Signal() +} + +func TestCondBroadcast(t *testing.T) { + var m Mutex + c := NewCond(&m) + n := 200 + running := make(chan int, n) + awake := make(chan int, n) + exit := false + for i := 0; i < n; i++ { + go func(g int) { + m.Lock() + for !exit { + running <- g + c.Wait() + awake <- g + } + m.Unlock() + }(i) + } + for i := 0; i < n; i++ { + for i := 0; i < n; i++ { + <-running // Will deadlock unless n are running. + } + if i == n-1 { + m.Lock() + exit = true + m.Unlock() + } + select { + case <-awake: + t.Fatal("goroutine not asleep") + default: + } + m.Lock() + c.Broadcast() + m.Unlock() + seen := make([]bool, n) + for i := 0; i < n; i++ { + g := <-awake + if seen[g] { + t.Fatal("goroutine woke up twice") + } + seen[g] = true + } + } + select { + case <-running: + t.Fatal("goroutine did not exit") + default: + } + c.Broadcast() +} diff --git a/libgo/go/sync/mutex.go b/libgo/go/sync/mutex.go index 9a2bb2bb4f5..da565d38def 100644 --- a/libgo/go/sync/mutex.go +++ b/libgo/go/sync/mutex.go @@ -3,43 +3,36 @@ // license that can be found in the LICENSE file. // The sync package provides basic synchronization primitives -// such as mutual exclusion locks. Other than the Once type, -// most are intended for use by low-level library routines. -// Higher-level synchronization is better done via channels -// and communication. +// such as mutual exclusion locks. Other than the Once and +// WaitGroup types, most are intended for use by low-level +// library routines. Higher-level synchronization is better +// done via channels and communication. package sync -import "runtime" - -func cas(val *uint32, old, new uint32) bool +import ( + "runtime" + "sync/atomic" +) // A Mutex is a mutual exclusion lock. // Mutexes can be created as part of other structures; // the zero value for a Mutex is an unlocked mutex. type Mutex struct { - key uint32 + key int32 sema uint32 } -// Add delta to *val, and return the new *val in a thread-safe way. If multiple -// goroutines call xadd on the same val concurrently, the changes will be -// serialized, and all the deltas will be added in an undefined order. -func xadd(val *uint32, delta int32) (new uint32) { - for { - v := *val - nv := v + uint32(delta) - if cas(val, v, nv) { - return nv - } - } - panic("unreached") +// A Locker represents an object that can be locked and unlocked. +type Locker interface { + Lock() + Unlock() } // Lock locks m. // If the lock is already in use, the calling goroutine // blocks until the mutex is available. func (m *Mutex) Lock() { - if xadd(&m.key, 1) == 1 { + if atomic.AddInt32(&m.key, 1) == 1 { // changed from 0 to 1; we hold lock return } @@ -53,9 +46,14 @@ func (m *Mutex) Lock() { // It is allowed for one goroutine to lock a Mutex and then // arrange for another goroutine to unlock it. func (m *Mutex) Unlock() { - if xadd(&m.key, -1) == 0 { + switch v := atomic.AddInt32(&m.key, -1); { + case v == 0: // changed from 1 to 0; no contention return + case v == -1: + // changed from 0 to -1: wasn't locked + // (or there are 4 billion goroutines waiting) + panic("sync: unlock of unlocked mutex") } runtime.Semrelease(&m.sema) } diff --git a/libgo/go/sync/mutex_test.go b/libgo/go/sync/mutex_test.go index d0e048ed7a4..f5c20ca49b4 100644 --- a/libgo/go/sync/mutex_test.go +++ b/libgo/go/sync/mutex_test.go @@ -89,3 +89,16 @@ func BenchmarkContendedMutex(b *testing.B) { <-c <-c } + +func TestMutexPanic(t *testing.T) { + defer func() { + if recover() == nil { + t.Fatalf("unlock of unlocked mutex did not panic") + } + }() + + var mu Mutex + mu.Lock() + mu.Unlock() + mu.Unlock() +} diff --git a/libgo/go/sync/once.go b/libgo/go/sync/once.go index 8c877cdec21..b6f5f5a8726 100644 --- a/libgo/go/sync/once.go +++ b/libgo/go/sync/once.go @@ -13,7 +13,7 @@ type Once struct { // Do calls the function f if and only if the method is being called for the // first time with this receiver. In other words, given // var once Once -// if Do(f) is called multiple times, only the first call will invoke f, +// if once.Do(f) is called multiple times, only the first call will invoke f, // even if f has a different value in each invocation. A new instance of // Once is required for each function to execute. // diff --git a/libgo/go/sync/rwmutex.go b/libgo/go/sync/rwmutex.go index 06fd0b0ffb2..9248b4b0370 100644 --- a/libgo/go/sync/rwmutex.go +++ b/libgo/go/sync/rwmutex.go @@ -4,6 +4,8 @@ package sync +import "sync/atomic" + // An RWMutex is a reader/writer mutual exclusion lock. // The lock can be held by an arbitrary number of readers // or a single writer. @@ -14,9 +16,9 @@ package sync // Writers take priority over Readers: no new RLocks // are granted while a blocked Lock call is waiting. type RWMutex struct { - w Mutex // held if there are pending readers or writers - r Mutex // held if the w is being rd - readerCount uint32 // number of pending readers + w Mutex // held if there are pending readers or writers + r Mutex // held if the w is being rd + readerCount int32 // number of pending readers } // RLock locks rw for reading. @@ -33,7 +35,7 @@ func (rw *RWMutex) RLock() { // B: rw.RUnlock() // ... (new readers come and go indefinitely, W is starving) rw.r.Lock() - if xadd(&rw.readerCount, 1) == 1 { + if atomic.AddInt32(&rw.readerCount, 1) == 1 { // The first reader locks rw.w, so writers will be blocked // while the readers have the RLock. rw.w.Lock() @@ -46,7 +48,7 @@ func (rw *RWMutex) RLock() { // It is a run-time error if rw is not locked for reading // on entry to RUnlock. func (rw *RWMutex) RUnlock() { - if xadd(&rw.readerCount, -1) == 0 { + if atomic.AddInt32(&rw.readerCount, -1) == 0 { // last reader finished, enable writers rw.w.Unlock() } @@ -64,12 +66,21 @@ func (rw *RWMutex) Lock() { rw.r.Unlock() } -// Unlock unlocks rw for writing. -// It is a run-time error if rw is not locked for writing -// on entry to Unlock. +// Unlock unlocks rw for writing. It is a run-time error if rw is +// not locked for writing on entry to Unlock. // -// Like for Mutexes, -// a locked RWMutex is not associated with a particular goroutine. -// It is allowed for one goroutine to RLock (Lock) an RWMutex and then +// As with Mutexes, a locked RWMutex is not associated with a particular +// goroutine. One goroutine may RLock (Lock) an RWMutex and then // arrange for another goroutine to RUnlock (Unlock) it. func (rw *RWMutex) Unlock() { rw.w.Unlock() } + +// RLocker returns a Locker interface that implements +// the Lock and Unlock methods by calling rw.RLock and rw.RUnlock. +func (rw *RWMutex) RLocker() Locker { + return (*rlocker)(rw) +} + +type rlocker RWMutex + +func (r *rlocker) Lock() { (*RWMutex)(r).RLock() } +func (r *rlocker) Unlock() { (*RWMutex)(r).RUnlock() } diff --git a/libgo/go/sync/rwmutex_test.go b/libgo/go/sync/rwmutex_test.go index 111bca1e389..405079270dc 100644 --- a/libgo/go/sync/rwmutex_test.go +++ b/libgo/go/sync/rwmutex_test.go @@ -10,6 +10,7 @@ import ( "fmt" "runtime" . "sync" + "sync/atomic" "testing" ) @@ -49,31 +50,31 @@ func TestParallelReaders(t *testing.T) { doTestParallelReaders(4, 2) } -func reader(rwm *RWMutex, num_iterations int, activity *uint32, cdone chan bool) { +func reader(rwm *RWMutex, num_iterations int, activity *int32, cdone chan bool) { for i := 0; i < num_iterations; i++ { rwm.RLock() - n := Xadd(activity, 1) + n := atomic.AddInt32(activity, 1) if n < 1 || n >= 10000 { panic(fmt.Sprintf("wlock(%d)\n", n)) } for i := 0; i < 100; i++ { } - Xadd(activity, -1) + atomic.AddInt32(activity, -1) rwm.RUnlock() } cdone <- true } -func writer(rwm *RWMutex, num_iterations int, activity *uint32, cdone chan bool) { +func writer(rwm *RWMutex, num_iterations int, activity *int32, cdone chan bool) { for i := 0; i < num_iterations; i++ { rwm.Lock() - n := Xadd(activity, 10000) + n := atomic.AddInt32(activity, 10000) if n != 10000 { panic(fmt.Sprintf("wlock(%d)\n", n)) } for i := 0; i < 100; i++ { } - Xadd(activity, -10000) + atomic.AddInt32(activity, -10000) rwm.Unlock() } cdone <- true @@ -82,7 +83,7 @@ func writer(rwm *RWMutex, num_iterations int, activity *uint32, cdone chan bool) func HammerRWMutex(gomaxprocs, numReaders, num_iterations int) { runtime.GOMAXPROCS(gomaxprocs) // Number of active readers + 10000 * number of active writers. - var activity uint32 + var activity int32 var rwm RWMutex cdone := make(chan bool) go writer(&rwm, num_iterations, &activity, cdone) @@ -112,3 +113,38 @@ func TestRWMutex(t *testing.T) { HammerRWMutex(10, 10, 1000) HammerRWMutex(10, 5, 10000) } + +func TestRLocker(t *testing.T) { + var wl RWMutex + var rl Locker + wlocked := make(chan bool, 1) + rlocked := make(chan bool, 1) + rl = wl.RLocker() + n := 10 + go func() { + for i := 0; i < n; i++ { + rl.Lock() + rl.Lock() + rlocked <- true + wl.Lock() + wlocked <- true + } + }() + for i := 0; i < n; i++ { + <-rlocked + rl.Unlock() + select { + case <-wlocked: + t.Fatal("RLocker() didn't read-lock it") + default: + } + rl.Unlock() + <-wlocked + select { + case <-rlocked: + t.Fatal("RLocker() didn't respect the write lock") + default: + } + wl.Unlock() + } +} diff --git a/libgo/go/sync/waitgroup.go b/libgo/go/sync/waitgroup.go new file mode 100644 index 00000000000..68e1d509f48 --- /dev/null +++ b/libgo/go/sync/waitgroup.go @@ -0,0 +1,86 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sync + +import "runtime" + +// A WaitGroup waits for a collection of goroutines to finish. +// The main goroutine calls Add to set the number of +// goroutines to wait for. Then each of the goroutines +// runs and calls Done when finished. At the same time, +// Wait can be used to block until all goroutines have finished. +// +// For example: +// +// for i := 0; i < n; i++ { +// if !condition(i) { +// continue +// } +// wg.Add(1) +// go func() { +// // Do something. +// wg.Done() +// } +// } +// wg.Wait() +// +type WaitGroup struct { + m Mutex + counter int + waiters int + sema *uint32 +} + +// WaitGroup creates a new semaphore each time the old semaphore +// is released. This is to avoid the following race: +// +// G1: Add(1) +// G1: go G2() +// G1: Wait() // Context switch after Unlock() and before Semacquire(). +// G2: Done() // Release semaphore: sema == 1, waiters == 0. G1 doesn't run yet. +// G3: Wait() // Finds counter == 0, waiters == 0, doesn't block. +// G3: Add(1) // Makes counter == 1, waiters == 0. +// G3: go G4() +// G3: Wait() // G1 still hasn't run, G3 finds sema == 1, unblocked! Bug. + +// Add adds delta, which may be negative, to the WaitGroup counter. +// If the counter becomes zero, all goroutines blocked on Wait() are released. +func (wg *WaitGroup) Add(delta int) { + wg.m.Lock() + if delta < -wg.counter { + wg.m.Unlock() + panic("sync: negative WaitGroup count") + } + wg.counter += delta + if wg.counter == 0 && wg.waiters > 0 { + for i := 0; i < wg.waiters; i++ { + runtime.Semrelease(wg.sema) + } + wg.waiters = 0 + wg.sema = nil + } + wg.m.Unlock() +} + +// Done decrements the WaitGroup counter. +func (wg *WaitGroup) Done() { + wg.Add(-1) +} + +// Wait blocks until the WaitGroup counter is zero. +func (wg *WaitGroup) Wait() { + wg.m.Lock() + if wg.counter == 0 { + wg.m.Unlock() + return + } + wg.waiters++ + if wg.sema == nil { + wg.sema = new(uint32) + } + s := wg.sema + wg.m.Unlock() + runtime.Semacquire(s) +} diff --git a/libgo/go/sync/waitgroup_test.go b/libgo/go/sync/waitgroup_test.go new file mode 100644 index 00000000000..fe35732e7a6 --- /dev/null +++ b/libgo/go/sync/waitgroup_test.go @@ -0,0 +1,60 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sync_test + +import ( + . "sync" + "testing" +) + +func testWaitGroup(t *testing.T, wg1 *WaitGroup, wg2 *WaitGroup) { + n := 16 + wg1.Add(n) + wg2.Add(n) + exited := make(chan bool, n) + for i := 0; i != n; i++ { + go func(i int) { + wg1.Done() + wg2.Wait() + exited <- true + }(i) + } + wg1.Wait() + for i := 0; i != n; i++ { + select { + case <-exited: + t.Fatal("WaitGroup released group too soon") + default: + } + wg2.Done() + } + for i := 0; i != n; i++ { + <-exited // Will block if barrier fails to unlock someone. + } +} + +func TestWaitGroup(t *testing.T) { + wg1 := &WaitGroup{} + wg2 := &WaitGroup{} + + // Run the same test a few times to ensure barrier is in a proper state. + for i := 0; i != 8; i++ { + testWaitGroup(t, wg1, wg2) + } +} + +func TestWaitGroupMisuse(t *testing.T) { + defer func() { + err := recover() + if err != "sync: negative WaitGroup count" { + t.Fatalf("Unexpected panic: %#v", err) + } + }() + wg := &WaitGroup{} + wg.Add(1) + wg.Done() + wg.Done() + t.Fatal("Should panic") +} diff --git a/libgo/go/sync/xadd_test.go b/libgo/go/sync/xadd_test.go deleted file mode 100644 index 8b2ef76e6ba..00000000000 --- a/libgo/go/sync/xadd_test.go +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package sync - -func Xadd(val *uint32, delta int32) (new uint32) { - return xadd(val, delta) -} diff --git a/libgo/go/syslog/syslog_test.go b/libgo/go/syslog/syslog_test.go index 063ab71b446..2958bcb1f8e 100644 --- a/libgo/go/syslog/syslog_test.go +++ b/libgo/go/syslog/syslog_test.go @@ -28,7 +28,7 @@ func runSyslog(c net.PacketConn, done chan<- string) { func startServer(done chan<- string) { c, e := net.ListenPacket("udp", "127.0.0.1:0") if e != nil { - log.Exitf("net.ListenPacket failed udp :0 %v", e) + log.Fatalf("net.ListenPacket failed udp :0 %v", e) } serverAddr = c.LocalAddr().String() c.SetReadTimeout(100e6) // 100ms diff --git a/libgo/go/template/template.go b/libgo/go/template/template.go index a67dbf8ad24..c3cb8901a09 100644 --- a/libgo/go/template/template.go +++ b/libgo/go/template/template.go @@ -24,7 +24,8 @@ - The result of invoking a niladic single-valued method with that name (result = data.field()) - Major constructs ({} are metacharacters; [] marks optional elements): + Major constructs ({} are the default delimiters for template actions; + [] are the notation in this comment for optional elements): {# comment } @@ -47,12 +48,20 @@ {field1 field2 ...} {field|formatter} {field1 field2...|formatter} + {field|formatter1|formatter2} Insert the value of the fields into the output. Each field is first looked for in the cursor, as in .section and .repeated. If it is not found, the search continues in outer sections until the top level is reached. + If the field value is a pointer, leading asterisks indicate + that the value to be inserted should be evaluated through the + pointer. For example, if x.p is of type *int, {x.p} will + insert the value of the pointer but {*x.p} will insert the + value of the underlying integer. If the value is nil or not a + pointer, asterisks have no effect. + If a formatter is specified, it must be named in the formatter map passed to the template set up routines or in the default set ("html","str","") and is used to process the data for @@ -62,10 +71,20 @@ values at the instantiation, and formatter is its name at the invocation site. The default formatter just concatenates the string representations of the fields. + + Multiple formatters separated by the pipeline character | are + executed sequentially, with each formatter receiving the bytes + emitted by the one to its left. + + The delimiter strings get their default value, "{" and "}", from + JSON-template. They may be set to any non-empty, space-free + string using the SetDelims method. Their value can be printed + in the output using {.meta-left} and {.meta-right}. */ package template import ( + "bytes" "container/vector" "fmt" "io" @@ -131,9 +150,9 @@ type literalElement struct { // A variable invocation to be evaluated type variableElement struct { - linenum int - word []string // The fields in the invocation. - formatter string // TODO(r): implement pipelines + linenum int + word []string // The fields in the invocation. + fmts []string // Names of formatters to apply. len(fmts) > 0 } // A .section block, possibly with a .or @@ -169,13 +188,14 @@ type Template struct { // the data item descends into the fields associated with sections, etc. // Parent is used to walk upwards to find variables higher in the tree. type state struct { - parent *state // parent in hierarchy - data reflect.Value // the driver data for this section etc. - wr io.Writer // where to send output + parent *state // parent in hierarchy + data reflect.Value // the driver data for this section etc. + wr io.Writer // where to send output + buf [2]bytes.Buffer // alternating buffers used when chaining formatters } func (parent *state) clone(data reflect.Value) *state { - return &state{parent, data, parent.wr} + return &state{parent: parent, data: data, wr: parent.wr} } // New creates a new template with the specified formatter map (which @@ -402,38 +422,43 @@ func (t *Template) analyze(item []byte) (tok int, w []string) { return } +// formatter returns the Formatter with the given name in the Template, or nil if none exists. +func (t *Template) formatter(name string) func(io.Writer, string, ...interface{}) { + if t.fmap != nil { + if fn := t.fmap[name]; fn != nil { + return fn + } + } + return builtins[name] +} + // -- Parsing // Allocate a new variable-evaluation element. -func (t *Template) newVariable(words []string) (v *variableElement) { - // The words are tokenized elements from the {item}. The last one may be of - // the form "|fmt". For example: {a b c|d} - formatter := "" +func (t *Template) newVariable(words []string) *variableElement { + // After the final space-separated argument, formatters may be specified separated + // by pipe symbols, for example: {a b c|d|e} + + // Until we learn otherwise, formatters contains a single name: "", the default formatter. + formatters := []string{""} lastWord := words[len(words)-1] - bar := strings.Index(lastWord, "|") + bar := strings.IndexRune(lastWord, '|') if bar >= 0 { words[len(words)-1] = lastWord[0:bar] - formatter = lastWord[bar+1:] + formatters = strings.Split(lastWord[bar+1:], "|", -1) } - // Probably ok, so let's build it. - v = &variableElement{t.linenum, words, formatter} // We could remember the function address here and avoid the lookup later, // but it's more dynamic to let the user change the map contents underfoot. // We do require the name to be present, though. // Is it in user-supplied map? - if t.fmap != nil { - if _, ok := t.fmap[formatter]; ok { - return + for _, f := range formatters { + if t.formatter(f) == nil { + t.parseError("unknown formatter: %q", f) } } - // Is it in builtin map? - if _, ok := builtins[formatter]; ok { - return - } - t.parseError("unknown formatter: %s", formatter) - return + return &variableElement{t.linenum, words, formatters} } // Grab the next item. If it's simple, just append it to the template. @@ -622,7 +647,10 @@ func (t *Template) lookup(st *state, v reflect.Value, name string) reflect.Value } return av.FieldByName(name) case *reflect.MapValue: - return av.Elem(reflect.NewValue(name)) + if v := av.Elem(reflect.NewValue(name)); v != nil { + return v + } + return reflect.MakeZero(typ.(*reflect.MapType).Elem()) default: return nil } @@ -630,6 +658,23 @@ func (t *Template) lookup(st *state, v reflect.Value, name string) reflect.Value return v } +// indirectPtr returns the item numLevels levels of indirection below the value. +// It is forgiving: if the value is not a pointer, it returns it rather than giving +// an error. If the pointer is nil, it is returned as is. +func indirectPtr(v reflect.Value, numLevels int) reflect.Value { + for i := numLevels; v != nil && i > 0; i++ { + if p, ok := v.(*reflect.PtrValue); ok { + if p.IsNil() { + return v + } + v = p.Elem() + } else { + break + } + } + return v +} + // Walk v through pointers and interfaces, extracting the elements within. func indirect(v reflect.Value) reflect.Value { loop: @@ -651,12 +696,16 @@ loop: // The special name "@" (the "cursor") denotes the current data. // The value coming in (st.data) might need indirecting to reach // a struct while the return value is not indirected - that is, -// it represents the actual named field. +// it represents the actual named field. Leading stars indicate +// levels of indirection to be applied to the value. func (t *Template) findVar(st *state, s string) reflect.Value { + data := st.data + flattenedName := strings.TrimLeft(s, "*") + numStars := len(s) - len(flattenedName) + s = flattenedName if s == "@" { - return st.data + return indirectPtr(data, numStars) } - data := st.data for _, elem := range strings.Split(s, ".", -1) { // Look up field; data must be a struct or map. data = t.lookup(st, data, elem) @@ -664,7 +713,7 @@ func (t *Template) findVar(st *state, s string) reflect.Value { return nil } } - return data + return indirectPtr(data, numStars) } // Is there no data to look at? @@ -702,28 +751,31 @@ func (t *Template) varValue(name string, st *state) reflect.Value { return field } +func (t *Template) format(wr io.Writer, fmt string, val []interface{}, v *variableElement, st *state) { + fn := t.formatter(fmt) + if fn == nil { + t.execError(st, v.linenum, "missing formatter %s for variable %s", fmt, v.word[0]) + } + fn(wr, fmt, val...) +} + // Evaluate a variable, looking up through the parent if necessary. // If it has a formatter attached ({var|formatter}) run that too. func (t *Template) writeVariable(v *variableElement, st *state) { - formatter := v.formatter // Turn the words of the invocation into values. val := make([]interface{}, len(v.word)) for i, word := range v.word { val[i] = t.varValue(word, st).Interface() } - // is it in user-supplied map? - if t.fmap != nil { - if fn, ok := t.fmap[formatter]; ok { - fn(st.wr, formatter, val...) - return - } - } - // is it in builtin map? - if fn, ok := builtins[formatter]; ok { - fn(st.wr, formatter, val...) - return + + for i, fmt := range v.fmts[:len(v.fmts)-1] { + b := &st.buf[i&1] + b.Reset() + t.format(b, fmt, val, v, st) + val = val[0:1] + val[0] = b.Bytes() } - t.execError(st, v.linenum, "missing formatter %s for variable %s", formatter, v.word[0]) + t.format(st.wr, v.fmts[len(v.fmts)-1], val, v, st) } // Execute element i. Return next index to execute. @@ -926,12 +978,12 @@ func (t *Template) ParseFile(filename string) (err os.Error) { // Execute applies a parsed template to the specified data object, // generating output to wr. -func (t *Template) Execute(data interface{}, wr io.Writer) (err os.Error) { +func (t *Template) Execute(wr io.Writer, data interface{}) (err os.Error) { // Extract the driver data. val := reflect.NewValue(data) defer checkError(&err) t.p = 0 - t.execute(0, t.elems.Len(), &state{nil, val, wr}) + t.execute(0, t.elems.Len(), &state{parent: nil, data: val, wr: wr}) return nil } diff --git a/libgo/go/template/template_test.go b/libgo/go/template/template_test.go index 57f297e8f0a..d21a5397a15 100644 --- a/libgo/go/template/template_test.go +++ b/libgo/go/template/template_test.go @@ -31,8 +31,10 @@ type U struct { type S struct { Header string + HeaderPtr *string Integer int - Raw string + IntegerPtr *int + NilPtr *int InnerT T InnerPointerT *T Data []T @@ -47,7 +49,7 @@ type S struct { JSON interface{} Innermap U Stringmap map[string]string - Bytes []byte + Ptrmap map[string]*string Iface interface{} Ifaceptr interface{} } @@ -118,6 +120,24 @@ var tests = []*Test{ out: "Header=77\n", }, + &Test{ + in: "Pointers: {*HeaderPtr}={*IntegerPtr}\n", + + out: "Pointers: Header=77\n", + }, + + &Test{ + in: "Stars but not pointers: {*Header}={*Integer}\n", + + out: "Stars but not pointers: Header=77\n", + }, + + &Test{ + in: "nil pointer: {*NilPtr}={*Integer}\n", + + out: "nil pointer: <nil>=77\n", + }, + // Method at top level &Test{ in: "ptrmethod={PointerMethod}\n", @@ -312,38 +332,6 @@ var tests = []*Test{ out: "ItemNumber1=ValueNumber1\n", }, - - // Formatters - &Test{ - in: "{.section Pdata }\n" + - "{Header|uppercase}={Integer|+1}\n" + - "{Header|html}={Integer|str}\n" + - "{.end}\n", - - out: "HEADER=78\n" + - "Header=77\n", - }, - - &Test{ - in: "{.section Pdata }\n" + - "{Header|uppercase}={Integer Header|multiword}\n" + - "{Header|html}={Header Integer|multiword}\n" + - "{Header|html}={Header Integer}\n" + - "{.end}\n", - - out: "HEADER=<77><Header>\n" + - "Header=<Header><77>\n" + - "Header=Header77\n", - }, - - &Test{ - in: "{Raw}\n" + - "{Raw|html}\n", - - out: "&<>!@ #$%^\n" + - "&<>!@ #$%^\n", - }, - &Test{ in: "{.section Emptystring}emptystring{.end}\n" + "{.section Header}header{.end}\n", @@ -358,12 +346,6 @@ var tests = []*Test{ out: "1\n4\n", }, - &Test{ - in: "{Bytes}", - - out: "hello", - }, - // Maps &Test{ @@ -407,6 +389,20 @@ var tests = []*Test{ out: "\tstringresult\n" + "\tstringresult\n", }, + &Test{ + in: "{*Ptrmap.stringkey1}\n", + + out: "pointedToString\n", + }, + &Test{ + in: "{.repeated section Ptrmap}\n" + + "{*@}\n" + + "{.end}", + + out: "pointedToString\n" + + "pointedToString\n", + }, + // Interface values @@ -460,8 +456,9 @@ func testAll(t *testing.T, parseFunc func(*Test) (*Template, os.Error)) { s := new(S) // initialized by hand for clarity. s.Header = "Header" + s.HeaderPtr = &s.Header s.Integer = 77 - s.Raw = "&<>!@ #$%^" + s.IntegerPtr = &s.Integer s.InnerT = t1 s.Data = []T{t1, t2} s.Pdata = []*T{&t1, &t2} @@ -480,7 +477,10 @@ func testAll(t *testing.T, parseFunc func(*Test) (*Template, os.Error)) { s.Stringmap = make(map[string]string) s.Stringmap["stringkey1"] = "stringresult" // the same value so repeated section is order-independent s.Stringmap["stringkey2"] = "stringresult" - s.Bytes = []byte("hello") + s.Ptrmap = make(map[string]*string) + x := "pointedToString" + s.Ptrmap["stringkey1"] = &x // the same value so repeated section is order-independent + s.Ptrmap["stringkey2"] = &x s.Iface = []int{1, 2, 3} s.Ifaceptr = &T{"Item", "Value"} @@ -492,7 +492,7 @@ func testAll(t *testing.T, parseFunc func(*Test) (*Template, os.Error)) { t.Error("unexpected parse error: ", err) continue } - err = tmpl.Execute(s, &buf) + err = tmpl.Execute(&buf, s) if test.err == "" { if err != nil { t.Error("unexpected execute error:", err) @@ -517,14 +517,32 @@ func TestMapDriverType(t *testing.T) { t.Error("unexpected parse error:", err) } var b bytes.Buffer - err = tmpl.Execute(mp, &b) + err = tmpl.Execute(&b, mp) if err != nil { t.Error("unexpected execute error:", err) } s := b.String() - expected := "template: Ahoy!" - if s != expected { - t.Errorf("failed passing string as data: expected %q got %q", "template: Ahoy!", s) + expect := "template: Ahoy!" + if s != expect { + t.Errorf("failed passing string as data: expected %q got %q", expect, s) + } +} + +func TestMapNoEntry(t *testing.T) { + mp := make(map[string]int) + tmpl, err := Parse("template: {notthere}!", nil) + if err != nil { + t.Error("unexpected parse error:", err) + } + var b bytes.Buffer + err = tmpl.Execute(&b, mp) + if err != nil { + t.Error("unexpected execute error:", err) + } + s := b.String() + expect := "template: 0!" + if s != expect { + t.Errorf("failed passing string as data: expected %q got %q", expect, s) } } @@ -534,13 +552,14 @@ func TestStringDriverType(t *testing.T) { t.Error("unexpected parse error:", err) } var b bytes.Buffer - err = tmpl.Execute("hello", &b) + err = tmpl.Execute(&b, "hello") if err != nil { t.Error("unexpected execute error:", err) } s := b.String() - if s != "template: hello" { - t.Errorf("failed passing string as data: expected %q got %q", "template: hello", s) + expect := "template: hello" + if s != expect { + t.Errorf("failed passing string as data: expected %q got %q", expect, s) } } @@ -550,23 +569,23 @@ func TestTwice(t *testing.T) { t.Error("unexpected parse error:", err) } var b bytes.Buffer - err = tmpl.Execute("hello", &b) + err = tmpl.Execute(&b, "hello") if err != nil { t.Error("unexpected parse error:", err) } s := b.String() - text := "template: hello" - if s != text { - t.Errorf("failed passing string as data: expected %q got %q", text, s) + expect := "template: hello" + if s != expect { + t.Errorf("failed passing string as data: expected %q got %q", expect, s) } - err = tmpl.Execute("hello", &b) + err = tmpl.Execute(&b, "hello") if err != nil { t.Error("unexpected parse error:", err) } s = b.String() - text += text - if s != text { - t.Errorf("failed passing string as data: expected %q got %q", text, s) + expect += expect + if s != expect { + t.Errorf("failed passing string as data: expected %q got %q", expect, s) } } @@ -595,7 +614,7 @@ func TestCustomDelims(t *testing.T) { continue } var b bytes.Buffer - err = tmpl.Execute("hello", &b) + err = tmpl.Execute(&b, "hello") s := b.String() if s != "template: hello"+ldelim+rdelim { t.Errorf("failed delim check(%q %q) %q got %q", ldelim, rdelim, text, s) @@ -616,7 +635,7 @@ func TestVarIndirection(t *testing.T) { if err != nil { t.Fatal("unexpected parse error:", err) } - err = tmpl.Execute(s, &buf) + err = tmpl.Execute(&buf, s) if err != nil { t.Fatal("unexpected execute error:", err) } @@ -650,7 +669,7 @@ func TestReferenceToUnexported(t *testing.T) { if err != nil { t.Fatal("unexpected parse error:", err) } - err = tmpl.Execute(u, &buf) + err = tmpl.Execute(&buf, u) if err == nil { t.Fatal("expected execute error, got none") } @@ -658,3 +677,87 @@ func TestReferenceToUnexported(t *testing.T) { t.Fatal("expected unexported error; got", err) } } + +var formatterTests = []Test{ + { + in: "{Header|uppercase}={Integer|+1}\n" + + "{Header|html}={Integer|str}\n", + + out: "HEADER=78\n" + + "Header=77\n", + }, + + { + in: "{Header|uppercase}={Integer Header|multiword}\n" + + "{Header|html}={Header Integer|multiword}\n" + + "{Header|html}={Header Integer}\n", + + out: "HEADER=<77><Header>\n" + + "Header=<Header><77>\n" + + "Header=Header77\n", + }, + { + in: "{Raw}\n" + + "{Raw|html}\n", + + out: "a <&> b\n" + + "a <&> b\n", + }, + { + in: "{Bytes}", + out: "hello", + }, + { + in: "{Raw|uppercase|html|html}", + out: "A &lt;&amp;&gt; B", + }, + { + in: "{Header Integer|multiword|html}", + out: "<Header><77>", + }, + { + in: "{Integer|no_formatter|html}", + err: `unknown formatter: "no_formatter"`, + }, + { + in: "{Integer|||||}", // empty string is a valid formatter + out: "77", + }, +} + +func TestFormatters(t *testing.T) { + data := map[string]interface{}{ + "Header": "Header", + "Integer": 77, + "Raw": "a <&> b", + "Bytes": []byte("hello"), + } + for _, c := range formatterTests { + tmpl, err := Parse(c.in, formatters) + if err != nil { + if c.err == "" { + t.Error("unexpected parse error:", err) + continue + } + if strings.Index(err.String(), c.err) < 0 { + t.Errorf("unexpected error: expected %q, got %q", c.err, err.String()) + continue + } + } else { + if c.err != "" { + t.Errorf("For %q, expected error, got none.", c.in) + continue + } + buf := bytes.NewBuffer(nil) + err = tmpl.Execute(buf, data) + if err != nil { + t.Error("unexpected Execute error: ", err) + continue + } + actual := buf.String() + if actual != c.out { + t.Errorf("for %q: expected %q but got %q.", c.in, c.out, actual) + } + } + } +} diff --git a/libgo/go/testing/benchmark.go b/libgo/go/testing/benchmark.go index ad938027d3e..cf73e2b48f8 100644 --- a/libgo/go/testing/benchmark.go +++ b/libgo/go/testing/benchmark.go @@ -8,10 +8,11 @@ import ( "flag" "fmt" "os" + "runtime" "time" ) -var matchBenchmarks = flag.String("benchmarks", "", "regular expression to select benchmarks to run") +var matchBenchmarks = flag.String("test.bench", "", "regular expression to select benchmarks to run") // An internal type but exported because it is cross-package; part of the implementation // of gotest. @@ -64,6 +65,9 @@ func (b *B) nsPerOp() int64 { // runN runs a single benchmark for the specified number of iterations. func (b *B) runN(n int) { + // Try to get a comparable environment for each run + // by clearing garbage from previous runs. + runtime.GC() b.N = n b.ResetTimer() b.StartTimer() @@ -175,7 +179,7 @@ func RunBenchmarks(matchString func(pat, str string) (bool, os.Error), benchmark for _, Benchmark := range benchmarks { matched, err := matchString(*matchBenchmarks, Benchmark.Name) if err != nil { - println("invalid regexp for -benchmarks:", err.String()) + println("invalid regexp for -test.bench:", err.String()) os.Exit(1) } if !matched { diff --git a/libgo/go/testing/testing.go b/libgo/go/testing/testing.go index 0e04935ce44..324b5a70e1f 100644 --- a/libgo/go/testing/testing.go +++ b/libgo/go/testing/testing.go @@ -12,7 +12,7 @@ // // Functions of the form // func BenchmarkXxx(*testing.B) -// are considered benchmarks, and are executed by gotest when the -benchmarks +// are considered benchmarks, and are executed by gotest when the -test.bench // flag is provided. // // A sample benchmark function looks like this: @@ -43,11 +43,12 @@ import ( "fmt" "os" "runtime" + "time" ) // Report as tests are run; default is silent for success. -var chatty = flag.Bool("v", false, "verbose: print additional output") -var match = flag.String("match", "", "regular expression to select tests to run") +var chatty = flag.Bool("test.v", false, "verbose: print additional output") +var match = flag.String("test.run", "", "regular expression to select tests to run") // Insert final newline if needed and tabs after internal newlines. @@ -91,7 +92,7 @@ func (t *T) FailNow() { // and records the text in the error log. func (t *T) Log(args ...interface{}) { t.errors += "\t" + tabify(fmt.Sprintln(args...)) } -// Log formats its arguments according to the format, analogous to Printf(), +// Logf formats its arguments according to the format, analogous to Printf(), // and records the text in the error log. func (t *T) Logf(format string, args ...interface{}) { t.errors += "\t" + tabify(fmt.Sprintf(format, args...)) @@ -144,7 +145,7 @@ func Main(matchString func(pat, str string) (bool, os.Error), tests []InternalTe for i := 0; i < len(tests); i++ { matched, err := matchString(*match, tests[i].Name) if err != nil { - println("invalid regexp for -match:", err.String()) + println("invalid regexp for -test.run:", err.String()) os.Exit(1) } if !matched { @@ -153,16 +154,19 @@ func Main(matchString func(pat, str string) (bool, os.Error), tests []InternalTe if *chatty { println("=== RUN ", tests[i].Name) } + ns := -time.Nanoseconds() t := new(T) t.ch = make(chan *T) go tRunner(t, &tests[i]) <-t.ch + ns += time.Nanoseconds() + tstr := fmt.Sprintf("(%.1f seconds)", float64(ns)/1e9) if t.failed { - println("--- FAIL:", tests[i].Name) + println("--- FAIL:", tests[i].Name, tstr) print(t.errors) ok = false } else if *chatty { - println("--- PASS:", tests[i].Name) + println("--- PASS:", tests[i].Name, tstr) print(t.errors) } } diff --git a/libgo/go/time/sleep.go b/libgo/go/time/sleep.go index 3538775adfb..833552d6841 100644 --- a/libgo/go/time/sleep.go +++ b/libgo/go/time/sleep.go @@ -11,20 +11,40 @@ import ( "container/heap" ) -// The event type represents a single After or AfterFunc event. -type event struct { - t int64 // The absolute time that the event should fire. - f func(int64) // The function to call when the event fires. - sleeping bool // A sleeper is sleeping for this event. +// The Timer type represents a single event. +// When the Timer expires, the current time will be sent on C +// unless the Timer represents an AfterFunc event. +type Timer struct { + C <-chan int64 + t int64 // The absolute time that the event should fire. + f func(int64) // The function to call when the event fires. + i int // The event's index inside eventHeap. } -type eventHeap []*event +type timerHeap []*Timer -var events eventHeap -var eventMutex sync.Mutex +// forever is the absolute time (in ns) of an event that is forever away. +const forever = 1 << 62 + +// maxSleepTime is the maximum length of time that a sleeper +// sleeps for before checking if it is defunct. +const maxSleepTime = 1e9 + +var ( + // timerMutex guards the variables inside this var group. + timerMutex sync.Mutex + + // timers holds a binary heap of pending events, terminated with a sentinel. + timers timerHeap + + // currentSleeper is an ever-incrementing counter which represents + // the current sleeper. It allows older sleepers to detect that they are + // defunct and exit. + currentSleeper int64 +) func init() { - events.Push(&event{1 << 62, nil, true}) // sentinel + timers.Push(&Timer{t: forever}) // sentinel } // Sleep pauses the current goroutine for at least ns nanoseconds. @@ -51,101 +71,133 @@ func sleep(t, ns int64) (int64, os.Error) { return t, nil } +// NewTimer creates a new Timer that will send +// the current time on its channel after at least ns nanoseconds. +func NewTimer(ns int64) *Timer { + c := make(chan int64, 1) + e := after(ns, func(t int64) { c <- t }) + e.C = c + return e +} + // After waits at least ns nanoseconds before sending the current time // on the returned channel. +// It is equivalent to NewTimer(ns).C. func After(ns int64) <-chan int64 { - c := make(chan int64, 1) - after(ns, func(t int64) { c <- t }) - return c + return NewTimer(ns).C } // AfterFunc waits at least ns nanoseconds before calling f -// in its own goroutine. -func AfterFunc(ns int64, f func()) { - after(ns, func(_ int64) { +// in its own goroutine. It returns a Timer that can +// be used to cancel the call using its Stop method. +func AfterFunc(ns int64, f func()) *Timer { + return after(ns, func(_ int64) { go f() }) } +// Stop prevents the Timer from firing. +// It returns true if the call stops the timer, false if the timer has already +// expired or stopped. +func (e *Timer) Stop() (ok bool) { + timerMutex.Lock() + // Avoid removing the first event in the queue so that + // we don't start a new sleeper unnecessarily. + if e.i > 0 { + heap.Remove(timers, e.i) + } + ok = e.f != nil + e.f = nil + timerMutex.Unlock() + return +} + // after is the implementation of After and AfterFunc. // When the current time is after ns, it calls f with the current time. // It assumes that f will not block. -func after(ns int64, f func(int64)) { +func after(ns int64, f func(int64)) (e *Timer) { + now := Nanoseconds() t := Nanoseconds() + ns - eventMutex.Lock() - t0 := events[0].t - heap.Push(events, &event{t, f, false}) - if t < t0 { - go sleeper() + if ns > 0 && t < now { + panic("time: time overflow") } - eventMutex.Unlock() + timerMutex.Lock() + t0 := timers[0].t + e = &Timer{nil, t, f, -1} + heap.Push(timers, e) + // Start a new sleeper if the new event is before + // the first event in the queue. If the length of time + // until the new event is at least maxSleepTime, + // then we're guaranteed that the sleeper will wake up + // in time to service it, so no new sleeper is needed. + if t0 > t && (t0 == forever || ns < maxSleepTime) { + currentSleeper++ + go sleeper(currentSleeper) + } + timerMutex.Unlock() + return } -// sleeper continually looks at the earliest event in the queue, marks it -// as sleeping, waits until it happens, then removes any events -// in the queue that are due. It stops when it finds an event that is -// already marked as sleeping. When an event is inserted before the first item, -// a new sleeper is started. -// -// Scheduling vagaries mean that sleepers may not wake up in -// exactly the order of the events that they are waiting for, -// but this does not matter as long as there are at least as -// many sleepers as events marked sleeping (invariant). This ensures that -// there is always a sleeper to service the remaining events. -// -// A sleeper will remove at least the event it has been waiting for -// unless the event has already been removed by another sleeper. Both -// cases preserve the invariant described above. -func sleeper() { - eventMutex.Lock() - e := events[0] - for !e.sleeping { - t := Nanoseconds() +// sleeper continually looks at the earliest event in the queue, waits until it happens, +// then removes any events in the queue that are due. It stops when the queue +// is empty or when another sleeper has been started. +func sleeper(sleeperId int64) { + timerMutex.Lock() + e := timers[0] + t := Nanoseconds() + for e.t != forever { if dt := e.t - t; dt > 0 { - e.sleeping = true - eventMutex.Unlock() - if nt, err := sleep(t, dt); err != nil { - // If sleep has encountered an error, - // there's not much we can do. We pretend - // that time really has advanced by the required - // amount and lie to the rest of the system. - t = e.t - } else { - t = nt + if dt > maxSleepTime { + dt = maxSleepTime + } + timerMutex.Unlock() + syscall.Sleep(dt) + timerMutex.Lock() + if currentSleeper != sleeperId { + // Another sleeper has been started, making this one redundant. + break } - eventMutex.Lock() - e = events[0] } + e = timers[0] + t = Nanoseconds() for t >= e.t { - e.f(t) - heap.Pop(events) - e = events[0] + if e.f != nil { + e.f(t) + e.f = nil + } + heap.Pop(timers) + e = timers[0] } } - eventMutex.Unlock() + timerMutex.Unlock() } -func (eventHeap) Len() int { - return len(events) +func (timerHeap) Len() int { + return len(timers) } -func (eventHeap) Less(i, j int) bool { - return events[i].t < events[j].t +func (timerHeap) Less(i, j int) bool { + return timers[i].t < timers[j].t } -func (eventHeap) Swap(i, j int) { - events[i], events[j] = events[j], events[i] +func (timerHeap) Swap(i, j int) { + timers[i], timers[j] = timers[j], timers[i] + timers[i].i = i + timers[j].i = j } -func (eventHeap) Push(x interface{}) { - events = append(events, x.(*event)) +func (timerHeap) Push(x interface{}) { + e := x.(*Timer) + e.i = len(timers) + timers = append(timers, e) } -func (eventHeap) Pop() interface{} { +func (timerHeap) Pop() interface{} { // TODO: possibly shrink array. - n := len(events) - 1 - e := events[n] - events[n] = nil - events = events[0:n] + n := len(timers) - 1 + e := timers[n] + timers[n] = nil + timers = timers[0:n] + e.i = -1 return e } diff --git a/libgo/go/time/sleep_test.go b/libgo/go/time/sleep_test.go index 9e36288f886..8bf599c3e1a 100644 --- a/libgo/go/time/sleep_test.go +++ b/libgo/go/time/sleep_test.go @@ -64,6 +64,18 @@ func BenchmarkAfterFunc(b *testing.B) { <-c } +func BenchmarkAfter(b *testing.B) { + for i := 0; i < b.N; i++ { + <-After(1) + } +} + +func BenchmarkStop(b *testing.B) { + for i := 0; i < b.N; i++ { + NewTimer(1e9).Stop() + } +} + func TestAfter(t *testing.T) { const delay = int64(100e6) start := Nanoseconds() @@ -94,6 +106,32 @@ func TestAfterTick(t *testing.T) { } } +func TestAfterStop(t *testing.T) { + const msec = 1e6 + AfterFunc(100*msec, func() {}) + t0 := NewTimer(50 * msec) + c1 := make(chan bool, 1) + t1 := AfterFunc(150*msec, func() { c1 <- true }) + c2 := After(200 * msec) + if !t0.Stop() { + t.Fatalf("failed to stop event 0") + } + if !t1.Stop() { + t.Fatalf("failed to stop event 1") + } + <-c2 + select { + case <-t0.C: + t.Fatalf("event 0 was not stopped") + case <-c1: + t.Fatalf("event 1 was not stopped") + default: + } + if t1.Stop() { + t.Fatalf("Stop returned true twice") + } +} + var slots = []int{5, 3, 6, 6, 6, 1, 1, 2, 7, 9, 4, 8, 0} type afterResult struct { diff --git a/libgo/go/time/tick.go b/libgo/go/time/tick.go index ddd72727027..6c21bf19b92 100644 --- a/libgo/go/time/tick.go +++ b/libgo/go/time/tick.go @@ -22,8 +22,12 @@ type Ticker struct { // Stop turns off a ticker. After Stop, no more ticks will be sent. func (t *Ticker) Stop() { - // Make it non-blocking so multiple Stops don't block. - _ = t.shutdown <- true + select { + case t.shutdown <- true: + // ok + default: + // Stop in progress already + } } // Tick is a convenience wrapper for NewTicker providing access to the ticking @@ -106,7 +110,8 @@ func tickerLoop() { // that need it and determining the next wake time. // TODO(r): list should be sorted in time order. for t := tickers; t != nil; t = t.next { - if _, ok := <-t.shutdown; ok { + select { + case <-t.shutdown: // Ticker is done; remove it from list. if prev == nil { tickers = t.next @@ -114,6 +119,7 @@ func tickerLoop() { prev.next = t.next } continue + default: } if t.nextTick <= now { if len(t.c) == 0 { diff --git a/libgo/go/time/tick_test.go b/libgo/go/time/tick_test.go index 2a63a0f2b3b..4dcb63956b2 100644 --- a/libgo/go/time/tick_test.go +++ b/libgo/go/time/tick_test.go @@ -29,9 +29,11 @@ func TestTicker(t *testing.T) { } // Now test that the ticker stopped Sleep(2 * Delta) - _, received := <-ticker.C - if received { + select { + case <-ticker.C: t.Fatal("Ticker did not shut down") + default: + // ok } } @@ -43,3 +45,14 @@ func TestTeardown(t *testing.T) { ticker.Stop() } } + +func BenchmarkTicker(b *testing.B) { + ticker := NewTicker(1) + b.ResetTimer() + b.StartTimer() + for i := 0; i < b.N; i++ { + <-ticker.C + } + b.StopTimer() + ticker.Stop() +} diff --git a/libgo/go/unicode/letter_test.go b/libgo/go/unicode/letter_test.go index b8ef648270d..432ffb67130 100644 --- a/libgo/go/unicode/letter_test.go +++ b/libgo/go/unicode/letter_test.go @@ -56,6 +56,7 @@ var letterTest = []int{ 0xf9, 0x2ec, 0x535, + 0x620, 0x6e6, 0x93d, 0xa15, @@ -85,7 +86,7 @@ var notletterTest = []int{ 0x20, 0x35, 0x375, - 0x620, + 0x619, 0x700, 0xfffe, 0x1ffff, diff --git a/libgo/go/unicode/script_test.go b/libgo/go/unicode/script_test.go index ffdc40dc07d..ff452b75ced 100644 --- a/libgo/go/unicode/script_test.go +++ b/libgo/go/unicode/script_test.go @@ -14,7 +14,7 @@ type T struct { script string } -// Hand-chosen tests from Unicode 5.1.0, mostly to discover when new +// Hand-chosen tests from Unicode 5.1.0 & 6.0..0, mostly to discover when new // scripts and categories arise. var inTest = []T{ {0x06e2, "Arabic"}, @@ -22,11 +22,13 @@ var inTest = []T{ {0x10b20, "Avestan"}, {0x1b37, "Balinese"}, {0xa6af, "Bamum"}, + {0x1be1, "Batak"}, {0x09c2, "Bengali"}, {0x3115, "Bopomofo"}, {0x282d, "Braille"}, {0x1a1a, "Buginese"}, {0x1747, "Buhid"}, + {0x11011, "Brahmi"}, {0x156d, "Canadian_Aboriginal"}, {0x102a9, "Carian"}, {0xaa4d, "Cham"}, @@ -72,6 +74,7 @@ var inTest = []T{ {0x10290, "Lycian"}, {0x10930, "Lydian"}, {0x0d42, "Malayalam"}, + {0x0843, "Mandaic"}, {0xabd0, "Meetei_Mayek"}, {0x1822, "Mongolian"}, {0x104c, "Myanmar"}, @@ -204,7 +207,7 @@ func TestScripts(t *testing.T) { } } for k := range notTested { - t.Error("not tested:", k) + t.Error("script not tested:", k) } } @@ -223,7 +226,7 @@ func TestCategories(t *testing.T) { notTested[test.script] = false, false } for k := range notTested { - t.Error("not tested:", k) + t.Error("category not tested:", k) } } @@ -242,6 +245,6 @@ func TestProperties(t *testing.T) { notTested[test.script] = false, false } for k := range notTested { - t.Error("not tested:", k) + t.Error("property not tested:", k) } } diff --git a/libgo/go/unicode/tables.go b/libgo/go/unicode/tables.go index b56c9bd0387..89fd99411bc 100644 --- a/libgo/go/unicode/tables.go +++ b/libgo/go/unicode/tables.go @@ -1,11 +1,11 @@ // Generated by running -// maketables --tables=all --data=http://www.unicode.org/Public/5.2.0/ucd/UnicodeData.txt +// maketables --tables=all --data=http://www.unicode.org/Public/6.0.0/ucd/UnicodeData.txt // DO NOT EDIT package unicode // Version is the Unicode edition from which the tables are derived. -const Version = "5.2.0" +const Version = "6.0.0" // Categories is the set of Unicode data tables. var Categories = map[string][]Range{ @@ -61,7 +61,7 @@ var _Lm = []Range{ {0x1d78, 0x1d9b, 35}, {0x1d9c, 0x1dbf, 1}, {0x2071, 0x207f, 14}, - {0x2090, 0x2094, 1}, + {0x2090, 0x209c, 1}, {0x2c7d, 0x2d6f, 242}, {0x2e2f, 0x3005, 470}, {0x3031, 0x3035, 1}, @@ -129,7 +129,7 @@ var _Ll = []Range{ {0x0461, 0x0481, 2}, {0x048b, 0x04bf, 2}, {0x04c2, 0x04ce, 2}, - {0x04cf, 0x0525, 2}, + {0x04cf, 0x0527, 2}, {0x0561, 0x0587, 1}, {0x1d00, 0x1d2b, 1}, {0x1d62, 0x1d77, 1}, @@ -174,8 +174,7 @@ var _Ll = []Range{ {0x2ce4, 0x2cec, 8}, {0x2cee, 0x2d00, 18}, {0x2d01, 0x2d25, 1}, - {0xa641, 0xa65f, 2}, - {0xa663, 0xa66d, 2}, + {0xa641, 0xa66d, 2}, {0xa681, 0xa697, 2}, {0xa723, 0xa72f, 2}, {0xa730, 0xa731, 1}, @@ -183,7 +182,10 @@ var _Ll = []Range{ {0xa772, 0xa778, 1}, {0xa77a, 0xa77c, 2}, {0xa77f, 0xa787, 2}, - {0xa78c, 0xfb00, 21364}, + {0xa78c, 0xa78e, 2}, + {0xa791, 0xa7a1, 16}, + {0xa7a3, 0xa7a9, 2}, + {0xa7fa, 0xfb00, 21254}, {0xfb01, 0xfb06, 1}, {0xfb13, 0xfb17, 1}, {0xff41, 0xff5a, 1}, @@ -220,19 +222,18 @@ var _Ll = []Range{ var _Me = []Range{ {0x0488, 0x0489, 1}, - {0x06de, 0x20dd, 6655}, - {0x20de, 0x20e0, 1}, + {0x20dd, 0x20e0, 1}, {0x20e2, 0x20e4, 1}, {0xa670, 0xa672, 1}, } var _Mc = []Range{ - {0x0903, 0x093e, 59}, - {0x093f, 0x0940, 1}, + {0x0903, 0x093b, 56}, + {0x093e, 0x0940, 1}, {0x0949, 0x094c, 1}, - {0x094e, 0x0982, 52}, - {0x0983, 0x09be, 59}, - {0x09bf, 0x09c0, 1}, + {0x094e, 0x094f, 1}, + {0x0982, 0x0983, 1}, + {0x09be, 0x09c0, 1}, {0x09c7, 0x09c8, 1}, {0x09cb, 0x09cc, 1}, {0x09d7, 0x0a03, 44}, @@ -299,7 +300,10 @@ var _Mc = []Range{ {0x1b43, 0x1b44, 1}, {0x1b82, 0x1ba1, 31}, {0x1ba6, 0x1ba7, 1}, - {0x1baa, 0x1c24, 122}, + {0x1baa, 0x1be7, 61}, + {0x1bea, 0x1bec, 1}, + {0x1bee, 0x1bf2, 4}, + {0x1bf3, 0x1c24, 49}, {0x1c25, 0x1c2b, 1}, {0x1c34, 0x1c35, 1}, {0x1ce1, 0x1cf2, 17}, @@ -318,7 +322,8 @@ var _Mc = []Range{ {0xabe3, 0xabe4, 1}, {0xabe6, 0xabe7, 1}, {0xabe9, 0xabea, 1}, - {0xabec, 0x11082, 25750}, + {0xabec, 0x11000, 25620}, + {0x11002, 0x11082, 128}, {0x110b0, 0x110b2, 1}, {0x110b7, 0x110b8, 1}, {0x1d165, 0x1d166, 1}, @@ -333,7 +338,7 @@ var _Mn = []Range{ {0x05c2, 0x05c4, 2}, {0x05c5, 0x05c7, 2}, {0x0610, 0x061a, 1}, - {0x064b, 0x065e, 1}, + {0x064b, 0x065f, 1}, {0x0670, 0x06d6, 102}, {0x06d7, 0x06dc, 1}, {0x06df, 0x06e4, 1}, @@ -347,11 +352,12 @@ var _Mn = []Range{ {0x081b, 0x0823, 1}, {0x0825, 0x0827, 1}, {0x0829, 0x082d, 1}, + {0x0859, 0x085b, 1}, {0x0900, 0x0902, 1}, - {0x093c, 0x0941, 5}, - {0x0942, 0x0948, 1}, + {0x093a, 0x093c, 2}, + {0x0941, 0x0948, 1}, {0x094d, 0x0951, 4}, - {0x0952, 0x0955, 1}, + {0x0952, 0x0957, 1}, {0x0962, 0x0963, 1}, {0x0981, 0x09bc, 59}, {0x09c1, 0x09c4, 1}, @@ -400,7 +406,7 @@ var _Mn = []Range{ {0x0f71, 0x0f7e, 1}, {0x0f80, 0x0f84, 1}, {0x0f86, 0x0f87, 1}, - {0x0f90, 0x0f97, 1}, + {0x0f8d, 0x0f97, 1}, {0x0f99, 0x0fbc, 1}, {0x0fc6, 0x102d, 103}, {0x102e, 0x1030, 1}, @@ -412,7 +418,8 @@ var _Mn = []Range{ {0x1071, 0x1074, 1}, {0x1082, 0x1085, 3}, {0x1086, 0x108d, 7}, - {0x109d, 0x135f, 706}, + {0x109d, 0x135d, 704}, + {0x135e, 0x135f, 1}, {0x1712, 0x1714, 1}, {0x1732, 0x1734, 1}, {0x1752, 0x1753, 1}, @@ -442,6 +449,9 @@ var _Mn = []Range{ {0x1b80, 0x1b81, 1}, {0x1ba2, 0x1ba5, 1}, {0x1ba8, 0x1ba9, 1}, + {0x1be6, 0x1be8, 2}, + {0x1be9, 0x1bed, 4}, + {0x1bef, 0x1bf1, 1}, {0x1c2c, 0x1c33, 1}, {0x1c36, 0x1c37, 1}, {0x1cd0, 0x1cd2, 1}, @@ -449,12 +459,13 @@ var _Mn = []Range{ {0x1ce2, 0x1ce8, 1}, {0x1ced, 0x1dc0, 211}, {0x1dc1, 0x1de6, 1}, - {0x1dfd, 0x1dff, 1}, + {0x1dfc, 0x1dff, 1}, {0x20d0, 0x20dc, 1}, {0x20e1, 0x20e5, 4}, {0x20e6, 0x20f0, 1}, {0x2cef, 0x2cf1, 1}, - {0x2de0, 0x2dff, 1}, + {0x2d7f, 0x2de0, 97}, + {0x2de1, 0x2dff, 1}, {0x302a, 0x302f, 1}, {0x3099, 0x309a, 1}, {0xa66f, 0xa67c, 13}, @@ -488,9 +499,10 @@ var _Mn = []Range{ {0x10a05, 0x10a06, 1}, {0x10a0c, 0x10a0f, 1}, {0x10a38, 0x10a3a, 1}, - {0x10a3f, 0x11080, 1601}, - {0x11081, 0x110b3, 50}, - {0x110b4, 0x110b6, 1}, + {0x10a3f, 0x11001, 1474}, + {0x11038, 0x11046, 1}, + {0x11080, 0x11081, 1}, + {0x110b3, 0x110b6, 1}, {0x110b9, 0x110ba, 1}, {0x1d167, 0x1d169, 1}, {0x1d17b, 0x1d182, 1}, @@ -524,13 +536,13 @@ var letter = []Range{ {0x038f, 0x03a1, 1}, {0x03a3, 0x03f5, 1}, {0x03f7, 0x0481, 1}, - {0x048a, 0x0525, 1}, + {0x048a, 0x0527, 1}, {0x0531, 0x0556, 1}, {0x0559, 0x0561, 8}, {0x0562, 0x0587, 1}, {0x05d0, 0x05ea, 1}, {0x05f0, 0x05f2, 1}, - {0x0621, 0x064a, 1}, + {0x0620, 0x064a, 1}, {0x066e, 0x066f, 1}, {0x0671, 0x06d3, 1}, {0x06d5, 0x06e5, 16}, @@ -546,11 +558,12 @@ var letter = []Range{ {0x07fa, 0x0800, 6}, {0x0801, 0x0815, 1}, {0x081a, 0x0824, 10}, - {0x0828, 0x0904, 220}, - {0x0905, 0x0939, 1}, + {0x0828, 0x0840, 24}, + {0x0841, 0x0858, 1}, + {0x0904, 0x0939, 1}, {0x093d, 0x0950, 19}, {0x0958, 0x0961, 1}, - {0x0971, 0x0972, 1}, + {0x0971, 0x0977, 1}, {0x0979, 0x097f, 1}, {0x0985, 0x098c, 1}, {0x098f, 0x0990, 1}, @@ -615,13 +628,13 @@ var letter = []Range{ {0x0cb5, 0x0cb9, 1}, {0x0cbd, 0x0cde, 33}, {0x0ce0, 0x0ce1, 1}, + {0x0cf1, 0x0cf2, 1}, {0x0d05, 0x0d0c, 1}, {0x0d0e, 0x0d10, 1}, - {0x0d12, 0x0d28, 1}, - {0x0d2a, 0x0d39, 1}, - {0x0d3d, 0x0d60, 35}, - {0x0d61, 0x0d7a, 25}, - {0x0d7b, 0x0d7f, 1}, + {0x0d12, 0x0d3a, 1}, + {0x0d3d, 0x0d4e, 17}, + {0x0d60, 0x0d61, 1}, + {0x0d7a, 0x0d7f, 1}, {0x0d85, 0x0d96, 1}, {0x0d9a, 0x0db1, 1}, {0x0db3, 0x0dbb, 1}, @@ -647,7 +660,7 @@ var letter = []Range{ {0x0edd, 0x0f00, 35}, {0x0f40, 0x0f47, 1}, {0x0f49, 0x0f6c, 1}, - {0x0f88, 0x0f8b, 1}, + {0x0f88, 0x0f8c, 1}, {0x1000, 0x102a, 1}, {0x103f, 0x1050, 17}, {0x1051, 0x1055, 1}, @@ -706,6 +719,7 @@ var letter = []Range{ {0x1b45, 0x1b4b, 1}, {0x1b83, 0x1ba0, 1}, {0x1bae, 0x1baf, 1}, + {0x1bc0, 0x1be5, 1}, {0x1c00, 0x1c23, 1}, {0x1c4d, 0x1c4f, 1}, {0x1c5a, 0x1c7d, 1}, @@ -730,7 +744,7 @@ var letter = []Range{ {0x1ff2, 0x1ff4, 1}, {0x1ff6, 0x1ffc, 1}, {0x2071, 0x207f, 14}, - {0x2090, 0x2094, 1}, + {0x2090, 0x209c, 1}, {0x2102, 0x2107, 5}, {0x210a, 0x2113, 1}, {0x2115, 0x2119, 4}, @@ -768,7 +782,7 @@ var letter = []Range{ {0x30fc, 0x30ff, 1}, {0x3105, 0x312d, 1}, {0x3131, 0x318e, 1}, - {0x31a0, 0x31b7, 1}, + {0x31a0, 0x31ba, 1}, {0x31f0, 0x31ff, 1}, {0x3400, 0x4db5, 1}, {0x4e00, 0x9fcb, 1}, @@ -777,14 +791,15 @@ var letter = []Range{ {0xa500, 0xa60c, 1}, {0xa610, 0xa61f, 1}, {0xa62a, 0xa62b, 1}, - {0xa640, 0xa65f, 1}, - {0xa662, 0xa66e, 1}, + {0xa640, 0xa66e, 1}, {0xa67f, 0xa697, 1}, {0xa6a0, 0xa6e5, 1}, {0xa717, 0xa71f, 1}, {0xa722, 0xa788, 1}, - {0xa78b, 0xa78c, 1}, - {0xa7fb, 0xa801, 1}, + {0xa78b, 0xa78e, 1}, + {0xa790, 0xa791, 1}, + {0xa7a0, 0xa7a9, 1}, + {0xa7fa, 0xa801, 1}, {0xa803, 0xa805, 1}, {0xa807, 0xa80a, 1}, {0xa80c, 0xa822, 1}, @@ -808,6 +823,11 @@ var letter = []Range{ {0xaaba, 0xaabd, 1}, {0xaac0, 0xaac2, 2}, {0xaadb, 0xaadd, 1}, + {0xab01, 0xab06, 1}, + {0xab09, 0xab0e, 1}, + {0xab11, 0xab16, 1}, + {0xab20, 0xab26, 1}, + {0xab28, 0xab2e, 1}, {0xabc0, 0xabe2, 1}, {0xac00, 0xd7a3, 1}, {0xd7b0, 0xd7c6, 1}, @@ -871,9 +891,12 @@ var letter = []Range{ {0x10b40, 0x10b55, 1}, {0x10b60, 0x10b72, 1}, {0x10c00, 0x10c48, 1}, + {0x11003, 0x11037, 1}, {0x11083, 0x110af, 1}, {0x12000, 0x1236e, 1}, {0x13000, 0x1342e, 1}, + {0x16800, 0x16a38, 1}, + {0x1b000, 0x1b001, 1}, {0x1d400, 0x1d454, 1}, {0x1d456, 0x1d49c, 1}, {0x1d49e, 0x1d49f, 1}, @@ -906,6 +929,7 @@ var letter = []Range{ {0x1d7c4, 0x1d7cb, 1}, {0x20000, 0x2a6d6, 1}, {0x2a700, 0x2b734, 1}, + {0x2b740, 0x2b81d, 1}, {0x2f800, 0x2fa1d, 1}, } @@ -976,13 +1000,14 @@ var _Po = []Range{ {0x0701, 0x070d, 1}, {0x07f7, 0x07f9, 1}, {0x0830, 0x083e, 1}, - {0x0964, 0x0965, 1}, - {0x0970, 0x0df4, 1156}, - {0x0e4f, 0x0e5a, 11}, - {0x0e5b, 0x0f04, 169}, - {0x0f05, 0x0f12, 1}, + {0x085e, 0x0964, 262}, + {0x0965, 0x0970, 11}, + {0x0df4, 0x0e4f, 91}, + {0x0e5a, 0x0e5b, 1}, + {0x0f04, 0x0f12, 1}, {0x0f85, 0x0fd0, 75}, {0x0fd1, 0x0fd4, 1}, + {0x0fd9, 0x0fda, 1}, {0x104a, 0x104f, 1}, {0x10fb, 0x1361, 614}, {0x1362, 0x1368, 1}, @@ -994,11 +1019,11 @@ var _Po = []Range{ {0x1800, 0x1805, 1}, {0x1807, 0x180a, 1}, {0x1944, 0x1945, 1}, - {0x19de, 0x19df, 1}, {0x1a1e, 0x1a1f, 1}, {0x1aa0, 0x1aa6, 1}, {0x1aa8, 0x1aad, 1}, {0x1b5a, 0x1b60, 1}, + {0x1bfc, 0x1bff, 1}, {0x1c3b, 0x1c3f, 1}, {0x1c7e, 0x1c7f, 1}, {0x1cd3, 0x2016, 835}, @@ -1012,8 +1037,9 @@ var _Po = []Range{ {0x2056, 0x205e, 1}, {0x2cf9, 0x2cfc, 1}, {0x2cfe, 0x2cff, 1}, - {0x2e00, 0x2e01, 1}, - {0x2e06, 0x2e08, 1}, + {0x2d70, 0x2e00, 144}, + {0x2e01, 0x2e06, 5}, + {0x2e07, 0x2e08, 1}, {0x2e0b, 0x2e0e, 3}, {0x2e0f, 0x2e16, 1}, {0x2e18, 0x2e19, 1}, @@ -1060,6 +1086,7 @@ var _Po = []Range{ {0x10a50, 0x10a58, 1}, {0x10a7f, 0x10b39, 186}, {0x10b3a, 0x10b3f, 1}, + {0x11047, 0x1104d, 1}, {0x110bb, 0x110bc, 1}, {0x110be, 0x110c1, 1}, {0x12470, 0x12473, 1}, @@ -1174,7 +1201,7 @@ var _Nd = []Range{ {0x17e0, 0x17e9, 1}, {0x1810, 0x1819, 1}, {0x1946, 0x194f, 1}, - {0x19d0, 0x19da, 1}, + {0x19d0, 0x19d9, 1}, {0x1a80, 0x1a89, 1}, {0x1a90, 0x1a99, 1}, {0x1b50, 0x1b59, 1}, @@ -1189,6 +1216,7 @@ var _Nd = []Range{ {0xabf0, 0xabf9, 1}, {0xff10, 0xff19, 1}, {0x104a0, 0x104a9, 1}, + {0x11066, 0x1106f, 1}, {0x1d7ce, 0x1d7ff, 1}, } @@ -1211,14 +1239,15 @@ var _No = []Range{ {0x00b9, 0x00bc, 3}, {0x00bd, 0x00be, 1}, {0x09f4, 0x09f9, 1}, + {0x0b72, 0x0b77, 1}, {0x0bf0, 0x0bf2, 1}, {0x0c78, 0x0c7e, 1}, {0x0d70, 0x0d75, 1}, {0x0f2a, 0x0f33, 1}, {0x1369, 0x137c, 1}, {0x17f0, 0x17f9, 1}, - {0x2070, 0x2074, 4}, - {0x2075, 0x2079, 1}, + {0x19da, 0x2070, 1686}, + {0x2074, 0x2079, 1}, {0x2080, 0x2089, 1}, {0x2150, 0x215f, 1}, {0x2189, 0x2460, 727}, @@ -1243,6 +1272,7 @@ var _No = []Range{ {0x10b58, 0x10b5f, 1}, {0x10b78, 0x10b7f, 1}, {0x10e60, 0x10e7e, 1}, + {0x11052, 0x11065, 1}, {0x1d360, 0x1d371, 1}, {0x1f100, 0x1f10a, 1}, } @@ -1252,13 +1282,12 @@ var _So = []Range{ {0x00a9, 0x00ae, 5}, {0x00b0, 0x00b6, 6}, {0x0482, 0x060e, 396}, - {0x060f, 0x06e9, 218}, - {0x06fd, 0x06fe, 1}, - {0x07f6, 0x09fa, 516}, - {0x0b70, 0x0bf3, 131}, - {0x0bf4, 0x0bf8, 1}, + {0x060f, 0x06de, 207}, + {0x06e9, 0x06fd, 20}, + {0x06fe, 0x07f6, 248}, + {0x09fa, 0x0b70, 374}, + {0x0bf3, 0x0bf8, 1}, {0x0bfa, 0x0c7f, 133}, - {0x0cf1, 0x0cf2, 1}, {0x0d79, 0x0f01, 392}, {0x0f02, 0x0f03, 1}, {0x0f13, 0x0f17, 1}, @@ -1271,16 +1300,16 @@ var _So = []Range{ {0x109e, 0x109f, 1}, {0x1360, 0x1390, 48}, {0x1391, 0x1399, 1}, - {0x1940, 0x19e0, 160}, - {0x19e1, 0x19ff, 1}, + {0x1940, 0x19de, 158}, + {0x19df, 0x19ff, 1}, {0x1b61, 0x1b6a, 1}, {0x1b74, 0x1b7c, 1}, {0x2100, 0x2101, 1}, {0x2103, 0x2106, 1}, {0x2108, 0x2109, 1}, {0x2114, 0x2116, 2}, - {0x2117, 0x2118, 1}, - {0x211e, 0x2123, 1}, + {0x2117, 0x211e, 7}, + {0x211f, 0x2123, 1}, {0x2125, 0x2129, 2}, {0x212e, 0x213a, 12}, {0x213b, 0x214a, 15}, @@ -1301,7 +1330,7 @@ var _So = []Range{ {0x232b, 0x237b, 1}, {0x237d, 0x239a, 1}, {0x23b4, 0x23db, 1}, - {0x23e2, 0x23e8, 1}, + {0x23e2, 0x23f3, 1}, {0x2400, 0x2426, 1}, {0x2440, 0x244a, 1}, {0x249c, 0x24e9, 1}, @@ -1309,21 +1338,9 @@ var _So = []Range{ {0x25b8, 0x25c0, 1}, {0x25c2, 0x25f7, 1}, {0x2600, 0x266e, 1}, - {0x2670, 0x26cd, 1}, - {0x26cf, 0x26e1, 1}, - {0x26e3, 0x26e8, 5}, - {0x26e9, 0x26ff, 1}, - {0x2701, 0x2704, 1}, - {0x2706, 0x2709, 1}, - {0x270c, 0x2727, 1}, - {0x2729, 0x274b, 1}, - {0x274d, 0x274f, 2}, - {0x2750, 0x2752, 1}, - {0x2756, 0x275e, 1}, - {0x2761, 0x2767, 1}, - {0x2794, 0x2798, 4}, - {0x2799, 0x27af, 1}, - {0x27b1, 0x27be, 1}, + {0x2670, 0x26ff, 1}, + {0x2701, 0x2767, 1}, + {0x2794, 0x27bf, 1}, {0x2800, 0x28ff, 1}, {0x2b00, 0x2b2f, 1}, {0x2b45, 0x2b46, 1}, @@ -1372,18 +1389,43 @@ var _So = []Range{ {0x1d301, 0x1d356, 1}, {0x1f000, 0x1f02b, 1}, {0x1f030, 0x1f093, 1}, + {0x1f0a0, 0x1f0ae, 1}, + {0x1f0b1, 0x1f0be, 1}, + {0x1f0c1, 0x1f0cf, 1}, + {0x1f0d1, 0x1f0df, 1}, {0x1f110, 0x1f12e, 1}, - {0x1f131, 0x1f13d, 12}, - {0x1f13f, 0x1f142, 3}, - {0x1f146, 0x1f14a, 4}, - {0x1f14b, 0x1f14e, 1}, - {0x1f157, 0x1f15f, 8}, - {0x1f179, 0x1f17b, 2}, - {0x1f17c, 0x1f17f, 3}, - {0x1f18a, 0x1f18d, 1}, - {0x1f190, 0x1f200, 112}, - {0x1f210, 0x1f231, 1}, + {0x1f130, 0x1f169, 1}, + {0x1f170, 0x1f19a, 1}, + {0x1f1e6, 0x1f202, 1}, + {0x1f210, 0x1f23a, 1}, {0x1f240, 0x1f248, 1}, + {0x1f250, 0x1f251, 1}, + {0x1f300, 0x1f320, 1}, + {0x1f330, 0x1f335, 1}, + {0x1f337, 0x1f37c, 1}, + {0x1f380, 0x1f393, 1}, + {0x1f3a0, 0x1f3c4, 1}, + {0x1f3c6, 0x1f3ca, 1}, + {0x1f3e0, 0x1f3f0, 1}, + {0x1f400, 0x1f43e, 1}, + {0x1f440, 0x1f442, 2}, + {0x1f443, 0x1f4f7, 1}, + {0x1f4f9, 0x1f4fc, 1}, + {0x1f500, 0x1f53d, 1}, + {0x1f550, 0x1f567, 1}, + {0x1f5fb, 0x1f5ff, 1}, + {0x1f601, 0x1f610, 1}, + {0x1f612, 0x1f614, 1}, + {0x1f616, 0x1f61c, 2}, + {0x1f61d, 0x1f61e, 1}, + {0x1f620, 0x1f625, 1}, + {0x1f628, 0x1f62b, 1}, + {0x1f62d, 0x1f630, 3}, + {0x1f631, 0x1f633, 1}, + {0x1f635, 0x1f640, 1}, + {0x1f645, 0x1f64f, 1}, + {0x1f680, 0x1f6c5, 1}, + {0x1f700, 0x1f773, 1}, } var _Sm = []Range{ @@ -1397,7 +1439,8 @@ var _Sm = []Range{ {0x2044, 0x2052, 14}, {0x207a, 0x207c, 1}, {0x208a, 0x208c, 1}, - {0x2140, 0x2144, 1}, + {0x2118, 0x2140, 40}, + {0x2141, 0x2144, 1}, {0x214b, 0x2190, 69}, {0x2191, 0x2194, 1}, {0x219a, 0x219b, 1}, @@ -1416,8 +1459,8 @@ var _Sm = []Range{ {0x266f, 0x27c0, 337}, {0x27c1, 0x27c4, 1}, {0x27c7, 0x27ca, 1}, - {0x27cc, 0x27d0, 4}, - {0x27d1, 0x27e5, 1}, + {0x27cc, 0x27ce, 2}, + {0x27cf, 0x27e5, 1}, {0x27f0, 0x27ff, 1}, {0x2900, 0x2982, 1}, {0x2999, 0x29d7, 1}, @@ -1459,6 +1502,7 @@ var _Sk = []Range{ {0xa700, 0xa716, 1}, {0xa720, 0xa721, 1}, {0xa789, 0xa78a, 1}, + {0xfbb2, 0xfbc1, 1}, {0xff3e, 0xff40, 2}, {0xffe3, 0xffe3, 1}, } @@ -1470,7 +1514,7 @@ var _Sc = []Range{ {0x09f3, 0x09fb, 8}, {0x0af1, 0x0bf9, 264}, {0x0e3f, 0x17db, 2460}, - {0x20a0, 0x20b8, 1}, + {0x20a0, 0x20b9, 1}, {0xa838, 0xfdfc, 21956}, {0xfe69, 0xff04, 155}, {0xffe0, 0xffe1, 1}, @@ -1528,7 +1572,7 @@ var _Lu = []Range{ {0x0460, 0x0480, 2}, {0x048a, 0x04c0, 2}, {0x04c1, 0x04cd, 2}, - {0x04d0, 0x0524, 2}, + {0x04d0, 0x0526, 2}, {0x0531, 0x0556, 1}, {0x10a0, 0x10c5, 1}, {0x1e00, 0x1e94, 2}, @@ -1564,15 +1608,16 @@ var _Lu = []Range{ {0x2c7e, 0x2c80, 1}, {0x2c82, 0x2ce2, 2}, {0x2ceb, 0x2ced, 2}, - {0xa640, 0xa65e, 2}, - {0xa662, 0xa66c, 2}, + {0xa640, 0xa66c, 2}, {0xa680, 0xa696, 2}, {0xa722, 0xa72e, 2}, {0xa732, 0xa76e, 2}, {0xa779, 0xa77d, 2}, {0xa77e, 0xa786, 2}, - {0xa78b, 0xff21, 22422}, - {0xff22, 0xff3a, 1}, + {0xa78b, 0xa78d, 2}, + {0xa790, 0xa7a0, 16}, + {0xa7a2, 0xa7a8, 2}, + {0xff21, 0xff3a, 1}, {0x10400, 0x10427, 1}, {0x1d400, 0x1d419, 1}, {0x1d434, 0x1d44d, 1}, @@ -1622,7 +1667,7 @@ var _Lo = []Range{ {0x0294, 0x05d0, 828}, {0x05d1, 0x05ea, 1}, {0x05f0, 0x05f2, 1}, - {0x0621, 0x063f, 1}, + {0x0620, 0x063f, 1}, {0x0641, 0x064a, 1}, {0x066e, 0x066f, 1}, {0x0671, 0x06d3, 1}, @@ -1635,11 +1680,12 @@ var _Lo = []Range{ {0x07b1, 0x07ca, 25}, {0x07cb, 0x07ea, 1}, {0x0800, 0x0815, 1}, + {0x0840, 0x0858, 1}, {0x0904, 0x0939, 1}, {0x093d, 0x0950, 19}, {0x0958, 0x0961, 1}, - {0x0972, 0x0979, 7}, - {0x097a, 0x097f, 1}, + {0x0972, 0x0977, 1}, + {0x0979, 0x097f, 1}, {0x0985, 0x098c, 1}, {0x098f, 0x0990, 1}, {0x0993, 0x09a8, 1}, @@ -1703,13 +1749,13 @@ var _Lo = []Range{ {0x0cb5, 0x0cb9, 1}, {0x0cbd, 0x0cde, 33}, {0x0ce0, 0x0ce1, 1}, + {0x0cf1, 0x0cf2, 1}, {0x0d05, 0x0d0c, 1}, {0x0d0e, 0x0d10, 1}, - {0x0d12, 0x0d28, 1}, - {0x0d2a, 0x0d39, 1}, - {0x0d3d, 0x0d60, 35}, - {0x0d61, 0x0d7a, 25}, - {0x0d7b, 0x0d7f, 1}, + {0x0d12, 0x0d3a, 1}, + {0x0d3d, 0x0d4e, 17}, + {0x0d60, 0x0d61, 1}, + {0x0d7a, 0x0d7f, 1}, {0x0d85, 0x0d96, 1}, {0x0d9a, 0x0db1, 1}, {0x0db3, 0x0dbb, 1}, @@ -1735,7 +1781,7 @@ var _Lo = []Range{ {0x0f00, 0x0f40, 64}, {0x0f41, 0x0f47, 1}, {0x0f49, 0x0f6c, 1}, - {0x0f88, 0x0f8b, 1}, + {0x0f88, 0x0f8c, 1}, {0x1000, 0x102a, 1}, {0x103f, 0x1050, 17}, {0x1051, 0x1055, 1}, @@ -1792,6 +1838,7 @@ var _Lo = []Range{ {0x1b45, 0x1b4b, 1}, {0x1b83, 0x1ba0, 1}, {0x1bae, 0x1baf, 1}, + {0x1bc0, 0x1be5, 1}, {0x1c00, 0x1c23, 1}, {0x1c4d, 0x1c4f, 1}, {0x1c5a, 0x1c77, 1}, @@ -1815,7 +1862,7 @@ var _Lo = []Range{ {0x30ff, 0x3105, 6}, {0x3106, 0x312d, 1}, {0x3131, 0x318e, 1}, - {0x31a0, 0x31b7, 1}, + {0x31a0, 0x31ba, 1}, {0x31f0, 0x31ff, 1}, {0x3400, 0x4db5, 1}, {0x4e00, 0x9fcb, 1}, @@ -1851,6 +1898,11 @@ var _Lo = []Range{ {0xaaba, 0xaabd, 1}, {0xaac0, 0xaac2, 2}, {0xaadb, 0xaadc, 1}, + {0xab01, 0xab06, 1}, + {0xab09, 0xab0e, 1}, + {0xab11, 0xab16, 1}, + {0xab20, 0xab26, 1}, + {0xab28, 0xab2e, 1}, {0xabc0, 0xabe2, 1}, {0xac00, 0xd7a3, 1}, {0xd7b0, 0xd7c6, 1}, @@ -1912,11 +1964,15 @@ var _Lo = []Range{ {0x10b40, 0x10b55, 1}, {0x10b60, 0x10b72, 1}, {0x10c00, 0x10c48, 1}, + {0x11003, 0x11037, 1}, {0x11083, 0x110af, 1}, {0x12000, 0x1236e, 1}, {0x13000, 0x1342e, 1}, + {0x16800, 0x16a38, 1}, + {0x1b000, 0x1b001, 1}, {0x20000, 0x2a6d6, 1}, {0x2a700, 0x2b734, 1}, + {0x2b740, 0x2b81d, 1}, {0x2f800, 0x2fa1d, 1}, } @@ -1958,7 +2014,7 @@ var ( ) // Generated by running -// maketables --scripts=all --url=http://www.unicode.org/Public/5.2.0/ucd/ +// maketables --scripts=all --url=http://www.unicode.org/Public/6.0.0/ucd/ // DO NOT EDIT // Scripts is the set of Unicode script tables. @@ -1977,6 +2033,7 @@ var Scripts = map[string][]Range{ "Bopomofo": Bopomofo, "Imperial_Aramaic": Imperial_Aramaic, "Kaithi": Kaithi, + "Mandaic": Mandaic, "Old_South_Arabian": Old_South_Arabian, "Kayah_Li": Kayah_Li, "New_Tai_Lue": New_Tai_Lue, @@ -1987,6 +2044,7 @@ var Scripts = map[string][]Range{ "Old_Turkic": Old_Turkic, "Tamil": Tamil, "Tagalog": Tagalog, + "Brahmi": Brahmi, "Arabic": Arabic, "Tagbanwa": Tagbanwa, "Canadian_Aboriginal": Canadian_Aboriginal, @@ -2019,6 +2077,7 @@ var Scripts = map[string][]Range{ "Vai": Vai, "Cherokee": Cherokee, "Ogham": Ogham, + "Batak": Batak, "Syriac": Syriac, "Gurmukhi": Gurmukhi, "Tai_Tham": Tai_Tham, @@ -2065,17 +2124,17 @@ var _Katakana = []Range{ {0x3300, 0x3357, 1}, {0xff66, 0xff6f, 1}, {0xff71, 0xff9d, 1}, + {0x1b000, 0x1b000, 1}, } var _Malayalam = []Range{ {0x0d02, 0x0d03, 1}, {0x0d05, 0x0d0c, 1}, {0x0d0e, 0x0d10, 1}, - {0x0d12, 0x0d28, 1}, - {0x0d2a, 0x0d39, 1}, + {0x0d12, 0x0d3a, 1}, {0x0d3d, 0x0d44, 1}, {0x0d46, 0x0d48, 1}, - {0x0d4a, 0x0d4d, 1}, + {0x0d4a, 0x0d4e, 1}, {0x0d57, 0x0d57, 1}, {0x0d60, 0x0d63, 1}, {0x0d66, 0x0d75, 1}, @@ -2108,15 +2167,17 @@ var _Latin = []Range{ {0x1e00, 0x1eff, 1}, {0x2071, 0x2071, 1}, {0x207f, 0x207f, 1}, - {0x2090, 0x2094, 1}, + {0x2090, 0x209c, 1}, {0x212a, 0x212b, 1}, {0x2132, 0x2132, 1}, {0x214e, 0x214e, 1}, {0x2160, 0x2188, 1}, {0x2c60, 0x2c7f, 1}, {0xa722, 0xa787, 1}, - {0xa78b, 0xa78c, 1}, - {0xa7fb, 0xa7ff, 1}, + {0xa78b, 0xa78e, 1}, + {0xa790, 0xa791, 1}, + {0xa7a0, 0xa7a9, 1}, + {0xa7fa, 0xa7ff, 1}, {0xfb00, 0xfb06, 1}, {0xff21, 0xff3a, 1}, {0xff41, 0xff5a, 1}, @@ -2143,6 +2204,7 @@ var _Inherited = []Range{ {0x0300, 0x036f, 1}, {0x0485, 0x0486, 1}, {0x064b, 0x0655, 1}, + {0x065f, 0x065f, 1}, {0x0670, 0x0670, 1}, {0x0951, 0x0952, 1}, {0x1cd0, 0x1cd2, 1}, @@ -2150,10 +2212,10 @@ var _Inherited = []Range{ {0x1ce2, 0x1ce8, 1}, {0x1ced, 0x1ced, 1}, {0x1dc0, 0x1de6, 1}, - {0x1dfd, 0x1dff, 1}, + {0x1dfc, 0x1dff, 1}, {0x200c, 0x200d, 1}, {0x20d0, 0x20f0, 1}, - {0x302a, 0x302f, 1}, + {0x302a, 0x302d, 1}, {0x3099, 0x309a, 1}, {0xfe00, 0xfe0f, 1}, {0xfe20, 0xfe26, 1}, @@ -2188,8 +2250,9 @@ var _Samaritan = []Range{ } var _Bopomofo = []Range{ + {0x02ea, 0x02eb, 1}, {0x3105, 0x312d, 1}, - {0x31a0, 0x31b7, 1}, + {0x31a0, 0x31ba, 1}, } var _Imperial_Aramaic = []Range{ @@ -2201,6 +2264,11 @@ var _Kaithi = []Range{ {0x11080, 0x110c1, 1}, } +var _Mandaic = []Range{ + {0x0840, 0x085b, 1}, + {0x085e, 0x085e, 1}, +} + var _Old_South_Arabian = []Range{ {0x10a60, 0x10a7f, 1}, } @@ -2241,13 +2309,13 @@ var _Common = []Range{ {0x00d7, 0x00d7, 1}, {0x00f7, 0x00f7, 1}, {0x02b9, 0x02df, 1}, - {0x02e5, 0x02ff, 1}, + {0x02e5, 0x02e9, 1}, + {0x02ec, 0x02ff, 1}, {0x0374, 0x0374, 1}, {0x037e, 0x037e, 1}, {0x0385, 0x0385, 1}, {0x0387, 0x0387, 1}, {0x0589, 0x0589, 1}, - {0x0600, 0x0603, 1}, {0x060c, 0x060c, 1}, {0x061b, 0x061b, 1}, {0x061f, 0x061f, 1}, @@ -2256,7 +2324,6 @@ var _Common = []Range{ {0x06dd, 0x06dd, 1}, {0x0964, 0x0965, 1}, {0x0970, 0x0970, 1}, - {0x0cf1, 0x0cf2, 1}, {0x0e3f, 0x0e3f, 1}, {0x0fd5, 0x0fd8, 1}, {0x10fb, 0x10fb, 1}, @@ -2273,33 +2340,20 @@ var _Common = []Range{ {0x206a, 0x2070, 1}, {0x2074, 0x207e, 1}, {0x2080, 0x208e, 1}, - {0x20a0, 0x20b8, 1}, + {0x20a0, 0x20b9, 1}, {0x2100, 0x2125, 1}, {0x2127, 0x2129, 1}, {0x212c, 0x2131, 1}, {0x2133, 0x214d, 1}, {0x214f, 0x215f, 1}, {0x2189, 0x2189, 1}, - {0x2190, 0x23e8, 1}, + {0x2190, 0x23f3, 1}, {0x2400, 0x2426, 1}, {0x2440, 0x244a, 1}, - {0x2460, 0x26cd, 1}, - {0x26cf, 0x26e1, 1}, - {0x26e3, 0x26e3, 1}, - {0x26e8, 0x26ff, 1}, - {0x2701, 0x2704, 1}, - {0x2706, 0x2709, 1}, - {0x270c, 0x2727, 1}, - {0x2729, 0x274b, 1}, - {0x274d, 0x274d, 1}, - {0x274f, 0x2752, 1}, - {0x2756, 0x275e, 1}, - {0x2761, 0x2794, 1}, - {0x2798, 0x27af, 1}, - {0x27b1, 0x27be, 1}, - {0x27c0, 0x27ca, 1}, + {0x2460, 0x26ff, 1}, + {0x2701, 0x27ca, 1}, {0x27cc, 0x27cc, 1}, - {0x27d0, 0x27ff, 1}, + {0x27ce, 0x27ff, 1}, {0x2900, 0x2b4c, 1}, {0x2b50, 0x2b59, 1}, {0x2e00, 0x2e31, 1}, @@ -2373,23 +2427,47 @@ var _Common = []Range{ {0x1d7ce, 0x1d7ff, 1}, {0x1f000, 0x1f02b, 1}, {0x1f030, 0x1f093, 1}, + {0x1f0a0, 0x1f0ae, 1}, + {0x1f0b1, 0x1f0be, 1}, + {0x1f0c1, 0x1f0cf, 1}, + {0x1f0d1, 0x1f0df, 1}, {0x1f100, 0x1f10a, 1}, {0x1f110, 0x1f12e, 1}, - {0x1f131, 0x1f131, 1}, - {0x1f13d, 0x1f13d, 1}, - {0x1f13f, 0x1f13f, 1}, - {0x1f142, 0x1f142, 1}, - {0x1f146, 0x1f146, 1}, - {0x1f14a, 0x1f14e, 1}, - {0x1f157, 0x1f157, 1}, - {0x1f15f, 0x1f15f, 1}, - {0x1f179, 0x1f179, 1}, - {0x1f17b, 0x1f17c, 1}, - {0x1f17f, 0x1f17f, 1}, - {0x1f18a, 0x1f18d, 1}, - {0x1f190, 0x1f190, 1}, - {0x1f210, 0x1f231, 1}, + {0x1f130, 0x1f169, 1}, + {0x1f170, 0x1f19a, 1}, + {0x1f1e6, 0x1f1ff, 1}, + {0x1f201, 0x1f202, 1}, + {0x1f210, 0x1f23a, 1}, {0x1f240, 0x1f248, 1}, + {0x1f250, 0x1f251, 1}, + {0x1f300, 0x1f320, 1}, + {0x1f330, 0x1f335, 1}, + {0x1f337, 0x1f37c, 1}, + {0x1f380, 0x1f393, 1}, + {0x1f3a0, 0x1f3c4, 1}, + {0x1f3c6, 0x1f3ca, 1}, + {0x1f3e0, 0x1f3f0, 1}, + {0x1f400, 0x1f43e, 1}, + {0x1f440, 0x1f440, 1}, + {0x1f442, 0x1f4f7, 1}, + {0x1f4f9, 0x1f4fc, 1}, + {0x1f500, 0x1f53d, 1}, + {0x1f550, 0x1f567, 1}, + {0x1f5fb, 0x1f5ff, 1}, + {0x1f601, 0x1f610, 1}, + {0x1f612, 0x1f614, 1}, + {0x1f616, 0x1f616, 1}, + {0x1f618, 0x1f618, 1}, + {0x1f61a, 0x1f61a, 1}, + {0x1f61c, 0x1f61e, 1}, + {0x1f620, 0x1f625, 1}, + {0x1f628, 0x1f62b, 1}, + {0x1f62d, 0x1f62d, 1}, + {0x1f630, 0x1f633, 1}, + {0x1f635, 0x1f640, 1}, + {0x1f645, 0x1f64f, 1}, + {0x1f680, 0x1f6c5, 1}, + {0x1f700, 0x1f773, 1}, {0xe0001, 0xe0001, 1}, {0xe0020, 0xe007f, 1}, } @@ -2408,6 +2486,7 @@ var _Kannada = []Range{ {0x0cde, 0x0cde, 1}, {0x0ce0, 0x0ce3, 1}, {0x0ce6, 0x0cef, 1}, + {0x0cf1, 0x0cf2, 1}, } var _Old_Turkic = []Range{ @@ -2438,18 +2517,24 @@ var _Tagalog = []Range{ {0x170e, 0x1714, 1}, } +var _Brahmi = []Range{ + {0x11000, 0x1104d, 1}, + {0x11052, 0x1106f, 1}, +} + var _Arabic = []Range{ + {0x0600, 0x0603, 1}, {0x0606, 0x060b, 1}, {0x060d, 0x061a, 1}, {0x061e, 0x061e, 1}, - {0x0621, 0x063f, 1}, + {0x0620, 0x063f, 1}, {0x0641, 0x064a, 1}, {0x0656, 0x065e, 1}, {0x066a, 0x066f, 1}, {0x0671, 0x06dc, 1}, {0x06de, 0x06ff, 1}, {0x0750, 0x077f, 1}, - {0xfb50, 0xfbb1, 1}, + {0xfb50, 0xfbc1, 1}, {0xfbd3, 0xfd3d, 1}, {0xfd50, 0xfd8f, 1}, {0xfd92, 0xfdc7, 1}, @@ -2473,11 +2558,11 @@ var _Canadian_Aboriginal = []Range{ var _Tibetan = []Range{ {0x0f00, 0x0f47, 1}, {0x0f49, 0x0f6c, 1}, - {0x0f71, 0x0f8b, 1}, - {0x0f90, 0x0f97, 1}, + {0x0f71, 0x0f97, 1}, {0x0f99, 0x0fbc, 1}, {0x0fbe, 0x0fcc, 1}, {0x0fce, 0x0fd4, 1}, + {0x0fd9, 0x0fda, 1}, } var _Coptic = []Range{ @@ -2489,6 +2574,7 @@ var _Coptic = []Range{ var _Hiragana = []Range{ {0x3041, 0x3096, 1}, {0x309d, 0x309f, 1}, + {0x1b001, 0x1b001, 1}, {0x1f200, 0x1f200, 1}, } @@ -2620,6 +2706,7 @@ var _Saurashtra = []Range{ var _Hangul = []Range{ {0x1100, 0x11ff, 1}, + {0x302e, 0x302f, 1}, {0x3131, 0x318e, 1}, {0x3200, 0x321e, 1}, {0x3260, 0x327e, 1}, @@ -2671,7 +2758,7 @@ var _Oriya = []Range{ {0x0b56, 0x0b57, 1}, {0x0b5c, 0x0b5d, 1}, {0x0b5f, 0x0b63, 1}, - {0x0b66, 0x0b71, 1}, + {0x0b66, 0x0b77, 1}, } var _Buhid = []Range{ @@ -2695,7 +2782,7 @@ var _Ethiopic = []Range{ {0x12d8, 0x1310, 1}, {0x1312, 0x1315, 1}, {0x1318, 0x135a, 1}, - {0x135f, 0x137c, 1}, + {0x135d, 0x137c, 1}, {0x1380, 0x1399, 1}, {0x2d80, 0x2d96, 1}, {0x2da0, 0x2da6, 1}, @@ -2706,6 +2793,11 @@ var _Ethiopic = []Range{ {0x2dc8, 0x2dce, 1}, {0x2dd0, 0x2dd6, 1}, {0x2dd8, 0x2dde, 1}, + {0xab01, 0xab06, 1}, + {0xab09, 0xab0e, 1}, + {0xab11, 0xab16, 1}, + {0xab20, 0xab26, 1}, + {0xab28, 0xab2e, 1}, } var _Javanese = []Range{ @@ -2730,6 +2822,11 @@ var _Ogham = []Range{ {0x1680, 0x169c, 1}, } +var _Batak = []Range{ + {0x1bc0, 0x1bf3, 1}, + {0x1bfc, 0x1bff, 1}, +} + var _Syriac = []Range{ {0x0700, 0x070d, 1}, {0x070f, 0x074a, 1}, @@ -2796,6 +2893,7 @@ var _Buginese = []Range{ var _Bamum = []Range{ {0xa6a0, 0xa6f7, 1}, + {0x16800, 0x16a38, 1}, } var _Lepcha = []Range{ @@ -2890,13 +2988,10 @@ var _Tai_Viet = []Range{ } var _Devanagari = []Range{ - {0x0900, 0x0939, 1}, - {0x093c, 0x094e, 1}, - {0x0950, 0x0950, 1}, - {0x0953, 0x0955, 1}, - {0x0958, 0x0963, 1}, + {0x0900, 0x0950, 1}, + {0x0953, 0x0963, 1}, {0x0966, 0x096f, 1}, - {0x0971, 0x0972, 1}, + {0x0971, 0x0977, 1}, {0x0979, 0x097f, 1}, {0xa8e0, 0xa8fb, 1}, } @@ -2908,7 +3003,8 @@ var _Lydian = []Range{ var _Tifinagh = []Range{ {0x2d30, 0x2d65, 1}, - {0x2d6f, 0x2d6f, 1}, + {0x2d6f, 0x2d70, 1}, + {0x2d7f, 0x2d7f, 1}, } var _Ugaritic = []Range{ @@ -2923,12 +3019,11 @@ var _Thai = []Range{ var _Cyrillic = []Range{ {0x0400, 0x0484, 1}, - {0x0487, 0x0525, 1}, + {0x0487, 0x0527, 1}, {0x1d2b, 0x1d2b, 1}, {0x1d78, 0x1d78, 1}, {0x2de0, 0x2dff, 1}, - {0xa640, 0xa65f, 1}, - {0xa662, 0xa673, 1}, + {0xa640, 0xa673, 1}, {0xa67c, 0xa697, 1}, } @@ -2982,6 +3077,7 @@ var _Han = []Range{ {0xfa70, 0xfad9, 1}, {0x20000, 0x2a6d6, 1}, {0x2a700, 0x2b734, 1}, + {0x2b740, 0x2b81d, 1}, {0x2f800, 0x2fa1d, 1}, } @@ -2995,8 +3091,10 @@ var ( Avestan = _Avestan // Avestan is the set of Unicode characters in script Avestan. Balinese = _Balinese // Balinese is the set of Unicode characters in script Balinese. Bamum = _Bamum // Bamum is the set of Unicode characters in script Bamum. + Batak = _Batak // Batak is the set of Unicode characters in script Batak. Bengali = _Bengali // Bengali is the set of Unicode characters in script Bengali. Bopomofo = _Bopomofo // Bopomofo is the set of Unicode characters in script Bopomofo. + Brahmi = _Brahmi // Brahmi is the set of Unicode characters in script Brahmi. Braille = _Braille // Braille is the set of Unicode characters in script Braille. Buginese = _Buginese // Buginese is the set of Unicode characters in script Buginese. Buhid = _Buhid // Buhid is the set of Unicode characters in script Buhid. @@ -3044,6 +3142,7 @@ var ( Lycian = _Lycian // Lycian is the set of Unicode characters in script Lycian. Lydian = _Lydian // Lydian is the set of Unicode characters in script Lydian. Malayalam = _Malayalam // Malayalam is the set of Unicode characters in script Malayalam. + Mandaic = _Mandaic // Mandaic is the set of Unicode characters in script Mandaic. Meetei_Mayek = _Meetei_Mayek // Meetei_Mayek is the set of Unicode characters in script Meetei_Mayek. Mongolian = _Mongolian // Mongolian is the set of Unicode characters in script Mongolian. Myanmar = _Myanmar // Myanmar is the set of Unicode characters in script Myanmar. @@ -3085,7 +3184,7 @@ var ( ) // Generated by running -// maketables --props=all --url=http://www.unicode.org/Public/5.2.0/ucd/ +// maketables --props=all --url=http://www.unicode.org/Public/6.0.0/ucd/ // DO NOT EDIT // Properties is the set of Unicode property tables. @@ -3236,6 +3335,7 @@ var _ASCII_Hex_Digit = []Range{ var _Deprecated = []Range{ {0x0149, 0x0149, 1}, + {0x0673, 0x0673, 1}, {0x0f77, 0x0f77, 1}, {0x0f79, 0x0f79, 1}, {0x17a3, 0x17a4, 1}, @@ -3263,6 +3363,7 @@ var _Terminal_Punctuation = []Range{ {0x070c, 0x070c, 1}, {0x07f8, 0x07f9, 1}, {0x0830, 0x083e, 1}, + {0x085e, 0x085e, 1}, {0x0964, 0x0965, 1}, {0x0e5a, 0x0e5b, 1}, {0x0f08, 0x0f08, 1}, @@ -3309,6 +3410,7 @@ var _Terminal_Punctuation = []Range{ {0x10857, 0x10857, 1}, {0x1091f, 0x1091f, 1}, {0x10b3a, 0x10b3f, 1}, + {0x11047, 0x1104d, 1}, {0x110be, 0x110c1, 1}, {0x12470, 0x12473, 1}, } @@ -3332,6 +3434,7 @@ var _Other_ID_Continue = []Range{ {0x00b7, 0x00b7, 1}, {0x0387, 0x0387, 1}, {0x1369, 0x1371, 1}, + {0x19da, 0x19da, 1}, } var _Bidi_Control = []Range{ @@ -3383,6 +3486,7 @@ var _Other_Math = []Range{ {0x20e5, 0x20e6, 1}, {0x20eb, 0x20ef, 1}, {0x2102, 0x2102, 1}, + {0x2107, 0x2107, 1}, {0x210a, 0x2113, 1}, {0x2115, 0x2115, 1}, {0x2119, 0x211d, 1}, @@ -3480,6 +3584,7 @@ var _Unified_Ideograph = []Range{ {0xfa27, 0xfa29, 1}, {0x20000, 0x2a6d6, 1}, {0x2a700, 0x2b734, 1}, + {0x2b740, 0x2b81d, 1}, } var _Hyphen = []Range{ @@ -3535,9 +3640,11 @@ var _STerm = []Range{ {0x1362, 0x1362, 1}, {0x1367, 0x1368, 1}, {0x166e, 0x166e, 1}, + {0x1735, 0x1736, 1}, {0x1803, 0x1803, 1}, {0x1809, 0x1809, 1}, {0x1944, 0x1945, 1}, + {0x1aa8, 0x1aab, 1}, {0x1b5a, 0x1b5b, 1}, {0x1b5e, 0x1b5f, 1}, {0x1c3b, 0x1c3c, 1}, @@ -3562,6 +3669,8 @@ var _STerm = []Range{ {0xff0e, 0xff0e, 1}, {0xff1f, 0xff1f, 1}, {0xff61, 0xff61, 1}, + {0x10a56, 0x10a57, 1}, + {0x11047, 0x11048, 1}, {0x110be, 0x110c1, 1}, } @@ -3574,7 +3683,7 @@ var _Other_Alphabetic = []Range{ {0x05c7, 0x05c7, 1}, {0x0610, 0x061a, 1}, {0x064b, 0x0657, 1}, - {0x0659, 0x065e, 1}, + {0x0659, 0x065f, 1}, {0x0670, 0x0670, 1}, {0x06d6, 0x06dc, 1}, {0x06e1, 0x06e4, 1}, @@ -3588,9 +3697,10 @@ var _Other_Alphabetic = []Range{ {0x0825, 0x0827, 1}, {0x0829, 0x082c, 1}, {0x0900, 0x0903, 1}, + {0x093a, 0x093b, 1}, {0x093e, 0x094c, 1}, - {0x094e, 0x094e, 1}, - {0x0955, 0x0955, 1}, + {0x094e, 0x094f, 1}, + {0x0955, 0x0957, 1}, {0x0962, 0x0963, 1}, {0x0981, 0x0983, 1}, {0x09be, 0x09c4, 1}, @@ -3652,7 +3762,7 @@ var _Other_Alphabetic = []Range{ {0x0ebb, 0x0ebc, 1}, {0x0ecd, 0x0ecd, 1}, {0x0f71, 0x0f81, 1}, - {0x0f90, 0x0f97, 1}, + {0x0f8d, 0x0f97, 1}, {0x0f99, 0x0fbc, 1}, {0x102b, 0x1036, 1}, {0x1038, 0x1038, 1}, @@ -3682,6 +3792,7 @@ var _Other_Alphabetic = []Range{ {0x1b35, 0x1b43, 1}, {0x1b80, 0x1b82, 1}, {0x1ba1, 0x1ba9, 1}, + {0x1be7, 0x1bf1, 1}, {0x1c24, 0x1c35, 1}, {0x1cf2, 0x1cf2, 1}, {0x24b6, 0x24e9, 1}, @@ -3692,7 +3803,7 @@ var _Other_Alphabetic = []Range{ {0xa926, 0xa92a, 1}, {0xa947, 0xa952, 1}, {0xa980, 0xa983, 1}, - {0xa9b3, 0xa9bf, 1}, + {0xa9b4, 0xa9bf, 1}, {0xaa29, 0xaa36, 1}, {0xaa43, 0xaa43, 1}, {0xaa4c, 0xaa4d, 1}, @@ -3705,6 +3816,8 @@ var _Other_Alphabetic = []Range{ {0x10a01, 0x10a03, 1}, {0x10a05, 0x10a06, 1}, {0x10a0c, 0x10a0f, 1}, + {0x11000, 0x11002, 1}, + {0x11038, 0x11045, 1}, {0x11082, 0x11082, 1}, {0x110b0, 0x110b8, 1}, } @@ -3867,6 +3980,7 @@ var _Ideographic = []Range{ {0xfa70, 0xfad9, 1}, {0x20000, 0x2a6d6, 1}, {0x2a700, 0x2b734, 1}, + {0x2b740, 0x2b81d, 1}, {0x2f800, 0x2fa1d, 1}, } @@ -3978,7 +4092,7 @@ var ( ) // Generated by running -// maketables --data=http://www.unicode.org/Public/5.2.0/ucd/UnicodeData.txt +// maketables --data=http://www.unicode.org/Public/6.0.0/ucd/UnicodeData.txt // DO NOT EDIT // CaseRanges is the table describing case mappings for all letters with @@ -4078,6 +4192,7 @@ var _CaseRanges = []CaseRange{ {0x025B, 0x025B, d{-203, 0, -203}}, {0x0260, 0x0260, d{-205, 0, -205}}, {0x0263, 0x0263, d{-207, 0, -207}}, + {0x0265, 0x0265, d{42280, 0, 42280}}, {0x0268, 0x0268, d{-209, 0, -209}}, {0x0269, 0x0269, d{-211, 0, -211}}, {0x026B, 0x026B, d{10743, 0, 10743}}, @@ -4134,7 +4249,7 @@ var _CaseRanges = []CaseRange{ {0x04C0, 0x04C0, d{0, 15, 0}}, {0x04C1, 0x04CE, d{UpperLower, UpperLower, UpperLower}}, {0x04CF, 0x04CF, d{-15, 0, -15}}, - {0x04D0, 0x0525, d{UpperLower, UpperLower, UpperLower}}, + {0x04D0, 0x0527, d{UpperLower, UpperLower, UpperLower}}, {0x0531, 0x0556, d{0, 48, 0}}, {0x0561, 0x0586, d{-48, 0, -48}}, {0x10A0, 0x10C5, d{0, 7264, 0}}, @@ -4222,8 +4337,7 @@ var _CaseRanges = []CaseRange{ {0x2C80, 0x2CE3, d{UpperLower, UpperLower, UpperLower}}, {0x2CEB, 0x2CEE, d{UpperLower, UpperLower, UpperLower}}, {0x2D00, 0x2D25, d{-7264, 0, -7264}}, - {0xA640, 0xA65F, d{UpperLower, UpperLower, UpperLower}}, - {0xA662, 0xA66D, d{UpperLower, UpperLower, UpperLower}}, + {0xA640, 0xA66D, d{UpperLower, UpperLower, UpperLower}}, {0xA680, 0xA697, d{UpperLower, UpperLower, UpperLower}}, {0xA722, 0xA72F, d{UpperLower, UpperLower, UpperLower}}, {0xA732, 0xA76F, d{UpperLower, UpperLower, UpperLower}}, @@ -4231,6 +4345,9 @@ var _CaseRanges = []CaseRange{ {0xA77D, 0xA77D, d{0, -35332, 0}}, {0xA77E, 0xA787, d{UpperLower, UpperLower, UpperLower}}, {0xA78B, 0xA78C, d{UpperLower, UpperLower, UpperLower}}, + {0xA78D, 0xA78D, d{0, -42280, 0}}, + {0xA790, 0xA791, d{UpperLower, UpperLower, UpperLower}}, + {0xA7A0, 0xA7A9, d{UpperLower, UpperLower, UpperLower}}, {0xFF21, 0xFF3A, d{0, 32, 0}}, {0xFF41, 0xFF5A, d{-32, 0, -32}}, {0x10400, 0x10427, d{0, 40, 0}}, diff --git a/libgo/go/websocket/client.go b/libgo/go/websocket/client.go index 09134594405..d8a7aa0a266 100644 --- a/libgo/go/websocket/client.go +++ b/libgo/go/websocket/client.go @@ -245,20 +245,20 @@ func handshake(resourceName, host, origin, location, protocol string, br *bufio. } // Step 41. check websocket headers. - if resp.Header["Upgrade"] != "WebSocket" || - strings.ToLower(resp.Header["Connection"]) != "upgrade" { + if resp.Header.Get("Upgrade") != "WebSocket" || + strings.ToLower(resp.Header.Get("Connection")) != "upgrade" { return ErrBadUpgrade } - if resp.Header["Sec-Websocket-Origin"] != origin { + if resp.Header.Get("Sec-Websocket-Origin") != origin { return ErrBadWebSocketOrigin } - if resp.Header["Sec-Websocket-Location"] != location { + if resp.Header.Get("Sec-Websocket-Location") != location { return ErrBadWebSocketLocation } - if protocol != "" && resp.Header["Sec-Websocket-Protocol"] != protocol { + if protocol != "" && resp.Header.Get("Sec-Websocket-Protocol") != protocol { return ErrBadWebSocketProtocol } @@ -304,17 +304,17 @@ func draft75handshake(resourceName, host, origin, location, protocol string, br if resp.Status != "101 Web Socket Protocol Handshake" { return ErrBadStatus } - if resp.Header["Upgrade"] != "WebSocket" || - resp.Header["Connection"] != "Upgrade" { + if resp.Header.Get("Upgrade") != "WebSocket" || + resp.Header.Get("Connection") != "Upgrade" { return ErrBadUpgrade } - if resp.Header["Websocket-Origin"] != origin { + if resp.Header.Get("Websocket-Origin") != origin { return ErrBadWebSocketOrigin } - if resp.Header["Websocket-Location"] != location { + if resp.Header.Get("Websocket-Location") != location { return ErrBadWebSocketLocation } - if protocol != "" && resp.Header["Websocket-Protocol"] != protocol { + if protocol != "" && resp.Header.Get("Websocket-Protocol") != protocol { return ErrBadWebSocketProtocol } return diff --git a/libgo/go/websocket/server.go b/libgo/go/websocket/server.go index dd797f24e0e..25f057ba5b0 100644 --- a/libgo/go/websocket/server.go +++ b/libgo/go/websocket/server.go @@ -73,23 +73,23 @@ func (f Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) { } // HTTP version can be safely ignored. - if strings.ToLower(req.Header["Upgrade"]) != "websocket" || - strings.ToLower(req.Header["Connection"]) != "upgrade" { + if strings.ToLower(req.Header.Get("Upgrade")) != "websocket" || + strings.ToLower(req.Header.Get("Connection")) != "upgrade" { return } // TODO(ukai): check Host - origin, found := req.Header["Origin"] - if !found { + origin := req.Header.Get("Origin") + if origin == "" { return } - key1, found := req.Header["Sec-Websocket-Key1"] - if !found { + key1 := req.Header.Get("Sec-Websocket-Key1") + if key1 == "" { return } - key2, found := req.Header["Sec-Websocket-Key2"] - if !found { + key2 := req.Header.Get("Sec-Websocket-Key2") + if key2 == "" { return } key3 := make([]byte, 8) @@ -138,8 +138,8 @@ func (f Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) { buf.WriteString("Connection: Upgrade\r\n") buf.WriteString("Sec-WebSocket-Location: " + location + "\r\n") buf.WriteString("Sec-WebSocket-Origin: " + origin + "\r\n") - protocol, found := req.Header["Sec-Websocket-Protocol"] - if found { + protocol := strings.TrimSpace(req.Header.Get("Sec-Websocket-Protocol")) + if protocol != "" { buf.WriteString("Sec-WebSocket-Protocol: " + protocol + "\r\n") } // Step 12. send CRLF. @@ -167,18 +167,18 @@ func (f Draft75Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) { io.WriteString(w, "Unexpected request") return } - if req.Header["Upgrade"] != "WebSocket" { + if req.Header.Get("Upgrade") != "WebSocket" { w.WriteHeader(http.StatusBadRequest) io.WriteString(w, "missing Upgrade: WebSocket header") return } - if req.Header["Connection"] != "Upgrade" { + if req.Header.Get("Connection") != "Upgrade" { w.WriteHeader(http.StatusBadRequest) io.WriteString(w, "missing Connection: Upgrade header") return } - origin, found := req.Header["Origin"] - if !found { + origin := strings.TrimSpace(req.Header.Get("Origin")) + if origin == "" { w.WriteHeader(http.StatusBadRequest) io.WriteString(w, "missing Origin header") return @@ -205,9 +205,9 @@ func (f Draft75Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) { buf.WriteString("Connection: Upgrade\r\n") buf.WriteString("WebSocket-Origin: " + origin + "\r\n") buf.WriteString("WebSocket-Location: " + location + "\r\n") - protocol, found := req.Header["Websocket-Protocol"] + protocol := strings.TrimSpace(req.Header.Get("Websocket-Protocol")) // canonical header key of WebSocket-Protocol. - if found { + if protocol != "" { buf.WriteString("WebSocket-Protocol: " + protocol + "\r\n") } buf.WriteString("\r\n") diff --git a/libgo/go/websocket/websocket_test.go b/libgo/go/websocket/websocket_test.go index cc4b9dc1898..204a9de1e12 100644 --- a/libgo/go/websocket/websocket_test.go +++ b/libgo/go/websocket/websocket_test.go @@ -24,7 +24,7 @@ func echoServer(ws *Conn) { io.Copy(ws, ws) } func startServer() { l, e := net.Listen("tcp", "127.0.0.1:0") // any available address if e != nil { - log.Exitf("net.Listen tcp :0 %v", e) + log.Fatalf("net.Listen tcp :0 %v", e) } serverAddr = l.Addr().String() log.Print("Test WebSocket server listening on ", serverAddr) diff --git a/libgo/go/xml/read_test.go b/libgo/go/xml/read_test.go index 71ceddce4a4..a6b9a8ed18b 100644 --- a/libgo/go/xml/read_test.go +++ b/libgo/go/xml/read_test.go @@ -13,16 +13,16 @@ import ( func TestUnmarshalFeed(t *testing.T) { var f Feed - if err := Unmarshal(StringReader(rssFeedString), &f); err != nil { + if err := Unmarshal(StringReader(atomFeedString), &f); err != nil { t.Fatalf("Unmarshal: %s", err) } - if !reflect.DeepEqual(f, rssFeed) { - t.Fatalf("have %#v\nwant %#v", f, rssFeed) + if !reflect.DeepEqual(f, atomFeed) { + t.Fatalf("have %#v\nwant %#v", f, atomFeed) } } // hget http://codereview.appspot.com/rss/mine/rsc -const rssFeedString = ` +const atomFeedString = ` <?xml version="1.0" encoding="utf-8"?> <feed xmlns="http://www.w3.org/2005/Atom" xml:lang="en-us"><title>Code Review - My issues</title><link href="http://codereview.appspot.com/" rel="alternate"></link><li-nk href="http://codereview.appspot.com/rss/mine/rsc" rel="self"></li-nk><id>http://codereview.appspot.com/</id><updated>2009-10-04T01:35:58+00:00</updated><author><name>rietveld<></name></author><entry><title>rietveld: an attempt at pubsubhubbub </title><link hre-f="http://codereview.appspot.com/126085" rel="alternate"></link><updated>2009-10-04T01:35:58+00:00</updated><author><name>email-address-removed</name></author><id>urn:md5:134d9179c41f806be79b3a5f7877d19a</id><summary type="html"> @@ -115,7 +115,7 @@ type Text struct { type Time string -var rssFeed = Feed{ +var atomFeed = Feed{ XMLName: Name{"http://www.w3.org/2005/Atom", "feed"}, Title: "Code Review - My issues", Link: []Link{ diff --git a/libgo/go/xml/xml.go b/libgo/go/xml/xml.go index 4d9c672d273..691c13a1188 100644 --- a/libgo/go/xml/xml.go +++ b/libgo/go/xml/xml.go @@ -163,7 +163,7 @@ type Parser struct { // "quot": `"`, Entity map[string]string - r io.ReadByter + r io.ByteReader buf bytes.Buffer saved *bytes.Buffer stk *stack @@ -191,7 +191,7 @@ func NewParser(r io.Reader) *Parser { // Assume that if reader has its own // ReadByte, it's efficient enough. // Otherwise, use bufio. - if rb, ok := r.(io.ReadByter); ok { + if rb, ok := r.(io.ByteReader); ok { p.r = rb } else { p.r = bufio.NewReader(r) @@ -541,17 +541,36 @@ func (p *Parser) RawToken() (Token, os.Error) { } // Probably a directive: <!DOCTYPE ...>, <!ENTITY ...>, etc. - // We don't care, but accumulate for caller. + // We don't care, but accumulate for caller. Quoted angle + // brackets do not count for nesting. p.buf.Reset() p.buf.WriteByte(b) + inquote := uint8(0) + depth := 0 for { if b, ok = p.mustgetc(); !ok { return nil, p.err } - if b == '>' { + if inquote == 0 && b == '>' && depth == 0 { break } p.buf.WriteByte(b) + switch { + case b == inquote: + inquote = 0 + + case inquote != 0: + // in quotes, no special action + + case b == '\'' || b == '"': + inquote = b + + case b == '>' && inquote == 0: + depth-- + + case b == '<' && inquote == 0: + depth++ + } } return Directive(p.buf.Bytes()), nil } diff --git a/libgo/go/xml/xml_test.go b/libgo/go/xml/xml_test.go index 317ecabd90d..887bc3d140f 100644 --- a/libgo/go/xml/xml_test.go +++ b/libgo/go/xml/xml_test.go @@ -185,6 +185,52 @@ func TestRawToken(t *testing.T) { } } +// Ensure that directives (specifically !DOCTYPE) include the complete +// text of any nested directives, noting that < and > do not change +// nesting depth if they are in single or double quotes. + +var nestedDirectivesInput = ` +<!DOCTYPE [<!ENTITY rdf "http://www.w3.org/1999/02/22-rdf-syntax-ns#">]> +<!DOCTYPE [<!ENTITY xlt ">">]> +<!DOCTYPE [<!ENTITY xlt "<">]> +<!DOCTYPE [<!ENTITY xlt '>'>]> +<!DOCTYPE [<!ENTITY xlt '<'>]> +<!DOCTYPE [<!ENTITY xlt '">'>]> +<!DOCTYPE [<!ENTITY xlt "'<">]> +` + +var nestedDirectivesTokens = []Token{ + CharData([]byte("\n")), + Directive([]byte(`DOCTYPE [<!ENTITY rdf "http://www.w3.org/1999/02/22-rdf-syntax-ns#">]`)), + CharData([]byte("\n")), + Directive([]byte(`DOCTYPE [<!ENTITY xlt ">">]`)), + CharData([]byte("\n")), + Directive([]byte(`DOCTYPE [<!ENTITY xlt "<">]`)), + CharData([]byte("\n")), + Directive([]byte(`DOCTYPE [<!ENTITY xlt '>'>]`)), + CharData([]byte("\n")), + Directive([]byte(`DOCTYPE [<!ENTITY xlt '<'>]`)), + CharData([]byte("\n")), + Directive([]byte(`DOCTYPE [<!ENTITY xlt '">'>]`)), + CharData([]byte("\n")), + Directive([]byte(`DOCTYPE [<!ENTITY xlt "'<">]`)), + CharData([]byte("\n")), +} + +func TestNestedDirectives(t *testing.T) { + p := NewParser(StringReader(nestedDirectivesInput)) + + for i, want := range nestedDirectivesTokens { + have, err := p.Token() + if err != nil { + t.Fatalf("token %d: unexpected error: %s", i, err) + } + if !reflect.DeepEqual(have, want) { + t.Errorf("token %d = %#v want %#v", i, have, want) + } + } +} + func TestToken(t *testing.T) { p := NewParser(StringReader(testInput)) diff --git a/libgo/mksysinfo.sh b/libgo/mksysinfo.sh index e29febfa619..033c3ecbe3b 100755 --- a/libgo/mksysinfo.sh +++ b/libgo/mksysinfo.sh @@ -416,4 +416,12 @@ echo $msghdr | \ -e 's/msg_flags/Flags/' \ >> ${OUT} +# The ip_mreq struct +grep '^type _ip_mreq ' gen-sysinfo.go | \ + sed -e 's/_ip_mreq/IpMreq/' \ + -e 's/imr_multiaddr/Multiaddr/' \ + -e 's/imr_interface/Interface/' \ + -e 's/_in_addr/[4]byte/g' \ + >> ${OUT} + exit $? diff --git a/libgo/runtime/go-byte-array-to-string.c b/libgo/runtime/go-byte-array-to-string.c index 531730654d0..1b9ac2d7964 100644 --- a/libgo/runtime/go-byte-array-to-string.c +++ b/libgo/runtime/go-byte-array-to-string.c @@ -16,7 +16,7 @@ __go_byte_array_to_string (const void* p, size_t len) struct __go_string ret; bytes = (const unsigned char *) p; - retdata = runtime_mallocgc (len, RefNoPointers, 1, 0); + retdata = runtime_mallocgc (len, FlagNoPointers, 1, 0); __builtin_memcpy (retdata, bytes, len); ret.__data = retdata; ret.__length = len; diff --git a/libgo/runtime/go-int-array-to-string.c b/libgo/runtime/go-int-array-to-string.c index 46a33dafc2e..c16589f01a2 100644 --- a/libgo/runtime/go-int-array-to-string.c +++ b/libgo/runtime/go-int-array-to-string.c @@ -41,7 +41,7 @@ __go_int_array_to_string (const void* p, size_t len) slen += 4; } - retdata = runtime_mallocgc (slen, RefNoPointers, 1, 0); + retdata = runtime_mallocgc (slen, FlagNoPointers, 1, 0); ret.__data = retdata; ret.__length = slen; diff --git a/libgo/runtime/go-int-to-string.c b/libgo/runtime/go-int-to-string.c index 24d729cf893..af58015ed8f 100644 --- a/libgo/runtime/go-int-to-string.c +++ b/libgo/runtime/go-int-to-string.c @@ -51,7 +51,7 @@ __go_int_to_string (int v) } } - retdata = runtime_mallocgc (len, RefNoPointers, 1, 0); + retdata = runtime_mallocgc (len, FlagNoPointers, 1, 0); __builtin_memcpy (retdata, buf, len); ret.__data = retdata; ret.__length = len; diff --git a/libgo/runtime/go-new.c b/libgo/runtime/go-new.c index a592174e50f..8f25c5730ef 100644 --- a/libgo/runtime/go-new.c +++ b/libgo/runtime/go-new.c @@ -17,5 +17,5 @@ __go_new (size_t size) void * __go_new_nopointers (size_t size) { - return runtime_mallocgc (size, RefNoPointers, 1, 1); + return runtime_mallocgc (size, FlagNoPointers, 1, 1); } diff --git a/libgo/runtime/go-panic.c b/libgo/runtime/go-panic.c index 48d64416297..b684779cda8 100644 --- a/libgo/runtime/go-panic.c +++ b/libgo/runtime/go-panic.c @@ -112,7 +112,7 @@ __go_panic_msg (const char* msg) struct __go_empty_interface arg; len = __builtin_strlen (msg); - sdata = runtime_mallocgc (len, RefNoPointers, 0, 0); + sdata = runtime_mallocgc (len, FlagNoPointers, 0, 0); __builtin_memcpy (sdata, msg, len); s.__data = sdata; s.__length = len; diff --git a/libgo/runtime/go-string-to-byte-array.c b/libgo/runtime/go-string-to-byte-array.c index 3b646c81abe..10c565e39a5 100644 --- a/libgo/runtime/go-string-to-byte-array.c +++ b/libgo/runtime/go-string-to-byte-array.c @@ -15,7 +15,7 @@ __go_string_to_byte_array (struct __go_string str) unsigned char *data; struct __go_open_array ret; - data = (unsigned char *) runtime_mallocgc (str.__length, RefNoPointers, 1, 0); + data = (unsigned char *) runtime_mallocgc (str.__length, FlagNoPointers, 1, 0); __builtin_memcpy (data, str.__data, str.__length); ret.__values = (void *) data; ret.__count = str.__length; diff --git a/libgo/runtime/go-string-to-int-array.c b/libgo/runtime/go-string-to-int-array.c index 8d7f94f93ab..f59df6739f1 100644 --- a/libgo/runtime/go-string-to-int-array.c +++ b/libgo/runtime/go-string-to-int-array.c @@ -31,7 +31,7 @@ __go_string_to_int_array (struct __go_string str) p += __go_get_rune (p, pend - p, &rune); } - data = (uint32_t *) runtime_mallocgc (c * sizeof (uint32_t), RefNoPointers, + data = (uint32_t *) runtime_mallocgc (c * sizeof (uint32_t), FlagNoPointers, 1, 0); p = str.__data; pd = data; diff --git a/libgo/runtime/go-strplus.c b/libgo/runtime/go-strplus.c index c0cd356ca63..e4dea9c4690 100644 --- a/libgo/runtime/go-strplus.c +++ b/libgo/runtime/go-strplus.c @@ -21,7 +21,7 @@ __go_string_plus (struct __go_string s1, struct __go_string s2) return s1; len = s1.__length + s2.__length; - retdata = runtime_mallocgc (len, RefNoPointers, 1, 0); + retdata = runtime_mallocgc (len, FlagNoPointers, 1, 0); __builtin_memcpy (retdata, s1.__data, s1.__length); __builtin_memcpy (retdata + s1.__length, s2.__data, s2.__length); ret.__data = retdata; diff --git a/libgo/runtime/go-type.h b/libgo/runtime/go-type.h index b1f32850a00..76681217fdf 100644 --- a/libgo/runtime/go-type.h +++ b/libgo/runtime/go-type.h @@ -94,6 +94,10 @@ struct __go_type_descriptor /* A pointer to fields which are only used for some types. */ const struct __go_uncommon_type *__uncommon; + + /* The descriptor for the type which is a pointer to this type. + This may be NULL. */ + const struct __go_type_descriptor *__pointer_to_this; }; /* The information we store for each method of a type. */ diff --git a/libgo/runtime/go-unsafe-pointer.c b/libgo/runtime/go-unsafe-pointer.c index 804360f8a89..9ec18003f1f 100644 --- a/libgo/runtime/go-unsafe-pointer.c +++ b/libgo/runtime/go-unsafe-pointer.c @@ -51,6 +51,8 @@ const struct __go_type_descriptor unsafe_Pointer = /* __reflection */ &reflection_string, /* __uncommon */ + NULL, + /* __pointer_to_this */ NULL }; @@ -90,6 +92,8 @@ const struct __go_ptr_type pointer_unsafe_Pointer = /* __reflection */ &preflection_string, /* __uncommon */ + NULL, + /* __pointer_to_this */ NULL }, /* __element_type */ diff --git a/libgo/runtime/malloc.goc b/libgo/runtime/malloc.goc index d826d479f5c..5f99c4b73a4 100644 --- a/libgo/runtime/malloc.goc +++ b/libgo/runtime/malloc.goc @@ -45,14 +45,13 @@ fastrand1(void) // Small objects are allocated from the per-thread cache's free lists. // Large objects (> 32 kB) are allocated straight from the heap. void* -runtime_mallocgc(uintptr size, uint32 refflag, int32 dogc, int32 zeroed) +runtime_mallocgc(uintptr size, uint32 flag, int32 dogc, int32 zeroed) { int32 sizeclass, rate; MCache *c; uintptr npages; MSpan *s; void *v; - uint32 *ref; if(!__sync_bool_compare_and_swap(&m->mallocing, 0, 1)) runtime_throw("malloc/free - deadlock"); @@ -71,12 +70,6 @@ runtime_mallocgc(uintptr size, uint32 refflag, int32 dogc, int32 zeroed) mstats.alloc += size; mstats.total_alloc += size; mstats.by_size[sizeclass].nmalloc++; - - if(!runtime_mlookup(v, nil, nil, nil, &ref)) { - // runtime_printf("malloc %D; runtime_mlookup failed\n", (uint64)size); - runtime_throw("malloc runtime_mlookup"); - } - *ref = RefNone | refflag; } else { // TODO(rsc): Report tracebacks for very large allocations. @@ -93,14 +86,15 @@ runtime_mallocgc(uintptr size, uint32 refflag, int32 dogc, int32 zeroed) v = (void*)(s->start << PageShift); // setup for mark sweep - s->gcref0 = RefNone | refflag; - ref = &s->gcref0; + runtime_markspan(v, 0, 0, true); } + if(!(flag & FlagNoGC)) + runtime_markallocated(v, size, (flag&FlagNoPointers) != 0); __sync_bool_compare_and_swap(&m->mallocing, 1, 0); if(__sync_bool_compare_and_swap(&m->gcing, 1, 0)) { - if(!(refflag & RefNoProfiling)) + if(!(flag & FlagNoProfiling)) __go_run_goroutine_gc(0); else { // We are being called from the profiler. Tell it @@ -110,7 +104,7 @@ runtime_mallocgc(uintptr size, uint32 refflag, int32 dogc, int32 zeroed) } } - if(!(refflag & RefNoProfiling) && (rate = runtime_MemProfileRate) > 0) { + if(!(flag & FlagNoProfiling) && (rate = runtime_MemProfileRate) > 0) { if(size >= (uint32) rate) goto profile; if((uint32) m->mcache->next_sample > size) @@ -121,7 +115,7 @@ runtime_mallocgc(uintptr size, uint32 refflag, int32 dogc, int32 zeroed) rate = 0x3fffffff; m->mcache->next_sample = fastrand1() % (2*rate); profile: - *ref |= RefProfiled; + runtime_setblockspecial(v); runtime_MProf_Malloc(v, size); } } @@ -141,32 +135,37 @@ __go_alloc(uintptr size) void __go_free(void *v) { - int32 sizeclass, size; + int32 sizeclass; MSpan *s; MCache *c; - uint32 prof, *ref; + uint32 prof; + uintptr size; if(v == nil) return; + + // If you change this also change mgc0.c:/^sweepspan, + // which has a copy of the guts of free. if(!__sync_bool_compare_and_swap(&m->mallocing, 0, 1)) runtime_throw("malloc/free - deadlock"); - if(!runtime_mlookup(v, nil, nil, &s, &ref)) { + if(!runtime_mlookup(v, nil, nil, &s)) { // runtime_printf("free %p: not an allocated block\n", v); runtime_throw("free runtime_mlookup"); } - prof = *ref & RefProfiled; - *ref = RefFree; + prof = runtime_blockspecial(v); // Find size class for v. sizeclass = s->sizeclass; if(sizeclass == 0) { // Large object. - if(prof) - runtime_MProf_Free(v, s->npages<<PageShift); - mstats.alloc -= s->npages<<PageShift; - runtime_memclr(v, s->npages<<PageShift); + size = s->npages<<PageShift; + *(uintptr*)(s->start<<PageShift) = 1; // mark as "needs to be zeroed" + // Must mark v freed before calling unmarkspan and MHeap_Free: + // they might coalesce v into other spans and change the bitmap further. + runtime_markfreed(v, size); + runtime_unmarkspan(v, 1<<PageShift); runtime_MHeap_Free(&runtime_mheap, s, 1); } else { // Small object. @@ -174,12 +173,17 @@ __go_free(void *v) size = runtime_class_to_size[sizeclass]; if(size > (int32)sizeof(uintptr)) ((uintptr*)v)[1] = 1; // mark as "needs to be zeroed" - if(prof) - runtime_MProf_Free(v, size); - mstats.alloc -= size; + // Must mark v freed before calling MCache_Free: + // it might coalesce v and other blocks into a bigger span + // and change the bitmap further. + runtime_markfreed(v, size); mstats.by_size[sizeclass].nfree++; runtime_MCache_Free(c, v, sizeclass, size); } + mstats.alloc -= size; + if(prof) + runtime_MProf_Free(v, size); + __sync_bool_compare_and_swap(&m->mallocing, 1, 0); if(__sync_bool_compare_and_swap(&m->gcing, 1, 0)) @@ -187,23 +191,22 @@ __go_free(void *v) } int32 -runtime_mlookup(void *v, byte **base, uintptr *size, MSpan **sp, uint32 **ref) +runtime_mlookup(void *v, byte **base, uintptr *size, MSpan **sp) { - uintptr n, nobj, i; + uintptr n, i; byte *p; MSpan *s; mstats.nlookup++; - s = runtime_MHeap_LookupMaybe(&runtime_mheap, (uintptr)v>>PageShift); + s = runtime_MHeap_LookupMaybe(&runtime_mheap, v); if(sp) *sp = s; if(s == nil) { + runtime_checkfreed(v, 1); if(base) *base = nil; if(size) *size = 0; - if(ref) - *ref = 0; return 0; } @@ -214,14 +217,11 @@ runtime_mlookup(void *v, byte **base, uintptr *size, MSpan **sp, uint32 **ref) *base = p; if(size) *size = s->npages<<PageShift; - if(ref) - *ref = &s->gcref0; return 1; } - if((byte*)v >= (byte*)s->gcref) { - // pointers into the gc ref counts - // do not count as pointers. + if((byte*)v >= (byte*)s->limit) { + // pointers past the last block do not count as pointers. return 0; } @@ -232,21 +232,6 @@ runtime_mlookup(void *v, byte **base, uintptr *size, MSpan **sp, uint32 **ref) if(size) *size = n; - // good for error checking, but expensive - if(0) { - nobj = (s->npages << PageShift) / (n + RefcountOverhead); - if((byte*)s->gcref < p || (byte*)(s->gcref+nobj) > p+(s->npages<<PageShift)) { - // runtime_printf("odd span state=%d span=%p base=%p sizeclass=%d n=%D size=%D npages=%D\n", - // s->state, s, p, s->sizeclass, (uint64)nobj, (uint64)n, (uint64)s->npages); - // runtime_printf("s->base sizeclass %d v=%p base=%p gcref=%p blocksize=%D nobj=%D size=%D end=%p end=%p\n", - // s->sizeclass, v, p, s->gcref, (uint64)s->npages<<PageShift, - // (uint64)nobj, (uint64)n, s->gcref + nobj, p+(s->npages<<PageShift)); - runtime_throw("bad gcref"); - } - } - if(ref) - *ref = &s->gcref[i]; - return 1; } @@ -278,16 +263,90 @@ runtime_allocmcache(void) extern int32 runtime_sizeof_C_MStats __asm__ ("libgo_runtime.runtime.Sizeof_C_MStats"); +#define MaxArena32 (2U<<30) + void runtime_mallocinit(void) { - runtime_sizeof_C_MStats = sizeof(MStats); + byte *p; + uintptr arena_size, bitmap_size; + extern byte end[]; - runtime_initfintab(); - runtime_Mprof_Init(); + runtime_sizeof_C_MStats = sizeof(MStats); - runtime_SysMemInit(); runtime_InitSizes(); + + // Set up the allocation arena, a contiguous area of memory where + // allocated data will be found. The arena begins with a bitmap large + // enough to hold 4 bits per allocated word. + if(sizeof(void*) == 8) { + // On a 64-bit machine, allocate from a single contiguous reservation. + // 16 GB should be big enough for now. + // + // The code will work with the reservation at any address, but ask + // SysReserve to use 0x000000f800000000 if possible. + // Allocating a 16 GB region takes away 36 bits, and the amd64 + // doesn't let us choose the top 17 bits, so that leaves the 11 bits + // in the middle of 0x00f8 for us to choose. Choosing 0x00f8 means + // that the valid memory addresses will begin 0x00f8, 0x00f9, 0x00fa, 0x00fb. + // None of the bytes f8 f9 fa fb can appear in valid UTF-8, and + // they are otherwise as far from ff (likely a common byte) as possible. + // Choosing 0x00 for the leading 6 bits was more arbitrary, but it + // is not a common ASCII code point either. Using 0x11f8 instead + // caused out of memory errors on OS X during thread allocations. + // These choices are both for debuggability and to reduce the + // odds of the conservative garbage collector not collecting memory + // because some non-pointer block of memory had a bit pattern + // that matched a memory address. + // + // Actually we reserve 17 GB (because the bitmap ends up being 1 GB) + // but it hardly matters: fc is not valid UTF-8 either, and we have to + // allocate 15 GB before we get that far. + arena_size = (uintptr)(16LL<<30); + bitmap_size = arena_size / (sizeof(void*)*8/4); + p = runtime_SysReserve((void*)(0x00f8ULL<<32), bitmap_size + arena_size); + if(p == nil) + runtime_throw("runtime: cannot reserve arena virtual address space"); + } else { + // On a 32-bit machine, we can't typically get away + // with a giant virtual address space reservation. + // Instead we map the memory information bitmap + // immediately after the data segment, large enough + // to handle another 2GB of mappings (256 MB), + // along with a reservation for another 512 MB of memory. + // When that gets used up, we'll start asking the kernel + // for any memory anywhere and hope it's in the 2GB + // following the bitmap (presumably the executable begins + // near the bottom of memory, so we'll have to use up + // most of memory before the kernel resorts to giving out + // memory before the beginning of the text segment). + // + // Alternatively we could reserve 512 MB bitmap, enough + // for 4GB of mappings, and then accept any memory the + // kernel threw at us, but normally that's a waste of 512 MB + // of address space, which is probably too much in a 32-bit world. + bitmap_size = MaxArena32 / (sizeof(void*)*8/4); + arena_size = 512<<20; + + // SysReserve treats the address we ask for, end, as a hint, + // not as an absolute requirement. If we ask for the end + // of the data segment but the operating system requires + // a little more space before we can start allocating, it will + // give out a slightly higher pointer. That's fine. + // Run with what we get back. + p = runtime_SysReserve(end, bitmap_size + arena_size); + if(p == nil) + runtime_throw("runtime: cannot reserve arena virtual address space"); + } + if((uintptr)p & (((uintptr)1<<PageShift)-1)) + runtime_throw("runtime: SysReserve returned unaligned address"); + + runtime_mheap.bitmap = p; + runtime_mheap.arena_start = p + bitmap_size; + runtime_mheap.arena_used = runtime_mheap.arena_start; + runtime_mheap.arena_end = runtime_mheap.arena_start + arena_size; + + // Initialize the rest of the allocator. runtime_MHeap_Init(&runtime_mheap, runtime_SysAlloc); m->mcache = runtime_allocmcache(); @@ -295,6 +354,47 @@ runtime_mallocinit(void) runtime_free(runtime_malloc(1)); } +void* +runtime_MHeap_SysAlloc(MHeap *h, uintptr n) +{ + byte *p; + + if(n <= (uintptr)(h->arena_end - h->arena_used)) { + // Keep taking from our reservation. + p = h->arena_used; + runtime_SysMap(p, n); + h->arena_used += n; + runtime_MHeap_MapBits(h); + return p; + } + + // On 64-bit, our reservation is all we have. + if(sizeof(void*) == 8) + return nil; + + // On 32-bit, once the reservation is gone we can + // try to get memory at a location chosen by the OS + // and hope that it is in the range we allocated bitmap for. + p = runtime_SysAlloc(n); + if(p == nil) + return nil; + + if(p < h->arena_start || (uintptr)(p+n - h->arena_start) >= MaxArena32) { + runtime_printf("runtime: memory allocated by OS not in usable range"); + runtime_SysFree(p, n); + return nil; + } + + if(p+n > h->arena_used) { + h->arena_used = p+n; + if(h->arena_used > h->arena_end) + h->arena_end = h->arena_used; + runtime_MHeap_MapBits(h); + } + + return p; +} + // Runtime stubs. void* @@ -303,6 +403,10 @@ runtime_mal(uintptr n) return runtime_mallocgc(n, 0, 1, 1); } +func new(n uint32) (ret *uint8) { + ret = runtime_mal(n); +} + func Alloc(n uintptr) (p *byte) { p = runtime_malloc(n); } @@ -312,7 +416,7 @@ func Free(p *byte) { } func Lookup(p *byte) (base *byte, size uintptr) { - runtime_mlookup(p, &base, &size, nil, nil); + runtime_mlookup(p, &base, &size, nil); } func GC() { @@ -333,7 +437,7 @@ func SetFinalizer(obj Eface, finalizer Eface) { // runtime_printf("runtime.SetFinalizer: first argument is %S, not pointer\n", *obj.type->string); goto throw; } - if(!runtime_mlookup(obj.__object, &base, &size, nil, nil) || obj.__object != base) { + if(!runtime_mlookup(obj.__object, &base, &size, nil) || obj.__object != base) { // runtime_printf("runtime.SetFinalizer: pointer not at beginning of allocated block\n"); goto throw; } diff --git a/libgo/runtime/malloc.h b/libgo/runtime/malloc.h index 369f9b8e771..8131e964e49 100644 --- a/libgo/runtime/malloc.h +++ b/libgo/runtime/malloc.h @@ -19,7 +19,6 @@ // used to manage storage used by the allocator. // MHeap: the malloc heap, managed at page (4096-byte) granularity. // MSpan: a run of pages managed by the MHeap. -// MHeapMap: a mapping from page IDs to MSpans. // MCentral: a shared free list for a given size class. // MCache: a per-thread (in Go, per-M) cache for small objects. // MStats: allocation statistics. @@ -84,7 +83,6 @@ typedef struct FixAlloc FixAlloc; typedef struct MCentral MCentral; typedef struct MHeap MHeap; -typedef struct MHeapMap MHeapMap; typedef struct MSpan MSpan; typedef struct MStats MStats; typedef struct MLink MLink; @@ -99,8 +97,14 @@ typedef uintptr PageID; // address >> PageShift enum { + // Computed constant. The definition of MaxSmallSize and the + // algorithm in msize.c produce some number of different allocation + // size classes. NumSizeClasses is that number. It's needed here + // because there are static arrays of this length; when msize runs its + // size choosing algorithm it double-checks that NumSizeClasses agrees. + NumSizeClasses = 61, + // Tunable constants. - NumSizeClasses = 67, // Number of size classes (must match msize.c) MaxSmallSize = 32<<10, FixAllocChunk = 128<<10, // Chunk size for FixAlloc @@ -108,13 +112,16 @@ enum MaxMCacheSize = 2<<20, // Maximum bytes in one MCache MaxMHeapList = 1<<(20 - PageShift), // Maximum page length for fixed-size list in MHeap. HeapAllocChunk = 1<<20, // Chunk size for heap growth -}; + // Number of bits in page to span calculations (4k pages). + // On 64-bit, we limit the arena to 16G, so 22 bits suffices. + // On 32-bit, we don't bother limiting anything: 20 bits for 4G. #if __SIZEOF_POINTER__ == 8 -#include "mheapmap64.h" + MHeapMap_Bits = 22, #else -#include "mheapmap32.h" + MHeapMap_Bits = 20, #endif +}; // A generic linked list of blocks. (Typically the block is bigger than sizeof(MLink).) struct MLink @@ -124,7 +131,8 @@ struct MLink // SysAlloc obtains a large chunk of zeroed memory from the // operating system, typically on the order of a hundred kilobytes -// or a megabyte. +// or a megabyte. If the pointer argument is non-nil, the caller +// wants a mapping there or nowhere. // // SysUnused notifies the operating system that the contents // of the memory region are no longer needed and can be reused @@ -134,11 +142,19 @@ struct MLink // SysFree returns it unconditionally; this is only used if // an out-of-memory error has been detected midway through // an allocation. It is okay if SysFree is a no-op. +// +// SysReserve reserves address space without allocating memory. +// If the pointer passed to it is non-nil, the caller wants the +// reservation there, but SysReserve can still choose another +// location if that one is unavailable. +// +// SysMap maps previously reserved address space for use. void* runtime_SysAlloc(uintptr nbytes); void runtime_SysFree(void *v, uintptr nbytes); void runtime_SysUnused(void *v, uintptr nbytes); -void runtime_SysMemInit(void); +void runtime_SysMap(void *v, uintptr nbytes); +void* runtime_SysReserve(void *v, uintptr nbytes); // FixAlloc is a simple free-list allocator for fixed size objects. // Malloc uses a FixAlloc wrapped around SysAlloc to manages its @@ -194,7 +210,6 @@ struct MStats uint64 mspan_sys; uint64 mcache_inuse; // MCache structures uint64 mcache_sys; - uint64 heapmap_sys; // heap map uint64 buckhash_sys; // profiling bucket hash table // Statistics about garbage collector. @@ -281,10 +296,7 @@ struct MSpan uint32 ref; // number of allocated objects in this span uint32 sizeclass; // size class uint32 state; // MSpanInUse etc - union { - uint32 *gcref; // sizeclass > 0 - uint32 gcref0; // sizeclass == 0 - }; + byte *limit; // end of data in span }; void runtime_MSpan_Init(MSpan *span, PageID start, uintptr npages); @@ -323,11 +335,14 @@ struct MHeap MSpan *allspans; // span lookup - MHeapMap map; + MSpan *map[1<<MHeapMap_Bits]; // range of addresses we might see in the heap - byte *min; - byte *max; + byte *bitmap; + uintptr bitmap_mapped; + byte *arena_start; + byte *arena_used; + byte *arena_end; // central free lists for small size classes. // the union makes sure that the MCentrals are @@ -346,31 +361,31 @@ extern MHeap runtime_mheap; void runtime_MHeap_Init(MHeap *h, void *(*allocator)(uintptr)); MSpan* runtime_MHeap_Alloc(MHeap *h, uintptr npage, int32 sizeclass, int32 acct); void runtime_MHeap_Free(MHeap *h, MSpan *s, int32 acct); -MSpan* runtime_MHeap_Lookup(MHeap *h, PageID p); -MSpan* runtime_MHeap_LookupMaybe(MHeap *h, PageID p); -void runtime_MGetSizeClassInfo(int32 sizeclass, int32 *size, int32 *npages, int32 *nobj); +MSpan* runtime_MHeap_Lookup(MHeap *h, void *v); +MSpan* runtime_MHeap_LookupMaybe(MHeap *h, void *v); +void runtime_MGetSizeClassInfo(int32 sizeclass, uintptr *size, int32 *npages, int32 *nobj); +void* runtime_MHeap_SysAlloc(MHeap *h, uintptr n); +void runtime_MHeap_MapBits(MHeap *h); void* runtime_mallocgc(uintptr size, uint32 flag, int32 dogc, int32 zeroed); -int32 runtime_mlookup(void *v, byte **base, uintptr *size, MSpan **s, uint32 **ref); +int32 runtime_mlookup(void *v, byte **base, uintptr *size, MSpan **s); void runtime_gc(int32 force); - -void* runtime_SysAlloc(uintptr); -void runtime_SysUnused(void*, uintptr); -void runtime_SysFree(void*, uintptr); +void runtime_markallocated(void *v, uintptr n, bool noptr); +void runtime_checkallocated(void *v, uintptr n); +void runtime_markfreed(void *v, uintptr n); +void runtime_checkfreed(void *v, uintptr n); +int32 runtime_checking; +void runtime_markspan(void *v, uintptr size, uintptr n, bool leftover); +void runtime_unmarkspan(void *v, uintptr size); +bool runtime_blockspecial(void*); +void runtime_setblockspecial(void*); enum { - RefcountOverhead = 4, // one uint32 per object - - RefFree = 0, // must be zero - RefStack, // stack segment - don't free and don't scan for pointers - RefNone, // no references - RefSome, // some references - RefNoPointers = 0x80000000U, // flag - no pointers here - RefHasFinalizer = 0x40000000U, // flag - has finalizer - RefProfiled = 0x20000000U, // flag - is in profiling table - RefNoProfiling = 0x10000000U, // flag - must not profile - RefFlags = 0xFFFF0000U, + // flags to malloc + FlagNoPointers = 1<<0, // no pointers here + FlagNoProfiling = 1<<1, // must not profile + FlagNoGC = 1<<2, // must not free or scan for pointers }; void runtime_Mprof_Init(void); diff --git a/libgo/runtime/mcentral.c b/libgo/runtime/mcentral.c index 81e54b07dbf..cd3d6ca5ede 100644 --- a/libgo/runtime/mcentral.c +++ b/libgo/runtime/mcentral.c @@ -114,13 +114,11 @@ static void MCentral_Free(MCentral *c, void *v) { MSpan *s; - PageID page; - MLink *p, *next; + MLink *p; int32 size; // Find span for v. - page = (uintptr)v >> PageShift; - s = runtime_MHeap_Lookup(&runtime_mheap, page); + s = runtime_MHeap_Lookup(&runtime_mheap, v); if(s == nil || s->ref == 0) runtime_throw("invalid free"); @@ -140,16 +138,8 @@ MCentral_Free(MCentral *c, void *v) if(--s->ref == 0) { size = runtime_class_to_size[c->sizeclass]; runtime_MSpanList_Remove(s); - // The second word of each freed block indicates - // whether it needs to be zeroed. The first word - // is the link pointer and must always be cleared. - for(p=s->freelist; p; p=next) { - next = p->next; - if(size > (int32)sizeof(uintptr) && ((uintptr*)p)[1] != 0) - runtime_memclr((byte*)p, size); - else - p->next = nil; - } + runtime_unmarkspan((byte*)(s->start<<PageShift), s->npages<<PageShift); + *(uintptr*)(s->start<<PageShift) = 1; // needs zeroing s->freelist = nil; c->nfree -= (s->npages << PageShift) / size; runtime_unlock(c); @@ -159,7 +149,7 @@ MCentral_Free(MCentral *c, void *v) } void -runtime_MGetSizeClassInfo(int32 sizeclass, int32 *sizep, int32 *npagesp, int32 *nobj) +runtime_MGetSizeClassInfo(int32 sizeclass, uintptr *sizep, int32 *npagesp, int32 *nobj) { int32 size; int32 npages; @@ -168,7 +158,7 @@ runtime_MGetSizeClassInfo(int32 sizeclass, int32 *sizep, int32 *npagesp, int32 * size = runtime_class_to_size[sizeclass]; *npagesp = npages; *sizep = size; - *nobj = (npages << PageShift) / (size + RefcountOverhead); + *nobj = (npages << PageShift) / size; } // Fetch a new span from the heap and @@ -176,7 +166,8 @@ runtime_MGetSizeClassInfo(int32 sizeclass, int32 *sizep, int32 *npagesp, int32 * static bool MCentral_Grow(MCentral *c) { - int32 i, n, npages, size; + int32 i, n, npages; + uintptr size; MLink **tailp, *v; byte *p; MSpan *s; @@ -193,7 +184,7 @@ MCentral_Grow(MCentral *c) // Carve span into sequence of blocks. tailp = &s->freelist; p = (byte*)(s->start << PageShift); - s->gcref = (uint32*)(p + size*n); + s->limit = p + size*n; for(i=0; i<n; i++) { v = (MLink*)p; *tailp = v; @@ -201,6 +192,7 @@ MCentral_Grow(MCentral *c) p += size; } *tailp = nil; + runtime_markspan((byte*)(s->start<<PageShift), size, n, size*n < (s->npages<<PageShift)); runtime_lock(c); c->nfree += n; diff --git a/libgo/runtime/mem.c b/libgo/runtime/mem.c index 4d6c7420907..f62a4d37ee7 100644 --- a/libgo/runtime/mem.c +++ b/libgo/runtime/mem.c @@ -38,12 +38,11 @@ runtime_SysAlloc(uintptr n) p = runtime_mmap(nil, n, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_ANON|MAP_PRIVATE, fd, 0); if (p == MAP_FAILED) { if(errno == EACCES) { - printf("mmap: access denied\n"); - printf("If you're running SELinux, enable execmem for this process.\n"); - } else { - printf("mmap: errno=%d\n", errno); + printf("runtime: mmap: access denied\n"); + printf("if you're running SELinux, enable execmem for this process.\n"); + exit(2); } - exit(2); + return nil; } return p; } @@ -63,14 +62,61 @@ runtime_SysFree(void *v, uintptr n) runtime_munmap(v, n); } +void* +runtime_SysReserve(void *v, uintptr n) +{ + int fd = -1; + + // On 64-bit, people with ulimit -v set complain if we reserve too + // much address space. Instead, assume that the reservation is okay + // and check the assumption in SysMap. + if(sizeof(void*) == 8) + return v; + +#ifdef USE_DEV_ZERO + if (dev_zero == -1) { + dev_zero = open("/dev/zero", O_RDONLY); + if (dev_zero < 0) { + printf("open /dev/zero: errno=%d\n", errno); + exit(2); + } + } + fd = dev_zero; +#endif + + return runtime_mmap(v, n, PROT_NONE, MAP_ANON|MAP_PRIVATE, fd, 0); +} + void -runtime_SysMemInit(void) +runtime_SysMap(void *v, uintptr n) { - // Code generators assume that references to addresses - // on the first page will fault. Map the page explicitly with - // no permissions, to head off possible bugs like the system - // allocating that page as the virtual address space fills. - // Ignore any error, since other systems might be smart - // enough to never allow anything there. - runtime_mmap(nil, 4096, PROT_NONE, MAP_FIXED|MAP_ANON|MAP_PRIVATE, -1, 0); + void *p; + int fd = -1; + + mstats.sys += n; + +#ifdef USE_DEV_ZERO + if (dev_zero == -1) { + dev_zero = open("/dev/zero", O_RDONLY); + if (dev_zero < 0) { + printf("open /dev/zero: errno=%d\n", errno); + exit(2); + } + } + fd = dev_zero; +#endif + + // On 64-bit, we don't actually have v reserved, so tread carefully. + if(sizeof(void*) == 8) { + p = runtime_mmap(v, n, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_ANON|MAP_PRIVATE, fd, 0); + if(p != v) { + runtime_printf("runtime: address space conflict: map(%p) = %p\n", v, p); + runtime_throw("runtime: address space conflict"); + } + return; + } + + p = runtime_mmap(v, n, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_ANON|MAP_FIXED|MAP_PRIVATE, fd, 0); + if(p != v) + runtime_throw("runtime: cannot map pages in arena address space"); } diff --git a/libgo/runtime/mem_posix_memalign.c b/libgo/runtime/mem_posix_memalign.c index 3855dfcf185..2318be8da11 100644 --- a/libgo/runtime/mem_posix_memalign.c +++ b/libgo/runtime/mem_posix_memalign.c @@ -32,7 +32,13 @@ runtime_SysFree(void *v, uintptr n) free(v); } +void* +runtime_SysReserve(void *v, uintptr n) +{ + return runtime_SysAlloc(n); +} + void -runtime_SysMemInit(void) +runtime_SysMap(void *v, uintptr n) { } diff --git a/libgo/runtime/mfinal.c b/libgo/runtime/mfinal.c index 23c0d7a1663..04d58dddda9 100644 --- a/libgo/runtime/mfinal.c +++ b/libgo/runtime/mfinal.c @@ -5,6 +5,9 @@ #include "runtime.h" #include "malloc.h" +// Lock to protect finalizer data structures. +// Cannot reuse mheap.Lock because the finalizer +// maintenance requires allocation. static Lock finlock; void @@ -95,7 +98,6 @@ runtime_addfinalizer(void *p, void (*f)(void*), const struct __go_func_type *ft) { Fintab newtab; int32 i; - uint32 *ref; byte *base; Finalizer *e; @@ -110,25 +112,22 @@ runtime_addfinalizer(void *p, void (*f)(void*), const struct __go_func_type *ft) runtime_throw("finalizer deadlock"); runtime_lock(&finlock); - if(!runtime_mlookup(p, &base, nil, nil, &ref) || p != base) { + if(!runtime_mlookup(p, &base, nil, nil) || p != base) { runtime_unlock(&finlock); __sync_bool_compare_and_swap(&m->holds_finlock, 1, 0); runtime_throw("addfinalizer on invalid pointer"); } if(f == nil) { - if(*ref & RefHasFinalizer) { - lookfintab(&fintab, p, 1); - *ref &= ~RefHasFinalizer; - } + lookfintab(&fintab, p, 1); goto unlock; } - if(*ref & RefHasFinalizer) { + if(lookfintab(&fintab, p, 0)) { runtime_unlock(&finlock); __sync_bool_compare_and_swap(&m->holds_finlock, 1, 0); runtime_throw("double finalizer"); } - *ref |= RefHasFinalizer; + runtime_setblockspecial(p); if(fintab.nkey >= fintab.max/2+fintab.max/4) { // keep table at most 3/4 full: @@ -144,7 +143,7 @@ runtime_addfinalizer(void *p, void (*f)(void*), const struct __go_func_type *ft) newtab.max *= 3; } - newtab.key = runtime_mallocgc(newtab.max*sizeof newtab.key[0], RefNoPointers, 0, 1); + newtab.key = runtime_mallocgc(newtab.max*sizeof newtab.key[0], FlagNoPointers, 0, 1); newtab.val = runtime_mallocgc(newtab.max*sizeof newtab.val[0], 0, 0, 1); for(i=0; i<fintab.max; i++) { diff --git a/libgo/runtime/mgc0.c b/libgo/runtime/mgc0.c index f2703ab0263..27fc3cdcc4d 100644 --- a/libgo/runtime/mgc0.c +++ b/libgo/runtime/mgc0.c @@ -2,28 +2,65 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Garbage collector -- step 0. -// -// Stop the world, mark and sweep garbage collector. -// NOT INTENDED FOR PRODUCTION USE. -// -// A mark and sweep collector provides a way to exercise -// and test the memory allocator and the stack walking machinery -// without also needing to get reference counting -// exactly right. +// Garbage collector. #include "runtime.h" #include "malloc.h" enum { - Debug = 0 + Debug = 0, + UseCas = 1, + PtrSize = sizeof(void*), + + // Four bits per word (see #defines below). + wordsPerBitmapWord = sizeof(void*)*8/4, + bitShift = sizeof(void*)*8/4, }; -typedef struct BlockList BlockList; -struct BlockList +// Bits in per-word bitmap. +// #defines because enum might not be able to hold the values. +// +// Each word in the bitmap describes wordsPerBitmapWord words +// of heap memory. There are 4 bitmap bits dedicated to each heap word, +// so on a 64-bit system there is one bitmap word per 16 heap words. +// The bits in the word are packed together by type first, then by +// heap location, so each 64-bit bitmap word consists of, from top to bottom, +// the 16 bitSpecial bits for the corresponding heap words, then the 16 bitMarked bits, +// then the 16 bitNoPointers/bitBlockBoundary bits, then the 16 bitAllocated bits. +// This layout makes it easier to iterate over the bits of a given type. +// +// The bitmap starts at mheap.arena_start and extends *backward* from +// there. On a 64-bit system the off'th word in the arena is tracked by +// the off/16+1'th word before mheap.arena_start. (On a 32-bit system, +// the only difference is that the divisor is 8.) +// +// To pull out the bits corresponding to a given pointer p, we use: +// +// off = p - (uintptr*)mheap.arena_start; // word offset +// b = (uintptr*)mheap.arena_start - off/wordsPerBitmapWord - 1; +// shift = off % wordsPerBitmapWord +// bits = *b >> shift; +// /* then test bits & bitAllocated, bits & bitMarked, etc. */ +// +#define bitAllocated ((uintptr)1<<(bitShift*0)) +#define bitNoPointers ((uintptr)1<<(bitShift*1)) /* when bitAllocated is set */ +#define bitMarked ((uintptr)1<<(bitShift*2)) /* when bitAllocated is set */ +#define bitSpecial ((uintptr)1<<(bitShift*3)) /* when bitAllocated is set - has finalizer or being profiled */ +#define bitBlockBoundary ((uintptr)1<<(bitShift*1)) /* when bitAllocated is NOT set */ + +#define bitMask (bitBlockBoundary | bitAllocated | bitMarked | bitSpecial) + +static uint64 nlookup; +static uint64 nsizelookup; +static uint64 naddrlookup; +static int32 gctrace; + +typedef struct Workbuf Workbuf; +struct Workbuf { - byte *obj; - uintptr size; + Workbuf *next; + uintptr nw; + byte *w[2048-2]; }; static bool finstarted; @@ -31,83 +68,265 @@ static pthread_mutex_t finqlock = PTHREAD_MUTEX_INITIALIZER; static pthread_cond_t finqcond = PTHREAD_COND_INITIALIZER; static Finalizer *finq; static int32 fingwait; -static BlockList *bl, *ebl; static void runfinq(void*); - -enum { - PtrSize = sizeof(void*) -}; - +static Workbuf* getempty(Workbuf*); +static Workbuf* getfull(Workbuf*); + +// scanblock scans a block of n bytes starting at pointer b for references +// to other objects, scanning any it finds recursively until there are no +// unscanned objects left. Instead of using an explicit recursion, it keeps +// a work list in the Workbuf* structures and loops in the main function +// body. Keeping an explicit work list is easier on the stack allocator and +// more efficient. static void scanblock(byte *b, int64 n) { - int32 off; - void *obj; - uintptr size; - uint32 *refp, ref; + byte *obj, *arena_start, *p; void **vp; - int64 i; - BlockList *w; - - w = bl; - w->obj = b; - w->size = n; - w++; + uintptr size, *bitp, bits, shift, i, j, x, xbits, off; + MSpan *s; + PageID k; + void **bw, **w, **ew; + Workbuf *wbuf; - while(w > bl) { - w--; - b = w->obj; - n = w->size; + // Memory arena parameters. + arena_start = runtime_mheap.arena_start; + + wbuf = nil; // current work buffer + ew = nil; // end of work buffer + bw = nil; // beginning of work buffer + w = nil; // current pointer into work buffer + + // Align b to a word boundary. + off = (uintptr)b & (PtrSize-1); + if(off != 0) { + b += PtrSize - off; + n -= PtrSize - off; + } + for(;;) { + // Each iteration scans the block b of length n, queueing pointers in + // the work buffer. if(Debug > 1) runtime_printf("scanblock %p %lld\n", b, (long long) n); - off = (uint32)(uintptr)b & (PtrSize-1); - if(off) { - b += PtrSize - off; - n -= PtrSize - off; - } - + vp = (void**)b; n /= PtrSize; - for(i=0; i<n; i++) { - obj = vp[i]; - if(obj == nil) + for(i=0; i<(uintptr)n; i++) { + obj = (byte*)vp[i]; + + // Words outside the arena cannot be pointers. + if((byte*)obj < arena_start || (byte*)obj >= runtime_mheap.arena_used) continue; - if(runtime_mheap.min <= (byte*)obj && (byte*)obj < runtime_mheap.max) { - if(runtime_mlookup(obj, (byte**)&obj, &size, nil, &refp)) { - ref = *refp; - switch(ref & ~RefFlags) { - case RefNone: - if(Debug > 1) - runtime_printf("found at %p: ", &vp[i]); - *refp = RefSome | (ref & RefFlags); - if(!(ref & RefNoPointers)) { - if(w >= ebl) - runtime_throw("scanblock: garbage collection stack overflow"); - w->obj = obj; - w->size = size; - w++; - } - break; - } + + // obj may be a pointer to a live object. + // Try to find the beginning of the object. + + // Round down to word boundary. + obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-1)); + + // Find bits for this word. + off = (uintptr*)obj - (uintptr*)arena_start; + bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1; + shift = off % wordsPerBitmapWord; + xbits = *bitp; + bits = xbits >> shift; + + // Pointing at the beginning of a block? + if((bits & (bitAllocated|bitBlockBoundary)) != 0) + goto found; + + // Pointing just past the beginning? + // Scan backward a little to find a block boundary. + for(j=shift; j-->0; ) { + if(((xbits>>j) & (bitAllocated|bitBlockBoundary)) != 0) { + obj = (byte*)obj - (shift-j)*PtrSize; + shift = j; + bits = xbits>>shift; + goto found; } } + + // Otherwise consult span table to find beginning. + // (Manually inlined copy of MHeap_LookupMaybe.) + nlookup++; + naddrlookup++; + k = (uintptr)obj>>PageShift; + x = k; + if(sizeof(void*) == 8) + x -= (uintptr)arena_start>>PageShift; + s = runtime_mheap.map[x]; + if(s == nil || k < s->start || k - s->start >= s->npages || s->state != MSpanInUse) + continue; + p = (byte*)((uintptr)s->start<<PageShift); + if(s->sizeclass == 0) { + obj = p; + } else { + if((byte*)obj >= (byte*)s->limit) + continue; + size = runtime_class_to_size[s->sizeclass]; + int32 i = ((byte*)obj - p)/size; + obj = p+i*size; + } + + // Now that we know the object header, reload bits. + off = (uintptr*)obj - (uintptr*)arena_start; + bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1; + shift = off % wordsPerBitmapWord; + xbits = *bitp; + bits = xbits >> shift; + + found: + // Now we have bits, bitp, and shift correct for + // obj pointing at the base of the object. + // If not allocated or already marked, done. + if((bits & bitAllocated) == 0 || (bits & bitMarked) != 0) + continue; + *bitp |= bitMarked<<shift; + + // If object has no pointers, don't need to scan further. + if((bits & bitNoPointers) != 0) + continue; + + // If buffer is full, get a new one. + if(w >= ew) { + wbuf = getempty(wbuf); + bw = (void**)wbuf->w; + w = bw; + ew = bw + nelem(wbuf->w); + } + *w++ = obj; } + + // Done scanning [b, b+n). Prepare for the next iteration of + // the loop by setting b and n to the parameters for the next block. + + // Fetch b from the work buffers. + if(w <= bw) { + // Emptied our buffer: refill. + wbuf = getfull(wbuf); + if(wbuf == nil) + break; + bw = (void**)wbuf->w; + ew = (void**)(wbuf->w + nelem(wbuf->w)); + w = bw+wbuf->nw; + } + b = *--w; + + // Figure out n = size of b. Start by loading bits for b. + off = (uintptr*)b - (uintptr*)arena_start; + bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1; + shift = off % wordsPerBitmapWord; + xbits = *bitp; + bits = xbits >> shift; + + // Might be small; look for nearby block boundary. + // A block boundary is marked by either bitBlockBoundary + // or bitAllocated being set (see notes near their definition). + enum { + boundary = bitBlockBoundary|bitAllocated + }; + // Look for a block boundary both after and before b + // in the same bitmap word. + // + // A block boundary j words after b is indicated by + // bits>>j & boundary + // assuming shift+j < bitShift. (If shift+j >= bitShift then + // we'll be bleeding other bit types like bitMarked into our test.) + // Instead of inserting the conditional shift+j < bitShift into the loop, + // we can let j range from 1 to bitShift as long as we first + // apply a mask to keep only the bits corresponding + // to shift+j < bitShift aka j < bitShift-shift. + bits &= (boundary<<(bitShift-shift)) - boundary; + + // A block boundary j words before b is indicated by + // xbits>>(shift-j) & boundary + // (assuming shift >= j). There is no cleverness here + // avoid the test, because when j gets too large the shift + // turns negative, which is undefined in C. + + for(j=1; j<bitShift; j++) { + if(((bits>>j)&boundary) != 0 || (shift>=j && ((xbits>>(shift-j))&boundary) != 0)) { + n = j*PtrSize; + goto scan; + } + } + + // Fall back to asking span about size class. + // (Manually inlined copy of MHeap_Lookup.) + nlookup++; + nsizelookup++; + x = (uintptr)b>>PageShift; + if(sizeof(void*) == 8) + x -= (uintptr)arena_start>>PageShift; + s = runtime_mheap.map[x]; + if(s->sizeclass == 0) + n = s->npages<<PageShift; + else + n = runtime_class_to_size[s->sizeclass]; + scan:; + } +} + +static struct { + Workbuf *full; + Workbuf *empty; + byte *chunk; + uintptr nchunk; +} work; + +// Get an empty work buffer off the work.empty list, +// allocating new buffers as needed. +static Workbuf* +getempty(Workbuf *b) +{ + if(b != nil) { + b->nw = nelem(b->w); + b->next = work.full; + work.full = b; + } + b = work.empty; + if(b != nil) { + work.empty = b->next; + return b; + } + + if(work.nchunk < sizeof *b) { + work.nchunk = 1<<20; + work.chunk = runtime_SysAlloc(work.nchunk); } + b = (Workbuf*)work.chunk; + work.chunk += sizeof *b; + work.nchunk -= sizeof *b; + return b; } +// Get a full work buffer off the work.full list, or return nil. +static Workbuf* +getfull(Workbuf *b) +{ + if(b != nil) { + b->nw = 0; + b->next = work.empty; + work.empty = b; + } + b = work.full; + if(b != nil) + work.full = b->next; + return b; +} + +// Scanstack calls scanblock on each of gp's stack segments. static void markfin(void *v) { uintptr size; - uint32 *refp; size = 0; - refp = nil; - if(!runtime_mlookup(v, (byte**)&v, &size, nil, &refp) || !(*refp & RefHasFinalizer)) + if(!runtime_mlookup(v, (byte**)&v, &size, nil) || !runtime_blockspecial(v)) runtime_throw("mark - finalizer inconsistency"); - + // do not mark the finalizer block itself. just mark the things it points at. scanblock(v, size); } @@ -131,32 +350,12 @@ __go_register_gc_roots (struct root_list* r) roots = r; } +// Mark static void mark(void) { - uintptr blsize, nobj; struct root_list *pl; - // Figure out how big an object stack we need. - // Get a new one if we need more than we have - // or we need significantly less than we have. - nobj = mstats.heap_objects; - if(nobj > (uintptr)(ebl - bl) || nobj < (uintptr)(ebl-bl)/4) { - if(bl != nil) - runtime_SysFree(bl, (byte*)ebl - (byte*)bl); - - // While we're allocated a new object stack, - // add 20% headroom and also round up to - // the nearest page boundary, since mmap - // will anyway. - nobj = nobj * 12/10; - blsize = nobj * sizeof *bl; - blsize = (blsize + 4095) & ~4095; - nobj = blsize / sizeof *bl; - bl = runtime_SysAlloc(blsize); - ebl = bl + nobj; - } - for(pl = roots; pl != nil; pl = pl->next) { struct root* pr = &pl->roots[0]; while(1) { @@ -179,97 +378,85 @@ mark(void) runtime_walkfintab(markfin, scanblock); } -// free RefNone, free & queue finalizers for RefNone|RefHasFinalizer, reset RefSome +// Sweep frees or calls finalizers for blocks not marked in the mark phase. +// It clears the mark bits in preparation for the next GC round. static void -sweepspan(MSpan *s) +sweep(void) { - int32 n, npages, size; + MSpan *s; + int32 cl, n, npages; + uintptr size; byte *p; - uint32 ref, *gcrefp, *gcrefep; MCache *c; Finalizer *f; - p = (byte*)(s->start << PageShift); - if(s->sizeclass == 0) { - // Large block. - ref = s->gcref0; - switch(ref & ~(RefFlags^RefHasFinalizer)) { - case RefNone: - // Free large object. - mstats.alloc -= s->npages<<PageShift; - mstats.nfree++; - runtime_memclr(p, s->npages<<PageShift); - if(ref & RefProfiled) - runtime_MProf_Free(p, s->npages<<PageShift); - s->gcref0 = RefFree; - runtime_MHeap_Free(&runtime_mheap, s, 1); - break; - case RefNone|RefHasFinalizer: - f = runtime_getfinalizer(p, 1); - if(f == nil) - runtime_throw("finalizer inconsistency"); - f->arg = p; - f->next = finq; - finq = f; - ref &= ~RefHasFinalizer; - // fall through - case RefSome: - case RefSome|RefHasFinalizer: - s->gcref0 = RefNone | (ref&RefFlags); - break; + for(s = runtime_mheap.allspans; s != nil; s = s->allnext) { + if(s->state != MSpanInUse) + continue; + + p = (byte*)(s->start << PageShift); + cl = s->sizeclass; + if(cl == 0) { + size = s->npages<<PageShift; + n = 1; + } else { + // Chunk full of small blocks. + size = runtime_class_to_size[cl]; + npages = runtime_class_to_allocnpages[cl]; + n = (npages << PageShift) / size; } - return; - } + + // sweep through n objects of given size starting at p. + for(; n > 0; n--, p += size) { + uintptr off, *bitp, shift, bits; - // Chunk full of small blocks. - runtime_MGetSizeClassInfo(s->sizeclass, &size, &npages, &n); - gcrefp = s->gcref; - gcrefep = s->gcref + n; - for(; gcrefp < gcrefep; gcrefp++, p += size) { - ref = *gcrefp; - if(ref < RefNone) // RefFree or RefStack - continue; - switch(ref & ~(RefFlags^RefHasFinalizer)) { - case RefNone: - // Free small object. - if(ref & RefProfiled) + off = (uintptr*)p - (uintptr*)runtime_mheap.arena_start; + bitp = (uintptr*)runtime_mheap.arena_start - off/wordsPerBitmapWord - 1; + shift = off % wordsPerBitmapWord; + bits = *bitp>>shift; + + if((bits & bitAllocated) == 0) + continue; + + if((bits & bitMarked) != 0) { + *bitp &= ~(bitMarked<<shift); + continue; + } + + if((bits & bitSpecial) != 0) { + // Special means it has a finalizer or is being profiled. + f = runtime_getfinalizer(p, 1); + if(f != nil) { + f->arg = p; + f->next = finq; + finq = f; + continue; + } runtime_MProf_Free(p, size); - *gcrefp = RefFree; - c = m->mcache; - if(size > (int32)sizeof(uintptr)) - ((uintptr*)p)[1] = 1; // mark as "needs to be zeroed" + } + + // Mark freed; restore block boundary bit. + *bitp = (*bitp & ~(bitMask<<shift)) | (bitBlockBoundary<<shift); + + if(s->sizeclass == 0) { + // Free large span. + runtime_unmarkspan(p, 1<<PageShift); + *(uintptr*)p = 1; // needs zeroing + runtime_MHeap_Free(&runtime_mheap, s, 1); + } else { + // Free small object. + c = m->mcache; + if(size > sizeof(uintptr)) + ((uintptr*)p)[1] = 1; // mark as "needs to be zeroed" + mstats.by_size[s->sizeclass].nfree++; + runtime_MCache_Free(c, p, s->sizeclass, size); + } mstats.alloc -= size; mstats.nfree++; - mstats.by_size[s->sizeclass].nfree++; - runtime_MCache_Free(c, p, s->sizeclass, size); - break; - case RefNone|RefHasFinalizer: - f = runtime_getfinalizer(p, 1); - if(f == nil) - runtime_throw("finalizer inconsistency"); - f->arg = p; - f->next = finq; - finq = f; - ref &= ~RefHasFinalizer; - // fall through - case RefSome: - case RefSome|RefHasFinalizer: - *gcrefp = RefNone | (ref&RefFlags); - break; } } } -static void -sweep(void) -{ - MSpan *s; - - for(s = runtime_mheap.allspans; s != nil; s = s->allnext) - if(s->state == MSpanInUse) - sweepspan(s); -} - static pthread_mutex_t gcsema = PTHREAD_MUTEX_INITIALIZER; // Initialized from $GOGC. GOGC=off means no gc. @@ -286,7 +473,8 @@ static int32 gcpercent = -2; void runtime_gc(int32 force __attribute__ ((unused))) { - int64 t0, t1; + int64 t0, t1, t2, t3; + uint64 heap0, heap1, obj0, obj1; char *p; Finalizer *fp; @@ -309,29 +497,65 @@ runtime_gc(int32 force __attribute__ ((unused))) gcpercent = -1; else gcpercent = runtime_atoi(p); + + p = runtime_getenv("GOGCTRACE"); + if(p != nil) + gctrace = runtime_atoi(p); } if(gcpercent < 0) return; pthread_mutex_lock(&finqlock); pthread_mutex_lock(&gcsema); - m->locks++; // disable gc during the mallocs in newproc + if(!force && mstats.heap_alloc < mstats.next_gc) { + pthread_mutex_unlock(&gcsema); + pthread_mutex_unlock(&finqlock); + return; + } + t0 = runtime_nanotime(); + nlookup = 0; + nsizelookup = 0; + naddrlookup = 0; + + m->gcing = 1; runtime_stoptheworld(); - if(force || mstats.heap_alloc >= mstats.next_gc) { - __go_cachestats(); - mark(); - sweep(); - __go_stealcache(); - mstats.next_gc = mstats.heap_alloc+mstats.heap_alloc*gcpercent/100; - } + if(runtime_mheap.Lock.key != 0) + runtime_throw("runtime_mheap locked during gc"); + __go_cachestats(); + heap0 = mstats.heap_alloc; + obj0 = mstats.nmalloc - mstats.nfree; + + mark(); t1 = runtime_nanotime(); + sweep(); + t2 = runtime_nanotime(); + __go_stealcache(); + + mstats.next_gc = mstats.heap_alloc+mstats.heap_alloc*gcpercent/100; + m->gcing = 0; + + m->locks++; // disable gc during the mallocs in newproc + + heap1 = mstats.heap_alloc; + obj1 = mstats.nmalloc - mstats.nfree; + + t3 = runtime_nanotime(); + mstats.pause_ns[mstats.numgc%nelem(mstats.pause_ns)] = t3 - t0; + mstats.pause_total_ns += t3 - t0; mstats.numgc++; - mstats.pause_ns[mstats.numgc%nelem(mstats.pause_ns)] = t1 - t0; - mstats.pause_total_ns += t1 - t0; if(mstats.debuggc) - runtime_printf("pause %llu\n", (unsigned long long)t1-t0); + runtime_printf("pause %llu\n", (unsigned long long)t3-t0); + + if(gctrace) { + runtime_printf("gc%d: %llu+%llu+%llu ms %llu -> %llu MB %llu -> %llu (%llu-%llu) objects %llu pointer lookups (%llu size, %llu addr)\n", + mstats.numgc, (unsigned long long)(t1-t0)/1000000, (unsigned long long)(t2-t1)/1000000, (unsigned long long)(t3-t2)/1000000, + (unsigned long long)heap0>>20, (unsigned long long)heap1>>20, (unsigned long long)obj0, (unsigned long long)obj1, + (unsigned long long)mstats.nmalloc, (unsigned long long)mstats.nfree, + (unsigned long long)nlookup, (unsigned long long)nsizelookup, (unsigned long long)naddrlookup); + } + pthread_mutex_unlock(&gcsema); runtime_starttheworld(); @@ -350,6 +574,9 @@ runtime_gc(int32 force __attribute__ ((unused))) } m->locks--; pthread_mutex_unlock(&finqlock); + + if(gctrace > 1 && !force) + runtime_gc(1); } static void @@ -385,6 +612,202 @@ runfinq(void* dummy) } } +#define runtime_gomaxprocs 2 + +// mark the block at v of size n as allocated. +// If noptr is true, mark it as having no pointers. +void +runtime_markallocated(void *v, uintptr n, bool noptr) +{ + uintptr *b, obits, bits, off, shift; + + // if(0) + // runtime_printf("markallocated %p+%p\n", v, n); + + if((byte*)v+n > (byte*)runtime_mheap.arena_used || (byte*)v < runtime_mheap.arena_start) + runtime_throw("markallocated: bad pointer"); + + off = (uintptr*)v - (uintptr*)runtime_mheap.arena_start; // word offset + b = (uintptr*)runtime_mheap.arena_start - off/wordsPerBitmapWord - 1; + shift = off % wordsPerBitmapWord; + + for(;;) { + obits = *b; + bits = (obits & ~(bitMask<<shift)) | (bitAllocated<<shift); + if(noptr) + bits |= bitNoPointers<<shift; + if(runtime_gomaxprocs == 1) { + *b = bits; + break; + } else { + // gomaxprocs > 1: use atomic op + if(runtime_casp((void**)b, (void*)obits, (void*)bits)) + break; + } + } +} + +// mark the block at v of size n as freed. +void +runtime_markfreed(void *v, uintptr n) +{ + uintptr *b, obits, bits, off, shift; + + // if(0) + // runtime_printf("markallocated %p+%p\n", v, n); + + if((byte*)v+n > (byte*)runtime_mheap.arena_used || (byte*)v < runtime_mheap.arena_start) + runtime_throw("markallocated: bad pointer"); + + off = (uintptr*)v - (uintptr*)runtime_mheap.arena_start; // word offset + b = (uintptr*)runtime_mheap.arena_start - off/wordsPerBitmapWord - 1; + shift = off % wordsPerBitmapWord; + + for(;;) { + obits = *b; + bits = (obits & ~(bitMask<<shift)) | (bitBlockBoundary<<shift); + if(runtime_gomaxprocs == 1) { + *b = bits; + break; + } else { + // gomaxprocs > 1: use atomic op + if(runtime_casp((void**)b, (void*)obits, (void*)bits)) + break; + } + } +} + +// check that the block at v of size n is marked freed. +void +runtime_checkfreed(void *v, uintptr n) +{ + uintptr *b, bits, off, shift; + + if(!runtime_checking) + return; + + if((byte*)v+n > (byte*)runtime_mheap.arena_used || (byte*)v < runtime_mheap.arena_start) + return; // not allocated, so okay + + off = (uintptr*)v - (uintptr*)runtime_mheap.arena_start; // word offset + b = (uintptr*)runtime_mheap.arena_start - off/wordsPerBitmapWord - 1; + shift = off % wordsPerBitmapWord; + + bits = *b>>shift; + if((bits & bitAllocated) != 0) { + runtime_printf("checkfreed %p+%p: off=%p have=%p\n", + v, (void*)n, (void*)off, (void*)(bits & bitMask)); + runtime_throw("checkfreed: not freed"); + } +} + +// mark the span of memory at v as having n blocks of the given size. +// if leftover is true, there is left over space at the end of the span. +void +runtime_markspan(void *v, uintptr size, uintptr n, bool leftover) +{ + uintptr *b, off, shift; + byte *p; + + if((byte*)v+size*n > (byte*)runtime_mheap.arena_used || (byte*)v < runtime_mheap.arena_start) + runtime_throw("markspan: bad pointer"); + + p = v; + if(leftover) // mark a boundary just past end of last block too + n++; + for(; n-- > 0; p += size) { + // Okay to use non-atomic ops here, because we control + // the entire span, and each bitmap word has bits for only + // one span, so no other goroutines are changing these + // bitmap words. + off = (uintptr*)p - (uintptr*)runtime_mheap.arena_start; // word offset + b = (uintptr*)runtime_mheap.arena_start - off/wordsPerBitmapWord - 1; + shift = off % wordsPerBitmapWord; + *b = (*b & ~(bitMask<<shift)) | (bitBlockBoundary<<shift); + } +} + +// unmark the span of memory at v of length n bytes. +void +runtime_unmarkspan(void *v, uintptr n) +{ + uintptr *p, *b, off; + + if((byte*)v+n > (byte*)runtime_mheap.arena_used || (byte*)v < runtime_mheap.arena_start) + runtime_throw("markspan: bad pointer"); + + p = v; + off = p - (uintptr*)runtime_mheap.arena_start; // word offset + if(off % wordsPerBitmapWord != 0) + runtime_throw("markspan: unaligned pointer"); + b = (uintptr*)runtime_mheap.arena_start - off/wordsPerBitmapWord - 1; + n /= PtrSize; + if(n%wordsPerBitmapWord != 0) + runtime_throw("unmarkspan: unaligned length"); + // Okay to use non-atomic ops here, because we control + // the entire span, and each bitmap word has bits for only + // one span, so no other goroutines are changing these + // bitmap words. + n /= wordsPerBitmapWord; + while(n-- > 0) + *b-- = 0; +} + +bool +runtime_blockspecial(void *v) +{ + uintptr *b, off, shift; + + off = (uintptr*)v - (uintptr*)runtime_mheap.arena_start; + b = (uintptr*)runtime_mheap.arena_start - off/wordsPerBitmapWord - 1; + shift = off % wordsPerBitmapWord; + + return (*b & (bitSpecial<<shift)) != 0; +} + +void +runtime_setblockspecial(void *v) +{ + uintptr *b, off, shift, bits, obits; + + off = (uintptr*)v - (uintptr*)runtime_mheap.arena_start; + b = (uintptr*)runtime_mheap.arena_start - off/wordsPerBitmapWord - 1; + shift = off % wordsPerBitmapWord; + + for(;;) { + obits = *b; + bits = obits | (bitSpecial<<shift); + if(runtime_gomaxprocs == 1) { + *b = bits; + break; + } else { + // gomaxprocs > 1: use atomic op + if(runtime_casp((void**)b, (void*)obits, (void*)bits)) + break; + } + } +} + +void +runtime_MHeap_MapBits(MHeap *h) +{ + // Caller has added extra mappings to the arena. + // Add extra mappings of bitmap words as needed. + // We allocate extra bitmap pieces in chunks of bitmapChunk. + enum { + bitmapChunk = 8192 + }; + uintptr n; + + n = (h->arena_used - h->arena_start) / wordsPerBitmapWord; + n = (n+bitmapChunk-1) & ~(bitmapChunk-1); + if(h->bitmap_mapped >= n) + return; + + runtime_SysMap(h->arena_start - n, n - h->bitmap_mapped); + h->bitmap_mapped = n; +} + void __go_enable_gc() { diff --git a/libgo/runtime/mheap.c b/libgo/runtime/mheap.c index 52c6d8c1baa..b36df258818 100644 --- a/libgo/runtime/mheap.c +++ b/libgo/runtime/mheap.c @@ -42,7 +42,6 @@ runtime_MHeap_Init(MHeap *h, void *(*alloc)(uintptr)) runtime_initlock(h); runtime_FixAlloc_Init(&h->spanalloc, sizeof(MSpan), alloc, RecordSpan, h); runtime_FixAlloc_Init(&h->cachealloc, sizeof(MCache), alloc, nil, nil); - runtime_MHeapMap_Init(&h->map, alloc); // h->mapcache needs no init for(i=0; i<nelem(h->free); i++) runtime_MSpanList_Init(&h->free[i]); @@ -80,6 +79,7 @@ MHeap_AllocLocked(MHeap *h, uintptr npage, int32 sizeclass) { uintptr n; MSpan *s, *t; + PageID p; // Try in fixed-size lists up to max. for(n=npage; n < nelem(h->free); n++) { @@ -113,18 +113,29 @@ HaveSpan: mstats.mspan_sys = h->spanalloc.sys; runtime_MSpan_Init(t, s->start + npage, s->npages - npage); s->npages = npage; - runtime_MHeapMap_Set(&h->map, t->start - 1, s); - runtime_MHeapMap_Set(&h->map, t->start, t); - runtime_MHeapMap_Set(&h->map, t->start + t->npages - 1, t); + p = t->start; + if(sizeof(void*) == 8) + p -= ((uintptr)h->arena_start>>PageShift); + if(p > 0) + h->map[p-1] = s; + h->map[p] = t; + h->map[p+t->npages-1] = t; + *(uintptr*)(t->start<<PageShift) = *(uintptr*)(s->start<<PageShift); // copy "needs zeroing" mark t->state = MSpanInUse; MHeap_FreeLocked(h, t); } + if(*(uintptr*)(s->start<<PageShift) != 0) + runtime_memclr((byte*)(s->start<<PageShift), s->npages<<PageShift); + // Record span info, because gc needs to be // able to map interior pointer to containing span. s->sizeclass = sizeclass; + p = s->start; + if(sizeof(void*) == 8) + p -= ((uintptr)h->arena_start>>PageShift); for(n=0; n<npage; n++) - runtime_MHeapMap_Set(&h->map, s->start+n, s); + h->map[p+n] = s; return s; } @@ -162,6 +173,7 @@ MHeap_Grow(MHeap *h, uintptr npage) uintptr ask; void *v; MSpan *s; + PageID p; // Ask for a big chunk, to reduce the number of mappings // the operating system needs to track; also amortizes @@ -169,68 +181,72 @@ MHeap_Grow(MHeap *h, uintptr npage) // Allocate a multiple of 64kB (16 pages). npage = (npage+15)&~15; ask = npage<<PageShift; - if(ask < HeapAllocChunk) + if(ask > (uintptr)(h->arena_end - h->arena_used)) + return false; + if(ask < HeapAllocChunk && HeapAllocChunk <= h->arena_end - h->arena_used) ask = HeapAllocChunk; - v = runtime_SysAlloc(ask); + v = runtime_MHeap_SysAlloc(h, ask); if(v == nil) { if(ask > (npage<<PageShift)) { ask = npage<<PageShift; - v = runtime_SysAlloc(ask); + v = runtime_MHeap_SysAlloc(h, ask); } if(v == nil) return false; } mstats.heap_sys += ask; - if((byte*)v < h->min || h->min == nil) - h->min = v; - if((byte*)v+ask > h->max) - h->max = (byte*)v+ask; - - // NOTE(rsc): In tcmalloc, if we've accumulated enough - // system allocations, the heap map gets entirely allocated - // in 32-bit mode. (In 64-bit mode that's not practical.) - if(!runtime_MHeapMap_Preallocate(&h->map, ((uintptr)v>>PageShift) - 1, (ask>>PageShift) + 2)) { - runtime_SysFree(v, ask); - return false; - } - // Create a fake "in use" span and free it, so that the // right coalescing happens. s = runtime_FixAlloc_Alloc(&h->spanalloc); mstats.mspan_inuse = h->spanalloc.inuse; mstats.mspan_sys = h->spanalloc.sys; runtime_MSpan_Init(s, (uintptr)v>>PageShift, ask>>PageShift); - runtime_MHeapMap_Set(&h->map, s->start, s); - runtime_MHeapMap_Set(&h->map, s->start + s->npages - 1, s); + p = s->start; + if(sizeof(void*) == 8) + p -= ((uintptr)h->arena_start>>PageShift); + h->map[p] = s; + h->map[p + s->npages - 1] = s; s->state = MSpanInUse; MHeap_FreeLocked(h, s); return true; } -// Look up the span at the given page number. -// Page number is guaranteed to be in map +// Look up the span at the given address. +// Address is guaranteed to be in map // and is guaranteed to be start or end of span. MSpan* -runtime_MHeap_Lookup(MHeap *h, PageID p) +runtime_MHeap_Lookup(MHeap *h, void *v) { - return runtime_MHeapMap_Get(&h->map, p); + uintptr p; + + p = (uintptr)v; + if(sizeof(void*) == 8) + p -= (uintptr)h->arena_start; + return h->map[p >> PageShift]; } -// Look up the span at the given page number. -// Page number is *not* guaranteed to be in map +// Look up the span at the given address. +// Address is *not* guaranteed to be in map // and may be anywhere in the span. // Map entries for the middle of a span are only // valid for allocated spans. Free spans may have // other garbage in their middles, so we have to // check for that. MSpan* -runtime_MHeap_LookupMaybe(MHeap *h, PageID p) +runtime_MHeap_LookupMaybe(MHeap *h, void *v) { MSpan *s; + PageID p, q; - s = runtime_MHeapMap_GetMaybe(&h->map, p); + if((byte*)v < h->arena_start || (byte*)v >= h->arena_used) + return nil; + p = (uintptr)v>>PageShift; + q = p; + if(sizeof(void*) == 8) + q -= (uintptr)h->arena_start >> PageShift; + s = h->map[q]; if(s == nil || p < s->start || p - s->start >= s->npages) return nil; if(s->state != MSpanInUse) @@ -259,7 +275,9 @@ runtime_MHeap_Free(MHeap *h, MSpan *s, int32 acct) static void MHeap_FreeLocked(MHeap *h, MSpan *s) { + uintptr *sp, *tp; MSpan *t; + PageID p; if(s->state != MSpanInUse || s->ref != 0) { // runtime_printf("MHeap_FreeLocked - span %p ptr %p state %d ref %d\n", s, s->start<<PageShift, s->state, s->ref); @@ -267,21 +285,30 @@ MHeap_FreeLocked(MHeap *h, MSpan *s) } s->state = MSpanFree; runtime_MSpanList_Remove(s); + sp = (uintptr*)(s->start<<PageShift); // Coalesce with earlier, later spans. - if((t = runtime_MHeapMap_Get(&h->map, s->start - 1)) != nil && t->state != MSpanInUse) { + p = s->start; + if(sizeof(void*) == 8) + p -= (uintptr)h->arena_start >> PageShift; + if(p > 0 && (t = h->map[p-1]) != nil && t->state != MSpanInUse) { + tp = (uintptr*)(t->start<<PageShift); + *tp |= *sp; // propagate "needs zeroing" mark s->start = t->start; s->npages += t->npages; - runtime_MHeapMap_Set(&h->map, s->start, s); + p -= t->npages; + h->map[p] = s; runtime_MSpanList_Remove(t); t->state = MSpanDead; runtime_FixAlloc_Free(&h->spanalloc, t); mstats.mspan_inuse = h->spanalloc.inuse; mstats.mspan_sys = h->spanalloc.sys; } - if((t = runtime_MHeapMap_Get(&h->map, s->start + s->npages)) != nil && t->state != MSpanInUse) { + if(p+s->npages < nelem(h->map) && (t = h->map[p+s->npages]) != nil && t->state != MSpanInUse) { + tp = (uintptr*)(t->start<<PageShift); + *sp |= *tp; // propagate "needs zeroing" mark s->npages += t->npages; - runtime_MHeapMap_Set(&h->map, s->start + s->npages - 1, s); + h->map[p + s->npages - 1] = s; runtime_MSpanList_Remove(t); t->state = MSpanDead; runtime_FixAlloc_Free(&h->spanalloc, t); @@ -341,10 +368,14 @@ runtime_MSpanList_IsEmpty(MSpan *list) void runtime_MSpanList_Insert(MSpan *list, MSpan *span) { - if(span->next != nil || span->prev != nil) + if(span->next != nil || span->prev != nil) { + // runtime_printf("failed MSpanList_Insert %p %p %p\n", span, span->next, span->prev); runtime_throw("MSpanList_Insert"); + } span->next = list->next; span->prev = list; span->next->prev = span; span->prev->next = span; } + + diff --git a/libgo/runtime/mheapmap32.c b/libgo/runtime/mheapmap32.c deleted file mode 100644 index 547c602fe3f..00000000000 --- a/libgo/runtime/mheapmap32.c +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Heap map, 32-bit version -// See malloc.h and mheap.c for overview. - -#include "runtime.h" -#include "malloc.h" - -#if __SIZEOF_POINTER__ == 4 - -// 3-level radix tree mapping page ids to Span*. -void -runtime_MHeapMap_Init(MHeapMap *m, void *(*allocator)(uintptr)) -{ - m->allocator = allocator; -} - -MSpan* -runtime_MHeapMap_Get(MHeapMap *m, PageID k) -{ - int32 i1, i2; - - i2 = k & MHeapMap_Level2Mask; - k >>= MHeapMap_Level2Bits; - i1 = k & MHeapMap_Level1Mask; - k >>= MHeapMap_Level1Bits; - if(k != 0) - runtime_throw("MHeapMap_Get"); - - return m->p[i1]->s[i2]; -} - -MSpan* -runtime_MHeapMap_GetMaybe(MHeapMap *m, PageID k) -{ - int32 i1, i2; - MHeapMapNode2 *p2; - - i2 = k & MHeapMap_Level2Mask; - k >>= MHeapMap_Level2Bits; - i1 = k & MHeapMap_Level1Mask; - k >>= MHeapMap_Level1Bits; - if(k != 0) - runtime_throw("MHeapMap_Get"); - - p2 = m->p[i1]; - if(p2 == nil) - return nil; - return p2->s[i2]; -} - -void -runtime_MHeapMap_Set(MHeapMap *m, PageID k, MSpan *s) -{ - int32 i1, i2; - - i2 = k & MHeapMap_Level2Mask; - k >>= MHeapMap_Level2Bits; - i1 = k & MHeapMap_Level1Mask; - k >>= MHeapMap_Level1Bits; - if(k != 0) - runtime_throw("MHeapMap_Set"); - - m->p[i1]->s[i2] = s; -} - -// Allocate the storage required for entries [k, k+1, ..., k+len-1] -// so that Get and Set calls need not check for nil pointers. -bool -runtime_MHeapMap_Preallocate(MHeapMap *m, PageID k, uintptr len) -{ - uintptr end; - int32 i1; - MHeapMapNode2 *p2; - - end = k+len; - while(k < end) { - if((k >> MHeapMap_TotalBits) != 0) - return false; - i1 = (k >> MHeapMap_Level2Bits) & MHeapMap_Level1Mask; - - // first-level pointer - if(m->p[i1] == nil) { - p2 = m->allocator(sizeof *p2); - if(p2 == nil) - return false; - mstats.heapmap_sys += sizeof *p2; - m->p[i1] = p2; - } - - // advance key past this leaf node - k = ((k >> MHeapMap_Level2Bits) + 1) << MHeapMap_Level2Bits; - } - return true; -} - -#endif /* __SIZEOF_POINTER__ == 4 */ diff --git a/libgo/runtime/mheapmap32.h b/libgo/runtime/mheapmap32.h deleted file mode 100644 index 2861624690f..00000000000 --- a/libgo/runtime/mheapmap32.h +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Free(v) must be able to determine the MSpan containing v. -// The MHeapMap is a 2-level radix tree mapping page numbers to MSpans. - -typedef struct MHeapMapNode2 MHeapMapNode2; - -enum -{ - // 32 bit address - 12 bit page size = 20 bits to map - MHeapMap_Level1Bits = 10, - MHeapMap_Level2Bits = 10, - - MHeapMap_TotalBits = - MHeapMap_Level1Bits + - MHeapMap_Level2Bits, - - MHeapMap_Level1Mask = (1<<MHeapMap_Level1Bits) - 1, - MHeapMap_Level2Mask = (1<<MHeapMap_Level2Bits) - 1, -}; - -struct MHeapMap -{ - void *(*allocator)(uintptr); - MHeapMapNode2 *p[1<<MHeapMap_Level1Bits]; -}; - -struct MHeapMapNode2 -{ - MSpan *s[1<<MHeapMap_Level2Bits]; -}; - -void runtime_MHeapMap_Init(MHeapMap *m, void *(*allocator)(uintptr)); -bool runtime_MHeapMap_Preallocate(MHeapMap *m, PageID k, uintptr npages); -MSpan* runtime_MHeapMap_Get(MHeapMap *m, PageID k); -MSpan* runtime_MHeapMap_GetMaybe(MHeapMap *m, PageID k); -void runtime_MHeapMap_Set(MHeapMap *m, PageID k, MSpan *v); - - diff --git a/libgo/runtime/mheapmap64.c b/libgo/runtime/mheapmap64.c deleted file mode 100644 index d6305953ad5..00000000000 --- a/libgo/runtime/mheapmap64.c +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Heap map, 64-bit version -// See malloc.h and mheap.c for overview. - -#include "runtime.h" -#include "malloc.h" - -#if __SIZEOF_POINTER__ == 8 - -// 3-level radix tree mapping page ids to Span*. -void -runtime_MHeapMap_Init(MHeapMap *m, void *(*allocator)(uintptr)) -{ - m->allocator = allocator; -} - -MSpan* -runtime_MHeapMap_Get(MHeapMap *m, PageID k) -{ - int32 i1, i2, i3; - - i3 = k & MHeapMap_Level3Mask; - k >>= MHeapMap_Level3Bits; - i2 = k & MHeapMap_Level2Mask; - k >>= MHeapMap_Level2Bits; - i1 = k & MHeapMap_Level1Mask; - k >>= MHeapMap_Level1Bits; - if(k != 0) - runtime_throw("MHeapMap_Get"); - - return m->p[i1]->p[i2]->s[i3]; -} - -MSpan* -runtime_MHeapMap_GetMaybe(MHeapMap *m, PageID k) -{ - int32 i1, i2, i3; - MHeapMapNode2 *p2; - MHeapMapNode3 *p3; - - i3 = k & MHeapMap_Level3Mask; - k >>= MHeapMap_Level3Bits; - i2 = k & MHeapMap_Level2Mask; - k >>= MHeapMap_Level2Bits; - i1 = k & MHeapMap_Level1Mask; - k >>= MHeapMap_Level1Bits; - if(k != 0) - runtime_throw("MHeapMap_Get"); - - p2 = m->p[i1]; - if(p2 == nil) - return nil; - p3 = p2->p[i2]; - if(p3 == nil) - return nil; - return p3->s[i3]; -} - -void -runtime_MHeapMap_Set(MHeapMap *m, PageID k, MSpan *s) -{ - int32 i1, i2, i3; - - i3 = k & MHeapMap_Level3Mask; - k >>= MHeapMap_Level3Bits; - i2 = k & MHeapMap_Level2Mask; - k >>= MHeapMap_Level2Bits; - i1 = k & MHeapMap_Level1Mask; - k >>= MHeapMap_Level1Bits; - if(k != 0) - runtime_throw("MHeapMap_Set"); - - m->p[i1]->p[i2]->s[i3] = s; -} - -// Allocate the storage required for entries [k, k+1, ..., k+len-1] -// so that Get and Set calls need not check for nil pointers. -bool -runtime_MHeapMap_Preallocate(MHeapMap *m, PageID k, uintptr len) -{ - uintptr end; - int32 i1, i2; - MHeapMapNode2 *p2; - MHeapMapNode3 *p3; - - end = k+len; - while(k < end) { - if((k >> MHeapMap_TotalBits) != 0) - return false; - i2 = (k >> MHeapMap_Level3Bits) & MHeapMap_Level2Mask; - i1 = (k >> (MHeapMap_Level3Bits + MHeapMap_Level2Bits)) & MHeapMap_Level1Mask; - - // first-level pointer - if((p2 = m->p[i1]) == nil) { - p2 = m->allocator(sizeof *p2); - if(p2 == nil) - return false; - mstats.heapmap_sys += sizeof *p2; - m->p[i1] = p2; - } - - // second-level pointer - if(p2->p[i2] == nil) { - p3 = m->allocator(sizeof *p3); - if(p3 == nil) - return false; - mstats.heapmap_sys += sizeof *p3; - p2->p[i2] = p3; - } - - // advance key past this leaf node - k = ((k >> MHeapMap_Level3Bits) + 1) << MHeapMap_Level3Bits; - } - return true; -} - -#endif /* __SIZEOF_POINTER__ == 8 */ diff --git a/libgo/runtime/mheapmap64.h b/libgo/runtime/mheapmap64.h deleted file mode 100644 index be304cb2e8b..00000000000 --- a/libgo/runtime/mheapmap64.h +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Free(v) must be able to determine the MSpan containing v. -// The MHeapMap is a 3-level radix tree mapping page numbers to MSpans. -// -// NOTE(rsc): On a 32-bit platform (= 20-bit page numbers), -// we can swap in a 2-level radix tree. -// -// NOTE(rsc): We use a 3-level tree because tcmalloc does, but -// having only three levels requires approximately 1 MB per node -// in the tree, making the minimum map footprint 3 MB. -// Using a 4-level tree would cut the minimum footprint to 256 kB. -// On the other hand, it's just virtual address space: most of -// the memory is never going to be touched, thus never paged in. - -typedef struct MHeapMapNode2 MHeapMapNode2; -typedef struct MHeapMapNode3 MHeapMapNode3; - -enum -{ - // 64 bit address - 12 bit page size = 52 bits to map - MHeapMap_Level1Bits = 18, - MHeapMap_Level2Bits = 18, - MHeapMap_Level3Bits = 16, - - MHeapMap_TotalBits = - MHeapMap_Level1Bits + - MHeapMap_Level2Bits + - MHeapMap_Level3Bits, - - MHeapMap_Level1Mask = (1<<MHeapMap_Level1Bits) - 1, - MHeapMap_Level2Mask = (1<<MHeapMap_Level2Bits) - 1, - MHeapMap_Level3Mask = (1<<MHeapMap_Level3Bits) - 1, -}; - -struct MHeapMap -{ - void *(*allocator)(uintptr); - MHeapMapNode2 *p[1<<MHeapMap_Level1Bits]; -}; - -struct MHeapMapNode2 -{ - MHeapMapNode3 *p[1<<MHeapMap_Level2Bits]; -}; - -struct MHeapMapNode3 -{ - MSpan *s[1<<MHeapMap_Level3Bits]; -}; - -void runtime_MHeapMap_Init(MHeapMap *m, void *(*allocator)(uintptr)); -bool runtime_MHeapMap_Preallocate(MHeapMap *m, PageID k, uintptr npages); -MSpan* runtime_MHeapMap_Get(MHeapMap *m, PageID k); -MSpan* runtime_MHeapMap_GetMaybe(MHeapMap *m, PageID k); -void runtime_MHeapMap_Set(MHeapMap *m, PageID k, MSpan *v); - - diff --git a/libgo/runtime/mprof.goc b/libgo/runtime/mprof.goc index 6bd4ef72724..2e147edda02 100644 --- a/libgo/runtime/mprof.goc +++ b/libgo/runtime/mprof.goc @@ -67,7 +67,7 @@ stkbucket(uintptr *stk, int32 nstk) runtime_mcmp((byte*)b->stk, (byte*)stk, nstk*sizeof stk[0]) == 0) return b; - b = runtime_mallocgc(sizeof *b + nstk*sizeof stk[0], RefNoProfiling, 0, 1); + b = runtime_mallocgc(sizeof *b + nstk*sizeof stk[0], FlagNoProfiling, 0, 1); bucketmem += sizeof *b + nstk*sizeof stk[0]; runtime_memmove(b->stk, stk, nstk*sizeof stk[0]); b->hash = h; @@ -134,7 +134,7 @@ setaddrbucket(uintptr addr, Bucket *b) if(ah->addr == (addr>>20)) goto found; - ah = runtime_mallocgc(sizeof *ah, RefNoProfiling, 0, 1); + ah = runtime_mallocgc(sizeof *ah, FlagNoProfiling, 0, 1); addrmem += sizeof *ah; ah->next = addrhash[h]; ah->addr = addr>>20; @@ -142,7 +142,7 @@ setaddrbucket(uintptr addr, Bucket *b) found: if((e = addrfree) == nil) { - e = runtime_mallocgc(64*sizeof *e, RefNoProfiling, 0, 0); + e = runtime_mallocgc(64*sizeof *e, FlagNoProfiling, 0, 0); addrmem += 64*sizeof *e; for(i=0; i+1<64; i++) e[i].next = &e[i+1]; diff --git a/libgo/runtime/msize.c b/libgo/runtime/msize.c index 8b021a2b6b3..6e82885bab4 100644 --- a/libgo/runtime/msize.c +++ b/libgo/runtime/msize.c @@ -57,7 +57,7 @@ runtime_SizeToClass(int32 size) void runtime_InitSizes(void) { - int32 align, sizeclass, size, osize, nextsize, n; + int32 align, sizeclass, size, nextsize, n; uint32 i; uintptr allocsize, npages; @@ -81,8 +81,7 @@ runtime_InitSizes(void) // the leftover is less than 1/8 of the total, // so wasted space is at most 12.5%. allocsize = PageSize; - osize = size + RefcountOverhead; - while(allocsize%osize > (allocsize/8)) + while(allocsize%size > allocsize/8) allocsize += PageSize; npages = allocsize >> PageShift; @@ -93,7 +92,7 @@ runtime_InitSizes(void) // different sizes. if(sizeclass > 1 && (int32)npages == runtime_class_to_allocnpages[sizeclass-1] - && allocsize/osize == allocsize/(runtime_class_to_size[sizeclass-1]+RefcountOverhead)) { + && allocsize/size == allocsize/runtime_class_to_size[sizeclass-1]) { runtime_class_to_size[sizeclass-1] = size; continue; } diff --git a/libgo/runtime/runtime.h b/libgo/runtime/runtime.h index 95216e4a5ca..011ba7dab6b 100644 --- a/libgo/runtime/runtime.h +++ b/libgo/runtime/runtime.h @@ -185,6 +185,7 @@ void runtime_walkfintab(void (*fn)(void*), void (*scan)(byte *, int64)); #define runtime_mmap mmap #define runtime_munmap(p, s) munmap((p), (s)) #define runtime_cas(pval, old, new) __sync_bool_compare_and_swap (pval, old, new) +#define runtime_casp(pval, old, new) __sync_bool_compare_and_swap (pval, old, new) struct __go_func_type; void reflect_call(const struct __go_func_type *, const void *, _Bool, void **, diff --git a/libgo/runtime/sigqueue.goc b/libgo/runtime/sigqueue.goc index b5f2954bc8e..7cbd739e51e 100644 --- a/libgo/runtime/sigqueue.goc +++ b/libgo/runtime/sigqueue.goc @@ -102,7 +102,7 @@ func Signame(sig int32) (name String) { s = buf; } int32 len = __builtin_strlen(s); - unsigned char *data = runtime_mallocgc(len, RefNoPointers, 0, 0); + unsigned char *data = runtime_mallocgc(len, FlagNoPointers, 0, 0); __builtin_memcpy(data, s, len); name.__data = data; name.__length = len; diff --git a/libgo/syscalls/exec.go b/libgo/syscalls/exec.go index d0f56d3b923..7a9ee2825dc 100644 --- a/libgo/syscalls/exec.go +++ b/libgo/syscalls/exec.go @@ -12,7 +12,7 @@ import "unsafe" func libc_fcntl(fd int, cmd int, arg int) int __asm__ ("fcntl") func libc_fork() Pid_t __asm__ ("fork") -func libc_chdir(name *byte) int __asm__ ("chdir"); +func libc_chdir(name *byte) int __asm__ ("chdir") func libc_dup2(int, int) int __asm__ ("dup2") func libc_execve(*byte, **byte, **byte) int __asm__ ("execve") func libc_sysexit(int) __asm__ ("_exit") @@ -27,17 +27,17 @@ func libc_wait4(Pid_t, *int, int, *Rusage) Pid_t __asm__ ("wait4") func forkAndExecInChild(argv0 *byte, argv []*byte, envv []*byte, traceme bool, dir *byte, fd []int, pipe int) (pid int, err int) { // Declare all variables at top in case any // declarations require heap allocation (e.g., err1). - var r1, r2, err1 uintptr; - var nextfd int; - var i int; + var r1, r2, err1 uintptr + var nextfd int + var i int - darwin := OS == "darwin"; + darwin := OS == "darwin" // About to call fork. // No more allocation or calls of non-assembly functions. - child := libc_fork(); + child := libc_fork() if child == -1 { - return 0, GetErrno(); + return 0, GetErrno() } if child != 0 { @@ -50,41 +50,41 @@ func forkAndExecInChild(argv0 *byte, argv []*byte, envv []*byte, traceme bool, d // Enable tracing if requested. if traceme { if libc_ptrace(_PTRACE_TRACEME, 0, 0, nil) < 0 { - goto childerror; + goto childerror } } // Chdir if dir != nil { - r := libc_chdir(dir); + r := libc_chdir(dir) if r < 0 { - goto childerror; + goto childerror } } // Pass 1: look for fd[i] < i and move those up above len(fd) // so that pass 2 won't stomp on an fd it needs later. - nextfd = int(len(fd)); + nextfd = int(len(fd)) if pipe < nextfd { - r := libc_dup2(pipe, nextfd); + r := libc_dup2(pipe, nextfd) if r == -1 { - goto childerror; + goto childerror } - libc_fcntl(nextfd, F_SETFD, FD_CLOEXEC); - pipe = nextfd; - nextfd++; + libc_fcntl(nextfd, F_SETFD, FD_CLOEXEC) + pipe = nextfd + nextfd++ } for i = 0; i < len(fd); i++ { if fd[i] >= 0 && fd[i] < int(i) { - r := libc_dup2(fd[i], nextfd); + r := libc_dup2(fd[i], nextfd) if r == -1 { - goto childerror; + goto childerror } - libc_fcntl(nextfd, F_SETFD, FD_CLOEXEC); - fd[i] = nextfd; - nextfd++; - if nextfd == pipe { // don't stomp on pipe - nextfd++; + libc_fcntl(nextfd, F_SETFD, FD_CLOEXEC) + fd[i] = nextfd + nextfd++ + if nextfd == pipe { // don't stomp on pipe + nextfd++ } } } @@ -92,23 +92,23 @@ func forkAndExecInChild(argv0 *byte, argv []*byte, envv []*byte, traceme bool, d // Pass 2: dup fd[i] down onto i. for i = 0; i < len(fd); i++ { if fd[i] == -1 { - libc_close(i); - continue; + libc_close(i) + continue } if fd[i] == int(i) { // dup2(i, i) won't clear close-on-exec flag on Linux, // probably not elsewhere either. - r := libc_fcntl(fd[i], F_SETFD, 0); + r := libc_fcntl(fd[i], F_SETFD, 0) if r != 0 { - goto childerror; + goto childerror } - continue; + continue } // The new fd is created NOT close-on-exec, // which is exactly what we want. - r := libc_dup2(fd[i], i); + r := libc_dup2(fd[i], i) if r == -1 { - goto childerror; + goto childerror } } @@ -117,17 +117,17 @@ func forkAndExecInChild(argv0 *byte, argv []*byte, envv []*byte, traceme bool, d // Programs that know they inherit fds >= 3 will need // to set them close-on-exec. for i = len(fd); i < 3; i++ { - libc_close(i); + libc_close(i) } // Time to exec. - libc_execve(argv0, &argv[0], &envv[0]); + libc_execve(argv0, &argv[0], &envv[0]) childerror: // send error code on pipe - var e uintptr = uintptr(GetErrno()); + var e uintptr = uintptr(GetErrno()) libc_write(pipe, (*byte)(unsafe.Pointer(&e)), - Size_t(unsafe.Sizeof(err1))); + Size_t(unsafe.Sizeof(err1))) for { libc_sysexit(253) } @@ -135,79 +135,78 @@ childerror: // Calling panic is not actually safe, // but the for loop above won't break // and this shuts up the compiler. - panic("unreached"); + panic("unreached") } func forkExec(argv0 string, argv []string, envv []string, traceme bool, dir string, fd []int) (pid int, err int) { - var p [2]int; - var r1 int; - var err1 uintptr; - var wstatus WaitStatus; + var p [2]int + var r1 int + var err1 uintptr + var wstatus WaitStatus - p[0] = -1; - p[1] = -1; + p[0] = -1 + p[1] = -1 // Convert args to C form. - argv0p := StringBytePtr(argv0); - argvp := StringArrayPtr(argv); - envvp := StringArrayPtr(envv); - var dirp *byte; + argv0p := StringBytePtr(argv0) + argvp := StringArrayPtr(argv) + envvp := StringArrayPtr(envv) + var dirp *byte if len(dir) > 0 { - dirp = StringBytePtr(dir); + dirp = StringBytePtr(dir) } // Acquire the fork lock so that no other threads // create new fds that are not yet close-on-exec // before we fork. - ForkLock.Lock(); + ForkLock.Lock() // Allocate child status pipe close on exec. if err = Pipe(p[0:]); err != 0 { - goto error; + goto error } - var val int; - if val, err = fcntl(p[0], F_SETFD, FD_CLOEXEC); err != 0 { - goto error; + if _, err = fcntl(p[0], F_SETFD, FD_CLOEXEC); err != 0 { + goto error } - if val, err = fcntl(p[1], F_SETFD, FD_CLOEXEC); err != 0 { - goto error; + if _, err = fcntl(p[1], F_SETFD, FD_CLOEXEC); err != 0 { + goto error } // Kick off child. - pid, err = forkAndExecInChild(argv0p, argvp, envvp, traceme, dirp, fd, p[1]); + pid, err = forkAndExecInChild(argv0p, argvp, envvp, traceme, dirp, fd, p[1]) if err != 0 { error: if p[0] >= 0 { - Close(p[0]); - Close(p[1]); + Close(p[0]) + Close(p[1]) } - ForkLock.Unlock(); + ForkLock.Unlock() return 0, err } - ForkLock.Unlock(); + ForkLock.Unlock() // Read child error status from pipe. - Close(p[1]); + Close(p[1]) n := libc_read(p[0], (*byte)(unsafe.Pointer(&err1)), - Size_t(unsafe.Sizeof(err1))); - err = 0; + Size_t(unsafe.Sizeof(err1))) + err = 0 if n < 0 { - err = GetErrno(); + err = GetErrno() } - Close(p[0]); + Close(p[0]) if err != 0 || n != 0 { if int(n) == unsafe.Sizeof(err1) { - err = int(err1); + err = int(err1) } if err == 0 { - err = EPIPE; + err = EPIPE } // Child failed; wait for it to exit, to make sure // the zombies don't accumulate. - pid1, err1 := Wait4(pid, &wstatus, 0, nil); + _, err1 := Wait4(pid, &wstatus, 0, nil) for err1 == EINTR { - pid1, err1 = Wait4(pid, &wstatus, 0, nil); + _, err1 = Wait4(pid, &wstatus, 0, nil) } return 0, err } @@ -218,31 +217,37 @@ func forkExec(argv0 string, argv []string, envv []string, traceme bool, dir stri // Combination of fork and exec, careful to be thread safe. func ForkExec(argv0 string, argv []string, envv []string, dir string, fd []int) (pid int, err int) { - return forkExec(argv0, argv, envv, false, dir, fd); + return forkExec(argv0, argv, envv, false, dir, fd) } // PtraceForkExec is like ForkExec, but starts the child in a traced state. func PtraceForkExec(argv0 string, argv []string, envv []string, dir string, fd []int) (pid int, err int) { - return forkExec(argv0, argv, envv, true, dir, fd); + return forkExec(argv0, argv, envv, true, dir, fd) } // Ordinary exec. func Exec(argv0 string, argv []string, envv []string) (err int) { - argv_arg := StringArrayPtr(argv); - envv_arg := StringArrayPtr(envv); - libc_execve(StringBytePtr(argv0), &argv_arg[0], &envv_arg[0]); - return GetErrno(); + argv_arg := StringArrayPtr(argv) + envv_arg := StringArrayPtr(envv) + libc_execve(StringBytePtr(argv0), &argv_arg[0], &envv_arg[0]) + return GetErrno() +} + +// StartProcess wraps ForkExec for package os. +func StartProcess(argv0 string, argv []string, envv []string, dir string, fd []int) (pid, handle int, err int) { + pid, err = forkExec(argv0, argv, envv, false, dir, fd) + return pid, 0, err } func Wait4(pid int, wstatus *WaitStatus, options int, rusage *Rusage) (wpid int, errno int) { - var status int; - r := libc_wait4(Pid_t(pid), &status, options, rusage); - wpid = int(r); + var status int + r := libc_wait4(Pid_t(pid), &status, options, rusage) + wpid = int(r) if r < 0 { - errno = GetErrno(); + errno = GetErrno() } if wstatus != nil { - *wstatus = WaitStatus(status); + *wstatus = WaitStatus(status) } - return; + return } diff --git a/libgo/syscalls/socket.go b/libgo/syscalls/socket.go index 65c1916711c..e786653705b 100644 --- a/libgo/syscalls/socket.go +++ b/libgo/syscalls/socket.go @@ -232,6 +232,10 @@ func SetsockoptString(fd, level, opt int, s string) (errno int) { return setsockopt(fd, level, opt, uintptr(unsafe.Pointer(&[]byte(s)[0])), Socklen_t(len(s))) } +func SetsockoptIpMreq(fd, level, opt int, mreq *IpMreq) (errno int) { + return setsockopt(fd, level, opt, uintptr(unsafe.Pointer(mreq)), unsafe.Sizeof(*mreq)) +} + func Getsockname(fd int) (sa Sockaddr, errno int) { var rsa RawSockaddrAny; var len Socklen_t = SizeofSockaddrAny; diff --git a/libgo/testsuite/gotest b/libgo/testsuite/gotest index 28aba6e2dd8..93db3462e8a 100755 --- a/libgo/testsuite/gotest +++ b/libgo/testsuite/gotest @@ -131,6 +131,13 @@ esac # so that the tests do not have to refer to srcdir to find test data. ln -s $srcdir/* . +# Some tests refer to a ../testdata directory. +if test -e $srcdir/../testdata; then + rm -f ../testdata + abssrcdir=`cd $srcdir && pwd` + ln -s $abssrcdir/../testdata ../testdata +fi + # Copy the .go files because io/utils_test.go expects a regular file. case "x$gofiles" in x) |