diff options
Diffstat (limited to 'libgo/go/compress')
20 files changed, 850 insertions, 143 deletions
diff --git a/libgo/go/compress/bzip2/bit_reader.go b/libgo/go/compress/bzip2/bit_reader.go index ab1d6065143..32d1036ae1b 100644 --- a/libgo/go/compress/bzip2/bit_reader.go +++ b/libgo/go/compress/bzip2/bit_reader.go @@ -77,6 +77,14 @@ func (br *bitReader) ReadBit() bool { return n != 0 } +func (br *bitReader) TryReadBit() (bit byte, ok bool) { + if br.bits > 0 { + br.bits-- + return byte(br.n>>br.bits) & 1, true + } + return 0, false +} + func (br *bitReader) Err() error { return br.err } diff --git a/libgo/go/compress/bzip2/bzip2.go b/libgo/go/compress/bzip2/bzip2.go index 3dc8c620615..82e30c7c9d7 100644 --- a/libgo/go/compress/bzip2/bzip2.go +++ b/libgo/go/compress/bzip2/bzip2.go @@ -22,14 +22,17 @@ func (s StructuralError) Error() string { // A reader decompresses bzip2 compressed data. type reader struct { - br bitReader - setupDone bool // true if we have parsed the bzip2 header. - blockSize int // blockSize in bytes, i.e. 900 * 1024. - eof bool - buf []byte // stores Burrows-Wheeler transformed data. - c [256]uint // the `C' array for the inverse BWT. - tt []uint32 // mirrors the `tt' array in the bzip2 source and contains the P array in the upper 24 bits. - tPos uint32 // Index of the next output byte in tt. + br bitReader + fileCRC uint32 + blockCRC uint32 + wantBlockCRC uint32 + setupDone bool // true if we have parsed the bzip2 header. + blockSize int // blockSize in bytes, i.e. 900 * 1024. + eof bool + buf []byte // stores Burrows-Wheeler transformed data. + c [256]uint // the `C' array for the inverse BWT. + tt []uint32 // mirrors the `tt' array in the bzip2 source and contains the P array in the upper 24 bits. + tPos uint32 // Index of the next output byte in tt. preRLE []uint32 // contains the RLE data still to be processed. preRLEUsed int // number of entries of preRLE used. @@ -50,12 +53,14 @@ const bzip2BlockMagic = 0x314159265359 const bzip2FinalMagic = 0x177245385090 // setup parses the bzip2 header. -func (bz2 *reader) setup() error { +func (bz2 *reader) setup(needMagic bool) error { br := &bz2.br - magic := br.ReadBits(16) - if magic != bzip2FileMagic { - return StructuralError("bad magic value") + if needMagic { + magic := br.ReadBits(16) + if magic != bzip2FileMagic { + return StructuralError("bad magic value") + } } t := br.ReadBits(8) @@ -68,8 +73,11 @@ func (bz2 *reader) setup() error { return StructuralError("invalid compression level") } + bz2.fileCRC = 0 bz2.blockSize = 100 * 1024 * (int(level) - '0') - bz2.tt = make([]uint32, bz2.blockSize) + if bz2.blockSize > len(bz2.tt) { + bz2.tt = make([]uint32, bz2.blockSize) + } return nil } @@ -79,7 +87,7 @@ func (bz2 *reader) Read(buf []byte) (n int, err error) { } if !bz2.setupDone { - err = bz2.setup() + err = bz2.setup(true) brErr := bz2.br.Err() if brErr != nil { err = brErr @@ -98,14 +106,14 @@ func (bz2 *reader) Read(buf []byte) (n int, err error) { return } -func (bz2 *reader) read(buf []byte) (n int, err error) { +func (bz2 *reader) readFromBlock(buf []byte) int { // bzip2 is a block based compressor, except that it has a run-length // preprocessing step. The block based nature means that we can // preallocate fixed-size buffers and reuse them. However, the RLE // preprocessing would require allocating huge buffers to store the // maximum expansion. Thus we process blocks all at once, except for // the RLE which we decompress as required. - + n := 0 for (bz2.repeats > 0 || bz2.preRLEUsed < len(bz2.preRLE)) && n < len(buf) { // We have RLE data pending. @@ -148,34 +156,87 @@ func (bz2 *reader) read(buf []byte) (n int, err error) { n++ } - if n > 0 { - return - } + return n +} - // No RLE data is pending so we need to read a block. +func (bz2 *reader) read(buf []byte) (int, error) { + for { + n := bz2.readFromBlock(buf) + if n > 0 { + bz2.blockCRC = updateCRC(bz2.blockCRC, buf[:n]) + return n, nil + } - br := &bz2.br - magic := br.ReadBits64(48) - if magic == bzip2FinalMagic { - br.ReadBits64(32) // ignored CRC - bz2.eof = true - return 0, io.EOF - } else if magic != bzip2BlockMagic { - return 0, StructuralError("bad magic value found") - } + // End of block. Check CRC. + if bz2.blockCRC != bz2.wantBlockCRC { + bz2.br.err = StructuralError("block checksum mismatch") + return 0, bz2.br.err + } - err = bz2.readBlock() - if err != nil { - return 0, err - } + // Find next block. + br := &bz2.br + switch br.ReadBits64(48) { + default: + return 0, StructuralError("bad magic value found") + + case bzip2BlockMagic: + // Start of block. + err := bz2.readBlock() + if err != nil { + return 0, err + } - return bz2.read(buf) + case bzip2FinalMagic: + // Check end-of-file CRC. + wantFileCRC := uint32(br.ReadBits64(32)) + if br.err != nil { + return 0, br.err + } + if bz2.fileCRC != wantFileCRC { + br.err = StructuralError("file checksum mismatch") + return 0, br.err + } + + // Skip ahead to byte boundary. + // Is there a file concatenated to this one? + // It would start with BZ. + if br.bits%8 != 0 { + br.ReadBits(br.bits % 8) + } + b, err := br.r.ReadByte() + if err == io.EOF { + br.err = io.EOF + bz2.eof = true + return 0, io.EOF + } + if err != nil { + br.err = err + return 0, err + } + z, err := br.r.ReadByte() + if err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + br.err = err + return 0, err + } + if b != 'B' || z != 'Z' { + return 0, StructuralError("bad magic value in continuation file") + } + if err := bz2.setup(false); err != nil { + return 0, err + } + } + } } // readBlock reads a bzip2 block. The magic number should already have been consumed. func (bz2 *reader) readBlock() (err error) { br := &bz2.br - br.ReadBits64(32) // skip checksum. TODO: check it if we can figure out what it is. + bz2.wantBlockCRC = uint32(br.ReadBits64(32)) // skip checksum. TODO: check it if we can figure out what it is. + bz2.blockCRC = 0 + bz2.fileCRC = (bz2.fileCRC<<1 | bz2.fileCRC>>31) ^ bz2.wantBlockCRC randomized := br.ReadBits(1) if randomized != 0 { return StructuralError("deprecated randomized files") @@ -316,6 +377,9 @@ func (bz2 *reader) readBlock() (err error) { if repeat > 0 { // We have decoded a complete run-length so we need to // replicate the last output symbol. + if repeat > bz2.blockSize-bufIndex { + return StructuralError("repeats past end of block") + } for i := 0; i < repeat; i++ { b := byte(mtf.First()) bz2.tt[bufIndex] = uint32(b) @@ -339,6 +403,9 @@ func (bz2 *reader) readBlock() (err error) { // doesn't need to be encoded and we have |v-1| in the next // line. b := byte(mtf.Decode(int(v - 1))) + if bufIndex >= bz2.blockSize { + return StructuralError("data exceeds block size") + } bz2.tt[bufIndex] = uint32(b) bz2.c[b]++ bufIndex++ @@ -385,3 +452,33 @@ func inverseBWT(tt []uint32, origPtr uint, c []uint) uint32 { return tt[origPtr] >> 8 } + +// This is a standard CRC32 like in hash/crc32 except that all the shifts are reversed, +// causing the bits in the input to be processed in the reverse of the usual order. + +var crctab [256]uint32 + +func init() { + const poly = 0x04C11DB7 + for i := range crctab { + crc := uint32(i) << 24 + for j := 0; j < 8; j++ { + if crc&0x80000000 != 0 { + crc = (crc << 1) ^ poly + } else { + crc <<= 1 + } + } + crctab[i] = crc + } +} + +// updateCRC updates the crc value to incorporate the data in b. +// The initial value is 0. +func updateCRC(val uint32, b []byte) uint32 { + crc := ^val + for _, v := range b { + crc = crctab[byte(crc>>24)^v] ^ (crc << 8) + } + return ^crc +} diff --git a/libgo/go/compress/bzip2/bzip2_test.go b/libgo/go/compress/bzip2/bzip2_test.go index 7b227ac9f36..ada1f9a0016 100644 --- a/libgo/go/compress/bzip2/bzip2_test.go +++ b/libgo/go/compress/bzip2/bzip2_test.go @@ -6,6 +6,7 @@ package bzip2 import ( "bytes" + "encoding/base64" "encoding/hex" "io" "io/ioutil" @@ -62,6 +63,19 @@ func TestHelloWorldBZ2(t *testing.T) { } } +func TestConcat(t *testing.T) { + out, err := decompressHex(helloWorldBZ2Hex + helloWorldBZ2Hex) + if err != nil { + t.Errorf("error from Read: %s", err) + return + } + + hello2 := bytes.Repeat(helloWorld, 2) + if !bytes.Equal(hello2, out) { + t.Errorf("got %x, want %x", out, hello2) + } +} + func testZeros(t *testing.T, inHex string, n int) { out, err := decompressHex(inHex) if err != nil { @@ -155,3 +169,195 @@ const rand2Hex = "92d5652616ac444a4a04af1a8a3964aca0450d43d6cf233bd03233f4ba92f8 const rand3BZ2Hex = "425a68393141592653593be669d00000327ffffffffffffffffffffffffffffffffffff7ffffffffffffffffffffffffffffffc002b3b2b1b6e2bae400004c00132300004c0d268c004c08c0130026001a008683234c0684c34008c230261a04c0260064d07a8d00034000d27a1268c9931a8d327a3427a41faa69ea0da264c1a34219326869b51b49a6469a3268c689fa53269a62794687a9a68f5189994c9e487a8f534fd49a3d34043629e8c93d04da4f4648d30d4f44d3234c4d3023d0840680984d309934c234d3131a000640984f536a6132601300130130c8d00d04d1841ea7a8d31a02609b40023460010c01a34d4c1a0d04d3069306810034d0d0d4c0046130d034d0131a9a64d321804c68003400098344c13000991808c0001a00000000098004d3d4da4604c47a13012140aadf8d673c922c607ef6212a8c0403adea4b28aee578900e653b9cdeb8d11e6b838815f3ebaad5a01c5408d84a332170aff8734d4e06612d3c2889f31925fb89e33561f5100ae89b1f7047102e729373d3667e58d73aaa80fa7be368a1cc2dadd81d81ec8e1b504bd772ca31d03649269b01ceddaca07bf3d4eba24de141be3f86f93601e03714c0f64654671684f9f9528626fd4e1b76753dc0c54b842486b8d59d8ab314e86ca818e7a1f079463cbbd70d9b79b283c7edc419406311022e4be98c2c1374df9cdde2d008ce1d00e5f06ad1024baf555631f70831fc1023034e62be7c4bcb648caf276963ffa20e96bb50377fe1c113da0db4625b50741c35a058edb009c6ee5dbf93b8a6b060eec568180e8db791b82aab96cbf4326ca98361461379425ba8dcc347be670bdba7641883e5526ae3d833f6e9cb9bac9557747c79e206151072f7f0071dff3880411846f66bf4075c7462f302b53cb3400a74cf35652ad5641ed33572fd54e7ed7f85f58a0acba89327e7c6be5c58cb71528b99df2431f1d0358f8d28d81d95292da631fb06701decabb205fac59ff0fb1df536afc681eece6ea658c4d9eaa45f1342aa1ff70bdaff2ddaf25ec88c22f12829a0553db1ec2505554cb17d7b282e213a5a2aa30431ded2bce665bb199d023840832fedb2c0c350a27291407ff77440792872137df281592e82076a05c64c345ffb058c64f7f7c207ef78420b7010520610f17e302cc4dfcfaef72a0ed091aab4b541eb0531bbe941ca2f792bf7b31ca6162882b68054a8470115bc2c19f2df2023f7800432b39b04d3a304e8085ba3f1f0ca5b1ba4d38d339e6084de979cdea6d0e244c6c9fa0366bd890621e3d30846f5e8497e21597b8f29bbf52c961a485dfbea647600da0fc1f25ce4d203a8352ece310c39073525044e7ac46acf2ed9120bae1b4f6f02364abfe343f80b290983160c103557af1c68416480d024cc31b6c06cfec011456f1e95c420a12b48b1c3fe220c2879a982fb099948ac440db844b9a112a5188c7783fd3b19593290785f908d95c9db4b280bafe89c1313aeec24772046d9bc089645f0d182a21184e143823c5f52de50e5d7e98d3d7ab56f5413bbccd1415c9bcff707def475b643fb7f29842582104d4cc1dbaaca8f10a2f44273c339e0984f2b1e06ab2f0771db01fafa8142298345f3196f23e5847bda024034b6f59b11c29e981c881456e40d211929fd4f766200258aad8212016322bd5c605790dcfdf1bd2a93d99c9b8f498722d311d7eae7ff420496a31804c55f4759a7b13aaaf5f7ce006c3a8a998897d5e0a504398c2b627852545baf440798bcc5cc049357cf3f17d9771e4528a1af3d77dc794a11346e1bdf5efe37a405b127b4c43b616d61fbc5dc914e14240ef99a7400" const rand3Hex = "1744b384d68c042371244e13500d4bfb98c6244e3d71a5b700224420b59c593553f33bd786e3d0ce31626f511bc985f59d1a88aa38ba8ad6218d306abee60dd9172540232b95be1af146c69e72e5fde667a090dc3f93bdc5c5af0ab80acdbaa7a505f628c59dc0247b31a439cacf5010a94376d71521df08c178b02fb96fdb1809144ea38c68536187c53201fea8631fb0a880b4451ccdca7cc61f6aafca21cc7449d920599db61789ac3b1e164b3390124f95022aeea39ccca3ec1053f4fa10de2978e2861ea58e477085c2220021a0927aa94c5d0006b5055abba340e4f9eba22e969978dfd18e278a8b89d877328ae34268bc0174cfe211954c0036f078025217d1269fac1932a03b05a0b616012271bbe1fb554171c7a59b196d8a4479f45a77931b5d97aaf6c0c673cbe597b79b96e2a0c1eae2e66e46ccc8c85798e23ffe972ebdaa3f6caea243c004e60321eb47cd79137d78fd0613be606feacc5b3637bdc96a89c13746db8cad886f3ccf912b2178c823bcac395f06d28080269bdca2debf3419c66c690fd1adcfbd53e32e79443d7a42511a84cb22ca94fffad9149275a075b2f8ae0b021dcde9bf62b102db920733b897560518b06e1ad7f4b03458493ddaa7f4fa2c1609f7a1735aeeb1b3e2cea3ab45fc376323cc91873b7e9c90d07c192e38d3f5dfc9bfab1fd821c854da9e607ea596c391c7ec4161c6c4493929a8176badaa5a5af7211c623f29643a937677d3df0da9266181b7c4da5dd40376db677fe8f4a1dc456adf6f33c1e37cec471dd318c2647644fe52f93707a77da7d1702380a80e14cc0fdce7bf2eed48a529090bae0388ee277ce6c7018c5fb00b88362554362205c641f0d0fab94fd5b8357b5ff08b207fee023709bc126ec90cfb17c006754638f8186aaeb1265e80be0c1189ec07d01d5f6f96cb9ce82744147d18490de7dc72862f42f024a16968891a356f5e7e0e695d8c933ba5b5e43ad4c4ade5399bc2cae9bb6189b7870d7f22956194d277f28b10e01c10c6ffe3e065f7e2d6d056aa790db5649ca84dc64c35566c0af1b68c32b5b7874aaa66467afa44f40e9a0846a07ae75360a641dd2acc69d93219b2891f190621511e62a27f5e4fbe641ece1fa234fc7e9a74f48d2a760d82160d9540f649256b169d1fed6fbefdc491126530f3cbad7913e19fbd7aa53b1e243fbf28d5f38c10ebd77c8b986775975cc1d619efb27cdcd733fa1ca36cffe9c0a33cc9f02463c91a886601fd349efee85ef1462065ef9bd2c8f533220ad93138b8382d5938103ab25b2d9af8ae106e1211eb9b18793fba033900c809c02cd6d17e2f3e6fc84dae873411f8e87c3f0a8f1765b7825d185ce3730f299c3028d4a62da9ee95c2b870fb70c79370d485f9d5d9acb78926d20444033d960524d2776dc31988ec7c0dbf23b9905d" + +const ( + digits = iota + twain +) + +var testfiles = []string{ + // Digits is the digits of the irrational number e. Its decimal representation + // does not repeat, but there are only 10 posible digits, so it should be + // reasonably compressible. + digits: "testdata/e.txt.bz2", + // Twain is Project Gutenberg's edition of Mark Twain's classic English novel. + twain: "testdata/Mark.Twain-Tom.Sawyer.txt.bz2", +} + +func benchmarkDecode(b *testing.B, testfile int) { + compressed, err := ioutil.ReadFile(testfiles[testfile]) + if err != nil { + b.Fatal(err) + } + b.SetBytes(int64(len(compressed))) + for i := 0; i < b.N; i++ { + r := bytes.NewBuffer(compressed) + io.Copy(ioutil.Discard, NewReader(r)) + } +} + +func BenchmarkDecodeDigits(b *testing.B) { benchmarkDecode(b, digits) } +func BenchmarkDecodeTwain(b *testing.B) { benchmarkDecode(b, twain) } + +func TestBufferOverrun(t *testing.T) { + // Tests https://code.google.com/p/go/issues/detail?id=5747. + buffer := bytes.NewBuffer([]byte(bufferOverrunBase64)) + decoder := base64.NewDecoder(base64.StdEncoding, buffer) + decompressor := NewReader(decoder) + // This shouldn't panic. + ioutil.ReadAll(decompressor) +} + +var bufferOverrunBase64 string = ` +QlpoNTFBWSZTWTzyiGcACMP/////////////////////////////////3/7f3/// +////4N/fCZODak2Xo44GIHZgkGzDRbFAuwAAKoFV7T6AO6qwA6APb6s2rOoAkAAD +oACUoDtndh0iQAPkAAAAaPWihQoCgr5t97Obju21ChQB0NBm3RbA7apXrRoBooAA +AhA+IAHWl2Us3O7t9yieb3udvd76+4+fd33nd3HO1bVvfcGRne6+3vfPvfc++995 +w7k973eJhasLVec970tzDNXdX28LoPXZ3H3K9z0s5ufWAfes49d5594c3dUYtI+2 ++h1dvtpRa+uvrVEAG9bl893RVEN7cWvroSqWjPMGgAQi7Gq8TJSgKKdjKFBIB9Ae +LqWxleu715eXe7ml9e5098Z6G1vr7t1QZ6ot76YzPd3j7333t2ql2Chm7XrA9ICQ +VF77z3rVBWqkSXtlfb099hyezAr6USbGpICTSCFAaqHrKo+tUnm32rpE4Ue+t2mj +bKUeipEqwc93EdhhTwmQpOhhesC9iqDSPNTWYNSnUtBdm1nsA0nqqNd7OWwDXtFL +ONmmA6Ubke26I9UblvWIPR5VOWOnctai443URunnDy77uVC59OfRvezlDu33Z7Ly +3NNuuHW63088xu3t3NHZhkZbG7tXRlj00qOtbaXTJUUdspTbABR9R6EUwQAEAAAA +EMEwRpoAAAABMmhoAAjBNNAaCMhponpoGpgJpk9TEyp6niGKZkAaAEfqMQ09U80p ++pMGSCKngIAAAAgAAg0AAJhGgABGCEaaTyTKeNI1PE0wkj01GajMSNPSZGnqbU9T +anlPUNAHqGQ0DQAMg9TamgAAYRU/IAAICAmjQJgjQBMEwp5DTSaaYmhTeqfplPID +U1T9TynoU82pT1NPU/VP0j1NHqRpk9TTR7SnqaNNGmmQAaAD1Aeo0PSAAAAaaBiK +eBAQBGgIABGQA0AmBNNBoaAgaJmpglPEyYap6npiTT0agGjJjUaaDTQAAAAAAM1A +9QAaAAAADU8iEAQAEyAJk0NNNJgIZTJ5E00YSemiaZNGm1MpGNJ+lPU9qm9U2RDM +oY0EzJB6h6nqDID1NMBDDRpo1AGNAjCMmhkMgaYSJIgAAAQyAAEyBoATECCNhTT0 +U/IZAmCM1DSTxkzUE8p6NDaGiZGJqntTFHvUyU9qPQp7Kn5GgKNPU9QAGg9QAAA3 +wz0Pk/g/m/m9P9H4vxv2+dH3gCS8nhbbbbbYxtgNsBsG0m2MbG0NNtsbYNsaY0wb +bBibGmm22mxptNpsaGNDTY02JsG0MY0xg2MaYNNDbGwG0L5vsK/F9DO+EAA447Kq +p7Wdf6Y+5c20T7DfHyMXIzRKrZexw72uiQI+y55vOe52xpqbCLC2uR20JdER7Zvr +7ufuKb6zhiBxLuj0eA27v8RpMLucw9Ohwcizi2wrpt+yU1FdpM7ZYPcwS3XTef+A +Wzjxwhdrgw3aH1LeC1eZW900x8V9Nv4hTPXp4l067P/4ANVZFF/imOe/d5bdueam +/DFFokQWnFaU+ZqLBCM+d0PialJQWnLqRQZk/KhfbbYc2pCUTgffcSYbrCM1N+8l +HU6gSz+h2GJXs+tbrNviL83M97X0vcTn/F82P8wen8/3/h3sHY+sf9CSej9ThYTV +3lQ+FUHpfpGD4kv7dYMV995dpDX/y3xR8FoXx1bjUxBTNxuutwQ/h/Eedn9wpn6w +E3+ND8YhN1HSriIxRE/6uFyMv6/oC6Elarw3aHMMqHJkGiiz6tejmvnYLQa+Qm6G +deZ7jXTZV6NlpocgDnRdimS06bTYSkvPAL/xoWNLkX6N6VljU0dfKSBmm2uZE/xu +sutQ1EdP7GdjhglIq4xlOFUFEQpmX+xx7R8y6c0GSAaqusOjNZwxZRudOvmXm1tZ +T+YnbeB2ir9eiHNrtJNSLD/J/WDyuQpwBUtLKo0krccY/wIILP7f86teb9Z/9oyz +OX05qEWbObfhpRw+9+rCvp/35ML8KX3aHaI0n+tudbFRsV5FLW+Oa8ruLN4peyVL +DWjTHrXNthq/s7zAJYMeFJZkZt5mT9rfpH+5g3nc+piOSZ+J5nHtOnKI7Ff8Xl+j +0t76XTNucCHQ6whav1OHdF53TY5wuv5OzvrdnxoId8fTyUvERr0ERINu/8XxZZ5f +B5/kTZ8bBO0wv54Jp+ED/GQI8lZHzIQCP3vfQhwnCTj9TvITic7P4mYLDbH3fyzR +i+6EajCcpXLWSGf+ZXkOrWspDWDhXtEKas0v3UqWksqgY1rTj45krX4KihN+daXs +pZl5WPlta5p06CX6Xm2SfzqkMw12/3ix1bpnnZ+kFeBNX7A+E9zzG6OZaN78GOpl +9Ht/eZn9PqWdav852zr0zqkDK2H5IjdvNah+b1YVGdQGzwR4Nw+f13yEKnV+y66W +djfq7zWp7m5w+hzfv+Ly8O7oet5Vvd8/wQvO7qzOZ2vjf9X8Tj8PnMb/nc/nKqRR ++ml4UEhOOwfCeJEEI109CMYSh91iAJqPjMyH6KjrPD7W25llZVcREYNCTg6htbQt +M38wYoquCWP6tdKYlVIv14xTNUeUf4El/FunCf6csZkmv+9tfWx7t59wuKIa3saU +tZs9M+3HFOZtz3OLg/Unoaj9BYazYqA78xBU9tZzrtmF/rQL9CGJt90o/oYnSfcS +SL3haaw351LXWQ1XOsv1SmH3v6ymuxEpPPnEDmBELaTYsvvMIWJsmPZFFww++Kd7 +s/Jo0JFeUU7uNtI+gVosAIpVVuWfI/9tOIycz7I5Z7zjV+NR2OuZbYtW5F08KX4o +2k/xuJIchcNFPtxPfw9dkDgscRbMckyFMrzuZ3IvrcGzk0J6iI5ytrv37bGpAXMz +WK9mMMPebepNevmLjjo/QWoM968Sjv7ldlPS5AinHcXwsFv6dmmh8lJt7UOJWoKu +lMD1cB2ksIGpMdv8iuqR42Rn/kn+17BhhUZcwDBaUXVdX6bKW7fxlUYbq+mlqIcf +a9v8HF87M9ANbi9bq9onf9TD7nQ6Xf6vZci8TBPX+/GI0He6j31fTVQYW+NsQxvO +J8xrx+e58CCLQNjxeIyPt+F+qk/QMiXw+LyxGVkV/XcGQT9X03jSDP6beJ5QG1JW +9Q3qLv/YixWI7gPV9Mrhf2oRYTc/9KLFRhkE3SjKOTKuSSBKQ24fI+hEznamH71D +66Hwez8/0et7AtTv9zvamv2OD5He6fMV4k+ePl6+qPfO5CdHtK+eCDZL5+4f5yrl +gTcRFiq8fXbc5IaI5fbbc1KMM/2T0Mr7+Hwaco6FtXm0fmhCgTZRqY4pKiEIfmaz +QwHNOOCrtMJ2VwsyMumt7xsOolGnizRev6lILH43qPcczQM7Gc5zRin80YvFt1Qm +h/57Z0auR2h0fuX50MBO4XQ+26y5l6v4j902R66c0j3z2KHstKQ04J/h6LbuNQE4 +D6cu/lyfK69DxxX8wb8XaQkMUcJdo1LzqUGDAb3Kfn/A3P/JYc99MO9qv67+SxWb +wYTyqKdWTd+1KbR/Rcn0Io5zI/QquX7FA1bxfMytjQ/X+l0fh0Pf+Hx97meH4fQL +7/T8/sdTm9Tn8nELvedyhydLlPPTScINdXyLIq9wgIJr4fWPbp9ZhFh/56fdSgOG +HDXg+gkXsN2Rddr4HQ5P3u+RhLzmSjhzoqY5EsPC4QvRlX9JXjB84rPV5USR66qa +/kjw4156GJnzoXtydKJE53t6PHfZWO+3ujsfI6iAdshc7OFzGXiZB9PtItKodhYq +nABkTKdcpu4+TOpf9h5piX5slsaBjkeTnj/Ba02ilboQfcDVigxrYn/iTH5ySWUW +/lHtg78s5UZM8sErwhNe3N3w+6ZOMnU+5i86/xFNtqZfDdXTGy1H3PzGbdtZXYT+ +Ixx2vpwBYzbPVYHxKosM5rPiVmcTllI9nuoSfeh9ib4foFWauOpvdmhBDqpTpKTX +u8EO2l2Z195G2RIV7TlKSxGWjR5sl/nALu1uzBeLd9zpSujzMTd1uTX9Qk/Q1S+r +vaW6bm8qqPO4jb6Wx6XIkm321nrIF6Ae25d1+Dpv/P5G4NoLd2j6/EtENC3FeR5z +oo7bA+tI8yEQRhiF0z1FlJXLD5ZbhNNWQm/j/IbzRfh8JtOFZU7ruShLvHXysW9S +9V909tr9jn8/E/Hb5N/1NVNHnZu2HIUvJvHJiHd2ucmeI9PWUMnppmE65GQ5E9xV +ZRlGEH0X85EvmHyEupkMrCC0oMv9RCq+/H8gcfpe00Hs/S+regT5p58cyYomh93v +qvuw/A06BE/wzJESuYbN9pqYpoXqXFemW1NksHEJ2w+PYMJ27WJyD5FpaXB85VaW +qMOhDfO8E3QdH8ybyKt/UgI8/tDGpFbyOlaVdIv1FXJhoLp8soAA4Djg6/KZ066N +ZFYuS8WdjpSZGP4/Lw+1yaXlzNznc/k2uHe2uXP3uFuPcHx+Dm44utxldoO1uBPy ++jzOs14+MIgOjOHMVNqAbMd8fUedLlhJMCfMtm4uz01enLNKcMrtLlPIR37Yukh1 +YEMXYpm7eU4XU+j+Jj3pDyaXtXs+p1fWfTN/cy9/Oxs4umUXQ4uHh1kObtayDJ56 +/QMxiHobjHNKuKfMxsrYEwN+QVIyVjAwMDYuMjQ1AAA9IwJniiBLRkZDAAAXt0Ja +aDQxQVkmU1lZtwytAACLf///////////////////+//////v//////////bv78// +/+AXO133uwO2xB2UxIvbKXrCqCoURUBL2ytFI82AFdcOwMhVTHtk5rD3szEVNYD4 +aIQINCaMRoTaSn7SbSMJiYmEwieTEp+psqbMCp+VNPaFNpqbBNR7UmanlPUeKfqm +j1PU0/VPU08o9Q9EeKHlPJtKbYqeTCYhN6U9T1NH6mp+lPyoGNTI/Knkyg1MggAg +CaMEyQnqZoaaRtRtJpppppoDaTR6hpphGh6mmgHpMQBpkGTTEAAaAAAA00AZDag0 +ADIBkGgABqemiRNTI0k8aU0PRGRoAZlP0UAAAGgAAAyAADQaAAAaAAAAAAAAAAAA +AaAAAAM0kgRBJ5MlPFP1Gj0jTTTUaekxNAbUGjTQMgaZANNAAAAaAADTQAAAAAAA +ANAA0AAANADQ0QAAAAAAAAAaGgAAAAAAABoA0AAA0AAAAAAAAAAAAANAAAAAkSEI +aTRpomp5DUxNNDTJPTKaep6T09Kemmo2JG0aTQ9ENogaaGhkABo0NHqaBoDTI0DC +Gj0gNAMhoDQ9QMQNAGQAaDDwyMPIMlbG1vhRBTFo6JksSupgpAjPbY0ec02IGXjb +eS+FBsh01+O4ZOaD+srUZCFaT4DRjVDLx7uKIsFtESIDUg1ZkhyCSYov05C00MtR +BdNNa/AYPGOQZWcs+VegXOPrkushFbZ3mBoRD6WamClkpBaHZrUhUl02bIfRXX4w +b3/9cW9nHDVxh2qFBxqgRKfmq7/Jc/tdJk05nVrGbckGVy2PnIy30CDhpWmqrSot +K2bOnX0NbP1iy2cd0Na0ZmbRstm4MzMzbbMySTd35F7f+zPP8DC+NJLYcakkkkRd +NZlupJt3OMFoDAD2g+N3FAMCydhIpoRHRQAdFI5nNg4ugEXHCYxkMyGCwtaJmial +y0IMlpSYYM/weXNJAhFqS0GNmvaPEtYGjbvaucMdklOTmBX1vfVAkTYB1uXCSK64 +UNIixOqRKLuRCFtqIQtgwqaFrCkIYbbewErWABa+VGADWsJXJjfx5SJViLuwiGXq +Ru6vCuwmU5CJiJz3UiBpmLv0r2wskxUhY4tzPVGQ9RMXJl65eLSNwZVwaSyGZ9Cm +A3jztQUUpFeUryBTskW95iVwRMFrhBCwZBAFJBZvhMEMNoDJJlUoIhQkAkjbExp2 +YZio+ZYeAZUwmH1qUbdQixmxf0+61+aVgJ1hwxsO1yG3hFx4pfjc09ITVht0pG8u +FtVFhPa1KE0gTRUSVXywkITucqk0Waz5Fs6qJpVHYdNrbYRFxnFsQGY1qmsTLjK6 +4QX5Rddo6krM/Bx9CqIAKq4CzVQYHrmIAd2EBhYmwVYwLvhzKIUrc2EirnGIvyuD +O4YZDSwsVTA0BpVvUOjDErkCraBoSutcKwUSSLGhVvNYHLz3klgZD++wWsa/swLw +gvNDY2De+sncOv8X2lq4HD95ZdwPuTIMXCwSbg4RrIqv+L0y6F17pqDecyQYPEj3 +iN/0BBeWZlJAyBMi5U3Q1zAlsK8IlDhaXGmvZrgISq5CfNjmUgxDeMggOKqxu4sI +OrilS49Lkl1J3u3GjXTuH+rX+4ccyFAQnizCpPClcY77F59j63S6fr5vr+y99tuO +7Ox7Wg/ljwhdyaK4xMmXczeJbx7x07htJNtC4xcQfAtvzeznLrN6MN/ILIBOI65I +qIA2D5fHHj1XN4aN6TvOjWDaSbSWqxCSCvXUpzkNJAkWXAuTwF8k5uSJvQj/rVo0 +hAhEMEIYkCRGx9AX+byIuXWlLMbbVeliHNUL5AQYmNwLFu4SkmGD+UWtBMyVHQOQ +ss0ggoVKSKOBUgnVS6ljt7WE1qXqJJ4QA1pEwYNLEaguEE1LtPNoVr5WzjbSbWPk +V9OW3y9IneUDLoIV5pAkEFTEFGFVjeTFxtpzBBfGgycBxVCdz8eESBIzsamRchAa +TQunQH8DHnpfod9QuAuRvc7JBlKUCYmCjMvynLcxIFohxCaYrDvGw4QbXZB7oWQ7 +hpoGlz23ayDfB8NrRRzdilsEQyQniu9ASLQg7RrGZnoTr1ai12IbCEUCGdFq03P5 +nBnRFAGmisQGcyykV9gKtcVMWLhCuVmXg86dndn7slUpRNSSEAU20oaWIm1maFTu +E0DT4gTbg0nuhjtz3kNOz+i7sBm0bkXjxQWuLqlZEmp60ZTyRZJDUqKSEKg6hqcy +ERxdU22CSNOO10RYUUiDVpKhPNdKTOIE1thp02sBNoNTFSht8WJtaBQ09qN3jd5r +dOLX4IA5fevRyCCzDgRXfV4wzik4KROjmxmTMglBySlIMEzcXehnDXCRiZSlvwA2 +0YsIOROcm4UrIRFxJHctJH7OdN5u1aHVHb5UaLHpv48NgmFRE56KTSoaWunqm2st +S0mrAdOiqcR12PWVbdVRJKcQ0DQuhwlAPcRtpxN3D4kbXJjToSYJIFw406G2CSaK +jQMIJPZGlQmgyFhoCSzeGS1VSq5SKKQQxs5RqKUcVUNY57YUETb4mXzV84SPngKi +nsce0mXByZq5BKUA9puHZWLNwQIYuDaJUNgG+E01E3pDYVNLKYQ0hsVesgV5gZY0 +htDsRdGtm0+iGnkN6+Ea9YJtUZNAkx2GgSoix12nTW0avTUfxR3oYcpvZ7IdtABE +UhBcjG4qZtDZsS1JQHys243vhLaDTSvvTeBiJA2tmokqECTBcSOCAGkAxMKlVAva +4IsLRaBBqhxDbcGtgdw03mFcLUaFuhtKuuEIEkUleJQwby/zwu9uvvZK4xTV+ECM +a8lmzxKmqkBggYK1+xPdbmJclm6tSZhE/OSJtCEjs+unJIQkT9hCWgBJqGMS07Eh +AJNmBiuVEVdTyjkIJkavuZmx2sJF13htgEZUCC23lZFOE6gWbM9WyYNJTM8yCQrb +0Sx3OQvBML5cRATAQkSQkAJOAhoxpQkNi4ZiEVDbdtJAME0RXNDXGHA3M3Q0mm1o +IEwbWpaM1DQCSMbGRCAu3iRIQiT6RlBpT1n3tfwvUXz3gIVlx3mEximY/kZW1kNG +sgEJIrBisaEoGYPJ+1CQUYFBw+eGEHJQBpNHjErXUJY2iWHQ30hXwFBuMSxQ2lB5 +bg+/LX3euG6HsHUB1lFvBvaiaBrITVwkCTa1d0s9CHZCiDZjbWReKyrpPE2oSa7o +LPrR4BJvys9ttjUpzETSSMxh8vsr9dXTwKBtK+1xCTGDQmNIaE29HmHdS5GSxpya +MismcAUSEgSxHBrKtgsZzduG7vHZn16l3kFkVITtENIzS2JsiBwFTDlhgexsjBHv +5HXOYxHBzoSDCcPZ0ctvkY9aS5XpoQuFYkGJgCsqjJZeUMNUEpDSbKcnUc1PifIA +CbR2UoXawBlspkEBr9HBfvUi/MUakZVOf1WKYrqSaIXce62JOyhJLq3qJBloTA0F +VbILEtM+heFmNRCFt70GJrExVJri0ArYbCRbADSGDBpBXxxb/6fo+s3C7uaL7RjM +LV2IQBNrAJrKFeJwTsPnxbAsemirUx2lk1kaxschzdK4TQNJN5wQnolIFg401OZ4 +2na11LnT3lR+1k1TMJhiAjXMk0F1ooHnYlt9LKfJ3ZIOmeY+2l9bUQHWFNGyEyfj +EAcu3kpGLq0Ez7XOS+EpAASRQTAYMATfVQibHLTT30zG732+pNe9za1JNt8sNJYn +RjWuJ6jL5ILV0rcd9vT7X9fObvcXitpvJ2XBJE+PhX2HaTkyWeF9pwnlQNrTe9hV +tzhA+ihZrDrHNmLcQjZbnv/IMubqq8egxY80t5n6vZ6U5TR6U9uZJvai1xtqAyCR +NWkW52m00rDTEuO6BA4q2RHDWwbETF55rRsWLIgNW9qJCyMHPbTM/dMBmWMQSMxz +4M2pRzt47SICxA327UqSCEERqMFybmYi3nUxePtLgHYplqRiw4ynMbXd/kiQ0LE0 +PKJSSCXA42ymziCpAxNWflzpzQdJZusahRFr6t6m+4p273/Taj7k+hZyNgBAgXAY +8F7pTts6orLb8IA6o4TOwkwQYmKvKu9VwMrE7+GUhVIAgY9a8DyQMiDBkEAwh7S1 +KgCBfao8DK1CwSS8Z3WjL5MEgt93z2koUQCD/YxMBppiCMp7SDVSmkkIHptfGpeh +t+M13Ccv1tavIASFiaQl6rBz3K4N3DSGwNkCibrvEAC0fQirOWnc4NVbcLKpFG1l +NQXF/eqdT79wq1Mvlap3QSCLhcD2D3fCkKVWid4aSjtp9FOX1Uaf7P9eT93zd9Sv +mj2yNLRUGzyI/0oONNSzmmkvJ5Cq2X2CdldIWMGZO57RJ8oyATAWTQmRmNkfh0Sx +uuR/J9oUsomVy1AEntc0dlPivkqBkBqrxU3j5PnWkaI3ZRGc0gg9spCQEISh4xEU +pMhVrnmDQLfLP8Ouqpx917MAw7hkjQk6BJFTAbXDsz3LSHIxo/gB8qrA1vbvdZZh +LtR0frJdfdppX8nAQX/TAxOQ8+H6yw8a9i7/zJEfSYIhop59N/fhcWW2F14cj2Xc +fyHaZ04lTO4uPnly91jwuFPaREuZVp8AxImIhlkxkAN61tWdWG7tEbaCgszh6VIz +ThFnHo2Vi8SQXPrXCN7J9Tc9ZYiAYqoThV/u6SYsea5aZL8deOvKBQCgZZuIxX1z +4EnfcqG176vY4VqMBIC4pMJz0WcHJYqN+j7BiwGoMBwExrIdTB7q4XIFLotcIpS0 +1MqyVsesvoQq7WObmGQXdMliMirSLcDuSx8Qy+4pIBgGDIyMp1qbonnGdcHYvU8S +O0A8s/iua5oFdNZTWvbVI4FUH9sKcLiB3/fIAF+sB4n8q6L+UCfmbPcAo/crQ6b3 +HqhDBMY9J0q/jdz9GNYZ/1fbXdkUqAQKFePhtzJDRBZba27+LPQNMCcrHMq06F1T +4QmLmkHt7LxB2pAczUO+T2O9bHEw/HWw+dYf2MoRDUw= +` diff --git a/libgo/go/compress/bzip2/huffman.go b/libgo/go/compress/bzip2/huffman.go index f755019bb50..8f6b0c9cad3 100644 --- a/libgo/go/compress/bzip2/huffman.go +++ b/libgo/go/compress/bzip2/huffman.go @@ -33,14 +33,17 @@ const invalidNodeValue = 0xffff // Decode reads bits from the given bitReader and navigates the tree until a // symbol is found. -func (t huffmanTree) Decode(br *bitReader) (v uint16) { +func (t *huffmanTree) Decode(br *bitReader) (v uint16) { nodeIndex := uint16(0) // node 0 is the root of the tree. for { node := &t.nodes[nodeIndex] - bit := br.ReadBit() + bit, ok := br.TryReadBit() + if !ok && br.ReadBit() { + bit = 1 + } // bzip2 encodes left as a true bit. - if bit { + if bit != 0 { // left if node.left == invalidNodeValue { return node.leftValue diff --git a/libgo/go/compress/bzip2/move_to_front.go b/libgo/go/compress/bzip2/move_to_front.go index 0ed19dec39c..b7e75a700a1 100644 --- a/libgo/go/compress/bzip2/move_to_front.go +++ b/libgo/go/compress/bzip2/move_to_front.go @@ -15,10 +15,11 @@ type moveToFrontDecoder struct { // Rather than actually keep the list in memory, the symbols are stored // as a circular, double linked list with the symbol indexed by head // at the front of the list. - symbols []byte - next []uint8 - prev []uint8 + symbols [256]byte + next [256]uint8 + prev [256]uint8 head uint8 + len int } // newMTFDecoder creates a move-to-front decoder with an explicit initial list @@ -28,12 +29,9 @@ func newMTFDecoder(symbols []byte) *moveToFrontDecoder { panic("too many symbols") } - m := &moveToFrontDecoder{ - symbols: symbols, - next: make([]uint8, len(symbols)), - prev: make([]uint8, len(symbols)), - } - + m := new(moveToFrontDecoder) + copy(m.symbols[:], symbols) + m.len = len(symbols) m.threadLinkedList() return m } @@ -45,34 +43,29 @@ func newMTFDecoderWithRange(n int) *moveToFrontDecoder { panic("newMTFDecoderWithRange: cannot have > 256 symbols") } - m := &moveToFrontDecoder{ - symbols: make([]uint8, n), - next: make([]uint8, n), - prev: make([]uint8, n), - } - + m := new(moveToFrontDecoder) for i := 0; i < n; i++ { - m.symbols[i] = byte(i) + m.symbols[byte(i)] = byte(i) } - + m.len = n m.threadLinkedList() return m } // threadLinkedList creates the initial linked-list pointers. func (m *moveToFrontDecoder) threadLinkedList() { - if len(m.symbols) == 0 { + if m.len == 0 { return } - m.prev[0] = uint8(len(m.symbols) - 1) + m.prev[0] = uint8(m.len - 1) - for i := 0; i < len(m.symbols)-1; i++ { + for i := byte(0); int(i) < m.len-1; i++ { m.next[i] = uint8(i + 1) m.prev[i+1] = uint8(i) } - m.next[len(m.symbols)-1] = 0 + m.next[m.len-1] = 0 } func (m *moveToFrontDecoder) Decode(n int) (b byte) { diff --git a/libgo/go/compress/flate/copy.go b/libgo/go/compress/flate/copy.go index 06e5d2e66d9..a3200a8f49e 100644 --- a/libgo/go/compress/flate/copy.go +++ b/libgo/go/compress/flate/copy.go @@ -6,12 +6,27 @@ package flate // forwardCopy is like the built-in copy function except that it always goes // forward from the start, even if the dst and src overlap. -func forwardCopy(dst, src []byte) int { - if len(src) > len(dst) { - src = src[:len(dst)] +// It is equivalent to: +// for i := 0; i < n; i++ { +// mem[dst+i] = mem[src+i] +// } +func forwardCopy(mem []byte, dst, src, n int) { + if dst <= src { + copy(mem[dst:dst+n], mem[src:src+n]) + return } - for i, x := range src { - dst[i] = x + for { + if dst >= src+n { + copy(mem[dst:dst+n], mem[src:src+n]) + return + } + // There is some forward overlap. The destination + // will be filled with a repeated pattern of mem[src:src+k]. + // We copy one instance of the pattern here, then repeat. + // Each time around this loop k will double. + k := dst - src + copy(mem[dst:dst+k], mem[src:src+k]) + n -= k + dst += k } - return len(src) } diff --git a/libgo/go/compress/flate/copy_test.go b/libgo/go/compress/flate/copy_test.go index a9281d446e9..2011b1547c9 100644 --- a/libgo/go/compress/flate/copy_test.go +++ b/libgo/go/compress/flate/copy_test.go @@ -30,10 +30,12 @@ func TestForwardCopy(t *testing.T) { } for _, tc := range testCases { b := []byte("0123456789") - dst := b[tc.dst0:tc.dst1] - src := b[tc.src0:tc.src1] - n := forwardCopy(dst, src) - got := string(dst[:n]) + n := tc.dst1 - tc.dst0 + if tc.src1-tc.src0 < n { + n = tc.src1 - tc.src0 + } + forwardCopy(b, tc.dst0, tc.src0, n) + got := string(b[tc.dst0 : tc.dst0+n]) if got != tc.want { t.Errorf("dst=b[%d:%d], src=b[%d:%d]: got %q, want %q", tc.dst0, tc.dst1, tc.src0, tc.src1, got, tc.want) diff --git a/libgo/go/compress/flate/deflate.go b/libgo/go/compress/flate/deflate.go index d357fe361a5..8c79df0c607 100644 --- a/libgo/go/compress/flate/deflate.go +++ b/libgo/go/compress/flate/deflate.go @@ -416,6 +416,50 @@ func (d *compressor) init(w io.Writer, level int) (err error) { return nil } +var zeroes [32]int +var bzeroes [256]byte + +func (d *compressor) reset(w io.Writer) { + d.w.reset(w) + d.sync = false + d.err = nil + switch d.compressionLevel.chain { + case 0: + // level was NoCompression. + for i := range d.window { + d.window[i] = 0 + } + d.windowEnd = 0 + default: + d.chainHead = -1 + for s := d.hashHead; len(s) > 0; { + n := copy(s, zeroes[:]) + s = s[n:] + } + for s := d.hashPrev; len(s) > 0; s = s[len(zeroes):] { + copy(s, zeroes[:]) + } + d.hashOffset = 1 + + d.index, d.windowEnd = 0, 0 + for s := d.window; len(s) > 0; { + n := copy(s, bzeroes[:]) + s = s[n:] + } + d.blockStart, d.byteAvailable = 0, false + + d.tokens = d.tokens[:maxFlateBlockTokens+1] + for i := 0; i <= maxFlateBlockTokens; i++ { + d.tokens[i] = 0 + } + d.tokens = d.tokens[:0] + d.length = minMatchLength - 1 + d.offset = 0 + d.hash = 0 + d.maxInsertIndex = 0 + } +} + func (d *compressor) close() error { d.sync = true d.step(d) @@ -439,7 +483,6 @@ func (d *compressor) close() error { // If level is in the range [-1, 9] then the error returned will be nil. // Otherwise the error returned will be non-nil. func NewWriter(w io.Writer, level int) (*Writer, error) { - const logWindowSize = logMaxOffsetSize var dw Writer if err := dw.d.init(w, level); err != nil { return nil, err @@ -462,6 +505,7 @@ func NewWriterDict(w io.Writer, level int, dict []byte) (*Writer, error) { zw.Write(dict) zw.Flush() dw.enabled = true + zw.dict = append(zw.dict, dict...) // duplicate dictionary for Reset method. return zw, err } @@ -480,7 +524,8 @@ func (w *dictWriter) Write(b []byte) (n int, err error) { // A Writer takes data written to it and writes the compressed // form of that data to an underlying writer (see NewWriter). type Writer struct { - d compressor + d compressor + dict []byte } // Write writes data to w, which will eventually write the @@ -506,3 +551,21 @@ func (w *Writer) Flush() error { func (w *Writer) Close() error { return w.d.close() } + +// Reset discards the writer's state and makes it equivalent to +// the result of NewWriter or NewWriterDict called with dst +// and w's level and dictionary. +func (w *Writer) Reset(dst io.Writer) { + if dw, ok := w.d.w.w.(*dictWriter); ok { + // w was created with NewWriterDict + dw.w = dst + w.d.reset(dw) + dw.enabled = false + w.Write(w.dict) + w.Flush() + dw.enabled = true + } else { + // w was created with NewWriter + w.d.reset(dst) + } +} diff --git a/libgo/go/compress/flate/deflate_test.go b/libgo/go/compress/flate/deflate_test.go index 8c4a6d6b36f..730234c3850 100644 --- a/libgo/go/compress/flate/deflate_test.go +++ b/libgo/go/compress/flate/deflate_test.go @@ -9,6 +9,7 @@ import ( "fmt" "io" "io/ioutil" + "reflect" "sync" "testing" ) @@ -424,3 +425,66 @@ func TestRegression2508(t *testing.T) { } w.Close() } + +func TestWriterReset(t *testing.T) { + for level := 0; level <= 9; level++ { + if testing.Short() && level > 1 { + break + } + w, err := NewWriter(ioutil.Discard, level) + if err != nil { + t.Fatalf("NewWriter: %v", err) + } + buf := []byte("hello world") + for i := 0; i < 1024; i++ { + w.Write(buf) + } + w.Reset(ioutil.Discard) + + wref, err := NewWriter(ioutil.Discard, level) + if err != nil { + t.Fatalf("NewWriter: %v", err) + } + + // DeepEqual doesn't compare functions. + w.d.fill, wref.d.fill = nil, nil + w.d.step, wref.d.step = nil, nil + if !reflect.DeepEqual(w, wref) { + t.Errorf("level %d Writer not reset after Reset", level) + } + } + testResetOutput(t, func(w io.Writer) (*Writer, error) { return NewWriter(w, NoCompression) }) + testResetOutput(t, func(w io.Writer) (*Writer, error) { return NewWriter(w, DefaultCompression) }) + testResetOutput(t, func(w io.Writer) (*Writer, error) { return NewWriter(w, BestCompression) }) + dict := []byte("we are the world") + testResetOutput(t, func(w io.Writer) (*Writer, error) { return NewWriterDict(w, NoCompression, dict) }) + testResetOutput(t, func(w io.Writer) (*Writer, error) { return NewWriterDict(w, DefaultCompression, dict) }) + testResetOutput(t, func(w io.Writer) (*Writer, error) { return NewWriterDict(w, BestCompression, dict) }) +} + +func testResetOutput(t *testing.T, newWriter func(w io.Writer) (*Writer, error)) { + buf := new(bytes.Buffer) + w, err := newWriter(buf) + if err != nil { + t.Fatalf("NewWriter: %v", err) + } + b := []byte("hello world") + for i := 0; i < 1024; i++ { + w.Write(b) + } + w.Close() + out1 := buf.String() + + buf2 := new(bytes.Buffer) + w.Reset(buf2) + for i := 0; i < 1024; i++ { + w.Write(b) + } + w.Close() + out2 := buf2.String() + + if out1 != out2 { + t.Errorf("got %q, expected %q", out2, out1) + } + t.Logf("got %d bytes", len(out1)) +} diff --git a/libgo/go/compress/flate/flate_test.go b/libgo/go/compress/flate/flate_test.go index aba820a1f95..57fea5ab4dc 100644 --- a/libgo/go/compress/flate/flate_test.go +++ b/libgo/go/compress/flate/flate_test.go @@ -24,3 +24,39 @@ func TestUncompressedSource(t *testing.T) { t.Errorf("output[0] = %x, want 0x11", output[0]) } } + +// The following test should not panic. +func TestIssue5915(t *testing.T) { + bits := []int{4, 0, 0, 6, 4, 3, 2, 3, 3, 4, 4, 5, 0, 0, 0, 0, 5, 5, 6, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8, 6, 0, 11, 0, 8, 0, 6, 6, 10, 8} + h := new(huffmanDecoder) + ok := h.init(bits) + if ok == true { + t.Fatalf("Given sequence of bits is bad, and should not succeed.") + } +} + +// The following test should not panic. +func TestIssue5962(t *testing.T) { + bits := []int{4, 0, 0, 6, 4, 3, 2, 3, 3, 4, 4, 5, 0, 0, 0, 0, + 5, 5, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11} + h := new(huffmanDecoder) + ok := h.init(bits) + if ok == true { + t.Fatalf("Given sequence of bits is bad, and should not succeed.") + } +} + +// The following test should not panic. +func TestIssue6255(t *testing.T) { + bits1 := []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 11} + bits2 := []int{11, 13} + h := new(huffmanDecoder) + if !h.init(bits1) { + t.Fatalf("Given sequence of bits is good and should succeed.") + } + if h.init(bits2) { + t.Fatalf("Given sequence of bits is bad and should not succeed.") + } +} diff --git a/libgo/go/compress/flate/huffman_bit_writer.go b/libgo/go/compress/flate/huffman_bit_writer.go index 25e1da336aa..b182a710b9a 100644 --- a/libgo/go/compress/flate/huffman_bit_writer.go +++ b/libgo/go/compress/flate/huffman_bit_writer.go @@ -97,6 +97,31 @@ func newHuffmanBitWriter(w io.Writer) *huffmanBitWriter { } } +func (w *huffmanBitWriter) reset(writer io.Writer) { + w.w = writer + w.bits, w.nbits, w.nbytes, w.err = 0, 0, 0, nil + w.bytes = [64]byte{} + for i := range w.codegen { + w.codegen[i] = 0 + } + for _, s := range [...][]int32{w.literalFreq, w.offsetFreq, w.codegenFreq} { + for i := range s { + s[i] = 0 + } + } + for _, enc := range [...]*huffmanEncoder{ + w.literalEncoding, + w.offsetEncoding, + w.codegenEncoding} { + for i := range enc.code { + enc.code[i] = 0 + } + for i := range enc.codeBits { + enc.codeBits[i] = 0 + } + } +} + func (w *huffmanBitWriter) flushBits() { if w.err != nil { w.nbits = 0 diff --git a/libgo/go/compress/flate/huffman_code.go b/libgo/go/compress/flate/huffman_code.go index 009cce6267a..3b9fce466ed 100644 --- a/libgo/go/compress/flate/huffman_code.go +++ b/libgo/go/compress/flate/huffman_code.go @@ -19,23 +19,13 @@ type literalNode struct { freq int32 } -type chain struct { - // The sum of the leaves in this tree - freq int32 - - // The number of literals to the left of this item at this level - leafCount int32 - - // The right child of this chain in the previous level. - up *chain -} - +// A levelInfo describes the state of the constructed tree for a given depth. type levelInfo struct { // Our level. for better printing level int32 - // The most recent chain generated for this level - lastChain *chain + // The frequency of the last node at this level + lastFreq int32 // The frequency of the next character to add to this level nextCharFreq int32 @@ -47,12 +37,6 @@ type levelInfo struct { // The number of chains remaining to generate for this level before moving // up to the next level needed int32 - - // The levelInfo for level+1 - up *levelInfo - - // The levelInfo for level-1 - down *levelInfo } func maxNode() literalNode { return literalNode{math.MaxUint16, math.MaxInt32} } @@ -121,6 +105,8 @@ func (h *huffmanEncoder) bitLength(freq []int32) int64 { return total } +const maxBitsLimit = 16 + // Return the number of literals assigned to each bit size in the Huffman encoding // // This method is only called when list.length >= 3 @@ -131,9 +117,13 @@ func (h *huffmanEncoder) bitLength(freq []int32) int64 { // frequency, and has as its last element a special element with frequency // MaxInt32 // maxBits The maximum number of bits that should be used to encode any literal. +// Must be less than 16. // return An integer array in which array[i] indicates the number of literals // that should be encoded in i bits. func (h *huffmanEncoder) bitCounts(list []literalNode, maxBits int32) []int32 { + if maxBits >= maxBitsLimit { + panic("flate: maxBits too large") + } n := int32(len(list)) list = list[0 : n+1] list[n] = maxNode() @@ -148,53 +138,61 @@ func (h *huffmanEncoder) bitCounts(list []literalNode, maxBits int32) []int32 { // A bogus "Level 0" whose sole purpose is so that // level1.prev.needed==0. This makes level1.nextPairFreq // be a legitimate value that never gets chosen. - top := &levelInfo{needed: 0} - chain2 := &chain{list[1].freq, 2, new(chain)} + var levels [maxBitsLimit]levelInfo + // leafCounts[i] counts the number of literals at the left + // of ancestors of the rightmost node at level i. + // leafCounts[i][j] is the number of literals at the left + // of the level j ancestor. + var leafCounts [maxBitsLimit][maxBitsLimit]int32 + for level := int32(1); level <= maxBits; level++ { // For every level, the first two items are the first two characters. // We initialize the levels as if we had already figured this out. - top = &levelInfo{ + levels[level] = levelInfo{ level: level, - lastChain: chain2, + lastFreq: list[1].freq, nextCharFreq: list[2].freq, nextPairFreq: list[0].freq + list[1].freq, - down: top, } - top.down.up = top + leafCounts[level][level] = 2 if level == 1 { - top.nextPairFreq = math.MaxInt32 + levels[level].nextPairFreq = math.MaxInt32 } } // We need a total of 2*n - 2 items at top level and have already generated 2. - top.needed = 2*n - 4 + levels[maxBits].needed = 2*n - 4 - l := top + level := maxBits for { + l := &levels[level] if l.nextPairFreq == math.MaxInt32 && l.nextCharFreq == math.MaxInt32 { // We've run out of both leafs and pairs. // End all calculations for this level. - // To m sure we never come back to this level or any lower level, + // To make sure we never come back to this level or any lower level, // set nextPairFreq impossibly large. - l.lastChain = nil l.needed = 0 - l = l.up - l.nextPairFreq = math.MaxInt32 + levels[level+1].nextPairFreq = math.MaxInt32 + level++ continue } - prevFreq := l.lastChain.freq + prevFreq := l.lastFreq if l.nextCharFreq < l.nextPairFreq { // The next item on this row is a leaf node. - n := l.lastChain.leafCount + 1 - l.lastChain = &chain{l.nextCharFreq, n, l.lastChain.up} + n := leafCounts[level][level] + 1 + l.lastFreq = l.nextCharFreq + // Lower leafCounts are the same of the previous node. + leafCounts[level][level] = n l.nextCharFreq = list[n].freq } else { // The next item on this row is a pair from the previous row. // nextPairFreq isn't valid until we generate two // more values in the level below - l.lastChain = &chain{l.nextPairFreq, l.lastChain.leafCount, l.down.lastChain} - l.down.needed = 2 + l.lastFreq = l.nextPairFreq + // Take leaf counts from the lower level, except counts[level] remains the same. + copy(leafCounts[level][:level], leafCounts[level-1][:level]) + levels[l.level-1].needed = 2 } if l.needed--; l.needed == 0 { @@ -202,33 +200,33 @@ func (h *huffmanEncoder) bitCounts(list []literalNode, maxBits int32) []int32 { // Continue calculating one level up. Fill in nextPairFreq // of that level with the sum of the two nodes we've just calculated on // this level. - up := l.up - if up == nil { + if l.level == maxBits { // All done! break } - up.nextPairFreq = prevFreq + l.lastChain.freq - l = up + levels[l.level+1].nextPairFreq = prevFreq + l.lastFreq + level++ } else { // If we stole from below, move down temporarily to replenish it. - for l.down.needed > 0 { - l = l.down + for levels[level-1].needed > 0 { + level-- } } } // Somethings is wrong if at the end, the top level is null or hasn't used // all of the leaves. - if top.lastChain.leafCount != n { - panic("top.lastChain.leafCount != n") + if leafCounts[maxBits][maxBits] != n { + panic("leafCounts[maxBits][maxBits] != n") } bitCount := make([]int32, maxBits+1) bits := 1 - for chain := top.lastChain; chain.up != nil; chain = chain.up { + counts := &leafCounts[maxBits] + for level := maxBits; level > 0; level-- { // chain.leafCount gives the number of literals requiring at least "bits" // bits to encode. - bitCount[bits] = chain.leafCount - chain.up.leafCount + bitCount[bits] = counts[level] - counts[level-1] bits++ } return bitCount diff --git a/libgo/go/compress/flate/inflate.go b/libgo/go/compress/flate/inflate.go index beca34b4d8c..3eb3b2b83e6 100644 --- a/libgo/go/compress/flate/inflate.go +++ b/libgo/go/compress/flate/inflate.go @@ -91,6 +91,10 @@ type huffmanDecoder struct { // Initialize Huffman decoding tables from array of code lengths. func (h *huffmanDecoder) init(bits []int) bool { + if h.min != 0 { + *h = huffmanDecoder{} + } + // Count number of codes of each length, // compute min and max length. var count [maxCodeLen]int @@ -125,6 +129,9 @@ func (h *huffmanDecoder) init(bits []int) bool { if i == huffmanChunkBits+1 { // create link tables link := code >> 1 + if huffmanNumChunks < link { + return false + } h.links = make([][]uint32, huffmanNumChunks-link) for j := uint(link); j < huffmanNumChunks; j++ { reverse := int(reverseByte[j>>8]) | int(reverseByte[j&0xff])<<8 @@ -154,7 +161,11 @@ func (h *huffmanDecoder) init(bits []int) bool { h.chunks[off] = chunk } } else { - linktab := h.links[h.chunks[reverse&(huffmanNumChunks-1)]>>huffmanValueShift] + value := h.chunks[reverse&(huffmanNumChunks-1)] >> huffmanValueShift + if value >= uint32(len(h.links)) { + return false + } + linktab := h.links[value] reverse >>= huffmanChunkBits for off := reverse; off < numLinks; off += 1 << uint(n-huffmanChunkBits) { linktab[off] = chunk @@ -511,7 +522,7 @@ func (f *decompressor) copyHist() bool { if x := len(f.hist) - p; n > x { n = x } - forwardCopy(f.hist[f.hp:f.hp+n], f.hist[p:p+n]) + forwardCopy(f.hist[:], f.hp, p, n) p += n f.hp += n f.copyLen -= n @@ -633,6 +644,10 @@ func (f *decompressor) huffSym(h *huffmanDecoder) (int, error) { if n > huffmanChunkBits { chunk = h.links[chunk>>huffmanValueShift][(f.b>>huffmanChunkBits)&h.linkMask] n = uint(chunk & huffmanCountMask) + if n == 0 { + f.err = CorruptInputError(f.roffset) + return 0, f.err + } } if n <= f.nb { f.b >>= n diff --git a/libgo/go/compress/flate/reader_test.go b/libgo/go/compress/flate/reader_test.go index 54ed788dbd3..2a8ebbc9438 100644 --- a/libgo/go/compress/flate/reader_test.go +++ b/libgo/go/compress/flate/reader_test.go @@ -37,6 +37,7 @@ var testfiles = []string{ } func benchmarkDecode(b *testing.B, testfile, level, n int) { + b.ReportAllocs() b.StopTimer() b.SetBytes(int64(n)) buf0, err := ioutil.ReadFile(testfiles[testfile]) @@ -55,7 +56,7 @@ func benchmarkDecode(b *testing.B, testfile, level, n int) { if len(buf0) > n-i { buf0 = buf0[:n-i] } - io.Copy(w, bytes.NewBuffer(buf0)) + io.Copy(w, bytes.NewReader(buf0)) } w.Close() buf1 := compressed.Bytes() @@ -63,7 +64,7 @@ func benchmarkDecode(b *testing.B, testfile, level, n int) { runtime.GC() b.StartTimer() for i := 0; i < b.N; i++ { - io.Copy(ioutil.Discard, NewReader(bytes.NewBuffer(buf1))) + io.Copy(ioutil.Discard, NewReader(bytes.NewReader(buf1))) } } diff --git a/libgo/go/compress/gzip/gunzip_test.go b/libgo/go/compress/gzip/gunzip_test.go index a1333580dc0..572fb584885 100644 --- a/libgo/go/compress/gzip/gunzip_test.go +++ b/libgo/go/compress/gzip/gunzip_test.go @@ -7,7 +7,10 @@ package gzip import ( "bytes" "io" + "io/ioutil" + "os" "testing" + "time" ) type gunzipTest struct { @@ -302,3 +305,31 @@ func TestDecompressor(t *testing.T) { } } } + +func TestIssue6550(t *testing.T) { + f, err := os.Open("testdata/issue6550.gz") + if err != nil { + t.Fatal(err) + } + gzip, err := NewReader(f) + if err != nil { + t.Fatalf("NewReader(testdata/issue6550.gz): %v", err) + } + defer gzip.Close() + done := make(chan bool, 1) + go func() { + _, err := io.Copy(ioutil.Discard, gzip) + if err == nil { + t.Errorf("Copy succeeded") + } else { + t.Logf("Copy failed (correctly): %v", err) + } + done <- true + }() + select { + case <-time.After(1 * time.Second): + t.Errorf("Copy hung") + case <-done: + // ok + } +} diff --git a/libgo/go/compress/gzip/gzip.go b/libgo/go/compress/gzip/gzip.go index 45558b74289..fe32d6871ae 100644 --- a/libgo/go/compress/gzip/gzip.go +++ b/libgo/go/compress/gzip/gzip.go @@ -26,14 +26,15 @@ const ( // to its wrapped io.Writer. type Writer struct { Header - w io.Writer - level int - compressor *flate.Writer - digest hash.Hash32 - size uint32 - closed bool - buf [10]byte - err error + w io.Writer + level int + wroteHeader bool + compressor *flate.Writer + digest hash.Hash32 + size uint32 + closed bool + buf [10]byte + err error } // NewWriter creates a new Writer that satisfies writes by compressing data @@ -62,14 +63,39 @@ func NewWriterLevel(w io.Writer, level int) (*Writer, error) { if level < DefaultCompression || level > BestCompression { return nil, fmt.Errorf("gzip: invalid compression level: %d", level) } - return &Writer{ + z := new(Writer) + z.init(w, level) + return z, nil +} + +func (z *Writer) init(w io.Writer, level int) { + digest := z.digest + if digest != nil { + digest.Reset() + } else { + digest = crc32.NewIEEE() + } + compressor := z.compressor + if compressor != nil { + compressor.Reset(w) + } + *z = Writer{ Header: Header{ OS: 255, // unknown }, - w: w, - level: level, - digest: crc32.NewIEEE(), - }, nil + w: w, + level: level, + digest: digest, + compressor: compressor, + } +} + +// Reset discards the Writer z's state and makes it equivalent to the +// result of its original state from NewWriter or NewWriterLevel, but +// writing to w instead. This permits reusing a Writer rather than +// allocating a new one. +func (z *Writer) Reset(w io.Writer) { + z.init(w, z.level) } // GZIP (RFC 1952) is little-endian, unlike ZLIB (RFC 1950). @@ -138,7 +164,8 @@ func (z *Writer) Write(p []byte) (int, error) { } var n int // Write the GZIP header lazily. - if z.compressor == nil { + if !z.wroteHeader { + z.wroteHeader = true z.buf[0] = gzipID1 z.buf[1] = gzipID2 z.buf[2] = gzipDeflate @@ -183,7 +210,9 @@ func (z *Writer) Write(p []byte) (int, error) { return n, z.err } } - z.compressor, _ = flate.NewWriter(z.w, z.level) + if z.compressor == nil { + z.compressor, _ = flate.NewWriter(z.w, z.level) + } } z.size += uint32(len(p)) z.digest.Write(p) @@ -206,8 +235,11 @@ func (z *Writer) Flush() error { if z.closed { return nil } - if z.compressor == nil { + if !z.wroteHeader { z.Write(nil) + if z.err != nil { + return z.err + } } z.err = z.compressor.Flush() return z.err @@ -222,7 +254,7 @@ func (z *Writer) Close() error { return nil } z.closed = true - if z.compressor == nil { + if !z.wroteHeader { z.Write(nil) if z.err != nil { return z.err diff --git a/libgo/go/compress/gzip/gzip_test.go b/libgo/go/compress/gzip/gzip_test.go index 4d1af94381c..119be2e135b 100644 --- a/libgo/go/compress/gzip/gzip_test.go +++ b/libgo/go/compress/gzip/gzip_test.go @@ -197,3 +197,35 @@ func TestWriterFlush(t *testing.T) { t.Fatal("Flush didn't flush any data") } } + +// Multiple gzip files concatenated form a valid gzip file. +func TestConcat(t *testing.T) { + var buf bytes.Buffer + w := NewWriter(&buf) + w.Write([]byte("hello ")) + w.Close() + w = NewWriter(&buf) + w.Write([]byte("world\n")) + w.Close() + + r, err := NewReader(&buf) + data, err := ioutil.ReadAll(r) + if string(data) != "hello world\n" || err != nil { + t.Fatalf("ReadAll = %q, %v, want %q, nil", data, err, "hello world") + } +} + +func TestWriterReset(t *testing.T) { + buf := new(bytes.Buffer) + buf2 := new(bytes.Buffer) + z := NewWriter(buf) + msg := []byte("hello world") + z.Write(msg) + z.Close() + z.Reset(buf2) + z.Write(msg) + z.Close() + if buf.String() != buf2.String() { + t.Errorf("buf2 %q != original buf of %q", buf2.String(), buf.String()) + } +} diff --git a/libgo/go/compress/gzip/testdata/issue6550.gz b/libgo/go/compress/gzip/testdata/issue6550.gz Binary files differnew file mode 100644 index 00000000000..57972b63668 --- /dev/null +++ b/libgo/go/compress/gzip/testdata/issue6550.gz diff --git a/libgo/go/compress/zlib/writer.go b/libgo/go/compress/zlib/writer.go index cd8dea460a4..99ff6549acb 100644 --- a/libgo/go/compress/zlib/writer.go +++ b/libgo/go/compress/zlib/writer.go @@ -70,6 +70,23 @@ func NewWriterLevelDict(w io.Writer, level int, dict []byte) (*Writer, error) { }, nil } +// Reset clears the state of the Writer z such that it is equivalent to its +// initial state from NewWriterLevel or NewWriterLevelDict, but instead writing +// to w. +func (z *Writer) Reset(w io.Writer) { + z.w = w + // z.level and z.dict left unchanged. + if z.compressor != nil { + z.compressor.Reset(w) + } + if z.digest != nil { + z.digest.Reset() + } + z.err = nil + z.scratch = [4]byte{} + z.wroteHeader = false +} + // writeHeader writes the ZLIB header. func (z *Writer) writeHeader() (err error) { z.wroteHeader = true @@ -111,11 +128,15 @@ func (z *Writer) writeHeader() (err error) { return err } } - z.compressor, err = flate.NewWriterDict(z.w, z.level, z.dict) - if err != nil { - return err + if z.compressor == nil { + // Initialize deflater unless the Writer is being reused + // after a Reset call. + z.compressor, err = flate.NewWriterDict(z.w, z.level, z.dict) + if err != nil { + return err + } + z.digest = adler32.New() } - z.digest = adler32.New() return nil } diff --git a/libgo/go/compress/zlib/writer_test.go b/libgo/go/compress/zlib/writer_test.go index aee1a5c2f54..cf9c8325455 100644 --- a/libgo/go/compress/zlib/writer_test.go +++ b/libgo/go/compress/zlib/writer_test.go @@ -89,6 +89,56 @@ func testLevelDict(t *testing.T, fn string, b0 []byte, level int, d string) { } } +func testFileLevelDictReset(t *testing.T, fn string, level int, dict []byte) { + var b0 []byte + var err error + if fn != "" { + b0, err = ioutil.ReadFile(fn) + if err != nil { + t.Errorf("%s (level=%d): %v", fn, level, err) + return + } + } + + // Compress once. + buf := new(bytes.Buffer) + var zlibw *Writer + if dict == nil { + zlibw, err = NewWriterLevel(buf, level) + } else { + zlibw, err = NewWriterLevelDict(buf, level, dict) + } + if err == nil { + _, err = zlibw.Write(b0) + } + if err == nil { + err = zlibw.Close() + } + if err != nil { + t.Errorf("%s (level=%d): %v", fn, level, err) + return + } + out := buf.String() + + // Reset and comprses again. + buf2 := new(bytes.Buffer) + zlibw.Reset(buf2) + _, err = zlibw.Write(b0) + if err == nil { + err = zlibw.Close() + } + if err != nil { + t.Errorf("%s (level=%d): %v", fn, level, err) + return + } + out2 := buf2.String() + + if out2 != out { + t.Errorf("%s (level=%d): different output after reset (got %d bytes, expected %d", + fn, level, len(out2), len(out)) + } +} + func TestWriter(t *testing.T) { for i, s := range data { b := []byte(s) @@ -122,6 +172,21 @@ func TestWriterDict(t *testing.T) { } } +func TestWriterReset(t *testing.T) { + const dictionary = "0123456789." + for _, fn := range filenames { + testFileLevelDictReset(t, fn, NoCompression, nil) + testFileLevelDictReset(t, fn, DefaultCompression, nil) + testFileLevelDictReset(t, fn, NoCompression, []byte(dictionary)) + testFileLevelDictReset(t, fn, DefaultCompression, []byte(dictionary)) + if !testing.Short() { + for level := BestSpeed; level <= BestCompression; level++ { + testFileLevelDictReset(t, fn, level, nil) + } + } + } +} + func TestWriterDictIsUsed(t *testing.T) { var input = []byte("Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.") var buf bytes.Buffer |