libgo: update to Go 1.11

    
    Reviewed-on: https://go-review.googlesource.com/136435

gotools/:
	* Makefile.am (mostlyclean-local): Run chmod on check-go-dir to
	make sure it is writable.
	(check-go-tools): Likewise.
	(check-vet): Copy internal/objabi to check-vet-dir.
	* Makefile.in: Rebuild.

From-SVN: r264546

13 files changed, 201 insertions, 67 deletions

diff --git a/libgo/go/archive/tar/common.go b/libgo/go/archive/tar/common.go
index 4a2c173bf3a..dee9e47e4ae 100644
--- a/libgo/go/archive/tar/common.go
+++ b/libgo/go/archive/tar/common.go
@@ -56,7 +56,7 @@ func (he headerError) Error() string {
 const (
 	// Type '0' indicates a regular file.
 	TypeReg  = '0'
-	TypeRegA = '\x00' // For legacy support; use TypeReg instead
+	TypeRegA = '\x00' // Deprecated: Use TypeReg instead.
 
 	// Type '1' to '6' are header-only flags and may not have a data body.
 	TypeLink    = '1' // Hard link
@@ -138,7 +138,10 @@ var basicKeys = map[string]bool{
 // should do so by creating a new Header and copying the fields
 // that they are interested in preserving.
 type Header struct {
-	Typeflag byte // Type of header entry (should be TypeReg for most files)
+	// Typeflag is the type of header entry.
+	// The zero value is automatically promoted to either TypeReg or TypeDir
+	// depending on the presence of a trailing slash in Name.
+	Typeflag byte
 
 	Name     string // Name of file entry
 	Linkname string // Target name of link (valid for TypeLink or TypeSymlink)
@@ -184,7 +187,7 @@ type Header struct {
 	// The key and value should be non-empty UTF-8 strings.
 	//
 	// When Writer.WriteHeader is called, PAX records derived from the
-	// the other fields in Header take precedence over PAXRecords.
+	// other fields in Header take precedence over PAXRecords.
 	PAXRecords map[string]string
 
 	// Format specifies the format of the tar header.
diff --git a/libgo/go/archive/tar/format.go b/libgo/go/archive/tar/format.go
index 6e29698a14a..1f89d0c59a1 100644
--- a/libgo/go/archive/tar/format.go
+++ b/libgo/go/archive/tar/format.go
@@ -94,7 +94,7 @@ const (
 	// application can only parse GNU formatted archives.
 	//
 	// Reference:
-	//	http://www.gnu.org/software/tar/manual/html_node/Standard.html
+	//	https://www.gnu.org/software/tar/manual/html_node/Standard.html
 	FormatGNU
 
 	// Schily's tar format, which is incompatible with USTAR.
diff --git a/libgo/go/archive/tar/reader.go b/libgo/go/archive/tar/reader.go
index f4eeb557be9..39437185179 100644
--- a/libgo/go/archive/tar/reader.go
+++ b/libgo/go/archive/tar/reader.go
@@ -64,7 +64,6 @@ func (tr *Reader) next() (*Header, error) {
 	// normally be visible to the outside. As such, this loop iterates through
 	// one or more "header files" until it finds a "normal file".
 	format := FormatUSTAR | FormatPAX | FormatGNU
-loop:
 	for {
 		// Discard the remainder of the file and any padding.
 		if err := discard(tr.r, tr.curr.PhysicalRemaining()); err != nil {
@@ -102,7 +101,7 @@ loop:
 					Format:     format,
 				}, nil
 			}
-			continue loop // This is a meta header affecting the next header
+			continue // This is a meta header affecting the next header
 		case TypeGNULongName, TypeGNULongLink:
 			format.mayOnlyBe(FormatGNU)
 			realname, err := ioutil.ReadAll(tr)
@@ -117,7 +116,7 @@ loop:
 			case TypeGNULongLink:
 				gnuLongLink = p.parseString(realname)
 			}
-			continue loop // This is a meta header affecting the next header
+			continue // This is a meta header affecting the next header
 		default:
 			// The old GNU sparse format is handled here since it is technically
 			// just a regular file with additional attributes.
@@ -131,8 +130,12 @@ loop:
 			if gnuLongLink != "" {
 				hdr.Linkname = gnuLongLink
 			}
-			if hdr.Typeflag == TypeRegA && strings.HasSuffix(hdr.Name, "/") {
-				hdr.Typeflag = TypeDir // Legacy archives use trailing slash for directories
+			if hdr.Typeflag == TypeRegA {
+				if strings.HasSuffix(hdr.Name, "/") {
+					hdr.Typeflag = TypeDir // Legacy archives use trailing slash for directories
+				} else {
+					hdr.Typeflag = TypeReg
+				}
 			}
 
 			// The extended headers may have updated the size.
@@ -200,7 +203,7 @@ func (tr *Reader) handleSparseFile(hdr *Header, rawHdr *block) error {
 // readGNUSparsePAXHeaders checks the PAX headers for GNU sparse headers.
 // If they are found, then this function reads the sparse map and returns it.
 // This assumes that 0.0 headers have already been converted to 0.1 headers
-// by the the PAX header parsing logic.
+// by the PAX header parsing logic.
 func (tr *Reader) readGNUSparsePAXHeaders(hdr *Header) (sparseDatas, error) {
 	// Identify the version of GNU headers.
 	var is1x0 bool
diff --git a/libgo/go/archive/tar/reader_test.go b/libgo/go/archive/tar/reader_test.go
index a6832d33b1b..f153b668def 100644
--- a/libgo/go/archive/tar/reader_test.go
+++ b/libgo/go/archive/tar/reader_test.go
@@ -189,7 +189,7 @@ func TestReader(t *testing.T) {
 			Gid:      5000,
 			Size:     5,
 			ModTime:  time.Unix(1244593104, 0),
-			Typeflag: '\x00',
+			Typeflag: '0',
 		}, {
 			Name:     "small2.txt",
 			Mode:     0444,
@@ -197,7 +197,7 @@ func TestReader(t *testing.T) {
 			Gid:      5000,
 			Size:     11,
 			ModTime:  time.Unix(1244593104, 0),
-			Typeflag: '\x00',
+			Typeflag: '0',
 		}},
 	}, {
 		file: "testdata/pax.tar",
@@ -378,9 +378,9 @@ func TestReader(t *testing.T) {
 				"security.selinux": "unconfined_u:object_r:default_t:s0\x00",
 			},
 			PAXRecords: map[string]string{
-				"mtime": "1386065770.449252304",
-				"atime": "1389782991.41987522",
-				"ctime": "1386065770.449252304",
+				"mtime":                         "1386065770.449252304",
+				"atime":                         "1389782991.41987522",
+				"ctime":                         "1386065770.449252304",
 				"SCHILY.xattr.security.selinux": "unconfined_u:object_r:default_t:s0\x00",
 			},
 			Format: FormatPAX,
@@ -534,9 +534,10 @@ func TestReader(t *testing.T) {
 		// a buggy pre-Go1.8 tar.Writer.
 		file: "testdata/invalid-go17.tar",
 		headers: []*Header{{
-			Name:    "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/foo",
-			Uid:     010000000,
-			ModTime: time.Unix(0, 0),
+			Name:     "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/foo",
+			Uid:      010000000,
+			ModTime:  time.Unix(0, 0),
+			Typeflag: '0',
 		}},
 	}, {
 		// USTAR archive with a regular entry with non-zero device numbers.
diff --git a/libgo/go/archive/tar/tar_test.go b/libgo/go/archive/tar/tar_test.go
index af80d6e0c15..2676853122a 100644
--- a/libgo/go/archive/tar/tar_test.go
+++ b/libgo/go/archive/tar/tar_test.go
@@ -306,6 +306,7 @@ func TestRoundTrip(t *testing.T) {
 		ModTime:    time.Now().Round(time.Second),
 		PAXRecords: map[string]string{"uid": "2097152"},
 		Format:     FormatPAX,
+		Typeflag:   TypeReg,
 	}
 	if err := tw.WriteHeader(hdr); err != nil {
 		t.Fatalf("tw.WriteHeader: %v", err)
diff --git a/libgo/go/archive/tar/testdata/file-and-dir.tar b/libgo/go/archive/tar/testdata/file-and-dir.tar
new file mode 100644
index 00000000000..c18d4283e38
--- /dev/null
+++ b/libgo/go/archive/tar/testdata/file-and-dir.tar
diff --git a/libgo/go/archive/tar/testdata/trailing-slash.tar b/libgo/go/archive/tar/testdata/trailing-slash.tar
index bf1b2ec426b..93718b30348 100644
--- a/libgo/go/archive/tar/testdata/trailing-slash.tar
+++ b/libgo/go/archive/tar/testdata/trailing-slash.tar
diff --git a/libgo/go/archive/tar/writer.go b/libgo/go/archive/tar/writer.go
index 97d23f80388..e80498d03e3 100644
--- a/libgo/go/archive/tar/writer.go
+++ b/libgo/go/archive/tar/writer.go
@@ -5,7 +5,6 @@
 package tar
 
 import (
-	"bytes"
 	"fmt"
 	"io"
 	"path"
@@ -71,6 +70,16 @@ func (tw *Writer) WriteHeader(hdr *Header) error {
 	}
 	tw.hdr = *hdr // Shallow copy of Header
 
+	// Avoid usage of the legacy TypeRegA flag, and automatically promote
+	// it to use TypeReg or TypeDir.
+	if tw.hdr.Typeflag == TypeRegA {
+		if strings.HasSuffix(tw.hdr.Name, "/") {
+			tw.hdr.Typeflag = TypeDir
+		} else {
+			tw.hdr.Typeflag = TypeReg
+		}
+	}
+
 	// Round ModTime and ignore AccessTime and ChangeTime unless
 	// the format is explicitly chosen.
 	// This ensures nominal usage of WriteHeader (without specifying the format)
@@ -166,7 +175,7 @@ func (tw *Writer) writePAXHeader(hdr *Header, paxHdrs map[string]string) error {
 		sort.Strings(keys)
 
 		// Write each record to a buffer.
-		var buf bytes.Buffer
+		var buf strings.Builder
 		for _, k := range keys {
 			rec, err := formatPAXRecord(k, paxHdrs[k])
 			if err != nil {
diff --git a/libgo/go/archive/tar/writer_test.go b/libgo/go/archive/tar/writer_test.go
index 24e8da271c2..30556d27d02 100644
--- a/libgo/go/archive/tar/writer_test.go
+++ b/libgo/go/archive/tar/writer_test.go
@@ -461,6 +461,15 @@ func TestWriter(t *testing.T) {
 			testHeader{Header{Name: strings.Repeat("123456789/", 30)}, nil},
 			testClose{nil},
 		},
+	}, {
+		// Automatically promote zero value of Typeflag depending on the name.
+		file: "testdata/file-and-dir.tar",
+		tests: []testFnc{
+			testHeader{Header{Name: "small.txt", Size: 5}, nil},
+			testWrite{"Kilts", 5, nil},
+			testHeader{Header{Name: "dir/"}, nil},
+			testClose{nil},
+		},
 	}}
 
 	equalError := func(x, y error) bool {
@@ -809,8 +818,8 @@ func TestValidTypeflagWithPAXHeader(t *testing.T) {
 		if err != nil {
 			t.Fatalf("Failed to read header: %s", err)
 		}
-		if header.Typeflag != 0 {
-			t.Fatalf("Typeflag should've been 0, found %d", header.Typeflag)
+		if header.Typeflag != TypeReg {
+			t.Fatalf("Typeflag should've been %d, found %d", TypeReg, header.Typeflag)
 		}
 	}
 }
diff --git a/libgo/go/archive/zip/struct.go b/libgo/go/archive/zip/struct.go
index f613ebdc344..c90151d9d44 100644
--- a/libgo/go/archive/zip/struct.go
+++ b/libgo/go/archive/zip/struct.go
@@ -81,8 +81,17 @@ const (
 // See the zip spec for details.
 type FileHeader struct {
 	// Name is the name of the file.
-	// It must be a relative path, not start with a drive letter (e.g. C:),
-	// and must use forward slashes instead of back slashes.
+	//
+	// It must be a relative path, not start with a drive letter (such as "C:"),
+	// and must use forward slashes instead of back slashes. A trailing slash
+	// indicates that this file is a directory and should have no data.
+	//
+	// When reading zip files, the Name field is populated from
+	// the zip file directly and is not validated for correctness.
+	// It is the caller's responsibility to sanitize it as
+	// appropriate, including canonicalizing slash directions,
+	// validating that paths are relative, and preventing path
+	// traversal through filenames ("../../../").
 	Name string
 
 	// Comment is any arbitrary user-defined string shorter than 64KiB.
@@ -201,7 +210,7 @@ func timeZone(offset time.Duration) *time.Location {
 
 // msDosTimeToTime converts an MS-DOS date and time into a time.Time.
 // The resolution is 2s.
-// See: http://msdn.microsoft.com/en-us/library/ms724247(v=VS.85).aspx
+// See: https://msdn.microsoft.com/en-us/library/ms724247(v=VS.85).aspx
 func msDosTimeToTime(dosDate, dosTime uint16) time.Time {
 	return time.Date(
 		// date bits 0-4: day of month; 5-8: month; 9-15: years since 1980
@@ -221,7 +230,7 @@ func msDosTimeToTime(dosDate, dosTime uint16) time.Time {
 
 // timeToMsDosTime converts a time.Time to an MS-DOS date and time.
 // The resolution is 2s.
-// See: http://msdn.microsoft.com/en-us/library/ms724274(v=VS.85).aspx
+// See: https://msdn.microsoft.com/en-us/library/ms724274(v=VS.85).aspx
 func timeToMsDosTime(t time.Time) (fDate uint16, fTime uint16) {
 	fDate = uint16(t.Day() + int(t.Month())<<5 + (t.Year()-1980)<<9)
 	fTime = uint16(t.Second()/2 + t.Minute()<<5 + t.Hour()<<11)
diff --git a/libgo/go/archive/zip/writer.go b/libgo/go/archive/zip/writer.go
index 14a5ee48c11..5f0c0a1a555 100644
--- a/libgo/go/archive/zip/writer.go
+++ b/libgo/go/archive/zip/writer.go
@@ -11,6 +11,7 @@ import (
 	"hash"
 	"hash/crc32"
 	"io"
+	"strings"
 	"unicode/utf8"
 )
 
@@ -71,7 +72,7 @@ func (w *Writer) SetComment(comment string) error {
 }
 
 // Close finishes writing the zip file by writing the central directory.
-// It does not (and cannot) close the underlying writer.
+// It does not close the underlying writer.
 func (w *Writer) Close() error {
 	if w.last != nil && !w.last.closed {
 		if err := w.last.close(); err != nil {
@@ -209,7 +210,8 @@ func (w *Writer) Close() error {
 // The file contents will be compressed using the Deflate method.
 // The name must be a relative path: it must not start with a drive
 // letter (e.g. C:) or leading slash, and only forward slashes are
-// allowed.
+// allowed. To create a directory instead of a file, add a trailing
+// slash to the name.
 // The file's contents must be written to the io.Writer before the next
 // call to Create, CreateHeader, or Close.
 func (w *Writer) Create(name string) (io.Writer, error) {
@@ -261,8 +263,6 @@ func (w *Writer) CreateHeader(fh *FileHeader) (io.Writer, error) {
 		return nil, errors.New("archive/zip: invalid duplicate FileHeader")
 	}
 
-	fh.Flags |= 0x8 // we will write a data descriptor
-
 	// The ZIP format has a sad state of affairs regarding character encoding.
 	// Officially, the name and comment fields are supposed to be encoded
 	// in CP-437 (which is mostly compatible with ASCII), unless the UTF-8
@@ -319,35 +319,58 @@ func (w *Writer) CreateHeader(fh *FileHeader) (io.Writer, error) {
 		fh.Extra = append(fh.Extra, mbuf[:]...)
 	}
 
-	fw := &fileWriter{
-		zipw:      w.cw,
-		compCount: &countWriter{w: w.cw},
-		crc32:     crc32.NewIEEE(),
-	}
-	comp := w.compressor(fh.Method)
-	if comp == nil {
-		return nil, ErrAlgorithm
-	}
-	var err error
-	fw.comp, err = comp(fw.compCount)
-	if err != nil {
-		return nil, err
-	}
-	fw.rawCount = &countWriter{w: fw.comp}
-
+	var (
+		ow io.Writer
+		fw *fileWriter
+	)
 	h := &header{
 		FileHeader: fh,
 		offset:     uint64(w.cw.count),
 	}
-	w.dir = append(w.dir, h)
-	fw.header = h
 
+	if strings.HasSuffix(fh.Name, "/") {
+		// Set the compression method to Store to ensure data length is truly zero,
+		// which the writeHeader method always encodes for the size fields.
+		// This is necessary as most compression formats have non-zero lengths
+		// even when compressing an empty string.
+		fh.Method = Store
+		fh.Flags &^= 0x8 // we will not write a data descriptor
+
+		// Explicitly clear sizes as they have no meaning for directories.
+		fh.CompressedSize = 0
+		fh.CompressedSize64 = 0
+		fh.UncompressedSize = 0
+		fh.UncompressedSize64 = 0
+
+		ow = dirWriter{}
+	} else {
+		fh.Flags |= 0x8 // we will write a data descriptor
+
+		fw = &fileWriter{
+			zipw:      w.cw,
+			compCount: &countWriter{w: w.cw},
+			crc32:     crc32.NewIEEE(),
+		}
+		comp := w.compressor(fh.Method)
+		if comp == nil {
+			return nil, ErrAlgorithm
+		}
+		var err error
+		fw.comp, err = comp(fw.compCount)
+		if err != nil {
+			return nil, err
+		}
+		fw.rawCount = &countWriter{w: fw.comp}
+		fw.header = h
+		ow = fw
+	}
+	w.dir = append(w.dir, h)
 	if err := writeHeader(w.cw, fh); err != nil {
 		return nil, err
 	}
-
+	// If we're creating a directory, fw is nil.
 	w.last = fw
-	return fw, nil
+	return ow, nil
 }
 
 func writeHeader(w io.Writer, h *FileHeader) error {
@@ -400,6 +423,15 @@ func (w *Writer) compressor(method uint16) Compressor {
 	return comp
 }
 
+type dirWriter struct{}
+
+func (dirWriter) Write(b []byte) (int, error) {
+	if len(b) == 0 {
+		return 0, nil
+	}
+	return 0, errors.New("zip: write to directory")
+}
+
 type fileWriter struct {
 	*header
 	zipw      io.Writer
diff --git a/libgo/go/archive/zip/writer_test.go b/libgo/go/archive/zip/writer_test.go
index 38f32296fa8..1fedfd85e82 100644
--- a/libgo/go/archive/zip/writer_test.go
+++ b/libgo/go/archive/zip/writer_test.go
@@ -6,6 +6,7 @@ package zip
 
 import (
 	"bytes"
+	"encoding/binary"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -299,6 +300,59 @@ func TestWriterFlush(t *testing.T) {
 	}
 }
 
+func TestWriterDir(t *testing.T) {
+	w := NewWriter(ioutil.Discard)
+	dw, err := w.Create("dir/")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := dw.Write(nil); err != nil {
+		t.Errorf("Write(nil) to directory: got %v, want nil", err)
+	}
+	if _, err := dw.Write([]byte("hello")); err == nil {
+		t.Error(`Write("hello") to directory: got nil error, want non-nil`)
+	}
+}
+
+func TestWriterDirAttributes(t *testing.T) {
+	var buf bytes.Buffer
+	w := NewWriter(&buf)
+	if _, err := w.CreateHeader(&FileHeader{
+		Name:               "dir/",
+		Method:             Deflate,
+		CompressedSize64:   1234,
+		UncompressedSize64: 5678,
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if err := w.Close(); err != nil {
+		t.Fatal(err)
+	}
+	b := buf.Bytes()
+
+	var sig [4]byte
+	binary.LittleEndian.PutUint32(sig[:], uint32(fileHeaderSignature))
+
+	idx := bytes.Index(b, sig[:])
+	if idx == -1 {
+		t.Fatal("file header not found")
+	}
+	b = b[idx:]
+
+	if !bytes.Equal(b[6:10], []byte{0, 0, 0, 0}) { // FileHeader.Flags: 0, FileHeader.Method: 0
+		t.Errorf("unexpected method and flags: %v", b[6:10])
+	}
+
+	if !bytes.Equal(b[14:26], make([]byte, 12)) { // FileHeader.{CRC32,CompressSize,UncompressedSize} all zero.
+		t.Errorf("unexpected crc, compress and uncompressed size to be 0 was: %v", b[14:26])
+	}
+
+	binary.LittleEndian.PutUint32(sig[:], uint32(dataDescriptorSignature))
+	if bytes.Index(b, sig[:]) != -1 {
+		t.Error("there should be no data descriptor")
+	}
+}
+
 func testCreate(t *testing.T, w *Writer, wt *WriteTest) {
 	header := &FileHeader{
 		Name:   wt.Name,
diff --git a/libgo/go/archive/zip/zip_test.go b/libgo/go/archive/zip/zip_test.go
index 7e02cb0eeaa..50218a2bbd7 100644
--- a/libgo/go/archive/zip/zip_test.go
+++ b/libgo/go/archive/zip/zip_test.go
@@ -15,6 +15,7 @@ import (
 	"internal/testenv"
 	"io"
 	"io/ioutil"
+	"runtime"
 	"sort"
 	"strings"
 	"testing"
@@ -140,14 +141,7 @@ func (r *rleBuffer) Write(p []byte) (n int, err error) {
 		rp = &r.buf[len(r.buf)-1]
 		// Fast path, if p is entirely the same byte repeated.
 		if lastByte := rp.b; len(p) > 0 && p[0] == lastByte {
-			all := true
-			for _, b := range p {
-				if b != lastByte {
-					all = false
-					break
-				}
-			}
-			if all {
+			if bytes.Count(p, []byte{lastByte}) == len(p) {
 				rp.n += int64(len(p))
 				return len(p), nil
 			}
@@ -165,6 +159,25 @@ func (r *rleBuffer) Write(p []byte) (n int, err error) {
 	return len(p), nil
 }
 
+func min(x, y int) int {
+	if x < y {
+		return x
+	}
+	return y
+}
+
+func memset(a []byte, b byte) {
+	if len(a) == 0 {
+		return
+	}
+	// Double, until we reach power of 2 >= len(a), same as bytes.Repeat,
+	// but without allocation.
+	a[0] = b
+	for i, l := 1, len(a); i < l; i *= 2 {
+		copy(a[i:], a[:i])
+	}
+}
+
 func (r *rleBuffer) ReadAt(p []byte, off int64) (n int, err error) {
 	if len(p) == 0 {
 		return
@@ -176,16 +189,13 @@ func (r *rleBuffer) ReadAt(p []byte, off int64) (n int, err error) {
 	parts := r.buf[skipParts:]
 	if len(parts) > 0 {
 		skipBytes := off - parts[0].off
-		for len(parts) > 0 {
-			part := parts[0]
-			for i := skipBytes; i < part.n; i++ {
-				if n == len(p) {
-					return
-				}
-				p[n] = part.b
-				n++
+		for _, part := range parts {
+			repeat := min(int(part.n-skipBytes), len(p)-n)
+			memset(p[n:n+repeat], part.b)
+			n += repeat
+			if n == len(p) {
+				return
 			}
-			parts = parts[1:]
 			skipBytes = 0
 		}
 	}
@@ -452,6 +462,9 @@ func suffixIsZip64(t *testing.T, zip sizedReaderAt) bool {
 
 // Zip64 is required if the total size of the records is uint32max.
 func TestZip64LargeDirectory(t *testing.T) {
+	if runtime.GOARCH == "wasm" {
+		t.Skip("too slow on wasm")
+	}
 	if testing.Short() {
 		t.Skip("skipping in short mode")
 	}