procyberian/gcc
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

libgo: Update to current version of master library.

Browse source 
From-SVN: r193688
This commit is contained in:
Ian Lance Taylor 2012-11-21 07:03:38 +00:00
parent a51fb17f48
commit fabcaa8df3
321 changed files with 62096 additions and 19248 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
libgo/MERGE
View file

@ -1,4 +1,4 @@
291d9f1baf75
a070de932857
The first line of this file holds the Mercurial revision number of the
last merge done from the master library sources.

10
libgo/Makefile.am
View file

@ -773,7 +773,6 @@ go_net_files = \
go/net/lookup_unix.go \
go/net/mac.go \
go/net/net.go \
go/net/net_posix.go \
go/net/parse.go \
go/net/pipe.go \
go/net/port.go \
@ -1117,6 +1116,7 @@ go_crypto_x509_files = \
go/crypto/x509/pkcs8.go \
go/crypto/x509/root.go \
go/crypto/x509/root_unix.go \
go/crypto/x509/sec1.go \
go/crypto/x509/verify.go \
go/crypto/x509/x509.go
@ -1245,10 +1245,17 @@ go_exp_terminal_files = \
go/exp/terminal/terminal.go \
go/exp/terminal/util.go
go_exp_types_files = \
go/exp/types/builtins.go \
go/exp/types/check.go \
go/exp/types/const.go \
go/exp/types/conversions.go \
go/exp/types/errors.go \
go/exp/types/exportdata.go \
go/exp/types/expr.go \
go/exp/types/gcimporter.go \
go/exp/types/operand.go \
go/exp/types/predicates.go \
go/exp/types/stmt.go \
go/exp/types/types.go \
go/exp/types/universe.go
go_exp_utf8string_files = \
@ -1329,6 +1336,7 @@ go_image_jpeg_files = \
go/image/jpeg/huffman.go \
go/image/jpeg/idct.go \
go/image/jpeg/reader.go \
go/image/jpeg/scan.go \
go/image/jpeg/writer.go
go_image_png_files = \

10
libgo/Makefile.in
View file

@ -1027,7 +1027,6 @@ go_net_files = \
go/net/lookup_unix.go \
go/net/mac.go \
go/net/net.go \
go/net/net_posix.go \
go/net/parse.go \
go/net/pipe.go \
go/net/port.go \
@ -1312,6 +1311,7 @@ go_crypto_x509_files = \
go/crypto/x509/pkcs8.go \
go/crypto/x509/root.go \
go/crypto/x509/root_unix.go \
go/crypto/x509/sec1.go \
go/crypto/x509/verify.go \
go/crypto/x509/x509.go
@ -1463,10 +1463,17 @@ go_exp_terminal_files = \
go/exp/terminal/util.go
go_exp_types_files = \
go/exp/types/builtins.go \
go/exp/types/check.go \
go/exp/types/const.go \
go/exp/types/conversions.go \
go/exp/types/errors.go \
go/exp/types/exportdata.go \
go/exp/types/expr.go \
go/exp/types/gcimporter.go \
go/exp/types/operand.go \
go/exp/types/predicates.go \
go/exp/types/stmt.go \
go/exp/types/types.go \
go/exp/types/universe.go
@ -1557,6 +1564,7 @@ go_image_jpeg_files = \
go/image/jpeg/huffman.go \
go/image/jpeg/idct.go \
go/image/jpeg/reader.go \
go/image/jpeg/scan.go \
go/image/jpeg/writer.go
go_image_png_files = \

12
libgo/go/archive/tar/reader.go
View file

@ -72,6 +72,18 @@ func cString(b []byte) string {
}
func (tr *Reader) octal(b []byte) int64 {
// Check for binary format first.
if len(b) > 0 && b[0]&0x80 != 0 {
var x int64
for i, c := range b {
if i == 0 {
c &= 0x7f // ignore signal bit in first byte
}
x = x<<8 | int64(c)
}
return x
}
// Removing leading spaces.
for len(b) > 0 && b[0] == ' ' {
b = b[1:]

44
libgo/go/archive/tar/tar_test.go
View file

@ -5,7 +5,10 @@
package tar
import (
"bytes"
"io/ioutil"
"os"
"reflect"
"testing"
"time"
)
@ -54,3 +57,44 @@ func (symlink) Mode() os.FileMode { return os.ModeSymlink }
func (symlink) ModTime() time.Time { return time.Time{} }
func (symlink) IsDir() bool { return false }
func (symlink) Sys() interface{} { return nil }
func TestRoundTrip(t *testing.T) {
data := []byte("some file contents")
var b bytes.Buffer
tw := NewWriter(&b)
hdr := &Header{
Name: "file.txt",
Uid: 1 << 21, // too big for 8 octal digits
Size: int64(len(data)),
ModTime: time.Now(),
}
// tar only supports second precision.
hdr.ModTime = hdr.ModTime.Add(-time.Duration(hdr.ModTime.Nanosecond()) * time.Nanosecond)
if err := tw.WriteHeader(hdr); err != nil {
t.Fatalf("tw.WriteHeader: %v", err)
}
if _, err := tw.Write(data); err != nil {
t.Fatalf("tw.Write: %v", err)
}
if err := tw.Close(); err != nil {
t.Fatalf("tw.Close: %v", err)
}
// Read it back.
tr := NewReader(&b)
rHdr, err := tr.Next()
if err != nil {
t.Fatalf("tr.Next: %v", err)
}
if !reflect.DeepEqual(rHdr, hdr) {
t.Errorf("Header mismatch.\n got %+v\nwant %+v", rHdr, hdr)
}
rData, err := ioutil.ReadAll(tr)
if err != nil {
t.Fatalf("Read: %v", err)
}
if !bytes.Equal(rData, data) {
t.Errorf("Data mismatch.\n got %q\nwant %q", rData, data)
}
}

39
libgo/go/archive/tar/writer.go
View file

@ -12,6 +12,7 @@ import (
"fmt"
"io"
"strconv"
"time"
)
var (
@ -110,6 +111,12 @@ func (tw *Writer) numeric(b []byte, x int64) {
b[0] |= 0x80 // highest bit indicates binary format
}
var (
minTime = time.Unix(0, 0)
// There is room for 11 octal digits (33 bits) of mtime.
maxTime = minTime.Add((1<<33 - 1) * time.Second)
)
// WriteHeader writes hdr and prepares to accept the file's contents.
// WriteHeader calls Flush if it is not the first header.
// Calling after a Close will return ErrWriteAfterClose.
@ -133,19 +140,25 @@ func (tw *Writer) WriteHeader(hdr *Header) error {
// TODO(dsymonds): handle names longer than 100 chars
copy(s.next(100), []byte(hdr.Name))
tw.octal(s.next(8), hdr.Mode) // 100:108
tw.numeric(s.next(8), int64(hdr.Uid)) // 108:116
tw.numeric(s.next(8), int64(hdr.Gid)) // 116:124
tw.numeric(s.next(12), hdr.Size) // 124:136
tw.numeric(s.next(12), hdr.ModTime.Unix()) // 136:148
s.next(8) // chksum (148:156)
s.next(1)[0] = hdr.Typeflag // 156:157
tw.cString(s.next(100), hdr.Linkname) // linkname (157:257)
copy(s.next(8), []byte("ustar\x0000")) // 257:265
tw.cString(s.next(32), hdr.Uname) // 265:297
tw.cString(s.next(32), hdr.Gname) // 297:329
tw.numeric(s.next(8), hdr.Devmajor) // 329:337
tw.numeric(s.next(8), hdr.Devminor) // 337:345
// Handle out of range ModTime carefully.
var modTime int64
if !hdr.ModTime.Before(minTime) && !hdr.ModTime.After(maxTime) {
modTime = hdr.ModTime.Unix()
}
tw.octal(s.next(8), hdr.Mode) // 100:108
tw.numeric(s.next(8), int64(hdr.Uid)) // 108:116
tw.numeric(s.next(8), int64(hdr.Gid)) // 116:124
tw.numeric(s.next(12), hdr.Size) // 124:136
tw.numeric(s.next(12), modTime) // 136:148
s.next(8) // chksum (148:156)
s.next(1)[0] = hdr.Typeflag // 156:157
tw.cString(s.next(100), hdr.Linkname) // linkname (157:257)
copy(s.next(8), []byte("ustar\x0000")) // 257:265
tw.cString(s.next(32), hdr.Uname) // 265:297
tw.cString(s.next(32), hdr.Gname) // 297:329
tw.numeric(s.next(8), hdr.Devmajor) // 329:337
tw.numeric(s.next(8), hdr.Devminor) // 337:345
// Use the GNU magic instead of POSIX magic if we used any GNU extensions.
if tw.usedBinary {

9
libgo/go/archive/zip/reader.go
View file

@ -238,9 +238,12 @@ func readDirectoryHeader(f *File, r io.Reader) error {
if len(f.Extra) > 0 {
b := readBuf(f.Extra)
for len(b) > 0 {
for len(b) >= 4 { // need at least tag and size
tag := b.uint16()
size := b.uint16()
if int(size) > len(b) {
return ErrFormat
}
if tag == zip64ExtraId {
// update directory values from the zip64 extra block
eb := readBuf(b)
@ -256,6 +259,10 @@ func readDirectoryHeader(f *File, r io.Reader) error {
}
b = b[size:]
}
// Should have consumed the whole header.
if len(b) != 0 {
return ErrFormat
}
}
return nil
}

2
libgo/go/archive/zip/writer.go
View file

@ -174,7 +174,7 @@ func (w *Writer) Create(name string) (io.Writer, error) {
}
// CreateHeader adds a file to the zip file using the provided FileHeader
// for the file metadata.
// for the file metadata.
// It returns a Writer to which the file contents should be written.
// The file's contents must be written to the io.Writer before the next
// call to Create, CreateHeader, or Close.

82
libgo/go/archive/zip/zip_test.go
View file

@ -173,3 +173,85 @@ func TestZip64(t *testing.T) {
t.Errorf("UncompressedSize64 %d, want %d", got, want)
}
}
func testInvalidHeader(h *FileHeader, t *testing.T) {
var buf bytes.Buffer
z := NewWriter(&buf)
f, err := z.CreateHeader(h)
if err != nil {
t.Fatalf("error creating header: %v", err)
}
if _, err := f.Write([]byte("hi")); err != nil {
t.Fatalf("error writing content: %v", err)
}
if err := z.Close(); err != nil {
t.Fatalf("error closing zip writer: %v", err)
}
b := buf.Bytes()
if _, err = NewReader(bytes.NewReader(b), int64(len(b))); err != ErrFormat {
t.Fatalf("got %v, expected ErrFormat", err)
}
}
func testValidHeader(h *FileHeader, t *testing.T) {
var buf bytes.Buffer
z := NewWriter(&buf)
f, err := z.CreateHeader(h)
if err != nil {
t.Fatalf("error creating header: %v", err)
}
if _, err := f.Write([]byte("hi")); err != nil {
t.Fatalf("error writing content: %v", err)
}
if err := z.Close(); err != nil {
t.Fatalf("error closing zip writer: %v", err)
}
b := buf.Bytes()
if _, err = NewReader(bytes.NewReader(b), int64(len(b))); err != nil {
t.Fatalf("got %v, expected nil", err)
}
}
// Issue 4302.
func TestHeaderInvalidTagAndSize(t *testing.T) {
const timeFormat = "20060102T150405.000.txt"
ts := time.Now()
filename := ts.Format(timeFormat)
h := FileHeader{
Name: filename,
Method: Deflate,
Extra: []byte(ts.Format(time.RFC3339Nano)), // missing tag and len
}
h.SetModTime(ts)
testInvalidHeader(&h, t)
}
func TestHeaderTooShort(t *testing.T) {
h := FileHeader{
Name: "foo.txt",
Method: Deflate,
Extra: []byte{zip64ExtraId}, // missing size
}
testInvalidHeader(&h, t)
}
// Issue 4393. It is valid to have an extra data header
// which contains no body.
func TestZeroLengthHeader(t *testing.T) {
h := FileHeader{
Name: "extadata.txt",
Method: Deflate,
Extra: []byte{
85, 84, 5, 0, 3, 154, 144, 195, 77, // tag 21589 size 5
85, 120, 0, 0, // tag 30805 size 0
},
}
testValidHeader(&h, t)
}

30
libgo/go/bufio/bufio.go
View file

@ -567,6 +567,36 @@ func (b *Writer) WriteString(s string) (int, error) {
return nn, nil
}
// ReadFrom implements io.ReaderFrom.
func (b *Writer) ReadFrom(r io.Reader) (n int64, err error) {
if b.Buffered() == 0 {
if w, ok := b.wr.(io.ReaderFrom); ok {
return w.ReadFrom(r)
}
}
var m int
for {
m, err = r.Read(b.buf[b.n:])
if m == 0 {
break
}
b.n += m
n += int64(m)
if b.Available() == 0 {
if err1 := b.Flush(); err1 != nil {
return n, err1
}
}
if err != nil {
break
}
}
if err == io.EOF {
err = nil
}
return n, err
}
// buffered input and output
// ReadWriter stores pointers to a Reader and a Writer.

173
libgo/go/bufio/bufio_test.go
View file

@ -763,8 +763,8 @@ func testReadLineNewlines(t *testing.T, input string, expect []readLineResult) {
}
}
func TestReaderWriteTo(t *testing.T) {
input := make([]byte, 8192)
func createTestInput(n int) []byte {
input := make([]byte, n)
for i := range input {
// 101 and 251 are arbitrary prime numbers.
// The idea is to create an input sequence
@ -774,7 +774,12 @@ func TestReaderWriteTo(t *testing.T) {
input[i] ^= byte(i / 101)
}
}
r := NewReader(bytes.NewBuffer(input))
return input
}
func TestReaderWriteTo(t *testing.T) {
input := createTestInput(8192)
r := NewReader(onlyReader{bytes.NewBuffer(input)})
w := new(bytes.Buffer)
if n, err := r.WriteTo(w); err != nil || n != int64(len(input)) {
t.Fatalf("r.WriteTo(w) = %d, %v, want %d, nil", n, err, len(input))
@ -817,12 +822,129 @@ func TestReaderWriteToErrors(t *testing.T) {
}
}
func TestWriterReadFrom(t *testing.T) {
ws := []func(io.Writer) io.Writer{
func(w io.Writer) io.Writer { return onlyWriter{w} },
func(w io.Writer) io.Writer { return w },
}
rs := []func(io.Reader) io.Reader{
iotest.DataErrReader,
func(r io.Reader) io.Reader { return r },
}
for ri, rfunc := range rs {
for wi, wfunc := range ws {
input := createTestInput(8192)
b := new(bytes.Buffer)
w := NewWriter(wfunc(b))
r := rfunc(bytes.NewBuffer(input))
if n, err := w.ReadFrom(r); err != nil || n != int64(len(input)) {
t.Errorf("ws[%d],rs[%d]: w.ReadFrom(r) = %d, %v, want %d, nil", wi, ri, n, err, len(input))
continue
}
if got, want := b.String(), string(input); got != want {
t.Errorf("ws[%d], rs[%d]:\ngot %q\nwant %q\n", wi, ri, got, want)
}
}
}
}
type errorReaderFromTest struct {
rn, wn int
rerr, werr error
expected error
}
func (r errorReaderFromTest) Read(p []byte) (int, error) {
return len(p) * r.rn, r.rerr
}
func (w errorReaderFromTest) Write(p []byte) (int, error) {
return len(p) * w.wn, w.werr
}
var errorReaderFromTests = []errorReaderFromTest{
{0, 1, io.EOF, nil, nil},
{1, 1, io.EOF, nil, nil},
{0, 1, io.ErrClosedPipe, nil, io.ErrClosedPipe},
{0, 0, io.ErrClosedPipe, io.ErrShortWrite, io.ErrClosedPipe},
{1, 0, nil, io.ErrShortWrite, io.ErrShortWrite},
}
func TestWriterReadFromErrors(t *testing.T) {
for i, rw := range errorReaderFromTests {
w := NewWriter(rw)
if _, err := w.ReadFrom(rw); err != rw.expected {
t.Errorf("w.ReadFrom(errorReaderFromTests[%d]) = _, %v, want _,%v", i, err, rw.expected)
}
}
}
// TestWriterReadFromCounts tests that using io.Copy to copy into a
// bufio.Writer does not prematurely flush the buffer. For example, when
// buffering writes to a network socket, excessive network writes should be
// avoided.
func TestWriterReadFromCounts(t *testing.T) {
var w0 writeCountingDiscard
b0 := NewWriterSize(&w0, 1234)
b0.WriteString(strings.Repeat("x", 1000))
if w0 != 0 {
t.Fatalf("write 1000 'x's: got %d writes, want 0", w0)
}
b0.WriteString(strings.Repeat("x", 200))
if w0 != 0 {
t.Fatalf("write 1200 'x's: got %d writes, want 0", w0)
}
io.Copy(b0, onlyReader{strings.NewReader(strings.Repeat("x", 30))})
if w0 != 0 {
t.Fatalf("write 1230 'x's: got %d writes, want 0", w0)
}
io.Copy(b0, onlyReader{strings.NewReader(strings.Repeat("x", 9))})
if w0 != 1 {
t.Fatalf("write 1239 'x's: got %d writes, want 1", w0)
}
var w1 writeCountingDiscard
b1 := NewWriterSize(&w1, 1234)
b1.WriteString(strings.Repeat("x", 1200))
b1.Flush()
if w1 != 1 {
t.Fatalf("flush 1200 'x's: got %d writes, want 1", w1)
}
b1.WriteString(strings.Repeat("x", 89))
if w1 != 1 {
t.Fatalf("write 1200 + 89 'x's: got %d writes, want 1", w1)
}
io.Copy(b1, onlyReader{strings.NewReader(strings.Repeat("x", 700))})
if w1 != 1 {
t.Fatalf("write 1200 + 789 'x's: got %d writes, want 1", w1)
}
io.Copy(b1, onlyReader{strings.NewReader(strings.Repeat("x", 600))})
if w1 != 2 {
t.Fatalf("write 1200 + 1389 'x's: got %d writes, want 2", w1)
}
b1.Flush()
if w1 != 3 {
t.Fatalf("flush 1200 + 1389 'x's: got %d writes, want 3", w1)
}
}
// A writeCountingDiscard is like ioutil.Discard and counts the number of times
// Write is called on it.
type writeCountingDiscard int
func (w *writeCountingDiscard) Write(p []byte) (int, error) {
*w++
return len(p), nil
}
// An onlyReader only implements io.Reader, no matter what other methods the underlying implementation may have.
type onlyReader struct {
r io.Reader
}
func (r *onlyReader) Read(b []byte) (int, error) {
func (r onlyReader) Read(b []byte) (int, error) {
return r.r.Read(b)
}
@ -831,7 +953,7 @@ type onlyWriter struct {
w io.Writer
}
func (w *onlyWriter) Write(b []byte) (int, error) {
func (w onlyWriter) Write(b []byte) (int, error) {
return w.w.Write(b)
}
@ -840,7 +962,7 @@ func BenchmarkReaderCopyOptimal(b *testing.B) {
for i := 0; i < b.N; i++ {
b.StopTimer()
src := NewReader(bytes.NewBuffer(make([]byte, 8192)))
dst := &onlyWriter{new(bytes.Buffer)}
dst := onlyWriter{new(bytes.Buffer)}
b.StartTimer()
io.Copy(dst, src)
}
@ -850,8 +972,8 @@ func BenchmarkReaderCopyUnoptimal(b *testing.B) {
// Unoptimal case is where the underlying reader doesn't implement io.WriterTo
for i := 0; i < b.N; i++ {
b.StopTimer()
src := NewReader(&onlyReader{bytes.NewBuffer(make([]byte, 8192))})
dst := &onlyWriter{new(bytes.Buffer)}
src := NewReader(onlyReader{bytes.NewBuffer(make([]byte, 8192))})
dst := onlyWriter{new(bytes.Buffer)}
b.StartTimer()
io.Copy(dst, src)
}
@ -860,8 +982,39 @@ func BenchmarkReaderCopyUnoptimal(b *testing.B) {
func BenchmarkReaderCopyNoWriteTo(b *testing.B) {
for i := 0; i < b.N; i++ {
b.StopTimer()
src := &onlyReader{NewReader(bytes.NewBuffer(make([]byte, 8192)))}
dst := &onlyWriter{new(bytes.Buffer)}
src := onlyReader{NewReader(bytes.NewBuffer(make([]byte, 8192)))}
dst := onlyWriter{new(bytes.Buffer)}
b.StartTimer()
io.Copy(dst, src)
}
}
func BenchmarkWriterCopyOptimal(b *testing.B) {
// Optimal case is where the underlying writer implements io.ReaderFrom
for i := 0; i < b.N; i++ {
b.StopTimer()
src := onlyReader{bytes.NewBuffer(make([]byte, 8192))}
dst := NewWriter(new(bytes.Buffer))
b.StartTimer()
io.Copy(dst, src)
}
}
func BenchmarkWriterCopyUnoptimal(b *testing.B) {
for i := 0; i < b.N; i++ {
b.StopTimer()
src := onlyReader{bytes.NewBuffer(make([]byte, 8192))}
dst := NewWriter(onlyWriter{new(bytes.Buffer)})
b.StartTimer()
io.Copy(dst, src)
}
}
func BenchmarkWriterCopyNoReadFrom(b *testing.B) {
for i := 0; i < b.N; i++ {
b.StopTimer()
src := onlyReader{bytes.NewBuffer(make([]byte, 8192))}
dst := onlyWriter{NewWriter(new(bytes.Buffer))}
b.StartTimer()
io.Copy(dst, src)
}

4
libgo/go/bytes/buffer_test.go
View file

@ -251,10 +251,10 @@ func TestReadFrom(t *testing.T) {
func TestWriteTo(t *testing.T) {
var buf Buffer
for i := 3; i < 30; i += 3 {
s := fillBytes(t, "TestReadFrom (1)", &buf, "", 5, testBytes[0:len(testBytes)/i])
s := fillBytes(t, "TestWriteTo (1)", &buf, "", 5, testBytes[0:len(testBytes)/i])
var b Buffer
buf.WriteTo(&b)
empty(t, "TestReadFrom (2)", &b, s, make([]byte, len(data)))
empty(t, "TestWriteTo (2)", &b, s, make([]byte, len(data)))
}
}

2
libgo/go/bytes/bytes_test.go
View file

@ -1026,7 +1026,7 @@ func TestEqualFold(t *testing.T) {
var makeFieldsInput = func() []byte {
x := make([]byte, 1<<20)
// Input is ~10% space, ~10% 2-byte UTF-8, rest ASCII non-space.
// Input is ~10% space, ~10% 2-byte UTF-8, rest ASCII non-space.
for i := range x {
switch rand.Intn(10) {
case 0:

21
libgo/go/bytes/reader.go
View file

@ -10,7 +10,7 @@ import (
"unicode/utf8"
)
// A Reader implements the io.Reader, io.ReaderAt, io.Seeker,
// A Reader implements the io.Reader, io.ReaderAt, io.WriterTo, io.Seeker,
// io.ByteScanner, and io.RuneScanner interfaces by reading from
// a byte slice.
// Unlike a Buffer, a Reader is read-only and supports seeking.
@ -121,5 +121,24 @@ func (r *Reader) Seek(offset int64, whence int) (int64, error) {
return abs, nil
}
// WriteTo implements the io.WriterTo interface.
func (r *Reader) WriteTo(w io.Writer) (n int64, err error) {
r.prevRune = -1
if r.i >= len(r.s) {
return 0, io.EOF
}
b := r.s[r.i:]
m, err := w.Write(b)
if m > len(b) {
panic("bytes.Reader.WriteTo: invalid Write count")
}
r.i += m
n = int64(m)
if m != len(b) && err == nil {
err = io.ErrShortWrite
}
return
}
// NewReader returns a new Reader reading from b.
func NewReader(b []byte) *Reader { return &Reader{b, 0, -1} }

21
libgo/go/bytes/reader_test.go
View file

@ -86,3 +86,24 @@ func TestReaderAt(t *testing.T) {
}
}
}
func TestReaderWriteTo(t *testing.T) {
for i := 3; i < 30; i += 3 {
s := data[:len(data)/i]
r := NewReader(testBytes[:len(testBytes)/i])
var b Buffer
n, err := r.WriteTo(&b)
if expect := int64(len(s)); n != expect {
t.Errorf("got %v; want %v", n, expect)
}
if err != nil {
t.Errorf("got error = %v; want nil", err)
}
if b.String() != s {
t.Errorf("got string %q; want %q", b.String(), s)
}
if r.Len() != 0 {
t.Errorf("reader contains %v bytes; want 0", r.Len())
}
}
}

13
libgo/go/compress/bzip2/bit_reader.go
View file

@ -14,21 +14,16 @@ import (
// because the error handling was verbose. Instead, any error is kept and can
// be checked afterwards.
type bitReader struct {
r byteReader
r io.ByteReader
n uint64
bits uint
err error
}
// bitReader needs to read bytes from an io.Reader. We attempt to convert the
// given io.Reader to this interface and, if it doesn't already fit, we wrap in
// a bufio.Reader.
type byteReader interface {
ReadByte() (byte, error)
}
// newBitReader returns a new bitReader reading from r. If r is not
// already an io.ByteReader, it will be converted via a bufio.Reader.
func newBitReader(r io.Reader) bitReader {
byter, ok := r.(byteReader)
byter, ok := r.(io.ByteReader)
if !ok {
byter = bufio.NewReader(r)
}

8
libgo/go/compress/flate/inflate.go
View file

@ -208,8 +208,8 @@ type decompressor struct {
h1, h2 huffmanDecoder
// Length arrays used to define Huffman codes.
bits [maxLit + maxDist]int
codebits [numCodes]int
bits *[maxLit + maxDist]int
codebits *[numCodes]int
// Output history, buffer.
hist *[maxHist]byte
@ -692,6 +692,8 @@ func makeReader(r io.Reader) Reader {
// finished reading.
func NewReader(r io.Reader) io.ReadCloser {
var f decompressor
f.bits = new([maxLit + maxDist]int)
f.codebits = new([numCodes]int)
f.r = makeReader(r)
f.hist = new([maxHist]byte)
f.step = (*decompressor).nextBlock
@ -707,6 +709,8 @@ func NewReaderDict(r io.Reader, dict []byte) io.ReadCloser {
var f decompressor
f.r = makeReader(r)
f.hist = new([maxHist]byte)
f.bits = new([maxLit + maxDist]int)
f.codebits = new([numCodes]int)
f.step = (*decompressor).nextBlock
f.setDict(dict)
return &f

2
libgo/go/container/list/list.go
View file

@ -176,7 +176,7 @@ func (l *List) MoveToBack(e *Element) {
l.insert(l.remove(e), l.root.prev)
}
// PuchBackList inserts a copy of an other list at the back of list l.
// PushBackList inserts a copy of an other list at the back of list l.
// The lists l and other may be the same.
func (l *List) PushBackList(other *List) {
l.lazyInit()

6
libgo/go/crypto/cipher/cbc.go
View file

@ -33,6 +33,9 @@ type cbcEncrypter cbc
// mode, using the given Block. The length of iv must be the same as the
// Block's block size.
func NewCBCEncrypter(b Block, iv []byte) BlockMode {
if len(iv) != b.BlockSize() {
panic("cipher.NewCBCEncrypter: IV length must equal block size")
}
return (*cbcEncrypter)(newCBC(b, iv))
}
@ -58,6 +61,9 @@ type cbcDecrypter cbc
// mode, using the given Block. The length of iv must be the same as the
// Block's block size and must match the iv used to encrypt the data.
func NewCBCDecrypter(b Block, iv []byte) BlockMode {
if len(iv) != b.BlockSize() {
panic("cipher.NewCBCDecrypter: IV length must equal block size")
}
return (*cbcDecrypter)(newCBC(b, iv))
}

6
libgo/go/crypto/cipher/cfb.go
View file

@ -17,6 +17,9 @@ type cfb struct {
// using the given Block. The iv must be the same length as the Block's block
// size.
func NewCFBEncrypter(block Block, iv []byte) Stream {
if len(iv) != block.BlockSize() {
panic("cipher.NewCBFEncrypter: IV length must equal block size")
}
return newCFB(block, iv, false)
}
@ -24,6 +27,9 @@ func NewCFBEncrypter(block Block, iv []byte) Stream {
// using the given Block. The iv must be the same length as the Block's block
// size.
func NewCFBDecrypter(block Block, iv []byte) Stream {
if len(iv) != block.BlockSize() {
panic("cipher.NewCBFEncrypter: IV length must equal block size")
}
return newCFB(block, iv, true)
}

2
libgo/go/crypto/cipher/ctr.go
View file

@ -23,7 +23,7 @@ type ctr struct {
// counter mode. The length of iv must be the same as the Block's block size.
func NewCTR(block Block, iv []byte) Stream {
if len(iv) != block.BlockSize() {
panic("cipher.NewCTR: iv length must equal block size")
panic("cipher.NewCTR: IV length must equal block size")
}
return &ctr{

283
libgo/go/crypto/cipher/example_test.go Normal file
View file

@ -0,0 +1,283 @@
// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package cipher_test
import (
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"encoding/hex"
"fmt"
"io"
"os"
)
func ExampleNewCBCDecrypter() {
key := []byte("example key 1234")
ciphertext, _ := hex.DecodeString("f363f3ccdcb12bb883abf484ba77d9cd7d32b5baecb3d4b1b3e0e4beffdb3ded")
block, err := aes.NewCipher(key)
if err != nil {
panic(err)
}
// The IV needs to be unique, but not secure. Therefore it's common to
// include it at the beginning of the ciphertext.
if len(ciphertext) < aes.BlockSize {
panic("ciphertext too short")
}
iv := ciphertext[:aes.BlockSize]
ciphertext = ciphertext[aes.BlockSize:]
// CBC mode always works in whole blocks.
if len(ciphertext)%aes.BlockSize != 0 {
panic("ciphertext is not a multiple of the block size")
}
mode := cipher.NewCBCDecrypter(block, iv)
// CryptBlocks can work in-place if the two arguments are the same.
mode.CryptBlocks(ciphertext, ciphertext)
// If the original plaintext lengths are not a multiple of the block
// size, padding would have to be added when encrypting, which would be
// removed at this point. For an example, see
// https://tools.ietf.org/html/rfc5246#section-6.2.3.2. However, it's
// critical to note that ciphertexts must be authenticated (i.e. by
// using crypto/hmac) before being decrypted in order to avoid creating
// a padding oracle.
fmt.Printf("%s\n", ciphertext)
// Output: exampleplaintext
}
func ExampleNewCBCEncrypter() {
key := []byte("example key 1234")
plaintext := []byte("exampleplaintext")
// CBC mode works on blocks so plaintexts may need to be padded to the
// next whole block. For an example of such padding, see
// https://tools.ietf.org/html/rfc5246#section-6.2.3.2. Here we'll
// assume that the plaintext is already of the correct length.
if len(plaintext)%aes.BlockSize != 0 {
panic("plaintext is not a multiple of the block size")
}
block, err := aes.NewCipher(key)
if err != nil {
panic(err)
}
// The IV needs to be unique, but not secure. Therefore it's common to
// include it at the beginning of the ciphertext.
ciphertext := make([]byte, aes.BlockSize+len(plaintext))
iv := ciphertext[:aes.BlockSize]
if _, err := io.ReadFull(rand.Reader, iv); err != nil {
panic(err)
}
mode := cipher.NewCBCEncrypter(block, iv)
mode.CryptBlocks(ciphertext[aes.BlockSize:], plaintext)
// It's important to remember that ciphertexts must be authenticated
// (i.e. by using crypto/hmac) as well as being encrypted in order to
// be secure.
fmt.Printf("%x\n", ciphertext)
}
func ExampleNewCFBDecrypter() {
key := []byte("example key 1234")
ciphertext, _ := hex.DecodeString("22277966616d9bc47177bd02603d08c9a67d5380d0fe8cf3b44438dff7b9")
block, err := aes.NewCipher(key)
if err != nil {
panic(err)
}
// The IV needs to be unique, but not secure. Therefore it's common to
// include it at the beginning of the ciphertext.
if len(ciphertext) < aes.BlockSize {
panic("ciphertext too short")
}
iv := ciphertext[:aes.BlockSize]
ciphertext = ciphertext[aes.BlockSize:]
stream := cipher.NewCFBDecrypter(block, iv)
// XORKeyStream can work in-place if the two arguments are the same.
stream.XORKeyStream(ciphertext, ciphertext)
fmt.Printf("%s", ciphertext)
// Output: some plaintext
}
func ExampleNewCFBEncrypter() {
key := []byte("example key 1234")
plaintext := []byte("some plaintext")
block, err := aes.NewCipher(key)
if err != nil {
panic(err)
}
// The IV needs to be unique, but not secure. Therefore it's common to
// include it at the beginning of the ciphertext.
ciphertext := make([]byte, aes.BlockSize+len(plaintext))
iv := ciphertext[:aes.BlockSize]
if _, err := io.ReadFull(rand.Reader, iv); err != nil {
panic(err)
}
stream := cipher.NewCFBEncrypter(block, iv)
stream.XORKeyStream(ciphertext[aes.BlockSize:], plaintext)
// It's important to remember that ciphertexts must be authenticated
// (i.e. by using crypto/hmac) as well as being encrypted in order to
// be secure.
}
func ExampleNewCTR() {
key := []byte("example key 1234")
plaintext := []byte("some plaintext")
block, err := aes.NewCipher(key)
if err != nil {
panic(err)
}
// The IV needs to be unique, but not secure. Therefore it's common to
// include it at the beginning of the ciphertext.
ciphertext := make([]byte, aes.BlockSize+len(plaintext))
iv := ciphertext[:aes.BlockSize]
if _, err := io.ReadFull(rand.Reader, iv); err != nil {
panic(err)
}
stream := cipher.NewCTR(block, iv)
stream.XORKeyStream(ciphertext[aes.BlockSize:], plaintext)
// It's important to remember that ciphertexts must be authenticated
// (i.e. by using crypto/hmac) as well as being encrypted in order to
// be secure.
// CTR mode is the same for both encryption and decryption, so we can
// also decrypt that ciphertext with NewCTR.
plaintext2 := make([]byte, len(plaintext))
stream = cipher.NewCTR(block, iv)
stream.XORKeyStream(plaintext2, ciphertext[aes.BlockSize:])
fmt.Printf("%s\n", plaintext2)
// Output: some plaintext
}
func ExampleNewOFB() {
key := []byte("example key 1234")
plaintext := []byte("some plaintext")
block, err := aes.NewCipher(key)
if err != nil {
panic(err)
}
// The IV needs to be unique, but not secure. Therefore it's common to
// include it at the beginning of the ciphertext.
ciphertext := make([]byte, aes.BlockSize+len(plaintext))
iv := ciphertext[:aes.BlockSize]
if _, err := io.ReadFull(rand.Reader, iv); err != nil {
panic(err)
}
stream := cipher.NewOFB(block, iv)
stream.XORKeyStream(ciphertext[aes.BlockSize:], plaintext)
// It's important to remember that ciphertexts must be authenticated
// (i.e. by using crypto/hmac) as well as being encrypted in order to
// be secure.
// OFB mode is the same for both encryption and decryption, so we can
// also decrypt that ciphertext with NewOFB.
plaintext2 := make([]byte, len(plaintext))
stream = cipher.NewOFB(block, iv)
stream.XORKeyStream(plaintext2, ciphertext[aes.BlockSize:])
fmt.Printf("%s\n", plaintext2)
// Output: some plaintext
}
func ExampleStreamReader() {
key := []byte("example key 1234")
inFile, err := os.Open("encrypted-file")
if err != nil {
panic(err)
}
defer inFile.Close()
block, err := aes.NewCipher(key)
if err != nil {
panic(err)
}
// If the key is unique for each ciphertext, then it's ok to use a zero
// IV.
var iv [aes.BlockSize]byte
stream := cipher.NewOFB(block, iv[:])
outFile, err := os.OpenFile("decrypted-file", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
panic(err)
}
defer outFile.Close()
reader := &cipher.StreamReader{stream, inFile}
// Copy the input file to the output file, decrypting as we go.
if _, err := io.Copy(outFile, reader); err != nil {
panic(err)
}
// Note that this example is simplistic in that it omits any
// authentication of the encrypted data. It you were actually to use
// StreamReader in this manner, an attacker could flip arbitary bits in
// the output.
}
func ExampleStreamWriter() {
key := []byte("example key 1234")
inFile, err := os.Open("plaintext-file")
if err != nil {
panic(err)
}
defer inFile.Close()
block, err := aes.NewCipher(key)
if err != nil {
panic(err)
}
// If the key is unique for each ciphertext, then it's ok to use a zero
// IV.
var iv [aes.BlockSize]byte
stream := cipher.NewOFB(block, iv[:])
outFile, err := os.OpenFile("encrypted-file", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
panic(err)
}
defer outFile.Close()
writer := &cipher.StreamWriter{stream, outFile, nil}
// Copy the input file to the output file, encrypting as we go.
if _, err := io.Copy(writer, inFile); err != nil {
panic(err)
}
// Note that this example is simplistic in that it omits any
// authentication of the encrypted data. It you were actually to use
// StreamReader in this manner, an attacker could flip arbitary bits in
// the decrypted result.
}

32
libgo/go/crypto/hmac/hmac.go
View file

@ -2,13 +2,27 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package hmac implements the Keyed-Hash Message Authentication Code (HMAC) as
// defined in U.S. Federal Information Processing Standards Publication 198.
// An HMAC is a cryptographic hash that uses a key to sign a message.
// The receiver verifies the hash by recomputing it using the same key.
/*
Package hmac implements the Keyed-Hash Message Authentication Code (HMAC) as
defined in U.S. Federal Information Processing Standards Publication 198.
An HMAC is a cryptographic hash that uses a key to sign a message.
The receiver verifies the hash by recomputing it using the same key.
Receivers should be careful to use Equal to compare MACs in order to avoid
timing side-channels:
// CheckMAC returns true if messageMAC is a valid HMAC tag for message.
func CheckMAC(message, messageMAC, key []byte) bool {
mac := hmac.New(sha256.New, key)
mac.Write(message)
expectedMAC := mac.Sum(nil)
return hmac.Equal(messageMAC, expectedMAC)
}
*/
package hmac
import (
"crypto/subtle"
"hash"
)
@ -57,7 +71,7 @@ func (h *hmac) BlockSize() int { return h.blocksize }
func (h *hmac) Reset() {
h.inner.Reset()
h.tmpPad(0x36)
h.inner.Write(h.tmp[0:h.blocksize])
h.inner.Write(h.tmp[:h.blocksize])
}
// New returns a new HMAC hash using the given hash.Hash type and key.
@ -78,3 +92,11 @@ func New(h func() hash.Hash, key []byte) hash.Hash {
hm.Reset()
return hm
}
// Equal compares two MACs for equality without leaking timing information.
func Equal(mac1, mac2 []byte) bool {
// We don't have to be constant time if the lengths of the MACs are
// different as that suggests that a completely different hash function
// was used.
return len(mac1) == len(mac2) && subtle.ConstantTimeCompare(mac1, mac2) == 1
}

19
libgo/go/crypto/hmac/hmac_test.go
View file

@ -491,3 +491,22 @@ func TestHMAC(t *testing.T) {
}
}
}
func TestEqual(t *testing.T) {
a := []byte("test")
b := []byte("test1")
c := []byte("test2")
if !Equal(b, b) {
t.Error("Equal failed with equal arguments")
}
if Equal(a, b) {
t.Error("Equal accepted a prefix of the second argument")
}
if Equal(b, a) {
t.Error("Equal accepted a prefix of the first argument")
}
if Equal(b, c) {
t.Error("Equal accepted unequal slices")
}
}

2
libgo/go/crypto/md5/gen.go
View file

@ -203,6 +203,8 @@ func block(dig *digest, p []byte) {
// less code and run 1.3x faster if we take advantage of that.
// My apologies.
X = (*[16]uint32)(unsafe.Pointer(&p[0]))
} else if uintptr(unsafe.Pointer(&p[0]))&(unsafe.Alignof(uint32(0))-1) == 0 {
X = (*[16]uint32)(unsafe.Pointer(&p[0]))
} else {
X = &xbuf
j := 0

58
libgo/go/crypto/md5/md5_test.go
View file

@ -9,6 +9,7 @@ import (
"fmt"
"io"
"testing"
"unsafe"
)
type md5Test struct {
@ -54,13 +55,19 @@ func TestGolden(t *testing.T) {
for i := 0; i < len(golden); i++ {
g := golden[i]
c := md5.New()
for j := 0; j < 3; j++ {
buf := make([]byte, len(g.in)+4)
for j := 0; j < 3+4; j++ {
if j < 2 {
io.WriteString(c, g.in)
} else {
} else if j == 2 {
io.WriteString(c, g.in[0:len(g.in)/2])
c.Sum(nil)
io.WriteString(c, g.in[len(g.in)/2:])
} else if j > 2 {
// test unaligned write
buf = buf[1:]
copy(buf, g.in)
c.Write(buf[:len(g.in)])
}
s := fmt.Sprintf("%x", c.Sum(nil))
if s != g.out {
@ -80,26 +87,45 @@ func ExampleNew() {
}
var bench = md5.New()
var buf = makeBuf()
var buf = make([]byte, 8192+1)
var sum = make([]byte, bench.Size())
func makeBuf() []byte {
b := make([]byte, 8<<10)
for i := range b {
b[i] = byte(i)
func benchmarkSize(b *testing.B, size int, unaligned bool) {
b.SetBytes(int64(size))
buf := buf
if unaligned {
if uintptr(unsafe.Pointer(&buf[0]))&(unsafe.Alignof(uint32(0))-1) == 0 {
buf = buf[1:]
}
}
return b
b.ResetTimer()
for i := 0; i < b.N; i++ {
bench.Reset()
bench.Write(buf[:size])
bench.Sum(sum[:0])
}
}
func BenchmarkHash8Bytes(b *testing.B) {
benchmarkSize(b, 8, false)
}
func BenchmarkHash1K(b *testing.B) {
b.SetBytes(1024)
for i := 0; i < b.N; i++ {
bench.Write(buf[:1024])
}
benchmarkSize(b, 1024, false)
}
func BenchmarkHash8K(b *testing.B) {
b.SetBytes(int64(len(buf)))
for i := 0; i < b.N; i++ {
bench.Write(buf)
}
benchmarkSize(b, 8192, false)
}
func BenchmarkHash8BytesUnaligned(b *testing.B) {
benchmarkSize(b, 8, true)
}
func BenchmarkHash1KUnaligned(b *testing.B) {
benchmarkSize(b, 1024, true)
}
func BenchmarkHash8KUnaligned(b *testing.B) {
benchmarkSize(b, 8192, true)
}

2
libgo/go/crypto/md5/md5block.go
View file

@ -22,6 +22,8 @@ func block(dig *digest, p []byte) {
// less code and run 1.3x faster if we take advantage of that.
// My apologies.
X = (*[16]uint32)(unsafe.Pointer(&p[0]))
} else if uintptr(unsafe.Pointer(&p[0]))&(unsafe.Alignof(uint32(0))-1) == 0 {
X = (*[16]uint32)(unsafe.Pointer(&p[0]))
} else {
X = &xbuf
j := 0

2
libgo/go/crypto/rsa/rsa.go
View file

@ -61,7 +61,7 @@ type PrivateKey struct {
}
type PrecomputedValues struct {
Dp, Dq *big.Int // D mod (P-1) (or mod Q-1)
Dp, Dq *big.Int // D mod (P-1) (or mod Q-1)
Qinv *big.Int // Q^-1 mod Q
// CRTValues is used for the 3rd and subsequent primes. Due to a

4
libgo/go/crypto/rsa/rsa_test.go
View file

@ -116,7 +116,7 @@ func BenchmarkRSA2048Decrypt(b *testing.B) {
}
priv.Precompute()
c := fromBase10("1000")
c := fromBase10("8472002792838218989464636159316973636630013835787202418124758118372358261975764365740026024610403138425986214991379012696600761514742817632790916315594342398720903716529235119816755589383377471752116975374952783629225022962092351886861518911824745188989071172097120352727368980275252089141512321893536744324822590480751098257559766328893767334861211872318961900897793874075248286439689249972315699410830094164386544311554704755110361048571142336148077772023880664786019636334369759624917224888206329520528064315309519262325023881707530002540634660750469137117568199824615333883758410040459705787022909848740188613313")
b.StartTimer()
@ -141,7 +141,7 @@ func Benchmark3PrimeRSA2048Decrypt(b *testing.B) {
}
priv.Precompute()
c := fromBase10("1000")
c := fromBase10("8472002792838218989464636159316973636630013835787202418124758118372358261975764365740026024610403138425986214991379012696600761514742817632790916315594342398720903716529235119816755589383377471752116975374952783629225022962092351886861518911824745188989071172097120352727368980275252089141512321893536744324822590480751098257559766328893767334861211872318961900897793874075248286439689249972315699410830094164386544311554704755110361048571142336148077772023880664786019636334369759624917224888206329520528064315309519262325023881707530002540634660750469137117568199824615333883758410040459705787022909848740188613313")
b.StartTimer()

28
libgo/go/crypto/sha1/sha1_test.go
View file

@ -81,26 +81,26 @@ func ExampleNew() {
}
var bench = sha1.New()
var buf = makeBuf()
var buf = make([]byte, 8192)
func makeBuf() []byte {
b := make([]byte, 8<<10)
for i := range b {
b[i] = byte(i)
func benchmarkSize(b *testing.B, size int) {
b.SetBytes(int64(size))
sum := make([]byte, bench.Size())
for i := 0; i < b.N; i++ {
bench.Reset()
bench.Write(buf[:size])
bench.Sum(sum[:0])
}
return b
}
func BenchmarkHash8Bytes(b *testing.B) {
benchmarkSize(b, 8)
}
func BenchmarkHash1K(b *testing.B) {
b.SetBytes(1024)
for i := 0; i < b.N; i++ {
bench.Write(buf[:1024])
}
benchmarkSize(b, 1024)
}
func BenchmarkHash8K(b *testing.B) {
b.SetBytes(int64(len(buf)))
for i := 0; i < b.N; i++ {
bench.Write(buf)
}
benchmarkSize(b, 8192)
}

42
libgo/go/crypto/sha1/sha1block.go
View file

@ -16,7 +16,7 @@ const (
)
func block(dig *digest, p []byte) {
var w [80]uint32
var w [16]uint32
h0, h1, h2, h3, h4 := dig.h[0], dig.h[1], dig.h[2], dig.h[3], dig.h[4]
for len(p) >= chunk {
@ -26,42 +26,56 @@ func block(dig *digest, p []byte) {
j := i * 4
w[i] = uint32(p[j])<<24 | uint32(p[j+1])<<16 | uint32(p[j+2])<<8 | uint32(p[j+3])
}
for i := 16; i < 80; i++ {
tmp := w[i-3] ^ w[i-8] ^ w[i-14] ^ w[i-16]
w[i] = tmp<<1 | tmp>>(32-1)
}
a, b, c, d, e := h0, h1, h2, h3, h4
// Each of the four 20-iteration rounds
// differs only in the computation of f and
// the choice of K (_K0, _K1, etc).
for i := 0; i < 20; i++ {
i := 0
for ; i < 16; i++ {
f := b&c | (^b)&d
a5 := a<<5 | a>>(32-5)
b30 := b<<30 | b>>(32-30)
t := a5 + f + e + w[i] + _K0
t := a5 + f + e + w[i&0xf] + _K0
a, b, c, d, e = t, a, b30, c, d
}
for i := 20; i < 40; i++ {
for ; i < 20; i++ {
tmp := w[(i-3)&0xf] ^ w[(i-8)&0xf] ^ w[(i-14)&0xf] ^ w[(i)&0xf]
w[i&0xf] = tmp<<1 | tmp>>(32-1)
f := b&c | (^b)&d
a5 := a<<5 | a>>(32-5)
b30 := b<<30 | b>>(32-30)
t := a5 + f + e + w[i&0xf] + _K0
a, b, c, d, e = t, a, b30, c, d
}
for ; i < 40; i++ {
tmp := w[(i-3)&0xf] ^ w[(i-8)&0xf] ^ w[(i-14)&0xf] ^ w[(i)&0xf]
w[i&0xf] = tmp<<1 | tmp>>(32-1)
f := b ^ c ^ d
a5 := a<<5 | a>>(32-5)
b30 := b<<30 | b>>(32-30)
t := a5 + f + e + w[i] + _K1
t := a5 + f + e + w[i&0xf] + _K1
a, b, c, d, e = t, a, b30, c, d
}
for i := 40; i < 60; i++ {
f := b&c | b&d | c&d
for ; i < 60; i++ {
tmp := w[(i-3)&0xf] ^ w[(i-8)&0xf] ^ w[(i-14)&0xf] ^ w[(i)&0xf]
w[i&0xf] = tmp<<1 | tmp>>(32-1)
f := ((b | c) & d) | (b & c)
a5 := a<<5 | a>>(32-5)
b30 := b<<30 | b>>(32-30)
t := a5 + f + e + w[i] + _K2
t := a5 + f + e + w[i&0xf] + _K2
a, b, c, d, e = t, a, b30, c, d
}
for i := 60; i < 80; i++ {
for ; i < 80; i++ {
tmp := w[(i-3)&0xf] ^ w[(i-8)&0xf] ^ w[(i-14)&0xf] ^ w[(i)&0xf]
w[i&0xf] = tmp<<1 | tmp>>(32-1)
f := b ^ c ^ d
a5 := a<<5 | a>>(32-5)
b30 := b<<30 | b>>(32-30)
t := a5 + f + e + w[i] + _K3
t := a5 + f + e + w[i&0xf] + _K3
a, b, c, d, e = t, a, b30, c, d
}

28
libgo/go/crypto/sha256/sha256_test.go
View file

@ -125,26 +125,26 @@ func TestGolden(t *testing.T) {
}
var bench = New()
var buf = makeBuf()
var buf = make([]byte, 8192)
func makeBuf() []byte {
b := make([]byte, 8<<10)
for i := range b {
b[i] = byte(i)
func benchmarkSize(b *testing.B, size int) {
b.SetBytes(int64(size))
sum := make([]byte, bench.Size())
for i := 0; i < b.N; i++ {
bench.Reset()
bench.Write(buf[:size])
bench.Sum(sum[:0])
}
return b
}
func BenchmarkHash8Bytes(b *testing.B) {
benchmarkSize(b, 8)
}
func BenchmarkHash1K(b *testing.B) {
b.SetBytes(1024)
for i := 0; i < b.N; i++ {
bench.Write(buf[:1024])
}
benchmarkSize(b, 1024)
}
func BenchmarkHash8K(b *testing.B) {
b.SetBytes(int64(len(buf)))
for i := 0; i < b.N; i++ {
bench.Write(buf)
}
benchmarkSize(b, 8192)
}

28
libgo/go/crypto/sha512/sha512_test.go
View file

@ -125,26 +125,26 @@ func TestGolden(t *testing.T) {
}
var bench = New()
var buf = makeBuf()
var buf = make([]byte, 8192)
func makeBuf() []byte {
b := make([]byte, 8<<10)
for i := range b {
b[i] = byte(i)
func benchmarkSize(b *testing.B, size int) {
b.SetBytes(int64(size))
sum := make([]byte, bench.Size())
for i := 0; i < b.N; i++ {
bench.Reset()
bench.Write(buf[:size])
bench.Sum(sum[:0])
}
return b
}
func BenchmarkHash8Bytes(b *testing.B) {
benchmarkSize(b, 8)
}
func BenchmarkHash1K(b *testing.B) {
b.SetBytes(1024)
for i := 0; i < b.N; i++ {
bench.Write(buf[:1024])
}
benchmarkSize(b, 1024)
}
func BenchmarkHash8K(b *testing.B) {
b.SetBytes(int64(len(buf)))
for i := 0; i < b.N; i++ {
bench.Write(buf)
}
benchmarkSize(b, 8192)
}

8
libgo/go/crypto/tls/conn.go
View file

@ -513,7 +513,7 @@ Again:
// First message, be extra suspicious:
// this might not be a TLS client.
// Bail out before reading a full 'body', if possible.
// The current max version is 3.1.
// The current max version is 3.1.
// If the version is >= 16.0, it's probably not real.
// Similarly, a clientHello message encodes in
// well under a kilobyte. If the length is >= 12 kB,
@ -604,9 +604,11 @@ Again:
// sendAlert sends a TLS alert message.
// c.out.Mutex <= L.
func (c *Conn) sendAlertLocked(err alert) error {
c.tmp[0] = alertLevelError
if err == alertNoRenegotiation {
switch err {
case alertNoRenegotiation, alertCloseNotify:
c.tmp[0] = alertLevelWarning
default:
c.tmp[0] = alertLevelError
}
c.tmp[1] = byte(err)
c.writeRecord(recordTypeAlert, c.tmp[0:2])

100
libgo/go/crypto/tls/handshake_client_test.go
View file

@ -246,15 +246,15 @@ var ecdheAESClientScript = [][]byte{
},
{
0x16, 0x03, 0x01, 0x00, 0x54, 0x02, 0x00, 0x00,
0x50, 0x03, 0x01, 0x4f, 0x7f, 0x24, 0x25, 0x10,
0xa8, 0x9d, 0xb1, 0x33, 0xd6, 0x53, 0x81, 0xce,
0xb0, 0x69, 0xed, 0x1b, 0x9c, 0x5e, 0x40, 0x3a,
0x4d, 0x06, 0xbc, 0xc7, 0x84, 0x51, 0x5a, 0x30,
0x40, 0x50, 0x48, 0x20, 0xcd, 0x91, 0x80, 0x08,
0xff, 0x82, 0x38, 0xc6, 0x03, 0x2d, 0x45, 0x4c,
0x91, 0xbb, 0xcc, 0x27, 0x3d, 0x58, 0xff, 0x0d,
0x26, 0x34, 0x7b, 0x48, 0x7a, 0xce, 0x25, 0x20,
0x90, 0x0f, 0x35, 0x9f, 0xc0, 0x13, 0x00, 0x00,
0x50, 0x03, 0x01, 0x50, 0x77, 0x31, 0xf7, 0x5b,
0xdb, 0x3d, 0x7a, 0x62, 0x76, 0x70, 0x95, 0x33,
0x73, 0x71, 0x13, 0xfe, 0xa3, 0xb1, 0xd8, 0xb3,
0x4d, 0x0d, 0xdc, 0xfe, 0x58, 0x6e, 0x6a, 0x3a,
0xf9, 0xde, 0xdc, 0x20, 0x8e, 0xfa, 0x3d, 0x60,
0xd0, 0xda, 0xa4, 0x0e, 0x36, 0xf0, 0xde, 0xb6,
0x81, 0xb4, 0x80, 0x5e, 0xf9, 0xd2, 0x4c, 0xec,
0xd1, 0x9c, 0x2a, 0x81, 0xc3, 0x36, 0x0b, 0x0f,
0x4a, 0x3d, 0xdf, 0x75, 0xc0, 0x13, 0x00, 0x00,
0x08, 0x00, 0x0b, 0x00, 0x04, 0x03, 0x00, 0x01,
0x02, 0x16, 0x03, 0x01, 0x02, 0x39, 0x0b, 0x00,
0x02, 0x35, 0x00, 0x02, 0x32, 0x00, 0x02, 0x2f,
@ -329,23 +329,23 @@ var ecdheAESClientScript = [][]byte{
0xbb, 0x77, 0xba, 0xe4, 0x12, 0xbb, 0xf4, 0xc8,
0x5e, 0x9c, 0x81, 0xa8, 0x97, 0x60, 0x4c, 0x16,
0x03, 0x01, 0x00, 0x8b, 0x0c, 0x00, 0x00, 0x87,
0x03, 0x00, 0x17, 0x41, 0x04, 0x0b, 0xe5, 0x39,
0xde, 0x17, 0x7a, 0xaf, 0x96, 0xd5, 0x16, 0x01,
0xa8, 0x06, 0x80, 0x98, 0x75, 0x52, 0x56, 0x92,
0x15, 0xf9, 0x8d, 0xc0, 0x98, 0x62, 0xed, 0x54,
0xb7, 0xef, 0x03, 0x11, 0x34, 0x82, 0x65, 0xd1,
0xde, 0x25, 0x15, 0x4c, 0xf3, 0xdf, 0x4d, 0xbd,
0x6c, 0xed, 0x3d, 0xd6, 0x04, 0xcc, 0xd1, 0xf7,
0x6d, 0x32, 0xb1, 0x1c, 0x59, 0xca, 0xfb, 0xbc,
0x61, 0xeb, 0x4b, 0xe6, 0x00, 0x00, 0x40, 0x3e,
0xe6, 0x23, 0x54, 0x61, 0x3f, 0x63, 0x16, 0xeb,
0x5c, 0xc3, 0xba, 0x8a, 0x19, 0x13, 0x60, 0x9f,
0x23, 0xbf, 0x36, 0x1a, 0x32, 0x7a, 0xae, 0x34,
0x7f, 0x2f, 0x89, 0x85, 0xe1, 0x0e, 0x93, 0xd7,
0xf0, 0xab, 0xa1, 0x0d, 0x54, 0x95, 0x79, 0x0b,
0xb4, 0xf1, 0x1c, 0x1d, 0x0f, 0x8c, 0x16, 0xec,
0x82, 0x60, 0xee, 0xa3, 0x71, 0x2f, 0xaf, 0x3e,
0xf1, 0xbd, 0xb5, 0x1b, 0x7f, 0xe0, 0xd2, 0x16,
0x03, 0x00, 0x17, 0x41, 0x04, 0xec, 0x06, 0x1f,
0xa0, 0x5e, 0x29, 0x49, 0x71, 0x8b, 0x04, 0x9f,
0x47, 0x87, 0xb1, 0xcb, 0xae, 0x57, 0x8f, 0xd7,
0xf6, 0xf8, 0x59, 0x74, 0x64, 0x5d, 0x3a, 0x08,
0xaf, 0x20, 0xc6, 0xd9, 0xfc, 0x5e, 0x36, 0x8b,
0x62, 0x0e, 0xdb, 0xee, 0xd8, 0xcd, 0xef, 0x25,
0x8a, 0x38, 0x88, 0x2d, 0x5c, 0x71, 0x50, 0x22,
0xda, 0x3f, 0x94, 0x06, 0xc9, 0x68, 0x5b, 0x78,
0x3d, 0x95, 0xca, 0x54, 0x44, 0x00, 0x40, 0x36,
0xcf, 0x10, 0x81, 0xb4, 0x32, 0x45, 0x3c, 0xa5,
0x2d, 0x3e, 0xb0, 0xf8, 0xf4, 0x51, 0xf5, 0x28,
0x09, 0x85, 0x71, 0xa6, 0x79, 0x71, 0x4b, 0x4e,
0xda, 0x32, 0x5a, 0xc7, 0xb3, 0x57, 0xfd, 0xe8,
0x12, 0xab, 0xd8, 0x29, 0xfb, 0x8b, 0x43, 0x8f,
0x7e, 0x27, 0x63, 0x91, 0x84, 0x9c, 0x51, 0x0c,
0x26, 0x7e, 0x36, 0x3b, 0x37, 0x8d, 0x8f, 0x9e,
0xe2, 0x82, 0x62, 0xbb, 0xe5, 0xdf, 0xfc, 0x16,
0x03, 0x01, 0x00, 0x04, 0x0e, 0x00, 0x00, 0x00,
},
{
@ -359,34 +359,34 @@ var ecdheAESClientScript = [][]byte{
0xe2, 0x32, 0x42, 0xe9, 0x58, 0xb6, 0xd7, 0x49,
0xa6, 0xb5, 0x68, 0x1a, 0x41, 0x03, 0x56, 0x6b,
0xdc, 0x5a, 0x89, 0x14, 0x03, 0x01, 0x00, 0x01,
0x01, 0x16, 0x03, 0x01, 0x00, 0x30, 0x09, 0xac,
0xbe, 0x94, 0x75, 0x4d, 0x73, 0x45, 0xbd, 0xa8,
0x0c, 0xe3, 0x5f, 0x72, 0x0b, 0x40, 0x4f, 0xd0,
0xd2, 0xcb, 0x16, 0x50, 0xfe, 0xdd, 0x1a, 0x33,
0x5c, 0x18, 0x37, 0x98, 0x42, 0xfc, 0x25, 0x42,
0x33, 0xce, 0x60, 0xcf, 0x8e, 0x95, 0x6e, 0x48,
0xed, 0x00, 0x35, 0x50, 0x26, 0x7f,
0x01, 0x16, 0x03, 0x01, 0x00, 0x30, 0x9a, 0xaa,
0xca, 0x5b, 0x57, 0xae, 0x34, 0x92, 0x80, 0x45,
0x7f, 0xe6, 0xf9, 0x09, 0x19, 0xd0, 0xf0, 0x1e,
0x4b, 0xc3, 0xda, 0x71, 0xce, 0x34, 0x33, 0x56,
0x9f, 0x20, 0x9f, 0xf9, 0xa8, 0x62, 0x6c, 0x38,
0x1b, 0x41, 0xf5, 0x54, 0xf2, 0x79, 0x42, 0x6c,
0xb5, 0x0e, 0xe7, 0xe1, 0xbc, 0x54,
},
{
0x14, 0x03, 0x01, 0x00, 0x01, 0x01, 0x16, 0x03,
0x01, 0x00, 0x30, 0xf6, 0x6a, 0xdb, 0x83, 0xd4,
0x3c, 0x77, 0x52, 0xad, 0xc0, 0x0f, 0x3a, 0x2c,
0x42, 0xb9, 0x60, 0x4b, 0xb2, 0xf6, 0x84, 0xfd,
0x4e, 0x96, 0xfc, 0x15, 0xe7, 0x94, 0x25, 0xb0,
0x59, 0x9f, 0xdd, 0xb6, 0x58, 0x03, 0x13, 0x8d,
0xeb, 0xb0, 0xad, 0x30, 0x31, 0x58, 0x6c, 0xa0,
0x8f, 0x57, 0x50,
0x01, 0x00, 0x30, 0x62, 0x82, 0x41, 0x75, 0x2b,
0xee, 0x0f, 0xdc, 0x6c, 0x48, 0x5a, 0x63, 0xd6,
0xcb, 0x0a, 0xfd, 0x0a, 0x0e, 0xde, 0x8b, 0x41,
0x19, 0x0c, 0x13, 0x6b, 0x12, 0xd1, 0xc2, 0x53,
0xeb, 0x1e, 0xf3, 0x7a, 0xbf, 0x23, 0xc5, 0xa6,
0x81, 0xa1, 0xdb, 0xab, 0x2f, 0x2c, 0xbc, 0x35,
0x96, 0x72, 0x83,
},
{
0x17, 0x03, 0x01, 0x00, 0x20, 0xab, 0x64, 0x3d,
0x79, 0x69, 0x3e, 0xba, 0xc4, 0x24, 0x7b, 0xe5,
0xe5, 0x23, 0x66, 0x6f, 0x32, 0xdf, 0x50, 0x7c,
0x06, 0x2a, 0x02, 0x82, 0x79, 0x40, 0xdb, 0xb1,
0x04, 0xc0, 0x2b, 0xdc, 0x3a, 0x15, 0x03, 0x01,
0x00, 0x20, 0xf8, 0xad, 0xca, 0xd7, 0x96, 0xf0,
0xd6, 0xa3, 0x62, 0xe1, 0x03, 0x44, 0xdb, 0xd0,
0xc9, 0x63, 0x3e, 0x1b, 0x70, 0x41, 0x57, 0x0c,
0xd8, 0x8e, 0x71, 0x49, 0x68, 0xe3, 0x04, 0x53,
0x5a, 0xbe,
0x17, 0x03, 0x01, 0x00, 0x20, 0xaf, 0x5d, 0x35,
0x57, 0x10, 0x60, 0xb3, 0x25, 0x7c, 0x26, 0x0f,
0xf3, 0x5e, 0xb3, 0x0d, 0xad, 0x14, 0x53, 0xcc,
0x0c, 0x08, 0xd9, 0xa2, 0x67, 0xab, 0xf4, 0x03,
0x17, 0x20, 0xf1, 0x7e, 0xca, 0x15, 0x03, 0x01,
0x00, 0x20, 0x30, 0xd0, 0xc1, 0xfb, 0x5f, 0xa6,
0x1b, 0xb4, 0x48, 0xc2, 0x0b, 0x98, 0xa8, 0x88,
0x7a, 0xba, 0xdf, 0x36, 0x06, 0xd8, 0xcc, 0xe9,
0x34, 0xdd, 0x64, 0xc8, 0x73, 0xc5, 0xa2, 0x34,
0x64, 0xb7,
},
}

1666
libgo/go/crypto/tls/handshake_server_test.go
View file

File diff suppressed because it is too large Load diff

2
libgo/go/crypto/tls/prf_test.go
View file

@ -51,7 +51,7 @@ func TestKeysFromPreMasterSecret(t *testing.T) {
masterSecret := masterFromPreMasterSecret(test.version, in, clientRandom, serverRandom)
if s := hex.EncodeToString(masterSecret); s != test.masterSecret {
t.Errorf("#%d: bad master secret %s, want %s", s, test.masterSecret)
t.Errorf("#%d: bad master secret %s, want %s", i, s, test.masterSecret)
continue
}

72
libgo/go/crypto/tls/tls.go
View file

@ -6,6 +6,8 @@
package tls
import (
"crypto"
"crypto/ecdsa"
"crypto/rsa"
"crypto/x509"
"encoding/pem"
@ -153,30 +155,16 @@ func X509KeyPair(certPEMBlock, keyPEMBlock []byte) (cert Certificate, err error)
err = errors.New("crypto/tls: failed to parse key PEM data")
return
}
if keyDERBlock.Type != "CERTIFICATE" {
if strings.HasSuffix(keyDERBlock.Type, " PRIVATE KEY") {
break
}
}
// OpenSSL 0.9.8 generates PKCS#1 private keys by default, while
// OpenSSL 1.0.0 generates PKCS#8 keys. We try both.
var key *rsa.PrivateKey
if key, err = x509.ParsePKCS1PrivateKey(keyDERBlock.Bytes); err != nil {
var privKey interface{}
if privKey, err = x509.ParsePKCS8PrivateKey(keyDERBlock.Bytes); err != nil {
err = errors.New("crypto/tls: failed to parse key: " + err.Error())
return
}
var ok bool
if key, ok = privKey.(*rsa.PrivateKey); !ok {
err = errors.New("crypto/tls: found non-RSA private key in PKCS#8 wrapping")
return
}
cert.PrivateKey, err = parsePrivateKey(keyDERBlock.Bytes)
if err != nil {
return
}
cert.PrivateKey = key
// We don't need to parse the public key for TLS, but we so do anyway
// to check that it looks sane and matches the private key.
x509Cert, err := x509.ParseCertificate(cert.Certificate[0])
@ -184,10 +172,54 @@ func X509KeyPair(certPEMBlock, keyPEMBlock []byte) (cert Certificate, err error)
return
}
if x509Cert.PublicKeyAlgorithm != x509.RSA || x509Cert.PublicKey.(*rsa.PublicKey).N.Cmp(key.PublicKey.N) != 0 {
err = errors.New("crypto/tls: private key does not match public key")
switch pub := x509Cert.PublicKey.(type) {
case *rsa.PublicKey:
priv, ok := cert.PrivateKey.(*rsa.PrivateKey)
if !ok {
err = errors.New("crypto/tls: private key type does not match public key type")
return
}
if pub.N.Cmp(priv.N) != 0 {
err = errors.New("crypto/tls: private key does not match public key")
return
}
case *ecdsa.PublicKey:
priv, ok := cert.PrivateKey.(*ecdsa.PrivateKey)
if !ok {
err = errors.New("crypto/tls: private key type does not match public key type")
return
}
if pub.X.Cmp(priv.X) != 0 || pub.Y.Cmp(priv.Y) != 0 {
err = errors.New("crypto/tls: private key does not match public key")
return
}
default:
err = errors.New("crypto/tls: unknown public key algorithm")
return
}
return
}
// Attempt to parse the given private key DER block. OpenSSL 0.9.8 generates
// PKCS#1 private keys by default, while OpenSSL 1.0.0 generates PKCS#8 keys.
// OpenSSL ecparam generates SEC1 EC private keys for ECDSA. We try all three.
func parsePrivateKey(der []byte) (crypto.PrivateKey, error) {
if key, err := x509.ParsePKCS1PrivateKey(der); err == nil {
return key, nil
}
if key, err := x509.ParsePKCS8PrivateKey(der); err == nil {
switch key := key.(type) {
case *rsa.PrivateKey, *ecdsa.PrivateKey:
return key, nil
default:
return nil, errors.New("crypto/tls: found unknown private key type in PKCS#8 wrapping")
}
}
if key, err := x509.ParseECPrivateKey(der); err == nil {
return key, nil
}
return nil, errors.New("crypto/tls: failed to parse private key")
}

68
libgo/go/crypto/tls/tls_test.go
View file

@ -8,7 +8,7 @@ import (
"testing"
)
var certPEM = `-----BEGIN CERTIFICATE-----
var rsaCertPEM = `-----BEGIN CERTIFICATE-----
MIIB0zCCAX2gAwIBAgIJAI/M7BYjwB+uMA0GCSqGSIb3DQEBBQUAMEUxCzAJBgNV
BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX
aWRnaXRzIFB0eSBMdGQwHhcNMTIwOTEyMjE1MjAyWhcNMTUwOTEyMjE1MjAyWjBF
@ -22,7 +22,7 @@ r5QuVbpQhH6u+0UgcW0jp9QwpxoPTLTWGXEWBBBurxFwiCBhkQ+V
-----END CERTIFICATE-----
`
var keyPEM = `-----BEGIN RSA PRIVATE KEY-----
var rsaKeyPEM = `-----BEGIN RSA PRIVATE KEY-----
MIIBOwIBAAJBANLJhPHhITqQbPklG3ibCVxwGMRfp/v4XqhfdQHdcVfHap6NQ5Wo
k/4xIA+ui35/MmNartNuC+BdZ1tMuVCPFZcCAwEAAQJAEJ2N+zsR0Xn8/Q6twa4G
6OB1M1WO+k+ztnX/1SvNeWu8D6GImtupLTYgjZcHufykj09jiHmjHx8u8ZZB/o1N
@ -33,15 +33,61 @@ D2lWusoe2/nEqfDVVWGWlyJ7yOmqaVm/iNUN9B2N2g==
-----END RSA PRIVATE KEY-----
`
func TestX509KeyPair(t *testing.T) {
_, err := X509KeyPair([]byte(keyPEM+certPEM), []byte(keyPEM+certPEM))
if err != nil {
t.Errorf("Failed to load key followed by cert: %s", err)
}
var ecdsaCertPEM = `-----BEGIN CERTIFICATE-----
MIIB/jCCAWICCQDscdUxw16XFDAJBgcqhkjOPQQBMEUxCzAJBgNVBAYTAkFVMRMw
EQYDVQQIEwpTb21lLVN0YXRlMSEwHwYDVQQKExhJbnRlcm5ldCBXaWRnaXRzIFB0
eSBMdGQwHhcNMTIxMTE0MTI0MDQ4WhcNMTUxMTE0MTI0MDQ4WjBFMQswCQYDVQQG
EwJBVTETMBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UEChMYSW50ZXJuZXQgV2lk
Z2l0cyBQdHkgTHRkMIGbMBAGByqGSM49AgEGBSuBBAAjA4GGAAQBY9+my9OoeSUR
lDQdV/x8LsOuLilthhiS1Tz4aGDHIPwC1mlvnf7fg5lecYpMCrLLhauAc1UJXcgl
01xoLuzgtAEAgv2P/jgytzRSpUYvgLBt1UA0leLYBy6mQQbrNEuqT3INapKIcUv8
XxYP0xMEUksLPq6Ca+CRSqTtrd/23uTnapkwCQYHKoZIzj0EAQOBigAwgYYCQXJo
A7Sl2nLVf+4Iu/tAX/IF4MavARKC4PPHK3zfuGfPR3oCCcsAoz3kAzOeijvd0iXb
H5jBImIxPL4WxQNiBTexAkF8D1EtpYuWdlVQ80/h/f4pBcGiXPqX5h2PQSQY7hP1
+jwM1FGS4fREIOvlBYr/SzzQRtwrvrzGYxDEDbsC0ZGRnA==
-----END CERTIFICATE-----
`
_, err = X509KeyPair([]byte(certPEM+keyPEM), []byte(certPEM+keyPEM))
if err != nil {
t.Errorf("Failed to load cert followed by key: %s", err)
println(err.Error())
var ecdsaKeyPEM = `-----BEGIN EC PARAMETERS-----
BgUrgQQAIw==
-----END EC PARAMETERS-----
-----BEGIN EC PRIVATE KEY-----
MIHcAgEBBEIBrsoKp0oqcv6/JovJJDoDVSGWdirrkgCWxrprGlzB9o0X8fV675X0
NwuBenXFfeZvVcwluO7/Q9wkYoPd/t3jGImgBwYFK4EEACOhgYkDgYYABAFj36bL
06h5JRGUNB1X/Hwuw64uKW2GGJLVPPhoYMcg/ALWaW+d/t+DmV5xikwKssuFq4Bz
VQldyCXTXGgu7OC0AQCC/Y/+ODK3NFKlRi+AsG3VQDSV4tgHLqZBBus0S6pPcg1q
kohxS/xfFg/TEwRSSws+roJr4JFKpO2t3/be5OdqmQ==
-----END EC PRIVATE KEY-----
`
var keyPairTests = []struct {
algo string
cert *string
key *string
}{
{"ECDSA", &ecdsaCertPEM, &ecdsaKeyPEM},
{"RSA", &rsaCertPEM, &rsaKeyPEM},
}
func TestX509KeyPair(t *testing.T) {
var pem []byte
for _, test := range keyPairTests {
pem = []byte(*test.cert + *test.key)
if _, err := X509KeyPair(pem, pem); err != nil {
t.Errorf("Failed to load %s cert followed by %s key: %s", test.algo, test.algo, err)
}
pem = []byte(*test.key + *test.cert)
if _, err := X509KeyPair(pem, pem); err != nil {
t.Errorf("Failed to load %s key followed by %s cert: %s", test.algo, test.algo, err)
}
}
}
func TestX509MixedKeyPair(t *testing.T) {
if _, err := X509KeyPair([]byte(rsaCertPEM), []byte(ecdsaKeyPEM)); err == nil {
t.Error("Load of RSA certificate succeeded with ECDSA private key")
}
if _, err := X509KeyPair([]byte(ecdsaCertPEM), []byte(rsaKeyPEM)); err == nil {
t.Error("Load of ECDSA certificate succeeded with RSA private key")
}
}

2
libgo/go/crypto/x509/cert_pool.go
View file

@ -103,7 +103,7 @@ func (s *CertPool) AppendCertsFromPEM(pemCerts []byte) (ok bool) {
}
// Subjects returns a list of the DER-encoded subjects of
// all of the certificates in the pool.
// all of the certificates in the pool.
func (s *CertPool) Subjects() (res [][]byte) {
res = make([][]byte, len(s.certs))
for i, c := range s.certs {

158
libgo/go/crypto/x509/pem_decrypt.go
View file

@ -16,13 +16,64 @@ import (
"encoding/hex"
"encoding/pem"
"errors"
"io"
"strings"
)
// rfc1423Algos represents how to create a block cipher for a decryption mode.
type PEMCipher int
// Possible values for the EncryptPEMBlock encryption algorithm.
const (
_ PEMCipher = iota
PEMCipherDES
PEMCipher3DES
PEMCipherAES128
PEMCipherAES192
PEMCipherAES256
)
// rfc1423Algo holds a method for enciphering a PEM block.
type rfc1423Algo struct {
cipherFunc func([]byte) (cipher.Block, error)
cipher PEMCipher
name string
cipherFunc func(key []byte) (cipher.Block, error)
keySize int
blockSize int
}
// rfc1423Algos holds a slice of the possible ways to encrypt a PEM
// block. The ivSize numbers were taken from the OpenSSL source.
var rfc1423Algos = []rfc1423Algo{{
cipher: PEMCipherDES,
name: "DES-CBC",
cipherFunc: des.NewCipher,
keySize: 8,
blockSize: des.BlockSize,
}, {
cipher: PEMCipher3DES,
name: "DES-EDE3-CBC",
cipherFunc: des.NewTripleDESCipher,
keySize: 24,
blockSize: des.BlockSize,
}, {
cipher: PEMCipherAES128,
name: "AES-128-CBC",
cipherFunc: aes.NewCipher,
keySize: 16,
blockSize: aes.BlockSize,
}, {
cipher: PEMCipherAES192,
name: "AES-192-CBC",
cipherFunc: aes.NewCipher,
keySize: 24,
blockSize: aes.BlockSize,
}, {
cipher: PEMCipherAES256,
name: "AES-256-CBC",
cipherFunc: aes.NewCipher,
keySize: 32,
blockSize: aes.BlockSize,
},
}
// deriveKey uses a key derivation function to stretch the password into a key
@ -41,20 +92,9 @@ func (c rfc1423Algo) deriveKey(password, salt []byte) []byte {
digest = hash.Sum(digest[:0])
copy(out[i:], digest)
}
return out
}
// rfc1423Algos is a mapping of encryption algorithm to an rfc1423Algo that can
// create block ciphers for that mode.
var rfc1423Algos = map[string]rfc1423Algo{
"DES-CBC": {des.NewCipher, 8},
"DES-EDE3-CBC": {des.NewTripleDESCipher, 24},
"AES-128-CBC": {aes.NewCipher, 16},
"AES-192-CBC": {aes.NewCipher, 24},
"AES-256-CBC": {aes.NewCipher, 32},
}
// IsEncryptedPEMBlock returns if the PEM block is password encrypted.
func IsEncryptedPEMBlock(b *pem.Block) bool {
_, ok := b.Headers["DEK-Info"]
@ -81,17 +121,16 @@ func DecryptPEMBlock(b *pem.Block, password []byte) ([]byte, error) {
}
mode, hexIV := dek[:idx], dek[idx+1:]
ciph := cipherByName(mode)
if ciph == nil {
return nil, errors.New("x509: unknown encryption mode")
}
iv, err := hex.DecodeString(hexIV)
if err != nil {
return nil, err
}
if len(iv) < 8 {
return nil, errors.New("x509: not enough bytes in IV")
}
ciph, ok := rfc1423Algos[mode]
if !ok {
return nil, errors.New("x509: unknown encryption mode")
if len(iv) != ciph.blockSize {
return nil, errors.New("x509: incorrect IV size")
}
// Based on the OpenSSL implementation. The salt is the first 8 bytes
@ -107,27 +146,88 @@ func DecryptPEMBlock(b *pem.Block, password []byte) ([]byte, error) {
dec.CryptBlocks(data, b.Bytes)
// Blocks are padded using a scheme where the last n bytes of padding are all
// equal to n. It can pad from 1 to 8 bytes inclusive. See RFC 1423.
// equal to n. It can pad from 1 to blocksize bytes inclusive. See RFC 1423.
// For example:
// [x y z 2 2]
// [x y 7 7 7 7 7 7 7]
// If we detect a bad padding, we assume it is an invalid password.
dlen := len(data)
if dlen == 0 {
if dlen == 0 || dlen%ciph.blockSize != 0 {
return nil, errors.New("x509: invalid padding")
}
last := data[dlen-1]
if dlen < int(last) {
last := int(data[dlen-1])
if dlen < last {
return nil, IncorrectPasswordError
}
if last == 0 || last > 8 {
if last == 0 || last > ciph.blockSize {
return nil, IncorrectPasswordError
}
for _, val := range data[dlen-int(last):] {
if val != last {
for _, val := range data[dlen-last:] {
if int(val) != last {
return nil, IncorrectPasswordError
}
}
return data[:dlen-int(last)], nil
return data[:dlen-last], nil
}
// EncryptPEMBlock returns a PEM block of the specified type holding the
// given DER-encoded data encrypted with the specified algorithm and
// password.
func EncryptPEMBlock(rand io.Reader, blockType string, data, password []byte, alg PEMCipher) (*pem.Block, error) {
ciph := cipherByKey(alg)
if ciph == nil {
return nil, errors.New("x509: unknown encryption mode")
}
iv := make([]byte, ciph.blockSize)
if _, err := io.ReadFull(rand, iv); err != nil {
return nil, errors.New("x509: cannot generate IV: " + err.Error())
}
// The salt is the first 8 bytes of the initialization vector,
// matching the key derivation in DecryptPEMBlock.
key := ciph.deriveKey(password, iv[:8])
block, err := ciph.cipherFunc(key)
if err != nil {
return nil, err
}
enc := cipher.NewCBCEncrypter(block, iv)
pad := ciph.blockSize - len(data)%ciph.blockSize
encrypted := make([]byte, len(data), len(data)+pad)
// We could save this copy by encrypting all the whole blocks in
// the data separately, but it doesn't seem worth the additional
// code.
copy(encrypted, data)
// See RFC 1423, section 1.1
for i := 0; i < pad; i++ {
encrypted = append(encrypted, byte(pad))
}
enc.CryptBlocks(encrypted, encrypted)
return &pem.Block{
Type: blockType,
Headers: map[string]string{
"Proc-Type": "4,ENCRYPTED",
"DEK-Info": ciph.name + "," + hex.EncodeToString(iv),
},
Bytes: encrypted,
}, nil
}
func cipherByName(name string) *rfc1423Algo {
for i := range rfc1423Algos {
alg := &rfc1423Algos[i]
if alg.name == name {
return alg
}
}
return nil
}
func cipherByKey(key PEMCipher) *rfc1423Algo {
for i := range rfc1423Algos {
alg := &rfc1423Algos[i]
if alg.cipher == key {
return alg
}
}
return nil
}

124
libgo/go/crypto/x509/pem_decrypt_test.go
View file

@ -5,34 +5,79 @@
package x509
import (
"bytes"
"crypto/rand"
"encoding/base64"
"encoding/pem"
"testing"
)
func TestDecrypt(t *testing.T) {
for _, data := range testData {
for i, data := range testData {
t.Logf("test %d. %s", i, data.kind)
block, rest := pem.Decode(data.pemData)
if len(rest) > 0 {
t.Error(data.kind, "extra data")
t.Error("extra data")
}
der, err := DecryptPEMBlock(block, data.password)
if err != nil {
t.Error(data.kind, err)
t.Error("decrypt failed: ", err)
continue
}
if _, err := ParsePKCS1PrivateKey(der); err != nil {
t.Error(data.kind, "Invalid private key")
t.Error("invalid private key: ", err)
}
plainDER, err := base64.StdEncoding.DecodeString(data.plainDER)
if err != nil {
t.Fatal("cannot decode test DER data: ", err)
}
if !bytes.Equal(der, plainDER) {
t.Error("data mismatch")
}
}
}
func TestEncrypt(t *testing.T) {
for i, data := range testData {
t.Logf("test %d. %s", i, data.kind)
plainDER, err := base64.StdEncoding.DecodeString(data.plainDER)
if err != nil {
t.Fatal("cannot decode test DER data: ", err)
}
password := []byte("kremvax1")
block, err := EncryptPEMBlock(rand.Reader, "RSA PRIVATE KEY", plainDER, password, data.kind)
if err != nil {
t.Error("encrypt: ", err)
continue
}
if !IsEncryptedPEMBlock(block) {
t.Error("PEM block does not appear to be encrypted")
}
if block.Type != "RSA PRIVATE KEY" {
t.Errorf("unexpected block type; got %q want %q", block.Type, "RSA PRIVATE KEY")
}
if block.Headers["Proc-Type"] != "4,ENCRYPTED" {
t.Errorf("block does not have correct Proc-Type header")
}
der, err := DecryptPEMBlock(block, password)
if err != nil {
t.Error("decrypt: ", err)
continue
}
if !bytes.Equal(der, plainDER) {
t.Errorf("data mismatch")
}
}
}
var testData = []struct {
kind string
kind PEMCipher
password []byte
pemData []byte
plainDER string
}{
{
kind: "DES-CBC",
kind: PEMCipherDES,
password: []byte("asdf"),
pemData: []byte(`
-----BEGIN RSA PRIVATE KEY-----
@ -47,9 +92,17 @@ XOH9VfTjb52q/I8Suozq9coVQwg4tXfIoYUdT//O+mB7zJb9HI9Ps77b9TxDE6Gm
4C9brwZ3zg2vqXcwwV6QRZMtyll9rOpxkbw6NPlpfBqkc3xS51bbxivbO/Nve4KD
r12ymjFNF4stXCfJnNqKoZ50BHmEEUDu5Wb0fpVn82XrGw7CYc4iug==
-----END RSA PRIVATE KEY-----`),
plainDER: `
MIIBPAIBAAJBAPASZe+tCPU6p80AjHhDkVsLYa51D35e/YGa8QcZyooeZM8EHozo
KD0fNiKI+53bHdy07N+81VQ8/ejPcRoXPlsCAwEAAQJBAMTxIuSq27VpR+zZ7WJf
c6fvv1OBvpMZ0/d1pxL/KnOAgq2rD5hDtk9b0LGhTPgQAmrrMTKuSeGoIuYE+gKQ
QvkCIQD+GC1m+/do+QRurr0uo46Kx1LzLeSCrjBk34wiOp2+dwIhAPHfTLRXS2fv
7rljm0bYa4+eDZpz+E8RcXEgzhhvcQQ9AiAI5eHZJGOyml3MXnQjiPi55WcDOw0w
glcRgT6QCEtz2wIhANSyqaFtosIkHKqrDUGfz/bb5tqMYTAnBruVPaf/WEOBAiEA
9xORWeRG1tRpso4+dYy4KdDkuLPIO01KY6neYGm3BCM=`,
},
{
kind: "DES-EDE3-CBC",
kind: PEMCipher3DES,
password: []byte("asdf"),
pemData: []byte(`
-----BEGIN RSA PRIVATE KEY-----
@ -64,9 +117,17 @@ ldw5w7WC7d13x2LsRkwo8ZrDKgIV+Y9GNvhuCCkTzNP0V3gNeJpd201HZHR+9n3w
3z0VjR/MGqsfcy1ziEWMNOO53At3zlG6zP05aHMnMcZoVXadEK6L1gz++inSSDCq
gI0UJP4e3JVB7AkgYymYAwiYALAkoEIuanxoc50njJk=
-----END RSA PRIVATE KEY-----`),
plainDER: `
MIIBOwIBAAJBANOCXKdoNS/iP/MAbl9cf1/SF3P+Ns7ZeNL27CfmDh0O6Zduaax5
NBiumd2PmjkaCu7lQ5JOibHfWn+xJsc3kw0CAwEAAQJANX/W8d1Q/sCqzkuAn4xl
B5a7qfJWaLHndu1QRLNTRJPn0Ee7OKJ4H0QKOhQM6vpjRrz+P2u9thn6wUxoPsef
QQIhAP/jCkfejFcy4v15beqKzwz08/tslVjF+Yq41eJGejmxAiEA05pMoqfkyjcx
fyvGhpoOyoCp71vSGUfR2I9CR65oKh0CIC1Msjs66LlfJtQctRq6bCEtFCxEcsP+
eEjYo/Sk6WphAiEAxpgWPMJeU/shFT28gS+tmhjPZLpEoT1qkVlC14u0b3ECIQDX
tZZZxCtPAm7shftEib0VU77Lk8MsXJcx2C4voRsjEw==`,
},
{
kind: "AES-128-CBC",
kind: PEMCipherAES128,
password: []byte("asdf"),
pemData: []byte(`
-----BEGIN RSA PRIVATE KEY-----
@ -81,9 +142,17 @@ GZbBpf1jDH/pr0iGonuAdl2PCCZUiy+8eLsD2tyviHUkFLOB+ykYoJ5t8ngZ/B6D
080LzLHPCrXKdlr/f50yhNWq08ZxMWQFkui+FDHPDUaEELKAXV8/5PDxw80Rtybo
AVYoCVIbZXZCuCO81op8UcOgEpTtyU5Lgh3Mw5scQL0=
-----END RSA PRIVATE KEY-----`),
plainDER: `
MIIBOgIBAAJBAMBlj5FxYtqbcy8wY89d/S7n0+r5MzD9F63BA/Lpl78vQKtdJ5dT
cDGh/rBt1ufRrNp0WihcmZi7Mpl/3jHjiWECAwEAAQJABNOHYnKhtDIqFYj1OAJ3
k3GlU0OlERmIOoeY/cL2V4lgwllPBEs7r134AY4wMmZSBUj8UR/O4SNO668ElKPE
cQIhAOuqY7/115x5KCdGDMWi+jNaMxIvI4ETGwV40ykGzqlzAiEA0P9oEC3m9tHB
kbpjSTxaNkrXxDgdEOZz8X0uOUUwHNsCIAwzcSCiGLyYJTULUmP1ESERfW1mlV78
XzzESaJpIM/zAiBQkSTcl9VhcJreQqvjn5BnPZLP4ZHS4gPwJAGdsj5J4QIhAOVR
B3WlRNTXR2WsJ5JdByezg9xzdXzULqmga0OE339a`,
},
{
kind: "AES-192-CBC",
kind: PEMCipherAES192,
password: []byte("asdf"),
pemData: []byte(`
-----BEGIN RSA PRIVATE KEY-----
@ -98,9 +167,17 @@ ReUtTw8exmKsY4gsSjhkg5uiw7/ZB1Ihto0qnfQJgjGc680qGkT1d6JfvOfeYAk6
xn5RqS/h8rYAYm64KnepfC9vIujo4NqpaREDmaLdX5MJPQ+SlytITQvgUsUq3q/t
Ss85xjQEZH3hzwjQqdJvmA4hYP6SUjxYpBM+02xZ1Xw=
-----END RSA PRIVATE KEY-----`),
plainDER: `
MIIBOwIBAAJBAMGcRrZiNNmtF20zyS6MQ7pdGx17aFDl+lTl+qnLuJRUCMUG05xs
OmxmL/O1Qlf+bnqR8Bgg65SfKg21SYuLhiMCAwEAAQJBAL94uuHyO4wux2VC+qpj
IzPykjdU7XRcDHbbvksf4xokSeUFjjD3PB0Qa83M94y89ZfdILIqS9x5EgSB4/lX
qNkCIQD6cCIqLfzq/lYbZbQgAAjpBXeQVYsbvVtJrPrXJAlVVQIhAMXpDKMeFPMn
J0g2rbx1gngx0qOa5r5iMU5w/noN4W2XAiBjf+WzCG5yFvazD+dOx3TC0A8+4x3P
uZ3pWbaXf5PNuQIgAcdXarvhelH2w2piY1g3BPeFqhzBSCK/yLGxR82KIh8CIQDD
+qGKsd09NhQ/G27y/DARzOYtml1NvdmCQAgsDIIOLA==`,
},
{
kind: "AES-256-CBC",
kind: PEMCipherAES256,
password: []byte("asdf"),
pemData: []byte(`
-----BEGIN RSA PRIVATE KEY-----
@ -115,5 +192,32 @@ Pz3RZScwIuubzTGJ1x8EzdffYOsdCa9Mtgpp3L136+23dOd6L/qK2EG2fzrJSHs/
sv5Z/KwlX+3MDEpPQpUwGPlGGdLnjI3UZ+cjgqBcoMiNc6HfgbBgYJSU6aDSHuCk
clCwByxWkBNgJ2GrkwNrF26v+bGJJJNR4SKouY1jQf0=
-----END RSA PRIVATE KEY-----`),
plainDER: `
MIIBOgIBAAJBAKy3GFkstoCHIEeUU/qO8207m8WSrjksR+p9B4tf1w5k+2O1V/GY
AQ5WFCApItcOkQe/I0yZZJk/PmCqMzSxrc8CAwEAAQJAOCAz0F7AW9oNelVQSP8F
Sfzx7O1yom+qWyAQQJF/gFR11gpf9xpVnnyu1WxIRnDUh1LZwUsjwlDYb7MB74id
oQIhANPcOiLwOPT4sIUpRM5HG6BF1BI7L77VpyGVk8xNP7X/AiEA0LMHZtk4I+lJ
nClgYp4Yh2JZ1Znbu7IoQMCEJCjwKDECIGd8Dzm5tViTkUW6Hs3Tlf73nNs65duF
aRnSglss8I3pAiEAonEnKruawgD8RavDFR+fUgmQiPz4FnGGeVgfwpGG1JECIBYq
PXHYtPqxQIbD2pScR5qum7iGUh11lEUPkmt+2uqS`,
},
{
// generated with:
// openssl genrsa -aes128 -passout pass:asdf -out server.orig.key 128
kind: PEMCipherAES128,
password: []byte("asdf"),
pemData: []byte(`
-----BEGIN RSA PRIVATE KEY-----
Proc-Type: 4,ENCRYPTED
DEK-Info: AES-128-CBC,74611ABC2571AF11B1BF9B69E62C89E7
6ei/MlytjE0FFgZOGQ+jrwomKfpl8kdefeE0NSt/DMRrw8OacHAzBNi3pPEa0eX3
eND9l7C9meCirWovjj9QWVHrXyugFuDIqgdhQ8iHTgCfF3lrmcttVrbIfMDw+smD
hTP8O1mS/MHl92NE0nhv0w==
-----END RSA PRIVATE KEY-----`),
plainDER: `
MGMCAQACEQC6ssxmYuauuHGOCDAI54RdAgMBAAECEQCWIn6Yv2O+kBcDF7STctKB
AgkA8SEfu/2i3g0CCQDGNlXbBHX7kQIIK3Ww5o0cYbECCQDCimPb0dYGsQIIeQ7A
jryIst8=`,
},
}

20
libgo/go/crypto/x509/pkcs8.go
View file

@ -11,8 +11,9 @@ import (
"fmt"
)
// pkcs8 reflects an ASN.1, PKCS#8 PrivateKey. See
// ftp://ftp.rsasecurity.com/pub/pkcs/pkcs-8/pkcs-8v1_2.asn.
// pkcs8 reflects an ASN.1, PKCS#8 PrivateKey. See
// ftp://ftp.rsasecurity.com/pub/pkcs/pkcs-8/pkcs-8v1_2.asn
// and RFC5208.
type pkcs8 struct {
Version int
Algo pkix.AlgorithmIdentifier
@ -21,7 +22,7 @@ type pkcs8 struct {
}
// ParsePKCS8PrivateKey parses an unencrypted, PKCS#8 private key. See
// http://www.rsa.com/rsalabs/node.asp?id=2130
// http://www.rsa.com/rsalabs/node.asp?id=2130 and RFC5208.
func ParsePKCS8PrivateKey(der []byte) (key interface{}, err error) {
var privKey pkcs8
if _, err := asn1.Unmarshal(der, &privKey); err != nil {
@ -34,6 +35,19 @@ func ParsePKCS8PrivateKey(der []byte) (key interface{}, err error) {
return nil, errors.New("crypto/x509: failed to parse RSA private key embedded in PKCS#8: " + err.Error())
}
return key, nil
case privKey.Algo.Algorithm.Equal(oidPublicKeyECDSA):
bytes := privKey.Algo.Parameters.FullBytes
namedCurveOID := new(asn1.ObjectIdentifier)
if _, err := asn1.Unmarshal(bytes, namedCurveOID); err != nil {
namedCurveOID = nil
}
key, err = parseECPrivateKey(namedCurveOID, privKey.PrivateKey)
if err != nil {
return nil, errors.New("crypto/x509: failed to parse EC private key embedded in PKCS#8: " + err.Error())
}
return key, nil
default:
return nil, fmt.Errorf("crypto/x509: PKCS#8 wrapping contained private key with unknown algorithm: %v", privKey.Algo.Algorithm)
}

18
libgo/go/crypto/x509/pkcs8_test.go
View file

@ -9,12 +9,20 @@ import (
"testing"
)
var pkcs8PrivateKeyHex = `30820278020100300d06092a864886f70d0101010500048202623082025e02010002818100cfb1b5bf9685ffa97b4f99df4ff122b70e59ac9b992f3bc2b3dde17d53c1a34928719b02e8fd17839499bfbd515bd6ef99c7a1c47a239718fe36bfd824c0d96060084b5f67f0273443007a24dfaf5634f7772c9346e10eb294c2306671a5a5e719ae24b4de467291bc571014b0e02dec04534d66a9bb171d644b66b091780e8d020301000102818100b595778383c4afdbab95d2bfed12b3f93bb0a73a7ad952f44d7185fd9ec6c34de8f03a48770f2009c8580bcd275e9632714e9a5e3f32f29dc55474b2329ff0ebc08b3ffcb35bc96e6516b483df80a4a59cceb71918cbabf91564e64a39d7e35dce21cb3031824fdbc845dba6458852ec16af5dddf51a8397a8797ae0337b1439024100ea0eb1b914158c70db39031dd8904d6f18f408c85fbbc592d7d20dee7986969efbda081fdf8bc40e1b1336d6b638110c836bfdc3f314560d2e49cd4fbde1e20b024100e32a4e793b574c9c4a94c8803db5152141e72d03de64e54ef2c8ed104988ca780cd11397bc359630d01b97ebd87067c5451ba777cf045ca23f5912f1031308c702406dfcdbbd5a57c9f85abc4edf9e9e29153507b07ce0a7ef6f52e60dcfebe1b8341babd8b789a837485da6c8d55b29bbb142ace3c24a1f5b54b454d01b51e2ad03024100bd6a2b60dee01e1b3bfcef6a2f09ed027c273cdbbaf6ba55a80f6dcc64e4509ee560f84b4f3e076bd03b11e42fe71a3fdd2dffe7e0902c8584f8cad877cdc945024100aa512fa4ada69881f1d8bb8ad6614f192b83200aef5edf4811313d5ef30a86cbd0a90f7b025c71ea06ec6b34db6306c86b1040670fd8654ad7291d066d06d031`
var pkcs8RSAPrivateKeyHex = `30820278020100300d06092a864886f70d0101010500048202623082025e02010002818100cfb1b5bf9685ffa97b4f99df4ff122b70e59ac9b992f3bc2b3dde17d53c1a34928719b02e8fd17839499bfbd515bd6ef99c7a1c47a239718fe36bfd824c0d96060084b5f67f0273443007a24dfaf5634f7772c9346e10eb294c2306671a5a5e719ae24b4de467291bc571014b0e02dec04534d66a9bb171d644b66b091780e8d020301000102818100b595778383c4afdbab95d2bfed12b3f93bb0a73a7ad952f44d7185fd9ec6c34de8f03a48770f2009c8580bcd275e9632714e9a5e3f32f29dc55474b2329ff0ebc08b3ffcb35bc96e6516b483df80a4a59cceb71918cbabf91564e64a39d7e35dce21cb3031824fdbc845dba6458852ec16af5dddf51a8397a8797ae0337b1439024100ea0eb1b914158c70db39031dd8904d6f18f408c85fbbc592d7d20dee7986969efbda081fdf8bc40e1b1336d6b638110c836bfdc3f314560d2e49cd4fbde1e20b024100e32a4e793b574c9c4a94c8803db5152141e72d03de64e54ef2c8ed104988ca780cd11397bc359630d01b97ebd87067c5451ba777cf045ca23f5912f1031308c702406dfcdbbd5a57c9f85abc4edf9e9e29153507b07ce0a7ef6f52e60dcfebe1b8341babd8b789a837485da6c8d55b29bbb142ace3c24a1f5b54b454d01b51e2ad03024100bd6a2b60dee01e1b3bfcef6a2f09ed027c273cdbbaf6ba55a80f6dcc64e4509ee560f84b4f3e076bd03b11e42fe71a3fdd2dffe7e0902c8584f8cad877cdc945024100aa512fa4ada69881f1d8bb8ad6614f192b83200aef5edf4811313d5ef30a86cbd0a90f7b025c71ea06ec6b34db6306c86b1040670fd8654ad7291d066d06d031`
// Generated using:
// openssl ecparam -genkey -name secp521r1 | openssl pkcs8 -topk8 -nocrypt
var pkcs8ECPrivateKeyHex = `3081ed020100301006072a8648ce3d020106052b810400230481d53081d20201010441850d81618c5da1aec74c2eed608ba816038506975e6427237c2def150c96a3b13efbfa1f89f1be15cdf4d0ac26422e680e65a0ddd4ad3541ad76165fbf54d6e34ba18189038186000400da97bcedba1eb6d30aeb93c9f9a1454598fa47278df27d6f60ea73eb672d8dc528a9b67885b5b5dcef93c9824f7449ab512ee6a27e76142f56b94b474cfd697e810046c8ca70419365245c1d7d44d0db82c334073835d002232714548abbae6e5700f5ef315ee08b929d8581383dcf2d1c98c2f8a9fccbf79c9579f7b2fd8a90115ac2`
func TestPKCS8(t *testing.T) {
derBytes, _ := hex.DecodeString(pkcs8PrivateKeyHex)
_, err := ParsePKCS8PrivateKey(derBytes)
if err != nil {
t.Errorf("failed to decode PKCS8 key: %s", err)
derBytes, _ := hex.DecodeString(pkcs8RSAPrivateKeyHex)
if _, err := ParsePKCS8PrivateKey(derBytes); err != nil {
t.Errorf("failed to decode PKCS8 with RSA private key: %s", err)
}
derBytes, _ = hex.DecodeString(pkcs8ECPrivateKeyHex)
if _, err := ParsePKCS8PrivateKey(derBytes); err != nil {
t.Errorf("failed to decode PKCS8 with EC private key: %s", err)
}
}

69
libgo/go/crypto/x509/sec1.go Normal file
View file

@ -0,0 +1,69 @@
// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package x509
import (
"crypto/ecdsa"
"crypto/elliptic"
"encoding/asn1"
"errors"
"fmt"
"math/big"
)
const ecPrivKeyVersion = 1
// ecPrivateKey reflects an ASN.1 Elliptic Curve Private Key Structure.
// References:
// RFC5915
// SEC1 - http://www.secg.org/download/aid-780/sec1-v2.pdf
// Per RFC5915 the NamedCurveOID is marked as ASN.1 OPTIONAL, however in
// most cases it is not.
type ecPrivateKey struct {
Version int
PrivateKey []byte
NamedCurveOID asn1.ObjectIdentifier `asn1:"optional,explicit,tag:0"`
PublicKey asn1.BitString `asn1:"optional,explicit,tag:1"`
}
// ParseECPrivateKey parses an ASN.1 Elliptic Curve Private Key Structure.
func ParseECPrivateKey(der []byte) (key *ecdsa.PrivateKey, err error) {
return parseECPrivateKey(nil, der)
}
// parseECPrivateKey parses an ASN.1 Elliptic Curve Private Key Structure.
// The OID for the named curve may be provided from another source (such as
// the PKCS8 container) - if it is provided then use this instead of the OID
// that may exist in the EC private key structure.
func parseECPrivateKey(namedCurveOID *asn1.ObjectIdentifier, der []byte) (key *ecdsa.PrivateKey, err error) {
var privKey ecPrivateKey
if _, err := asn1.Unmarshal(der, &privKey); err != nil {
return nil, errors.New("crypto/x509: failed to parse EC private key: " + err.Error())
}
if privKey.Version != ecPrivKeyVersion {
return nil, fmt.Errorf("crypto/x509: unknown EC private key version %d", privKey.Version)
}
var curve elliptic.Curve
if namedCurveOID != nil {
curve = namedCurveFromOID(*namedCurveOID)
} else {
curve = namedCurveFromOID(privKey.NamedCurveOID)
}
if curve == nil {
return nil, errors.New("crypto/x509: unknown elliptic curve")
}
k := new(big.Int).SetBytes(privKey.PrivateKey)
if k.Cmp(curve.Params().N) >= 0 {
return nil, errors.New("crypto/x509: invalid elliptic curve private key value")
}
priv := new(ecdsa.PrivateKey)
priv.Curve = curve
priv.D = k
priv.X, priv.Y = curve.ScalarBaseMult(privKey.PrivateKey)
return priv, nil
}

22
libgo/go/crypto/x509/sec1_test.go Normal file
View file

@ -0,0 +1,22 @@
// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package x509
import (
"encoding/hex"
"testing"
)
// Generated using:
// openssl ecparam -genkey -name secp384r1 -outform PEM
var ecPrivateKeyHex = `3081a40201010430bdb9839c08ee793d1157886a7a758a3c8b2a17a4df48f17ace57c72c56b4723cf21dcda21d4e1ad57ff034f19fcfd98ea00706052b81040022a16403620004feea808b5ee2429cfcce13c32160e1c960990bd050bb0fdf7222f3decd0a55008e32a6aa3c9062051c4cba92a7a3b178b24567412d43cdd2f882fa5addddd726fe3e208d2c26d733a773a597abb749714df7256ead5105fa6e7b3650de236b50`
func TestParseECPrivateKey(t *testing.T) {
derBytes, _ := hex.DecodeString(ecPrivateKeyHex)
_, err := ParseECPrivateKey(derBytes)
if err != nil {
t.Errorf("failed to decode EC private key: %s", err)
}
}

18
libgo/go/crypto/x509/x509.go
View file

@ -156,8 +156,8 @@ const (
//
// pkcs-1 OBJECT IDENTIFIER ::= {
// iso(1) member-body(2) us(840) rsadsi(113549) pkcs(1) 1 }
//
//
//
//
// RFC 3279 2.2.1 RSA Signature Algorithms
//
// md2WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 2 }
@ -165,19 +165,19 @@ const (
// md5WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 4 }
//
// sha-1WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 5 }
//
//
// dsaWithSha1 OBJECT IDENTIFIER ::= {
// iso(1) member-body(2) us(840) x9-57(10040) x9cm(4) 3 }
// iso(1) member-body(2) us(840) x9-57(10040) x9cm(4) 3 }
//
// RFC 3279 2.2.3 ECDSA Signature Algorithm
//
//
// ecdsa-with-SHA1 OBJECT IDENTIFIER ::= {
// iso(1) member-body(2) us(840) ansi-x962(10045)
// signatures(4) ecdsa-with-SHA1(1)}
//
//
// RFC 4055 5 PKCS #1 Version 1.5
//
//
// sha256WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 11 }
//
// sha384WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 12 }
@ -1224,7 +1224,7 @@ func CreateCertificate(rand io.Reader, template, parent *Certificate, pub interf
SerialNumber: template.SerialNumber,
SignatureAlgorithm: signatureAlgorithm,
Issuer: asn1.RawValue{FullBytes: asn1Issuer},
Validity: validity{template.NotBefore, template.NotAfter},
Validity: validity{template.NotBefore.UTC(), template.NotAfter.UTC()},
Subject: asn1.RawValue{FullBytes: asn1Subject},
PublicKey: publicKeyInfo{nil, publicKeyAlgorithm, encodedPublicKey},
Extensions: extensions,
@ -1314,8 +1314,8 @@ func (c *Certificate) CreateCRL(rand io.Reader, priv interface{}, revokedCerts [
Algorithm: oidSignatureSHA1WithRSA,
},
Issuer: c.Subject.ToRDNSequence(),
ThisUpdate: now,
NextUpdate: expiry,
ThisUpdate: now.UTC(),
NextUpdate: expiry.UTC(),
RevokedCertificates: revokedCerts,
}

12
libgo/go/debug/elf/file.go
View file

@ -417,10 +417,6 @@ func (f *File) getSymbols32(typ SectionType) ([]Symbol, []byte, error) {
return nil, nil, errors.New("cannot load string table section")
}
// The first entry is all zeros.
var skip [Sym32Size]byte
symtab.Read(skip[0:])
symbols := make([]Symbol, symtab.Len()/Sym32Size)
i := 0
@ -460,10 +456,6 @@ func (f *File) getSymbols64(typ SectionType) ([]Symbol, []byte, error) {
return nil, nil, errors.New("cannot load string table section")
}
// The first entry is all zeros.
var skip [Sym64Size]byte
symtab.Read(skip[0:])
symbols := make([]Symbol, symtab.Len()/Sym64Size)
i := 0
@ -708,8 +700,8 @@ func (f *File) gnuVersionInit(str []byte) {
// gnuVersion adds Library and Version information to sym,
// which came from offset i of the symbol table.
func (f *File) gnuVersion(i int, sym *ImportedSymbol) {
// Each entry is two bytes; skip undef entry at beginning.
i = (i + 1) * 2
// Each entry is two bytes.
i = i * 2
if i >= len(f.gnuVersym) {
return
}

60
libgo/go/debug/elf/file_test.go
View file

@ -175,23 +175,41 @@ func TestOpen(t *testing.T) {
}
}
type relocationTestEntry struct {
entryNumber int
entry *dwarf.Entry
}
type relocationTest struct {
file string
firstEntry *dwarf.Entry
file string
entries []relocationTestEntry
}
var relocationTests = []relocationTest{
{
"testdata/go-relocation-test-gcc441-x86-64.obj",
&dwarf.Entry{Offset: 0xb, Tag: dwarf.TagCompileUnit, Children: true, Field: []dwarf.Field{{Attr: dwarf.AttrProducer, Val: "GNU C 4.4.1"}, {Attr: dwarf.AttrLanguage, Val: int64(1)}, {Attr: dwarf.AttrName, Val: "go-relocation-test.c"}, {Attr: dwarf.AttrCompDir, Val: "/tmp"}, {Attr: dwarf.AttrLowpc, Val: uint64(0x0)}, {Attr: dwarf.AttrHighpc, Val: uint64(0x6)}, {Attr: dwarf.AttrStmtList, Val: int64(0)}}},
[]relocationTestEntry{
{0, &dwarf.Entry{Offset: 0xb, Tag: dwarf.TagCompileUnit, Children: true, Field: []dwarf.Field{{Attr: dwarf.AttrProducer, Val: "GNU C 4.4.1"}, {Attr: dwarf.AttrLanguage, Val: int64(1)}, {Attr: dwarf.AttrName, Val: "go-relocation-test.c"}, {Attr: dwarf.AttrCompDir, Val: "/tmp"}, {Attr: dwarf.AttrLowpc, Val: uint64(0x0)}, {Attr: dwarf.AttrHighpc, Val: uint64(0x6)}, {Attr: dwarf.AttrStmtList, Val: int64(0)}}}},
},
},
{
"testdata/go-relocation-test-gcc441-x86.obj",
&dwarf.Entry{Offset: 0xb, Tag: dwarf.TagCompileUnit, Children: true, Field: []dwarf.Field{{Attr: dwarf.AttrProducer, Val: "GNU C 4.4.1"}, {Attr: dwarf.AttrLanguage, Val: int64(1)}, {Attr: dwarf.AttrName, Val: "t.c"}, {Attr: dwarf.AttrCompDir, Val: "/tmp"}, {Attr: dwarf.AttrLowpc, Val: uint64(0x0)}, {Attr: dwarf.AttrHighpc, Val: uint64(0x5)}, {Attr: dwarf.AttrStmtList, Val: int64(0)}}},
[]relocationTestEntry{
{0, &dwarf.Entry{Offset: 0xb, Tag: dwarf.TagCompileUnit, Children: true, Field: []dwarf.Field{{Attr: dwarf.AttrProducer, Val: "GNU C 4.4.1"}, {Attr: dwarf.AttrLanguage, Val: int64(1)}, {Attr: dwarf.AttrName, Val: "t.c"}, {Attr: dwarf.AttrCompDir, Val: "/tmp"}, {Attr: dwarf.AttrLowpc, Val: uint64(0x0)}, {Attr: dwarf.AttrHighpc, Val: uint64(0x5)}, {Attr: dwarf.AttrStmtList, Val: int64(0)}}}},
},
},
{
"testdata/go-relocation-test-gcc424-x86-64.obj",
&dwarf.Entry{Offset: 0xb, Tag: dwarf.TagCompileUnit, Children: true, Field: []dwarf.Field{{Attr: dwarf.AttrProducer, Val: "GNU C 4.2.4 (Ubuntu 4.2.4-1ubuntu4)"}, {Attr: dwarf.AttrLanguage, Val: int64(1)}, {Attr: dwarf.AttrName, Val: "go-relocation-test-gcc424.c"}, {Attr: dwarf.AttrCompDir, Val: "/tmp"}, {Attr: dwarf.AttrLowpc, Val: uint64(0x0)}, {Attr: dwarf.AttrHighpc, Val: uint64(0x6)}, {Attr: dwarf.AttrStmtList, Val: int64(0)}}},
[]relocationTestEntry{
{0, &dwarf.Entry{Offset: 0xb, Tag: dwarf.TagCompileUnit, Children: true, Field: []dwarf.Field{{Attr: dwarf.AttrProducer, Val: "GNU C 4.2.4 (Ubuntu 4.2.4-1ubuntu4)"}, {Attr: dwarf.AttrLanguage, Val: int64(1)}, {Attr: dwarf.AttrName, Val: "go-relocation-test-gcc424.c"}, {Attr: dwarf.AttrCompDir, Val: "/tmp"}, {Attr: dwarf.AttrLowpc, Val: uint64(0x0)}, {Attr: dwarf.AttrHighpc, Val: uint64(0x6)}, {Attr: dwarf.AttrStmtList, Val: int64(0)}}}},
},
},
{
"testdata/gcc-amd64-openbsd-debug-with-rela.obj",
[]relocationTestEntry{
{203, &dwarf.Entry{Offset: 0xc62, Tag: dwarf.TagMember, Children: false, Field: []dwarf.Field{{Attr: dwarf.AttrName, Val: "it_interval"}, {Attr: dwarf.AttrDeclFile, Val: int64(7)}, {Attr: dwarf.AttrDeclLine, Val: int64(236)}, {Attr: dwarf.AttrType, Val: dwarf.Offset(0xb7f)}, {Attr: dwarf.AttrDataMemberLoc, Val: []byte{0x23, 0x0}}}}},
{204, &dwarf.Entry{Offset: 0xc70, Tag: dwarf.TagMember, Children: false, Field: []dwarf.Field{{Attr: dwarf.AttrName, Val: "it_value"}, {Attr: dwarf.AttrDeclFile, Val: int64(7)}, {Attr: dwarf.AttrDeclLine, Val: int64(237)}, {Attr: dwarf.AttrType, Val: dwarf.Offset(0xb7f)}, {Attr: dwarf.AttrDataMemberLoc, Val: []byte{0x23, 0x10}}}}},
},
},
}
@ -207,20 +225,24 @@ func TestDWARFRelocations(t *testing.T) {
t.Error(err)
continue
}
reader := dwarf.Reader()
// Checking only the first entry is sufficient since it has
// many different strings. If the relocation had failed, all
// the string offsets would be zero and all the strings would
// end up being the same.
firstEntry, err := reader.Next()
if err != nil {
t.Error(err)
continue
}
if !reflect.DeepEqual(test.firstEntry, firstEntry) {
t.Errorf("#%d: mismatch: got:%#v want:%#v", i, firstEntry, test.firstEntry)
continue
for _, testEntry := range test.entries {
reader := dwarf.Reader()
for j := 0; j < testEntry.entryNumber; j++ {
entry, err := reader.Next()
if entry == nil || err != nil {
t.Errorf("Failed to skip to entry %d: %v", testEntry.entryNumber, err)
continue
}
}
entry, err := reader.Next()
if err != nil {
t.Error(err)
continue
}
if !reflect.DeepEqual(testEntry.entry, entry) {
t.Errorf("#%d/%d: mismatch: got:%#v want:%#v", i, testEntry.entryNumber, entry, testEntry.entry)
continue
}
}
}
}

BIN
libgo/go/debug/elf/testdata/gcc-amd64-openbsd-debug-with-rela.obj vendored Normal file
View file

Binary file not shown.

5
libgo/go/encoding/asn1/marshal.go
View file

@ -296,8 +296,7 @@ func marshalTwoDigits(out *forkableWriter, v int) (err error) {
}
func marshalUTCTime(out *forkableWriter, t time.Time) (err error) {
utc := t.UTC()
year, month, day := utc.Date()
year, month, day := t.Date()
switch {
case 1950 <= year && year < 2000:
@ -321,7 +320,7 @@ func marshalUTCTime(out *forkableWriter, t time.Time) (err error) {
return
}
hour, min, sec := utc.Clock()
hour, min, sec := t.Clock()
err = marshalTwoDigits(out, hour)
if err != nil {

2
libgo/go/encoding/asn1/marshal_test.go
View file

@ -82,7 +82,7 @@ var marshalTests = []marshalTest{
{explicitTagTest{64}, "3005a503020140"},
{time.Unix(0, 0).UTC(), "170d3730303130313030303030305a"},
{time.Unix(1258325776, 0).UTC(), "170d3039313131353232353631365a"},
{time.Unix(1258325776, 0).In(PST), "17113039313131353232353631362d30383030"},
{time.Unix(1258325776, 0).In(PST), "17113039313131353134353631362d30383030"},
{BitString{[]byte{0x80}, 1}, "03020780"},
{BitString{[]byte{0x81, 0xf0}, 12}, "03030481f0"},
{ObjectIdentifier([]int{1, 2, 3, 4}), "06032a0304"},

49
libgo/go/encoding/binary/binary.go
View file

@ -125,6 +125,9 @@ func (bigEndian) GoString() string { return "binary.BigEndian" }
// of fixed-size values.
// Bytes read from r are decoded using the specified byte order
// and written to successive fields of the data.
// When reading into structs, the field data for fields with
// blank (_) field names is skipped; i.e., blank field names
// may be used for padding.
func Read(r io.Reader, order ByteOrder, data interface{}) error {
// Fast path for basic types.
if n := intDestSize(data); n != 0 {
@ -154,7 +157,7 @@ func Read(r io.Reader, order ByteOrder, data interface{}) error {
return nil
}
// Fallback to reflect-based.
// Fallback to reflect-based decoding.
var v reflect.Value
switch d := reflect.ValueOf(data); d.Kind() {
case reflect.Ptr:
@ -181,6 +184,8 @@ func Read(r io.Reader, order ByteOrder, data interface{}) error {
// values, or a pointer to such data.
// Bytes written to w are encoded using the specified byte order
// and read from successive fields of the data.
// When writing structs, zero values are are written for fields
// with blank (_) field names.
func Write(w io.Writer, order ByteOrder, data interface{}) error {
// Fast path for basic types.
var b [8]byte
@ -239,6 +244,8 @@ func Write(w io.Writer, order ByteOrder, data interface{}) error {
_, err := w.Write(bs)
return err
}
// Fallback to reflect-based encoding.
v := reflect.Indirect(reflect.ValueOf(data))
size := dataSize(v)
if size < 0 {
@ -300,15 +307,13 @@ func sizeof(t reflect.Type) int {
return -1
}
type decoder struct {
type coder struct {
order ByteOrder
buf []byte
}
type encoder struct {
order ByteOrder
buf []byte
}
type decoder coder
type encoder coder
func (d *decoder) uint8() uint8 {
x := d.buf[0]
@ -379,9 +384,19 @@ func (d *decoder) value(v reflect.Value) {
}
case reflect.Struct:
t := v.Type()
l := v.NumField()
for i := 0; i < l; i++ {
d.value(v.Field(i))
// Note: Calling v.CanSet() below is an optimization.
// It would be sufficient to check the field name,
// but creating the StructField info for each field is
// costly (run "go test -bench=ReadStruct" and compare
// results when making changes to this code).
if v := v.Field(i); v.CanSet() || t.Field(i).Name != "_" {
d.value(v)
} else {
d.skip(v)
}
}
case reflect.Slice:
@ -435,9 +450,15 @@ func (e *encoder) value(v reflect.Value) {
}
case reflect.Struct:
t := v.Type()
l := v.NumField()
for i := 0; i < l; i++ {
e.value(v.Field(i))
// see comment for corresponding code in decoder.value()
if v := v.Field(i); v.CanSet() || t.Field(i).Name != "_" {
e.value(v)
} else {
e.skip(v)
}
}
case reflect.Slice:
@ -492,6 +513,18 @@ func (e *encoder) value(v reflect.Value) {
}
}
func (d *decoder) skip(v reflect.Value) {
d.buf = d.buf[dataSize(v):]
}
func (e *encoder) skip(v reflect.Value) {
n := dataSize(v)
for i := range e.buf[0:n] {
e.buf[i] = 0
}
e.buf = e.buf[n:]
}
// intDestSize returns the size of the integer that ptrType points to,
// or 0 if the type is not supported.
func intDestSize(ptrType interface{}) int {

83
libgo/go/encoding/binary/binary_test.go
View file

@ -121,18 +121,14 @@ func testWrite(t *testing.T, order ByteOrder, b []byte, s1 interface{}) {
checkResult(t, "Write", order, err, buf.Bytes(), b)
}
func TestBigEndianRead(t *testing.T) { testRead(t, BigEndian, big, s) }
func TestLittleEndianRead(t *testing.T) { testRead(t, LittleEndian, little, s) }
func TestBigEndianWrite(t *testing.T) { testWrite(t, BigEndian, big, s) }
func TestLittleEndianWrite(t *testing.T) { testWrite(t, LittleEndian, little, s) }
func TestBigEndianPtrWrite(t *testing.T) { testWrite(t, BigEndian, big, &s) }
func TestLittleEndianRead(t *testing.T) { testRead(t, LittleEndian, little, s) }
func TestLittleEndianWrite(t *testing.T) { testWrite(t, LittleEndian, little, s) }
func TestLittleEndianPtrWrite(t *testing.T) { testWrite(t, LittleEndian, little, &s) }
func TestBigEndianRead(t *testing.T) { testRead(t, BigEndian, big, s) }
func TestBigEndianWrite(t *testing.T) { testWrite(t, BigEndian, big, s) }
func TestBigEndianPtrWrite(t *testing.T) { testWrite(t, BigEndian, big, &s) }
func TestReadSlice(t *testing.T) {
slice := make([]int32, 2)
err := Read(bytes.NewBuffer(src), BigEndian, slice)
@ -148,20 +144,75 @@ func TestWriteSlice(t *testing.T) {
func TestWriteT(t *testing.T) {
buf := new(bytes.Buffer)
ts := T{}
err := Write(buf, BigEndian, ts)
if err == nil {
t.Errorf("WriteT: have nil, want non-nil")
if err := Write(buf, BigEndian, ts); err == nil {
t.Errorf("WriteT: have err == nil, want non-nil")
}
tv := reflect.Indirect(reflect.ValueOf(ts))
for i, n := 0, tv.NumField(); i < n; i++ {
err = Write(buf, BigEndian, tv.Field(i).Interface())
if err == nil {
t.Errorf("WriteT.%v: have nil, want non-nil", tv.Field(i).Type())
if err := Write(buf, BigEndian, tv.Field(i).Interface()); err == nil {
t.Errorf("WriteT.%v: have err == nil, want non-nil", tv.Field(i).Type())
}
}
}
type BlankFields struct {
A uint32
_ int32
B float64
_ [4]int16
C byte
_ [7]byte
_ struct {
f [8]float32
}
}
type BlankFieldsProbe struct {
A uint32
P0 int32
B float64
P1 [4]int16
C byte
P2 [7]byte
P3 struct {
F [8]float32
}
}
func TestBlankFields(t *testing.T) {
buf := new(bytes.Buffer)
b1 := BlankFields{A: 1234567890, B: 2.718281828, C: 42}
if err := Write(buf, LittleEndian, &b1); err != nil {
t.Error(err)
}
// zero values must have been written for blank fields
var p BlankFieldsProbe
if err := Read(buf, LittleEndian, &p); err != nil {
t.Error(err)
}
// quick test: only check first value of slices
if p.P0 != 0 || p.P1[0] != 0 || p.P2[0] != 0 || p.P3.F[0] != 0 {
t.Errorf("non-zero values for originally blank fields: %#v", p)
}
// write p and see if we can probe only some fields
if err := Write(buf, LittleEndian, &p); err != nil {
t.Error(err)
}
// read should ignore blank fields in b2
var b2 BlankFields
if err := Read(buf, LittleEndian, &b2); err != nil {
t.Error(err)
}
if b1.A != b2.A || b1.B != b2.B || b1.C != b2.C {
t.Errorf("%#v != %#v", b1, b2)
}
}
type byteSliceReader struct {
remain []byte
}

2
libgo/go/encoding/binary/varint.go
View file

@ -123,7 +123,7 @@ func ReadUvarint(r io.ByteReader) (uint64, error) {
panic("unreachable")
}
// ReadVarint reads an encoded unsigned integer from r and returns it as an int64.
// ReadVarint reads an encoded signed integer from r and returns it as an int64.
func ReadVarint(r io.ByteReader) (int64, error) {
ux, err := ReadUvarint(r) // ok to continue in presence of error
x := int64(ux >> 1)

2
libgo/go/encoding/gob/encoder.go
View file

@ -132,7 +132,7 @@ func (enc *Encoder) sendActualType(w io.Writer, state *encoderState, ut *userTyp
return true
}
// sendType sends the type info to the other side, if necessary.
// sendType sends the type info to the other side, if necessary.
func (enc *Encoder) sendType(w io.Writer, state *encoderState, origt reflect.Type) (sent bool) {
ut := userType(origt)
if ut.isGobEncoder {

8
libgo/go/encoding/json/decode.go
View file

@ -67,8 +67,8 @@ func Unmarshal(data []byte, v interface{}) error {
// Unmarshaler is the interface implemented by objects
// that can unmarshal a JSON description of themselves.
// The input can be assumed to be a valid JSON object
// encoding. UnmarshalJSON must copy the JSON data
// The input can be assumed to be a valid encoding of
// a JSON value. UnmarshalJSON must copy the JSON data
// if it wishes to retain the data after returning.
type Unmarshaler interface {
UnmarshalJSON([]byte) error
@ -617,12 +617,10 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool
switch c := item[0]; c {
case 'n': // null
switch v.Kind() {
default:
d.saveError(&UnmarshalTypeError{"null", v.Type()})
case reflect.Interface, reflect.Ptr, reflect.Map, reflect.Slice:
v.Set(reflect.Zero(v.Type()))
// otherwise, ignore null for primitives/string
}
case 't', 'f': // true, false
value := c == 't'
switch v.Kind() {

47
libgo/go/encoding/json/decode_test.go
View file

@ -953,3 +953,50 @@ func TestInterfaceSet(t *testing.T) {
}
}
}
// JSON null values should be ignored for primitives and string values instead of resulting in an error.
// Issue 2540
func TestUnmarshalNulls(t *testing.T) {
jsonData := []byte(`{
"Bool" : null,
"Int" : null,
"Int8" : null,
"Int16" : null,
"Int32" : null,
"Int64" : null,
"Uint" : null,
"Uint8" : null,
"Uint16" : null,
"Uint32" : null,
"Uint64" : null,
"Float32" : null,
"Float64" : null,
"String" : null}`)
nulls := All{
Bool: true,
Int: 2,
Int8: 3,
Int16: 4,
Int32: 5,
Int64: 6,
Uint: 7,
Uint8: 8,
Uint16: 9,
Uint32: 10,
Uint64: 11,
Float32: 12.1,
Float64: 13.1,
String: "14"}
err := Unmarshal(jsonData, &nulls)
if err != nil {
t.Errorf("Unmarshal of null values failed: %v", err)
}
if !nulls.Bool || nulls.Int != 2 || nulls.Int8 != 3 || nulls.Int16 != 4 || nulls.Int32 != 5 || nulls.Int64 != 6 ||
nulls.Uint != 7 || nulls.Uint8 != 8 || nulls.Uint16 != 9 || nulls.Uint32 != 10 || nulls.Uint64 != 11 ||
nulls.Float32 != 12.1 || nulls.Float64 != 13.1 || nulls.String != "14" {
t.Errorf("Unmarshal of null values affected primitives")
}
}

2
libgo/go/encoding/json/encode.go
View file

@ -86,7 +86,7 @@ import (
//
// Anonymous struct fields are usually marshaled as if their inner exported fields
// were fields in the outer struct, subject to the usual Go visibility rules.
// An anonymous struct field with a name given in its JSON tag is treated as
// An anonymous struct field with a name given in its JSON tag is treated as
// having that name instead of as anonymous.
//
// Handling of anonymous struct fields is new in Go 1.1.

63
libgo/go/encoding/pem/pem.go
View file

@ -11,6 +11,7 @@ import (
"bytes"
"encoding/base64"
"io"
"sort"
)
// A Block represents a PEM encoded structure.
@ -209,26 +210,46 @@ func (l *lineBreaker) Close() (err error) {
return
}
func Encode(out io.Writer, b *Block) (err error) {
_, err = out.Write(pemStart[1:])
if err != nil {
return
func writeHeader(out io.Writer, k, v string) error {
_, err := out.Write([]byte(k + ": " + v + "\n"))
return err
}
func Encode(out io.Writer, b *Block) error {
if _, err := out.Write(pemStart[1:]); err != nil {
return err
}
_, err = out.Write([]byte(b.Type + "-----\n"))
if err != nil {
return
if _, err := out.Write([]byte(b.Type + "-----\n")); err != nil {
return err
}
if len(b.Headers) > 0 {
for k, v := range b.Headers {
_, err = out.Write([]byte(k + ": " + v + "\n"))
if err != nil {
return
const procType = "Proc-Type"
h := make([]string, 0, len(b.Headers))
hasProcType := false
for k := range b.Headers {
if k == procType {
hasProcType = true
continue
}
h = append(h, k)
}
// The Proc-Type header must be written first.
// See RFC 1421, section 4.6.1.1
if hasProcType {
if err := writeHeader(out, procType, b.Headers[procType]); err != nil {
return err
}
}
_, err = out.Write([]byte{'\n'})
if err != nil {
return
// For consistency of output, write other headers sorted by key.
sort.Strings(h)
for _, k := range h {
if err := writeHeader(out, k, b.Headers[k]); err != nil {
return err
}
}
if _, err := out.Write([]byte{'\n'}); err != nil {
return err
}
}
@ -236,19 +257,17 @@ func Encode(out io.Writer, b *Block) (err error) {
breaker.out = out
b64 := base64.NewEncoder(base64.StdEncoding, &breaker)
_, err = b64.Write(b.Bytes)
if err != nil {
return
if _, err := b64.Write(b.Bytes); err != nil {
return err
}
b64.Close()
breaker.Close()
_, err = out.Write(pemEnd[1:])
if err != nil {
return
if _, err := out.Write(pemEnd[1:]); err != nil {
return err
}
_, err = out.Write([]byte(b.Type + "-----\n"))
return
_, err := out.Write([]byte(b.Type + "-----\n"))
return err
}
func EncodeToMemory(b *Block) []byte {

104
libgo/go/encoding/pem/pem_test.go
View file

@ -43,7 +43,7 @@ func TestDecode(t *testing.T) {
if !reflect.DeepEqual(result, privateKey) {
t.Errorf("#1 got:%#v want:%#v", result, privateKey)
}
result, _ = Decode([]byte(pemPrivateKey))
result, _ = Decode([]byte(pemPrivateKey2))
if !reflect.DeepEqual(result, privateKey2) {
t.Errorf("#2 got:%#v want:%#v", result, privateKey2)
}
@ -51,8 +51,8 @@ func TestDecode(t *testing.T) {
func TestEncode(t *testing.T) {
r := EncodeToMemory(privateKey2)
if string(r) != pemPrivateKey {
t.Errorf("got:%s want:%s", r, pemPrivateKey)
if string(r) != pemPrivateKey2 {
t.Errorf("got:%s want:%s", r, pemPrivateKey2)
}
}
@ -341,50 +341,64 @@ var privateKey = &Block{Type: "RSA PRIVATE KEY",
},
}
var privateKey2 = &Block{Type: "RSA PRIVATE KEY",
Headers: map[string]string{},
Bytes: []uint8{0x30, 0x82, 0x1, 0x3a, 0x2, 0x1, 0x0, 0x2,
0x41, 0x0, 0xb2, 0x99, 0xf, 0x49, 0xc4, 0x7d, 0xfa, 0x8c,
0xd4, 0x0, 0xae, 0x6a, 0x4d, 0x1b, 0x8a, 0x3b, 0x6a, 0x13,
0x64, 0x2b, 0x23, 0xf2, 0x8b, 0x0, 0x3b, 0xfb, 0x97, 0x79,
0xa, 0xde, 0x9a, 0x4c, 0xc8, 0x2b, 0x8b, 0x2a, 0x81, 0x74,
0x7d, 0xde, 0xc0, 0x8b, 0x62, 0x96, 0xe5, 0x3a, 0x8, 0xc3,
0x31, 0x68, 0x7e, 0xf2, 0x5c, 0x4b, 0xf4, 0x93, 0x6b, 0xa1,
0xc0, 0xe6, 0x4, 0x1e, 0x9d, 0x15, 0x2, 0x3, 0x1, 0x0, 0x1,
0x2, 0x41, 0x0, 0x8a, 0xbd, 0x6a, 0x69, 0xf4, 0xd1, 0xa4,
0xb4, 0x87, 0xf0, 0xab, 0x8d, 0x7a, 0xae, 0xfd, 0x38, 0x60,
0x94, 0x5, 0xc9, 0x99, 0x98, 0x4e, 0x30, 0xf5, 0x67, 0xe1,
0xe8, 0xae, 0xef, 0xf4, 0x4e, 0x8b, 0x18, 0xbd, 0xb1, 0xec,
0x78, 0xdf, 0xa3, 0x1a, 0x55, 0xe3, 0x2a, 0x48, 0xd7, 0xfb,
0x13, 0x1f, 0x5a, 0xf1, 0xf4, 0x4d, 0x7d, 0x6b, 0x2c, 0xed,
0x2a, 0x9d, 0xf5, 0xe5, 0xae, 0x45, 0x35, 0x2, 0x21, 0x0,
0xda, 0xb2, 0xf1, 0x80, 0x48, 0xba, 0xa6, 0x8d, 0xe7, 0xdf,
0x4, 0xd2, 0xd3, 0x5d, 0x5d, 0x80, 0xe6, 0xe, 0x2d, 0xfa,
0x42, 0xd5, 0xa, 0x9b, 0x4, 0x21, 0x90, 0x32, 0x71, 0x5e,
0x46, 0xb3, 0x2, 0x21, 0x0, 0xd1, 0xf, 0x2e, 0x66, 0xb1,
0xd0, 0xc1, 0x3f, 0x10, 0xef, 0x99, 0x27, 0xbf, 0x53, 0x24,
0xa3, 0x79, 0xca, 0x21, 0x81, 0x46, 0xcb, 0xf9, 0xca, 0xfc,
0x79, 0x52, 0x21, 0xf1, 0x6a, 0x31, 0x17, 0x2, 0x20, 0x21,
0x2, 0x89, 0x79, 0x37, 0x81, 0x14, 0xca, 0xae, 0x88, 0xf7,
0xd, 0x6b, 0x61, 0xd8, 0x4f, 0x30, 0x6a, 0x4b, 0x7e, 0x4e,
0xc0, 0x21, 0x4d, 0xac, 0x9d, 0xf4, 0x49, 0xe8, 0xda, 0xb6,
0x9, 0x2, 0x20, 0x16, 0xb3, 0xec, 0x59, 0x10, 0xa4, 0x57,
0xe8, 0xe, 0x61, 0xc6, 0xa3, 0xf, 0x5e, 0xeb, 0x12, 0xa9,
0xae, 0x2e, 0xb7, 0x48, 0x45, 0xec, 0x69, 0x83, 0xc3, 0x75,
0xc, 0xe4, 0x97, 0xa0, 0x9f, 0x2, 0x20, 0x69, 0x52, 0xb4,
0x6, 0xe8, 0x50, 0x60, 0x71, 0x4c, 0x3a, 0xb7, 0x66, 0xba,
0xd, 0x8a, 0xc9, 0xb7, 0xd, 0xa3, 0x8, 0x6c, 0xa3, 0xf2,
0x62, 0xb0, 0x2a, 0x84, 0xaa, 0x2f, 0xd6, 0x1e, 0x55,
var privateKey2 = &Block{
Type: "RSA PRIVATE KEY",
Headers: map[string]string{
"Proc-Type": "4,ENCRYPTED",
"DEK-Info": "AES-128-CBC,BFCD243FEDBB40A4AA6DDAA1335473A4",
"Content-Domain": "RFC822",
},
Bytes: []uint8{
0xa8, 0x35, 0xcc, 0x2b, 0xb9, 0xcb, 0x21, 0xab, 0xc0,
0x9d, 0x76, 0x61, 0x0, 0xf4, 0x81, 0xad, 0x69, 0xd2,
0xc0, 0x42, 0x41, 0x3b, 0xe4, 0x3c, 0xaf, 0x59, 0x5e,
0x6d, 0x2a, 0x3c, 0x9c, 0xa1, 0xa4, 0x5e, 0x68, 0x37,
0xc4, 0x8c, 0x70, 0x1c, 0xa9, 0x18, 0xe6, 0xc2, 0x2b,
0x8a, 0x91, 0xdc, 0x2d, 0x1f, 0x8, 0x23, 0x39, 0xf1,
0x4b, 0x8b, 0x1b, 0x2f, 0x46, 0xb, 0xb2, 0x26, 0xba,
0x4f, 0x40, 0x80, 0x39, 0xc4, 0xb1, 0xcb, 0x3b, 0xb4,
0x65, 0x3f, 0x1b, 0xb2, 0xf7, 0x8, 0xd2, 0xc6, 0xd5,
0xa8, 0x9f, 0x23, 0x69, 0xb6, 0x3d, 0xf9, 0xac, 0x1c,
0xb3, 0x13, 0x87, 0x64, 0x4, 0x37, 0xdb, 0x40, 0xc8,
0x82, 0xc, 0xd0, 0xf8, 0x21, 0x7c, 0xdc, 0xbd, 0x9, 0x4,
0x20, 0x16, 0xb0, 0x97, 0xe2, 0x6d, 0x56, 0x1d, 0xe3,
0xec, 0xf0, 0xfc, 0xe2, 0x56, 0xad, 0xa4, 0x3, 0x70,
0x6d, 0x63, 0x3c, 0x1, 0xbe, 0x3e, 0x28, 0x38, 0x6f,
0xc0, 0xe6, 0xfd, 0x85, 0xd1, 0x53, 0xa8, 0x9b, 0xcb,
0xd4, 0x4, 0xb1, 0x73, 0xb9, 0x73, 0x32, 0xd6, 0x7a,
0xc6, 0x29, 0x25, 0xa5, 0xda, 0x17, 0x93, 0x7a, 0x10,
0xe8, 0x41, 0xfb, 0xa5, 0x17, 0x20, 0xf8, 0x4e, 0xe9,
0xe3, 0x8f, 0x51, 0x20, 0x13, 0xbb, 0xde, 0xb7, 0x93,
0xae, 0x13, 0x8a, 0xf6, 0x9, 0xf4, 0xa6, 0x41, 0xe0,
0x2b, 0x51, 0x1a, 0x30, 0x38, 0xd, 0xb1, 0x3b, 0x67,
0x87, 0x64, 0xf5, 0xca, 0x32, 0x67, 0xd1, 0xc8, 0xa5,
0x3d, 0x23, 0x72, 0xc4, 0x6, 0xaf, 0x8f, 0x7b, 0x26,
0xac, 0x3c, 0x75, 0x91, 0xa1, 0x0, 0x13, 0xc6, 0x5c,
0x49, 0xd5, 0x3c, 0xe7, 0xb2, 0xb2, 0x99, 0xe0, 0xd5,
0x25, 0xfa, 0xe2, 0x12, 0x80, 0x37, 0x85, 0xcf, 0x92,
0xca, 0x1b, 0x9f, 0xf3, 0x4e, 0xd8, 0x80, 0xef, 0x3c,
0xce, 0xcd, 0xf5, 0x90, 0x9e, 0xf9, 0xa7, 0xb2, 0xc,
0x49, 0x4, 0xf1, 0x9, 0x8f, 0xea, 0x63, 0xd2, 0x70,
0xbb, 0x86, 0xbf, 0x34, 0xab, 0xb2, 0x3, 0xb1, 0x59,
0x33, 0x16, 0x17, 0xb0, 0xdb, 0x77, 0x38, 0xf4, 0xb4,
0x94, 0xb, 0x25, 0x16, 0x7e, 0x22, 0xd4, 0xf9, 0x22,
0xb9, 0x78, 0xa3, 0x4, 0x84, 0x4, 0xd2, 0xda, 0x84,
0x2d, 0x63, 0xdd, 0xf8, 0x50, 0x6a, 0xf6, 0xe3, 0xf5,
0x65, 0x40, 0x7c, 0xa9,
},
}
var pemPrivateKey = `-----BEGIN RSA PRIVATE KEY-----
MIIBOgIBAAJBALKZD0nEffqM1ACuak0bijtqE2QrI/KLADv7l3kK3ppMyCuLKoF0
fd7Ai2KW5ToIwzFofvJcS/STa6HA5gQenRUCAwEAAQJBAIq9amn00aS0h/CrjXqu
/ThglAXJmZhOMPVn4eiu7/ROixi9sex436MaVeMqSNf7Ex9a8fRNfWss7Sqd9eWu
RTUCIQDasvGASLqmjeffBNLTXV2A5g4t+kLVCpsEIZAycV5GswIhANEPLmax0ME/
EO+ZJ79TJKN5yiGBRsv5yvx5UiHxajEXAiAhAol5N4EUyq6I9w1rYdhPMGpLfk7A
IU2snfRJ6Nq2CQIgFrPsWRCkV+gOYcajD17rEqmuLrdIRexpg8N1DOSXoJ8CIGlS
tAboUGBxTDq3ZroNism3DaMIbKPyYrAqhKov1h5V
var pemPrivateKey2 = `-----BEGIN RSA PRIVATE KEY-----
Proc-Type: 4,ENCRYPTED
Content-Domain: RFC822
DEK-Info: AES-128-CBC,BFCD243FEDBB40A4AA6DDAA1335473A4
qDXMK7nLIavAnXZhAPSBrWnSwEJBO+Q8r1lebSo8nKGkXmg3xIxwHKkY5sIripHc
LR8IIznxS4sbL0YLsia6T0CAOcSxyzu0ZT8bsvcI0sbVqJ8jabY9+awcsxOHZAQ3
20DIggzQ+CF83L0JBCAWsJfibVYd4+zw/OJWraQDcG1jPAG+Pig4b8Dm/YXRU6ib
y9QEsXO5czLWesYpJaXaF5N6EOhB+6UXIPhO6eOPUSATu963k64TivYJ9KZB4CtR
GjA4DbE7Z4dk9coyZ9HIpT0jcsQGr497Jqw8dZGhABPGXEnVPOeyspng1SX64hKA
N4XPksobn/NO2IDvPM7N9ZCe+aeyDEkE8QmP6mPScLuGvzSrsgOxWTMWF7Dbdzj0
tJQLJRZ+ItT5Irl4owSEBNLahC1j3fhQavbj9WVAfKk=
-----END RSA PRIVATE KEY-----
`

21
libgo/go/encoding/xml/marshal_test.go
View file

@ -687,6 +687,27 @@ var marshalTests = []struct {
Value: &IgnoreTest{},
UnmarshalOnly: true,
},
// Test escaping.
{
ExpectXML: `<a><nested><value>dquote: &#34;; squote: &#39;; ampersand: &amp;; less: &lt;; greater: &gt;;</value></nested></a>`,
Value: &AnyTest{
Nested: `dquote: "; squote: '; ampersand: &; less: <; greater: >;`,
},
},
{
ExpectXML: `<a><nested><value>newline: &#xA;; cr: &#xD;; tab: &#x9;;</value></nested></a>`,
Value: &AnyTest{
Nested: "newline: \n; cr: \r; tab: \t;",
},
},
{
ExpectXML: "<a><nested><value>1\r2\r\n3\n\r4\n5</value></nested></a>",
Value: &AnyTest{
Nested: "1\n2\n3\n\n4\n5",
},
UnmarshalOnly: true,
},
}
func TestMarshal(t *testing.T) {

243
libgo/go/encoding/xml/xml.go
View file

@ -181,7 +181,6 @@ type Decoder struct {
ns map[string]string
err error
line int
tmp [32]byte
}
// NewDecoder creates a new XML parser reading from r.
@ -877,94 +876,103 @@ Input:
// XML in all its glory allows a document to define and use
// its own character names with <!ENTITY ...> directives.
// Parsers are required to recognize lt, gt, amp, apos, and quot
// even if they have not been declared. That's all we allow.
var i int
var semicolon bool
var valid bool
for i = 0; i < len(d.tmp); i++ {
var ok bool
d.tmp[i], ok = d.getc()
if !ok {
if d.err == io.EOF {
d.err = d.syntaxError("unexpected EOF")
}
// even if they have not been declared.
before := d.buf.Len()
d.buf.WriteByte('&')
var ok bool
var text string
var haveText bool
if b, ok = d.mustgetc(); !ok {
return nil
}
if b == '#' {
d.buf.WriteByte(b)
if b, ok = d.mustgetc(); !ok {
return nil
}
c := d.tmp[i]
if c == ';' {
semicolon = true
valid = i > 0
break
}
if 'a' <= c && c <= 'z' ||
'A' <= c && c <= 'Z' ||
'0' <= c && c <= '9' ||
c == '_' || c == '#' {
continue
}
d.ungetc(c)
break
}
s := string(d.tmp[0:i])
if !valid {
if !d.Strict {
b0, b1 = 0, 0
d.buf.WriteByte('&')
d.buf.Write(d.tmp[0:i])
if semicolon {
d.buf.WriteByte(';')
base := 10
if b == 'x' {
base = 16
d.buf.WriteByte(b)
if b, ok = d.mustgetc(); !ok {
return nil
}
continue Input
}
semi := ";"
if !semicolon {
semi = " (no semicolon)"
start := d.buf.Len()
for '0' <= b && b <= '9' ||
base == 16 && 'a' <= b && b <= 'f' ||
base == 16 && 'A' <= b && b <= 'F' {
d.buf.WriteByte(b)
if b, ok = d.mustgetc(); !ok {
return nil
}
}
if i < len(d.tmp) {
d.err = d.syntaxError("invalid character entity &" + s + semi)
if b != ';' {
d.ungetc(b)
} else {
d.err = d.syntaxError("invalid character entity &" + s + "... too long")
}
return nil
}
var haveText bool
var text string
if i >= 2 && s[0] == '#' {
var n uint64
var err error
if i >= 3 && s[1] == 'x' {
n, err = strconv.ParseUint(s[2:], 16, 64)
} else {
n, err = strconv.ParseUint(s[1:], 10, 64)
}
if err == nil && n <= unicode.MaxRune {
text = string(n)
haveText = true
s := string(d.buf.Bytes()[start:])
d.buf.WriteByte(';')
n, err := strconv.ParseUint(s, base, 64)
if err == nil && n <= unicode.MaxRune {
text = string(n)
haveText = true
}
}
} else {
if r, ok := entity[s]; ok {
text = string(r)
haveText = true
} else if d.Entity != nil {
text, haveText = d.Entity[s]
d.ungetc(b)
if !d.readName() {
if d.err != nil {
return nil
}
ok = false
}
}
if !haveText {
if !d.Strict {
b0, b1 = 0, 0
d.buf.WriteByte('&')
d.buf.Write(d.tmp[0:i])
if b, ok = d.mustgetc(); !ok {
return nil
}
if b != ';' {
d.ungetc(b)
} else {
name := d.buf.Bytes()[before+1:]
d.buf.WriteByte(';')
continue Input
if isName(name) {
s := string(name)
if r, ok := entity[s]; ok {
text = string(r)
haveText = true
} else if d.Entity != nil {
text, haveText = d.Entity[s]
}
}
}
d.err = d.syntaxError("invalid character entity &" + s + ";")
return nil
}
d.buf.Write([]byte(text))
b0, b1 = 0, 0
continue Input
if haveText {
d.buf.Truncate(before)
d.buf.Write([]byte(text))
b0, b1 = 0, 0
continue Input
}
if !d.Strict {
b0, b1 = 0, 0
continue Input
}
ent := string(d.buf.Bytes()[before])
if ent[len(ent)-1] != ';' {
ent += " (no semicolon)"
}
d.err = d.syntaxError("invalid character entity " + ent)
return nil
}
d.buf.WriteByte(b)
// We must rewrite unescaped \r and \r\n into \n.
if b == '\r' {
d.buf.WriteByte('\n')
} else if b1 == '\r' && b == '\n' {
// Skip \r\n--we already wrote \n.
} else {
d.buf.WriteByte(b)
}
b0, b1 = b1, b
}
data := d.buf.Bytes()
@ -985,20 +993,7 @@ Input:
}
}
// Must rewrite \r and \r\n into \n.
w := 0
for r := 0; r < len(data); r++ {
b := data[r]
if b == '\r' {
if r+1 < len(data) && data[r+1] == '\n' {
continue
}
b = '\n'
}
data[w] = b
w++
}
return data[0:w]
return data
}
// Decide whether the given rune is in the XML Character Range, per
@ -1034,18 +1029,34 @@ func (d *Decoder) nsname() (name Name, ok bool) {
// Do not set d.err if the name is missing (unless unexpected EOF is received):
// let the caller provide better context.
func (d *Decoder) name() (s string, ok bool) {
d.buf.Reset()
if !d.readName() {
return "", false
}
// Now we check the characters.
s = d.buf.String()
if !isName([]byte(s)) {
d.err = d.syntaxError("invalid XML name: " + s)
return "", false
}
return s, true
}
// Read a name and append its bytes to d.buf.
// The name is delimited by any single-byte character not valid in names.
// All multi-byte characters are accepted; the caller must check their validity.
func (d *Decoder) readName() (ok bool) {
var b byte
if b, ok = d.mustgetc(); !ok {
return
}
// As a first approximation, we gather the bytes [A-Za-z_:.-\x80-\xFF]*
if b < utf8.RuneSelf && !isNameByte(b) {
d.ungetc(b)
return "", false
return false
}
d.buf.Reset()
d.buf.WriteByte(b)
for {
if b, ok = d.mustgetc(); !ok {
return
@ -1056,16 +1067,7 @@ func (d *Decoder) name() (s string, ok bool) {
}
d.buf.WriteByte(b)
}
// Then we check the characters.
s = d.buf.String()
for i, c := range s {
if !unicode.Is(first, c) && (i == 0 || !unicode.Is(second, c)) {
d.err = d.syntaxError("invalid XML name: " + s)
return "", false
}
}
return s, true
return true
}
func isNameByte(c byte) bool {
@ -1075,6 +1077,30 @@ func isNameByte(c byte) bool {
c == '_' || c == ':' || c == '.' || c == '-'
}
func isName(s []byte) bool {
if len(s) == 0 {
return false
}
c, n := utf8.DecodeRune(s)
if c == utf8.RuneError && n == 1 {
return false
}
if !unicode.Is(first, c) {
return false
}
for n < len(s) {
s = s[n:]
c, n = utf8.DecodeRune(s)
if c == utf8.RuneError && n == 1 {
return false
}
if !unicode.Is(first, c) && !unicode.Is(second, c) {
return false
}
}
return true
}
// These tables were generated by cut and paste from Appendix B of
// the XML spec at http://www.xml.com/axml/testaxml.htm
// and then reformatting. First corresponds to (Letter | '_' | ':')
@ -1689,6 +1715,9 @@ var (
esc_amp = []byte("&amp;")
esc_lt = []byte("&lt;")
esc_gt = []byte("&gt;")
esc_tab = []byte("&#x9;")
esc_nl = []byte("&#xA;")
esc_cr = []byte("&#xD;")
)
// Escape writes to w the properly escaped XML equivalent
@ -1708,6 +1737,12 @@ func Escape(w io.Writer, s []byte) {
esc = esc_lt
case '>':
esc = esc_gt
case '\t':
esc = esc_tab
case '\n':
esc = esc_nl
case '\r':
esc = esc_cr
default:
continue
}

35
libgo/go/encoding/xml/xml_test.go
View file

@ -19,6 +19,7 @@ const testInput = `
<body xmlns:foo="ns1" xmlns="ns2" xmlns:tag="ns3" ` +
"\r\n\t" + ` >
<hello lang="en">World &lt;&gt;&apos;&quot; &#x767d;&#40300;翔</hello>
<query>&何; &is-it;</query>
<goodbye />
<outer foo:attr="value" xmlns:tag="ns4">
<inner/>
@ -28,6 +29,8 @@ const testInput = `
</tag:name>
</body><!-- missing final newline -->`
var testEntity = map[string]string{"何": "What", "is-it": "is it?"}
var rawTokens = []Token{
CharData("\n"),
ProcInst{"xml", []byte(`version="1.0" encoding="UTF-8"`)},
@ -41,6 +44,10 @@ var rawTokens = []Token{
CharData("World <>'\" 白鵬翔"),
EndElement{Name{"", "hello"}},
CharData("\n "),
StartElement{Name{"", "query"}, []Attr{}},
CharData("What is it?"),
EndElement{Name{"", "query"}},
CharData("\n "),
StartElement{Name{"", "goodbye"}, []Attr{}},
EndElement{Name{"", "goodbye"}},
CharData("\n "),
@ -74,6 +81,10 @@ var cookedTokens = []Token{
CharData("World <>'\" 白鵬翔"),
EndElement{Name{"ns2", "hello"}},
CharData("\n "),
StartElement{Name{"ns2", "query"}, []Attr{}},
CharData("What is it?"),
EndElement{Name{"ns2", "query"}},
CharData("\n "),
StartElement{Name{"ns2", "goodbye"}, []Attr{}},
EndElement{Name{"ns2", "goodbye"}},
CharData("\n "),
@ -156,6 +167,7 @@ var xmlInput = []string{
func TestRawToken(t *testing.T) {
d := NewDecoder(strings.NewReader(testInput))
d.Entity = testEntity
testRawToken(t, d, rawTokens)
}
@ -164,8 +176,14 @@ const nonStrictInput = `
<tag>&unknown;entity</tag>
<tag>&#123</tag>
<tag>&#zzz;</tag>
<tag>&なまえ3;</tag>
<tag>&lt-gt;</tag>
<tag>&;</tag>
<tag>&0a;</tag>
`
var nonStringEntity = map[string]string{"": "oops!", "0a": "oops!"}
var nonStrictTokens = []Token{
CharData("\n"),
StartElement{Name{"", "tag"}, []Attr{}},
@ -184,6 +202,22 @@ var nonStrictTokens = []Token{
CharData("&#zzz;"),
EndElement{Name{"", "tag"}},
CharData("\n"),
StartElement{Name{"", "tag"}, []Attr{}},
CharData("&なまえ3;"),
EndElement{Name{"", "tag"}},
CharData("\n"),
StartElement{Name{"", "tag"}, []Attr{}},
CharData("&lt-gt;"),
EndElement{Name{"", "tag"}},
CharData("\n"),
StartElement{Name{"", "tag"}, []Attr{}},
CharData("&;"),
EndElement{Name{"", "tag"}},
CharData("\n"),
StartElement{Name{"", "tag"}, []Attr{}},
CharData("&0a;"),
EndElement{Name{"", "tag"}},
CharData("\n"),
}
func TestNonStrictRawToken(t *testing.T) {
@ -317,6 +351,7 @@ func TestNestedDirectives(t *testing.T) {
func TestToken(t *testing.T) {
d := NewDecoder(strings.NewReader(testInput))
d.Entity = testEntity
for i, want := range cookedTokens {
have, err := d.Token()

3
libgo/go/exp/gotype/gotype.go
View file

@ -176,8 +176,7 @@ func processPackage(fset *token.FileSet, files map[string]*ast.File) {
report(err)
return
}
_, err = types.Check(fset, pkg)
if err != nil {
if err = types.Check(fset, pkg, nil, nil); err != nil {
report(err)
}
}

6
libgo/go/exp/gotype/gotype_test.go
View file

@ -35,10 +35,12 @@ var tests = []struct {
// directories
{filepath.Join(runtime.GOROOT(), "src/pkg/go/ast"), "ast"},
{filepath.Join(runtime.GOROOT(), "src/pkg/go/build"), "build"},
{filepath.Join(runtime.GOROOT(), "src/pkg/go/doc"), "doc"},
{filepath.Join(runtime.GOROOT(), "src/pkg/go/token"), "scanner"},
{filepath.Join(runtime.GOROOT(), "src/pkg/go/scanner"), "scanner"},
{filepath.Join(runtime.GOROOT(), "src/pkg/go/parser"), "parser"},
{filepath.Join(runtime.GOROOT(), "src/pkg/go/printer"), "printer"},
{filepath.Join(runtime.GOROOT(), "src/pkg/go/scanner"), "scanner"},
{filepath.Join(runtime.GOROOT(), "src/pkg/go/token"), "token"},
{filepath.Join(runtime.GOROOT(), "src/pkg/exp/types"), "types"},
}

2
libgo/go/exp/html/doctype.go
View file

@ -9,7 +9,7 @@ import (
)
// parseDoctype parses the data from a DoctypeToken into a name,
// public identifier, and system identifier. It returns a Node whose Type
// public identifier, and system identifier. It returns a Node whose Type
// is DoctypeNode, whose Data is the name, and which has attributes
// named "system" and "public" for the two identifiers if they were present.
// quirks is whether the document should be parsed in "quirks mode".

4
libgo/go/exp/html/escape.go
View file

@ -10,7 +10,7 @@ import (
"unicode/utf8"
)
// These replacements permit compatibility with old numeric entities that
// These replacements permit compatibility with old numeric entities that
// assumed Windows-1252 encoding.
// http://www.whatwg.org/specs/web-apps/current-work/multipage/tokenization.html#consume-a-character-reference
var replacementTable = [...]rune{
@ -46,7 +46,7 @@ var replacementTable = [...]rune{
'\u009D',
'\u017E',
'\u0178', // Last entry is 0x9F.
// 0x00->'\uFFFD' is handled programmatically.
// 0x00->'\uFFFD' is handled programmatically.
// 0x0D->'\u000D' is a no-op.
}

2
libgo/go/exp/html/parse.go
View file

@ -2027,7 +2027,7 @@ func Parse(r io.Reader) (*Node, error) {
return p.doc, nil
}
// ParseFragment parses a fragment of HTML and returns the nodes that were
// ParseFragment parses a fragment of HTML and returns the nodes that were
// found. If the fragment is the InnerHTML for an existing element, pass that
// element in context.
func ParseFragment(r io.Reader, context *Node) ([]*Node, error) {

4
libgo/go/exp/html/render.go
View file

@ -35,7 +35,7 @@ type writer interface {
// children; the <a> is reparented to the <table>'s parent. However, calling
// Parse on "<a><table><a>" does not return an error, but the result has an <a>
// element with an <a> child, and is therefore not 'well-formed'.
//
//
// Programmatically constructed trees are typically also 'well-formed', but it
// is possible to construct a tree that looks innocuous but, when rendered and
// re-parsed, results in a different tree. A simple example is that a solitary
@ -53,7 +53,7 @@ func Render(w io.Writer, n *Node) error {
return buf.Flush()
}
// plaintextAbort is returned from render1 when a <plaintext> element
// plaintextAbort is returned from render1 when a <plaintext> element
// has been rendered. No more end tags should be rendered after that.
var plaintextAbort = errors.New("html: internal error (plaintext abort)")

2
libgo/go/exp/html/token.go
View file

@ -320,7 +320,7 @@ loop:
}
// readRawEndTag attempts to read a tag like "</foo>", where "foo" is z.rawTag.
// If it succeeds, it backs up the input position to reconsume the tag and
// If it succeeds, it backs up the input position to reconsume the tag and
// returns true. Otherwise it returns false. The opening "</" has already been
// consumed.
func (z *Tokenizer) readRawEndTag() bool {

177
libgo/go/exp/locale/collate/build/builder.go
View file

@ -25,7 +25,7 @@ import (
// compacted.
// - Compress secondary weights into 8 bits.
// - Some LDML specs specify a context element. Currently we simply concatenate
// those. Context can be implemented using the contraction trie. If Builder
// those. Context can be implemented using the contraction trie. If Builder
// could analyze and detect when using a context makes sense, there is no
// need to expose this construct in the API.
@ -58,7 +58,9 @@ type Tailoring struct {
id string
builder *Builder
index *ordering
// TODO: implement.
anchor *entry
before bool
}
// NewBuilder returns a new Builder.
@ -72,7 +74,7 @@ func NewBuilder() *Builder {
}
}
// Tailoring returns a Tailoring for the given locale. One should
// Tailoring returns a Tailoring for the given locale. One should
// have completed all calls to Add before calling Tailoring.
func (b *Builder) Tailoring(locale string) *Tailoring {
t := &Tailoring{
@ -80,11 +82,12 @@ func (b *Builder) Tailoring(locale string) *Tailoring {
builder: b,
index: b.root.clone(),
}
t.index.id = t.id
b.locale = append(b.locale, t)
return t
}
// Add adds an entry to the collation element table, mapping
// Add adds an entry to the collation element table, mapping
// a slice of runes to a sequence of collation elements.
// A collation element is specified as list of weights: []int{primary, secondary, ...}.
// The entries are typically obtained from a collation element table
@ -95,7 +98,6 @@ func (b *Builder) Tailoring(locale string) *Tailoring {
// a value for each colelem that is a variable. (See the reference above.)
func (b *Builder) Add(runes []rune, colelems [][]int, variables []int) error {
str := string(runes)
elems := make([][]int, len(colelems))
for i, ce := range colelems {
elems[i] = append(elems[i], ce...)
@ -127,7 +129,7 @@ func (b *Builder) Add(runes []rune, colelems [][]int, variables []int) error {
if ce[0] > b.varTop {
b.varTop = ce[0]
}
} else if ce[0] > 0 {
} else if ce[0] > 1 { // 1 is a special primary value reserved for FFFE
if ce[0] <= b.varTop {
return fmt.Errorf("primary value %X of non-variable is smaller than the highest variable %X", ce[0], b.varTop)
}
@ -144,6 +146,21 @@ func (b *Builder) Add(runes []rune, colelems [][]int, variables []int) error {
return nil
}
func (t *Tailoring) setAnchor(anchor string) error {
anchor = norm.NFD.String(anchor)
a := t.index.find(anchor)
if a == nil {
a = t.index.newEntry(anchor, nil)
a.implicit = true
for _, r := range []rune(anchor) {
e := t.index.find(string(r))
e.lock = true
}
}
t.anchor = a
return nil
}
// SetAnchor sets the point after which elements passed in subsequent calls to
// Insert will be inserted. It is equivalent to the reset directive in an LDML
// specification. See Insert for an example.
@ -151,14 +168,20 @@ func (b *Builder) Add(runes []rune, colelems [][]int, variables []int) error {
// <first_tertiary_ignorable/>, <last_teriary_ignorable/>, <first_primary_ignorable/>,
// and <last_non_ignorable/>.
func (t *Tailoring) SetAnchor(anchor string) error {
// TODO: implement.
if err := t.setAnchor(anchor); err != nil {
return err
}
t.before = false
return nil
}
// SetAnchorBefore is similar to SetAnchor, except that subsequent calls to
// Insert will insert entries before the anchor.
func (t *Tailoring) SetAnchorBefore(anchor string) error {
// TODO: implement.
if err := t.setAnchor(anchor); err != nil {
return err
}
t.before = true
return nil
}
@ -171,7 +194,7 @@ func (t *Tailoring) SetAnchorBefore(anchor string) error {
// See http://www.unicode.org/reports/tr10/#Tailoring_Example for details
// on parametric tailoring and http://unicode.org/reports/tr35/#Collation_Elements
// for full details on LDML.
//
//
// Examples: create a tailoring for Swedish, where "ä" is ordered after "z"
// at the primary sorting level:
// t := b.Tailoring("se")
@ -195,7 +218,112 @@ func (t *Tailoring) SetAnchorBefore(anchor string) error {
// t.SetAnchor("<last_primary_ignorable/>")
// t.Insert(collate.Primary, "0", "")
func (t *Tailoring) Insert(level collate.Level, str, extend string) error {
// TODO: implement.
if t.anchor == nil {
return fmt.Errorf("%s:Insert: no anchor point set for tailoring of %s", t.id, str)
}
str = norm.NFD.String(str)
e := t.index.find(str)
if e == nil {
e = t.index.newEntry(str, nil)
} else if e.logical != noAnchor {
return fmt.Errorf("%s:Insert: cannot reinsert logical reset position %q", t.id, e.str)
}
if e.lock {
return fmt.Errorf("%s:Insert: cannot reinsert element %q", t.id, e.str)
}
a := t.anchor
// Find the first element after the anchor which differs at a level smaller or
// equal to the given level. Then insert at this position.
// See http://unicode.org/reports/tr35/#Collation_Elements, Section 5.14.5 for details.
e.before = t.before
if t.before {
t.before = false
if a.prev == nil {
a.insertBefore(e)
} else {
for a = a.prev; a.level > level; a = a.prev {
}
a.insertAfter(e)
}
e.level = level
} else {
for ; a.level > level; a = a.next {
}
e.level = a.level
if a != e {
a.insertAfter(e)
a.level = level
} else {
// We don't set a to prev itself. This has the effect of the entry
// getting new collation elements that are an increment of itself.
// This is intentional.
a.prev.level = level
}
}
e.extend = norm.NFD.String(extend)
e.exclude = false
e.elems = nil
t.anchor = e
return nil
}
func (o *ordering) getWeight(e *entry) [][]int {
if len(e.elems) == 0 && e.logical == noAnchor {
if e.implicit {
for _, r := range e.runes {
e.elems = append(e.elems, o.getWeight(o.find(string(r)))...)
}
} else if e.before {
count := [collate.Identity + 1]int{}
a := e
for ; a.elems == nil && !a.implicit; a = a.next {
count[a.level]++
}
e.elems = append([][]int(nil), make([]int, len(a.elems[0])))
copy(e.elems[0], a.elems[0])
for i := collate.Primary; i < collate.Quaternary; i++ {
if count[i] != 0 {
e.elems[0][i] -= count[i]
break
}
}
if e.prev != nil {
o.verifyWeights(e.prev, e, e.prev.level)
}
} else {
prev := e.prev
e.elems = nextWeight(prev.level, o.getWeight(prev))
o.verifyWeights(e, e.next, e.level)
}
}
return e.elems
}
func (o *ordering) addExtension(e *entry) {
if ex := o.find(e.extend); ex != nil {
e.elems = append(e.elems, ex.elems...)
} else {
for _, r := range []rune(e.extend) {
e.elems = append(e.elems, o.find(string(r)).elems...)
}
}
e.extend = ""
}
func (o *ordering) verifyWeights(a, b *entry, level collate.Level) error {
if level == collate.Identity || b == nil || b.elems == nil || a.elems == nil {
return nil
}
for i := collate.Primary; i < level; i++ {
if a.elems[0][i] < b.elems[0][i] {
return nil
}
}
if a.elems[0][level] >= b.elems[0][level] {
err := fmt.Errorf("%s:overflow: collation elements of %q (%X) overflows those of %q (%X) at level %d (%X >= %X)", o.id, a.str, a.runes, b.str, b.runes, level, a.elems, b.elems)
log.Println(err)
// TODO: return the error instead, or better, fix the conflicting entry by making room.
}
return nil
}
@ -205,7 +333,19 @@ func (b *Builder) error(e error) {
}
}
func (b *Builder) errorID(locale string, e error) {
if e != nil {
b.err = fmt.Errorf("%s:%v", locale, e)
}
}
func (b *Builder) buildOrdering(o *ordering) {
for _, e := range o.ordered {
o.getWeight(e)
}
for _, e := range o.ordered {
o.addExtension(e)
}
o.sort()
simplify(o)
b.processExpansions(o) // requires simplify
@ -215,7 +355,7 @@ func (b *Builder) buildOrdering(o *ordering) {
for e := o.front(); e != nil; e, _ = e.nextIndexed() {
if !e.skip() {
ce, err := e.encode()
b.error(err)
b.errorID(o.id, err)
t.insert(e.runes[0], ce)
}
}
@ -252,7 +392,11 @@ func (b *Builder) Build() (*collate.Collator, error) {
if err != nil {
return nil, err
}
return collate.Init(t), nil
c := collate.Init(t)
if c == nil {
panic("generated table of incompatible type")
}
return c, nil
}
// Build builds a Collator for Tailoring t.
@ -308,6 +452,10 @@ func reproducibleFromNFKD(e *entry, exp, nfkd [][]int) bool {
if i >= 2 && ce[2] != maxTertiary {
return false
}
if _, err := makeCE(ce); err != nil {
// Simply return false. The error will be caught elsewhere.
return false
}
}
return true
}
@ -332,12 +480,11 @@ func simplify(o *ordering) {
e.remove()
}
}
// Tag entries for which the runes NFKD decompose to identical values.
for e := o.front(); e != nil; e, _ = e.nextIndexed() {
s := e.str
nfkd := norm.NFKD.String(s)
if len(e.runes) > 1 || keep[e.runes[0]] || nfkd == s {
if e.decompose || len(e.runes) > 1 || len(e.elems) == 1 || keep[e.runes[0]] || nfkd == s {
continue
}
if reproducibleFromNFKD(e, e.elems, o.genColElems(nfkd)) {
@ -459,7 +606,7 @@ func (b *Builder) processContractions(o *ordering) {
elems := []uint32{}
for _, e := range es {
ce, err := e.encodeBase()
b.error(err)
b.errorID(o.id, err)
elems = append(elems, ce)
}
key = fmt.Sprintf("%v", elems)

44
libgo/go/exp/locale/collate/build/colelem.go
View file

@ -26,7 +26,7 @@ const (
// For normal collation elements, we assume that a collation element either has
// a primary or non-default secondary value, not both.
// Collation elements with a primary value are of the form
// 010ppppp pppppppp pppppppp ssssssss
// 01pppppp pppppppp ppppppp0 ssssssss
// - p* is primary collation value
// - s* is the secondary collation value
// or
@ -67,10 +67,10 @@ func makeCE(weights []int) (uint32, error) {
if weights[1] >= 1<<maxSecondaryCompactBits {
return 0, fmt.Errorf("makeCE: secondary weight with non-zero primary out of bounds: %x >= %x", weights[1], 1<<maxSecondaryCompactBits)
}
ce = uint32(weights[0]<<maxSecondaryCompactBits + weights[1])
ce = uint32(weights[0]<<(maxSecondaryCompactBits+1) + weights[1])
ce |= isPrimary
} else {
d := weights[1] - defaultSecondary
d := weights[1] - defaultSecondary + 4
if d >= 1<<maxSecondaryDiffBits || d < 0 {
return 0, fmt.Errorf("makeCE: secondary weight diff out of bounds: %x < 0 || %x > %x", d, d, 1<<maxSecondaryDiffBits)
}
@ -132,7 +132,7 @@ func makeExpandIndex(index int) (uint32, error) {
return expandID + uint32(index), nil
}
// Each list of collation elements corresponding to an expansion starts with
// Each list of collation elements corresponding to an expansion starts with
// a header indicating the length of the sequence.
func makeExpansionHeader(n int) (uint32, error) {
return uint32(n), nil
@ -199,7 +199,7 @@ func implicitPrimary(r rune) int {
return int(r) + otherOffset
}
// convertLargeWeights converts collation elements with large
// convertLargeWeights converts collation elements with large
// primaries (either double primaries or for illegal runes)
// to our own representation.
// A CJK character C is represented in the DUCET as
@ -258,21 +258,31 @@ func convertLargeWeights(elems [][]int) (res [][]int, err error) {
// nextWeight computes the first possible collation weights following elems
// for the given level.
func nextWeight(level collate.Level, elems [][]int) [][]int {
nce := make([][]int, len(elems))
copy(nce, elems)
if level != collate.Identity {
nce[0] = make([]int, len(elems[0]))
copy(nce[0], elems[0])
nce[0][level]++
if level < collate.Secondary {
nce[0][collate.Secondary] = defaultSecondary
if level == collate.Identity {
next := make([][]int, len(elems))
copy(next, elems)
return next
}
next := [][]int{make([]int, len(elems[0]))}
copy(next[0], elems[0])
next[0][level]++
if level < collate.Secondary {
next[0][collate.Secondary] = defaultSecondary
}
if level < collate.Tertiary {
next[0][collate.Tertiary] = defaultTertiary
}
// Filter entries that cannot influence ordering.
for _, ce := range elems[1:] {
skip := true
for i := collate.Primary; i < level; i++ {
skip = skip && ce[i] == 0
}
if level < collate.Tertiary {
nce[0][collate.Tertiary] = defaultTertiary
if !skip {
next = append(next, ce)
}
}
return nce
return next
}
func nextVal(elems [][]int, i int, level collate.Level) (index, value int) {

26
libgo/go/exp/locale/collate/build/colelem_test.go
View file

@ -34,10 +34,10 @@ func decompCE(in []int) (ce uint32, err error) {
var ceTests = []ceTest{
{normalCE, []int{0, 0, 0}, 0x80000000},
{normalCE, []int{0, 0x28, 3}, 0x80002803},
{normalCE, []int{100, defaultSecondary, 3}, 0x0000C803},
{normalCE, []int{100, defaultSecondary, 3}, 0x0000C883},
// non-ignorable primary with non-default secondary
{normalCE, []int{100, 0x28, defaultTertiary}, 0x40006428},
{normalCE, []int{100, defaultSecondary + 8, 3}, 0x0000C903},
{normalCE, []int{100, 0x28, defaultTertiary}, 0x4000C828},
{normalCE, []int{100, defaultSecondary + 8, 3}, 0x0000C983},
{normalCE, []int{100, 0, 3}, 0xFFFF}, // non-ignorable primary with non-supported secondary
{normalCE, []int{100, 1, 3}, 0xFFFF},
{normalCE, []int{1 << maxPrimaryBits, defaultSecondary, 0}, 0xFFFF},
@ -114,18 +114,24 @@ var nextWeightTests = []weightsTest{
},
}
var extra = []int{200, 32, 8, 0}
var extra = [][]int{{200, 32, 8, 0}, {0, 32, 8, 0}, {0, 0, 8, 0}, {0, 0, 0, 0}}
func TestNextWeight(t *testing.T) {
for i, tt := range nextWeightTests {
test := func(tt weightsTest, a, gold [][]int) {
test := func(l collate.Level, tt weightsTest, a, gold [][]int) {
res := nextWeight(tt.level, a)
if !equalCEArrays(gold, res) {
t.Errorf("%d: expected weights %d; found %d", i, tt.b, res)
t.Errorf("%d:%d: expected weights %d; found %d", i, l, gold, res)
}
}
test(-1, tt, tt.a, tt.b)
for l := collate.Primary; l <= collate.Tertiary; l++ {
if tt.level <= l {
test(l, tt, append(tt.a, extra[l]), tt.b)
} else {
test(l, tt, append(tt.a, extra[l]), append(tt.b, extra[l]))
}
}
test(tt, tt.a, tt.b)
test(tt, append(tt.a, extra), append(tt.b, extra))
}
}
@ -137,7 +143,7 @@ var compareTests = []weightsTest{
0,
},
{
[][]int{{100, 20, 5, 0}, extra},
[][]int{{100, 20, 5, 0}, extra[0]},
[][]int{{100, 20, 5, 1}},
collate.Primary,
1,
@ -192,6 +198,6 @@ func TestCompareWeights(t *testing.T) {
}
}
test(tt, tt.a, tt.b)
test(tt, append(tt.a, extra), append(tt.b, extra))
test(tt, append(tt.a, extra[0]), append(tt.b, extra[0]))
}
}

12
libgo/go/exp/locale/collate/build/contract.go
View file

@ -15,7 +15,7 @@ import (
// This file contains code for detecting contractions and generating
// the necessary tables.
// Any Unicode Collation Algorithm (UCA) table entry that has more than
// one rune one the left-hand side is called a contraction.
// one rune one the left-hand side is called a contraction.
// See http://www.unicode.org/reports/tr10/#Contractions for more details.
//
// We define the following terms:
@ -27,7 +27,7 @@ import (
// A rune may be both a initial and a non-initial and may be so in
// many contractions. An initial may typically also appear by itself.
// In case of ambiguities, the UCA requires we match the longest
// contraction.
// contraction.
//
// Many contraction rules share the same set of possible suffixes.
// We store sets of suffixes in a trie that associates an index with
@ -39,14 +39,14 @@ import (
// is represented as a subsequence of ctEntries, where each entry corresponds to
// a possible match of a next character in the search string. An entry
// also includes the length and offset to the next sequence of entries
// to check in case of a match.
// to check in case of a match.
const (
final = 0
noIndex = 0xFF
)
// ctEntry associates to a matching byte an offset and/or next sequence of
// ctEntry associates to a matching byte an offset and/or next sequence of
// bytes to check. A ctEntry c is called final if a match means that the
// longest suffix has been found. An entry c is final if c.n == 0.
// A single final entry can match a range of characters to an offset.
@ -58,7 +58,7 @@ const (
// {'a', 1, 1, noIndex}, // 'a' by itself does not match, so i is 0xFF.
// {'b', 'c', 0, 1}, // "ab" -> 1, "ac" -> 2
// }
//
//
// The suffix strings "ab", "abc", "abd", and "abcd" can be represented as:
// []ctEntry{
// {'a', 1, 1, noIndex}, // 'a' must be followed by 'b'.
@ -72,7 +72,7 @@ type ctEntry struct {
l uint8 // non-final: byte value to match; final: lowest match in range.
h uint8 // non-final: relative index to next block; final: highest match in range.
n uint8 // non-final: length of next block; final: final
i uint8 // result offset. Will be noIndex if more bytes are needed to complete.
i uint8 // result offset. Will be noIndex if more bytes are needed to complete.
}
// contractTrieSet holds a set of contraction tries. The tries are stored

106
libgo/go/exp/locale/collate/build/order.go
View file

@ -26,16 +26,21 @@ const (
// Collation Element Table.
// See http://www.unicode.org/Public/UCA/6.0.0/allkeys.txt.
type entry struct {
runes []rune
elems [][]int // the collation elements for runes
str string // same as string(runes)
str string // same as string(runes)
runes []rune
elems [][]int // the collation elements
extend string // weights of extend to be appended to elems
before bool // weights relative to next instead of previous.
lock bool // entry is used in extension and can no longer be moved.
// prev, next, and level are used to keep track of tailorings.
prev, next *entry
level collate.Level // next differs at this level
skipRemove bool // do not unlink when removed
decompose bool // can use NFKD decomposition to generate elems
exclude bool // do not include in table
implicit bool // derived, is not included in the list
logical logicalAnchor
expansionIndex int // used to store index into expansion table
@ -44,8 +49,8 @@ type entry struct {
}
func (e *entry) String() string {
return fmt.Sprintf("%X -> %X (ch:%x; ci:%d, ei:%d)",
e.runes, e.elems, e.contractionHandle, e.contractionIndex, e.expansionIndex)
return fmt.Sprintf("%X (%q) -> %X (ch:%x; ci:%d, ei:%d)",
e.runes, e.str, e.elems, e.contractionHandle, e.contractionIndex, e.expansionIndex)
}
func (e *entry) skip() bool {
@ -71,7 +76,7 @@ func (e *entry) contractionStarter() bool {
// examples of entries that will not be indexed.
func (e *entry) nextIndexed() (*entry, collate.Level) {
level := e.level
for e = e.next; e != nil && e.exclude; e = e.next {
for e = e.next; e != nil && (e.exclude || len(e.elems) == 0); e = e.next {
if e.level < level {
level = e.level
}
@ -87,16 +92,20 @@ func (e *entry) remove() {
if e.logical != noAnchor {
log.Fatalf("may not remove anchor %q", e.str)
}
if e.prev != nil {
e.prev.next = e.next
}
if e.next != nil {
e.next.prev = e.prev
}
// TODO: need to set e.prev.level to e.level if e.level is smaller?
e.elems = nil
if !e.skipRemove {
if e.prev != nil {
e.prev.next = e.next
}
if e.next != nil {
e.next.prev = e.prev
}
}
e.skipRemove = false
}
// insertAfter inserts t after e.
// insertAfter inserts n after e.
func (e *entry) insertAfter(n *entry) {
if e == n {
panic("e == anchor")
@ -109,10 +118,31 @@ func (e *entry) insertAfter(n *entry) {
n.next = e.next
n.prev = e
e.next.prev = n
if e.next != nil {
e.next.prev = n
}
e.next = n
}
// insertBefore inserts n before e.
func (e *entry) insertBefore(n *entry) {
if e == n {
panic("e == anchor")
}
if e == nil {
panic("unexpected nil anchor")
}
n.remove()
n.decompose = false // redo decomposition test
n.prev = e.prev
n.next = e
if e.prev != nil {
e.prev.next = n
}
e.prev = n
}
func (e *entry) encodeBase() (ce uint32, err error) {
switch {
case e.expansion():
@ -178,6 +208,7 @@ func (s sortedEntries) Less(i, j int) bool {
}
type ordering struct {
id string
entryMap map[string]*entry
ordered []*entry
handle *trieHandle
@ -187,7 +218,14 @@ type ordering struct {
// Note that insert simply appends e to ordered. To reattain a sorted
// order, o.sort() should be called.
func (o *ordering) insert(e *entry) {
o.entryMap[e.str] = e
if e.logical == noAnchor {
o.entryMap[e.str] = e
} else {
// Use key format as used in UCA rules.
o.entryMap[fmt.Sprintf("[%s]", e.str)] = e
// Also add index entry for XML format.
o.entryMap[fmt.Sprintf("<%s/>", strings.Replace(e.str, " ", "_", -1))] = e
}
o.ordered = append(o.ordered, e)
}
@ -236,13 +274,13 @@ func makeRootOrdering() ordering {
entryMap: make(map[string]*entry),
}
insert := func(typ logicalAnchor, s string, ce []int) {
// Use key format as used in UCA rules.
e := o.newEntry(fmt.Sprintf("[%s]", s), [][]int{ce})
// Also add index entry for XML format.
o.entryMap[fmt.Sprintf("<%s/>", strings.Replace(s, " ", "_", -1))] = e
e.runes = nil
e.exclude = true
e.logical = typ
e := &entry{
elems: [][]int{ce},
str: s,
exclude: true,
logical: typ,
}
o.insert(e)
}
insert(firstAnchor, "first tertiary ignorable", []int{0, 0, 0, 0})
insert(lastAnchor, "last tertiary ignorable", []int{0, 0, 0, max})
@ -252,6 +290,29 @@ func makeRootOrdering() ordering {
return o
}
// patchForInsert eleminates entries from the list with more than one collation element.
// The next and prev fields of the eliminated entries still point to appropriate
// values in the newly created list.
// It requires that sort has been called.
func (o *ordering) patchForInsert() {
for i := 0; i < len(o.ordered)-1; {
e := o.ordered[i]
lev := e.level
n := e.next
for ; n != nil && len(n.elems) > 1; n = n.next {
if n.level < lev {
lev = n.level
}
n.skipRemove = true
}
for ; o.ordered[i] != n; i++ {
o.ordered[i].level = lev
o.ordered[i].next = n
o.ordered[i+1].prev = e
}
}
}
// clone copies all ordering of es into a new ordering value.
func (o *ordering) clone() *ordering {
o.sort()
@ -270,6 +331,7 @@ func (o *ordering) clone() *ordering {
oo.insert(ne)
}
oo.sort() // link all ordering.
oo.patchForInsert()
return &oo
}

31
libgo/go/exp/locale/collate/build/order_test.go
View file

@ -128,6 +128,9 @@ func TestInsertAfter(t *testing.T) {
last.insertAfter(es[i])
last = es[i]
}
for _, e := range es {
e.elems = es[0].elems
}
e := es[0]
for _, i := range perm {
e, _ = e.nextIndexed()
@ -139,6 +142,34 @@ func TestInsertAfter(t *testing.T) {
}
}
func TestInsertBefore(t *testing.T) {
const n = 5
orig := makeList(n)
perm := make([]int, n)
for i := range perm {
perm[i] = i + 1
}
for ok := true; ok; ok = nextPerm(perm) {
es := makeList(n)
last := es[len(es)-1]
for _, i := range perm {
last.insertBefore(es[i])
last = es[i]
}
for _, e := range es {
e.elems = es[0].elems
}
e := es[0]
for i := n - 1; i >= 0; i-- {
e, _ = e.nextIndexed()
if e.runes[0] != rune(perm[i]) {
t.Errorf("%d:%d: expected entry %X; found %X", perm, i, orig[i].runes, e.runes)
break
}
}
}
}
type entryLessTest struct {
a, b *entry
res bool

37
libgo/go/exp/locale/collate/build/trie.go
View file

@ -4,7 +4,7 @@
// The trie in this file is used to associate the first full character
// in a UTF-8 string to a collation element.
// All but the last byte in a UTF-8 byte sequence are
// All but the last byte in a UTF-8 byte sequence are
// used to look up offsets in the index table to be used for the next byte.
// The last byte is used to index into a table of collation elements.
// This file contains the code for the generation of the trie.
@ -35,10 +35,11 @@ type trie struct {
// trieNode is the intermediate trie structure used for generating a trie.
type trieNode struct {
index []*trieNode
value []uint32
b byte
ref uint16
index []*trieNode
value []uint32
b byte
refValue uint16
refIndex uint16
}
func newNode() *trieNode {
@ -108,18 +109,20 @@ func (b *trieBuilder) computeOffsets(n *trieNode) *trieNode {
hasher := fnv.New32()
if n.index != nil {
for i, nn := range n.index {
v := uint16(0)
var vi, vv uint16
if nn != nil {
nn = b.computeOffsets(nn)
n.index[i] = nn
v = nn.ref
vi = nn.refIndex
vv = nn.refValue
}
hasher.Write([]byte{byte(v >> 8), byte(v)})
hasher.Write([]byte{byte(vi >> 8), byte(vi)})
hasher.Write([]byte{byte(vv >> 8), byte(vv)})
}
h := hasher.Sum32()
nn, ok := b.lookupBlockIdx[h]
if !ok {
n.ref = uint16(len(b.lookupBlocks)) - blockOffset
n.refIndex = uint16(len(b.lookupBlocks)) - blockOffset
b.lookupBlocks = append(b.lookupBlocks, n)
b.lookupBlockIdx[h] = n
} else {
@ -132,7 +135,8 @@ func (b *trieBuilder) computeOffsets(n *trieNode) *trieNode {
h := hasher.Sum32()
nn, ok := b.valueBlockIdx[h]
if !ok {
n.ref = uint16(len(b.valueBlocks)) - blockOffset
n.refValue = uint16(len(b.valueBlocks)) - blockOffset
n.refIndex = n.refValue
b.valueBlocks = append(b.valueBlocks, n)
b.valueBlockIdx[h] = n
} else {
@ -150,7 +154,8 @@ func (b *trieBuilder) addStartValueBlock(n *trieNode) uint16 {
h := hasher.Sum32()
nn, ok := b.valueBlockIdx[h]
if !ok {
n.ref = uint16(len(b.valueBlocks))
n.refValue = uint16(len(b.valueBlocks))
n.refIndex = n.refValue
b.valueBlocks = append(b.valueBlocks, n)
// Add a dummy block to accommodate the double block size.
b.valueBlocks = append(b.valueBlocks, nil)
@ -158,7 +163,7 @@ func (b *trieBuilder) addStartValueBlock(n *trieNode) uint16 {
} else {
n = nn
}
return n.ref
return n.refValue
}
func genValueBlock(t *trie, n *trieNode) {
@ -173,7 +178,11 @@ func genLookupBlock(t *trie, n *trieNode) {
for _, nn := range n.index {
v := uint16(0)
if nn != nil {
v = nn.ref
if n.index != nil {
v = nn.refIndex
} else {
v = nn.refValue
}
}
t.index = append(t.index, v)
}
@ -192,7 +201,7 @@ func (b *trieBuilder) addTrie(n *trieNode) *trieHandle {
}
n = b.computeOffsets(n)
// Offset by one extra block as the first byte starts at 0xC0 instead of 0x80.
h.lookupStart = n.ref - 1
h.lookupStart = n.refIndex - 1
return h
}

2
libgo/go/exp/locale/collate/build/trie_test.go
View file

@ -10,7 +10,7 @@ import (
"testing"
)
// We take the smallest, largest and an arbitrary value for each
// We take the smallest, largest and an arbitrary value for each
// of the UTF-8 sequence lengths.
var testRunes = []rune{
0x01, 0x0C, 0x7F, // 1-byte sequences

110
libgo/go/exp/locale/collate/colelem.go
View file

@ -8,16 +8,6 @@ import (
"unicode"
)
// weights holds the decoded weights per collation level.
type weights struct {
primary uint32
secondary uint16
tertiary uint8
// TODO: compute quaternary on the fly or compress this value into 8 bits
// such that weights fit within 64bit.
quaternary uint32
}
const (
defaultSecondary = 0x20
defaultTertiary = 0x2
@ -69,7 +59,7 @@ func (ce colElem) ctype() ceType {
// For normal collation elements, we assume that a collation element either has
// a primary or non-default secondary value, not both.
// Collation elements with a primary value are of the form
// 010ppppp pppppppp pppppppp ssssssss
// 01pppppp pppppppp ppppppp0 ssssssss
// - p* is primary collation value
// - s* is the secondary collation value
// or
@ -82,25 +72,87 @@ func (ce colElem) ctype() ceType {
// - 16 BMP implicit -> weight
// - 8 bit s
// - default tertiary
func splitCE(ce colElem) weights {
const primaryMask = 0x40000000
const secondaryMask = 0x80000000
w := weights{}
if ce&primaryMask != 0 {
w.tertiary = defaultTertiary
w.secondary = uint16(uint8(ce))
w.primary = uint32((ce >> 8) & 0x1FFFFF)
} else if ce&secondaryMask == 0 {
w.tertiary = uint8(ce & 0x1F)
ce >>= 5
w.secondary = defaultSecondary + uint16(ce&0xF)
ce >>= 4
w.primary = uint32(ce)
} else {
w.tertiary = uint8(ce)
w.secondary = uint16(ce >> 8)
// 11qqqqqq qqqqqqqq qqqqqqq0 00000000
// - q* quaternary value
const (
ceTypeMask = 0xC0000000
ceType1 = 0x40000000
ceType2 = 0x00000000
ceType3 = 0x80000000
ceTypeQ = 0xC0000000
ceIgnore = ceType3
firstNonPrimary = 0x80000000
secondaryMask = 0x80000000
hasTertiaryMask = 0x40000000
primaryValueMask = 0x3FFFFE00
primaryShift = 9
compactSecondaryShift = 5
minCompactSecondary = defaultSecondary - 4
)
func makeImplicitCE(primary int) colElem {
return ceType1 | colElem(primary<<primaryShift) | defaultSecondary
}
func makeQuaternary(primary int) colElem {
return ceTypeQ | colElem(primary<<primaryShift)
}
func (ce colElem) primary() int {
if ce >= firstNonPrimary {
return 0
}
return w
return int(ce&primaryValueMask) >> primaryShift
}
func (ce colElem) secondary() int {
switch ce & ceTypeMask {
case ceType1:
return int(uint8(ce))
case ceType2:
return minCompactSecondary + int((ce>>compactSecondaryShift)&0xF)
case ceType3:
return int(uint16(ce >> 8))
case ceTypeQ:
return 0
}
panic("should not reach here")
}
func (ce colElem) tertiary() uint8 {
if ce&hasTertiaryMask == 0 {
if ce&ceType3 == 0 {
return uint8(ce & 0x1F)
}
return uint8(ce)
} else if ce&ceTypeMask == ceType1 {
return defaultTertiary
}
// ce is a quaternary value.
return 0
}
func (ce colElem) updateTertiary(t uint8) colElem {
if ce&ceTypeMask == ceType1 {
nce := ce & primaryValueMask
nce |= colElem(uint8(ce)-minCompactSecondary) << compactSecondaryShift
ce = nce
} else {
ce &= ^colElem(maxTertiary)
}
return ce | colElem(t)
}
// quaternary returns the quaternary value if explicitly specified,
// 0 if ce == ceIgnore, or maxQuaternary otherwise.
// Quaternary values are used only for shifted variants.
func (ce colElem) quaternary() int {
if ce&ceTypeMask == ceTypeQ {
return int(ce&primaryValueMask) >> primaryShift
} else if ce == ceIgnore {
return 0
}
return maxQuaternary
}
// For contractions, collation elements are of the form

29
libgo/go/exp/locale/collate/colelem_test.go
View file

@ -29,10 +29,10 @@ func makeCE(weights []int) colElem {
var ce colElem
if weights[0] != 0 {
if weights[2] == defaultTertiary {
ce = colElem(weights[0]<<maxSecondaryCompactBits + weights[1])
ce = colElem(weights[0]<<(maxSecondaryCompactBits+1) + weights[1])
ce |= isPrimary
} else {
d := weights[1] - defaultSecondary
d := weights[1] - defaultSecondary + 4
ce = colElem(weights[0]<<maxSecondaryDiffBits + d)
ce = ce<<maxTertiaryCompactBits + colElem(weights[2])
}
@ -68,10 +68,10 @@ func makeDecompose(t1, t2 int) colElem {
}
func normalCE(inout []int) (ce colElem, t ceType) {
w := splitCE(makeCE(inout))
inout[0] = int(w.primary)
inout[1] = int(w.secondary)
inout[2] = int(w.tertiary)
w := makeCE(inout)
inout[0] = w.primary()
inout[1] = w.secondary()
inout[2] = int(w.tertiary())
return ce, ceNormal
}
@ -167,3 +167,20 @@ func TestImplicit(t *testing.T) {
}
}
}
func TestUpdateTertiary(t *testing.T) {
tests := []struct {
in, out colElem
t uint8
}{
{0x4000FE20, 0x0000FE8A, 0x0A},
{0x4000FE21, 0x0000FEAA, 0x0A},
{0x0000FE8B, 0x0000FE83, 0x03},
{0x8000CC02, 0x8000CC1B, 0x1B},
}
for i, tt := range tests {
if out := tt.in.updateTertiary(tt.t); out != tt.out {
t.Errorf("%d: was %X; want %X", i, out, tt.out)
}
}
}

310
libgo/go/exp/locale/collate/collate.go
View file

@ -30,7 +30,7 @@ const (
// AlternateHandling identifies the various ways in which variables are handled.
// A rune with a primary weight lower than the variable top is considered a
// variable.
// variable.
// See http://www.unicode.org/reports/tr10/#Variable_Weighting for details.
type AlternateHandling int
@ -83,9 +83,17 @@ type Collator struct {
f norm.Form
t *table
_iter [2]iter
}
func (c *Collator) iter(i int) *iter {
// TODO: evaluate performance for making the second iterator optional.
return &c._iter[i]
}
// Locales returns the list of locales for which collating differs from its parent locale.
// The returned value should not be modified.
func Locales() []string {
return availableLocales
}
@ -99,11 +107,18 @@ func New(loc string) *Collator {
t = mainTable.indexedTable(idx)
}
}
return &Collator{
return newCollator(t)
}
func newCollator(t *table) *Collator {
c := &Collator{
Strength: Quaternary,
f: norm.NFD,
t: t,
}
c._iter[0].init(c)
c._iter[1].init(c)
return c
}
// SetVariableTop sets all runes with primary strength less than the primary
@ -112,63 +127,114 @@ func (c *Collator) SetVariableTop(r rune) {
// TODO: implement
}
// Buffer holds reusable buffers that can be used during collation.
// Reusing a Buffer for the various calls that accept it may avoid
// unnecessary memory allocations.
// Buffer holds keys generated by Key and KeyString.
type Buffer struct {
// TODO: try various parameters and techniques, such as using
// a chan of buffers for a pool.
ba [4096]byte
wa [512]weights
buf [4096]byte
key []byte
ce []weights
}
func (b *Buffer) init() {
if b.ce == nil {
b.ce = b.wa[:0]
b.key = b.ba[:0]
} else {
b.ce = b.ce[:0]
if b.key == nil {
b.key = b.buf[:0]
}
}
// ResetKeys clears the buffer used for generated keys. Calling ResetKeys
// invalidates keys previously obtained from Key or KeyFromString.
func (b *Buffer) ResetKeys() {
b.ce = b.ce[:0]
// Reset clears the buffer from previous results generated by Key and KeyString.
func (b *Buffer) Reset() {
b.key = b.key[:0]
}
// Compare returns an integer comparing the two byte slices.
// The result will be 0 if a==b, -1 if a < b, and +1 if a > b.
// Compare calls ResetKeys, thereby invalidating keys
// previously generated using Key or KeyFromString using buf.
func (c *Collator) Compare(buf *Buffer, a, b []byte) int {
// TODO: for now we simply compute keys and compare. Once we
// have good benchmarks, move to an implementation that works
// incrementally for the majority of cases.
// - Benchmark with long strings that only vary in modifiers.
buf.ResetKeys()
ka := c.Key(buf, a)
kb := c.Key(buf, b)
defer buf.ResetKeys()
return bytes.Compare(ka, kb)
func (c *Collator) Compare(a, b []byte) int {
// TODO: skip identical prefixes once we have a fast way to detect if a rune is
// part of a contraction. This would lead to roughly a 10% speedup for the colcmp regtest.
c.iter(0).setInput(c, a)
c.iter(1).setInput(c, b)
if res := c.compare(); res != 0 {
return res
}
if Identity == c.Strength {
return bytes.Compare(a, b)
}
return 0
}
// CompareString returns an integer comparing the two strings.
// The result will be 0 if a==b, -1 if a < b, and +1 if a > b.
// CompareString calls ResetKeys, thereby invalidating keys
// previously generated using Key or KeyFromString using buf.
func (c *Collator) CompareString(buf *Buffer, a, b string) int {
buf.ResetKeys()
ka := c.KeyFromString(buf, a)
kb := c.KeyFromString(buf, b)
defer buf.ResetKeys()
return bytes.Compare(ka, kb)
func (c *Collator) CompareString(a, b string) int {
// TODO: skip identical prefixes once we have a fast way to detect if a rune is
// part of a contraction. This would lead to roughly a 10% speedup for the colcmp regtest.
c.iter(0).setInputString(c, a)
c.iter(1).setInputString(c, b)
if res := c.compare(); res != 0 {
return res
}
if Identity == c.Strength {
if a < b {
return -1
} else if a > b {
return 1
}
}
return 0
}
func (c *Collator) Prefix(buf *Buffer, s, prefix []byte) int {
func compareLevel(f func(i *iter) int, a, b *iter) int {
a.pce = 0
b.pce = 0
for {
va := f(a)
vb := f(b)
if va != vb {
if va < vb {
return -1
}
return 1
} else if va == 0 {
break
}
}
return 0
}
func (c *Collator) compare() int {
ia, ib := c.iter(0), c.iter(1)
// Process primary level
if c.Alternate != AltShifted {
// TODO: implement script reordering
// TODO: special hiragana handling
if res := compareLevel((*iter).nextPrimary, ia, ib); res != 0 {
return res
}
} else {
// TODO: handle shifted
}
if Secondary <= c.Strength {
f := (*iter).nextSecondary
if c.Backwards {
f = (*iter).prevSecondary
}
if res := compareLevel(f, ia, ib); res != 0 {
return res
}
}
// TODO: special case handling (Danish?)
if Tertiary <= c.Strength || c.CaseLevel {
if res := compareLevel((*iter).nextTertiary, ia, ib); res != 0 {
return res
}
// TODO: Not needed for the default value of AltNonIgnorable?
if Quaternary <= c.Strength {
if res := compareLevel((*iter).nextQuaternary, ia, ib); res != 0 {
return res
}
}
}
return 0
}
func (c *Collator) Prefix(s, prefix []byte) int {
// iterate over s, track bytes consumed.
return 0
}
@ -176,12 +242,11 @@ func (c *Collator) Prefix(buf *Buffer, s, prefix []byte) int {
// Key returns the collation key for str.
// Passing the buffer buf may avoid memory allocations.
// The returned slice will point to an allocation in Buffer and will remain
// valid until the next call to buf.ResetKeys().
// valid until the next call to buf.Reset().
func (c *Collator) Key(buf *Buffer, str []byte) []byte {
// See http://www.unicode.org/reports/tr10/#Main_Algorithm for more details.
buf.init()
c.getColElems(buf, str)
return c.key(buf, buf.ce)
return c.key(buf, c.getColElems(str))
}
// KeyFromString returns the collation key for str.
@ -191,46 +256,73 @@ func (c *Collator) Key(buf *Buffer, str []byte) []byte {
func (c *Collator) KeyFromString(buf *Buffer, str string) []byte {
// See http://www.unicode.org/reports/tr10/#Main_Algorithm for more details.
buf.init()
c.getColElemsString(buf, str)
return c.key(buf, buf.ce)
return c.key(buf, c.getColElemsString(str))
}
func (c *Collator) key(buf *Buffer, w []weights) []byte {
func (c *Collator) key(buf *Buffer, w []colElem) []byte {
processWeights(c.Alternate, c.t.variableTop, w)
kn := len(buf.key)
c.keyFromElems(buf, w)
return buf.key[kn:]
}
func (c *Collator) getColElems(buf *Buffer, str []byte) {
i := c.iter()
i.src.SetInput(c.f, str)
func (c *Collator) getColElems(str []byte) []colElem {
i := c.iter(0)
i.setInput(c, str)
for !i.done() {
buf.ce = i.next(buf.ce)
i.next()
}
return i.ce
}
func (c *Collator) getColElemsString(buf *Buffer, str string) {
i := c.iter()
i.src.SetInputString(c.f, str)
func (c *Collator) getColElemsString(str string) []colElem {
i := c.iter(0)
i.setInputString(c, str)
for !i.done() {
buf.ce = i.next(buf.ce)
i.next()
}
return i.ce
}
type iter struct {
src norm.Iter
ba [1024]byte
norm [1024]byte
buf []byte
t *table
p int
minBufSize int
wa [512]colElem
ce []colElem
pce int
t *table
_done, eof bool
}
func (c *Collator) iter() iter {
i := iter{t: c.t, minBufSize: c.t.maxContractLen}
i.buf = i.ba[:0]
func (i *iter) init(c *Collator) {
i.t = c.t
i.minBufSize = c.t.maxContractLen
i.ce = i.wa[:0]
i.buf = i.norm[:0]
}
func (i *iter) reset() {
i.ce = i.ce[:0]
i.buf = i.buf[:0]
i.p = 0
i.eof = i.src.Done()
i._done = i.eof
}
func (i *iter) setInput(c *Collator, s []byte) *iter {
i.src.SetInput(c.f, s)
i.reset()
return i
}
func (i *iter) setInputString(c *Collator, s string) *iter {
i.src.SetInputString(c.f, s)
i.reset()
return i
}
@ -238,7 +330,7 @@ func (i *iter) done() bool {
return i._done
}
func (i *iter) next(ce []weights) []weights {
func (i *iter) next() {
if !i.eof && len(i.buf)-i.p < i.minBufSize {
// replenish buffer
n := copy(i.buf, i.buf[i.p:])
@ -249,14 +341,70 @@ func (i *iter) next(ce []weights) []weights {
}
if i.p == len(i.buf) {
i._done = true
return ce
return
}
ce, sz := i.t.appendNext(ce, i.buf[i.p:])
sz := 0
i.ce, sz = i.t.appendNext(i.ce, i.buf[i.p:])
i.p += sz
return ce
}
func appendPrimary(key []byte, p uint32) []byte {
func (i *iter) nextPrimary() int {
for {
for ; i.pce < len(i.ce); i.pce++ {
if v := i.ce[i.pce].primary(); v != 0 {
i.pce++
return v
}
}
if i.done() {
return 0
}
i.next()
}
panic("should not reach here")
}
func (i *iter) nextSecondary() int {
for ; i.pce < len(i.ce); i.pce++ {
if v := i.ce[i.pce].secondary(); v != 0 {
i.pce++
return v
}
}
return 0
}
func (i *iter) prevSecondary() int {
for ; i.pce < len(i.ce); i.pce++ {
if v := i.ce[len(i.ce)-i.pce-1].secondary(); v != 0 {
i.pce++
return v
}
}
return 0
}
func (i *iter) nextTertiary() int {
for ; i.pce < len(i.ce); i.pce++ {
if v := i.ce[i.pce].tertiary(); v != 0 {
i.pce++
return int(v)
}
}
return 0
}
func (i *iter) nextQuaternary() int {
for ; i.pce < len(i.ce); i.pce++ {
if v := i.ce[i.pce].quaternary(); v != 0 {
i.pce++
return v
}
}
return 0
}
func appendPrimary(key []byte, p int) []byte {
// Convert to variable length encoding; supports up to 23 bits.
if p <= 0x7FFF {
key = append(key, uint8(p>>8), uint8(p))
@ -268,9 +416,9 @@ func appendPrimary(key []byte, p uint32) []byte {
// keyFromElems converts the weights ws to a compact sequence of bytes.
// The result will be appended to the byte buffer in buf.
func (c *Collator) keyFromElems(buf *Buffer, ws []weights) {
func (c *Collator) keyFromElems(buf *Buffer, ws []colElem) {
for _, v := range ws {
if w := v.primary; w > 0 {
if w := v.primary(); w > 0 {
buf.key = appendPrimary(buf.key, w)
}
}
@ -279,13 +427,13 @@ func (c *Collator) keyFromElems(buf *Buffer, ws []weights) {
// TODO: we can use one 0 if we can guarantee that all non-zero weights are > 0xFF.
if !c.Backwards {
for _, v := range ws {
if w := v.secondary; w > 0 {
if w := v.secondary(); w > 0 {
buf.key = append(buf.key, uint8(w>>8), uint8(w))
}
}
} else {
for i := len(ws) - 1; i >= 0; i-- {
if w := ws[i].secondary; w > 0 {
if w := ws[i].secondary(); w > 0 {
buf.key = append(buf.key, uint8(w>>8), uint8(w))
}
}
@ -296,20 +444,20 @@ func (c *Collator) keyFromElems(buf *Buffer, ws []weights) {
if Tertiary <= c.Strength || c.CaseLevel {
buf.key = append(buf.key, 0, 0)
for _, v := range ws {
if w := v.tertiary; w > 0 {
buf.key = append(buf.key, w)
if w := v.tertiary(); w > 0 {
buf.key = append(buf.key, uint8(w))
}
}
// Derive the quaternary weights from the options and other levels.
// Note that we represent maxQuaternary as 0xFF. The first byte of the
// representation of a a primary weight is always smaller than 0xFF,
// so using this single byte value will compare correctly.
if Quaternary <= c.Strength {
if Quaternary <= c.Strength && c.Alternate >= AltShifted {
if c.Alternate == AltShiftTrimmed {
lastNonFFFF := len(buf.key)
buf.key = append(buf.key, 0)
for _, v := range ws {
if w := v.quaternary; w == maxQuaternary {
if w := v.quaternary(); w == maxQuaternary {
buf.key = append(buf.key, 0xFF)
} else if w > 0 {
buf.key = appendPrimary(buf.key, w)
@ -320,7 +468,7 @@ func (c *Collator) keyFromElems(buf *Buffer, ws []weights) {
} else {
buf.key = append(buf.key, 0)
for _, v := range ws {
if w := v.quaternary; w == maxQuaternary {
if w := v.quaternary(); w == maxQuaternary {
buf.key = append(buf.key, 0xFF)
} else if w > 0 {
buf.key = appendPrimary(buf.key, w)
@ -331,29 +479,27 @@ func (c *Collator) keyFromElems(buf *Buffer, ws []weights) {
}
}
func processWeights(vw AlternateHandling, top uint32, wa []weights) {
func processWeights(vw AlternateHandling, top uint32, wa []colElem) {
ignore := false
vtop := int(top)
switch vw {
case AltShifted, AltShiftTrimmed:
for i := range wa {
if p := wa[i].primary; p <= top && p != 0 {
wa[i] = weights{quaternary: p}
if p := wa[i].primary(); p <= vtop && p != 0 {
wa[i] = makeQuaternary(p)
ignore = true
} else if p == 0 {
if ignore {
wa[i] = weights{}
} else if wa[i].tertiary != 0 {
wa[i].quaternary = maxQuaternary
wa[i] = ceIgnore
}
} else {
wa[i].quaternary = maxQuaternary
ignore = false
}
}
case AltBlanked:
for i := range wa {
if p := wa[i].primary; p <= top && (ignore || p != 0) {
wa[i] = weights{}
if p := wa[i].primary(); p <= vtop && (ignore || p != 0) {
wa[i] = ceIgnore
ignore = true
} else {
ignore = false

8
libgo/go/exp/locale/collate/export.go
View file

@ -4,8 +4,6 @@
package collate
import "exp/norm"
// Init is used by type Builder in exp/locale/collate/build/
// to create Collator instances. It is for internal use only.
func Init(data interface{}) *Collator {
@ -24,11 +22,7 @@ func Init(data interface{}) *Collator {
t.contractElem = init.ContractElems()
t.maxContractLen = init.MaxContractLen()
t.variableTop = init.VariableTop()
return &Collator{
Strength: Quaternary,
f: norm.NFD,
t: t,
}
return newCollator(t)
}
type tableInitializer interface {

25
libgo/go/exp/locale/collate/export_test.go
View file

@ -24,6 +24,8 @@ func W(ce ...int) Weights {
}
if len(ce) > 3 {
w.Quaternary = ce[3]
} else if w.Tertiary != 0 {
w.Quaternary = maxQuaternary
}
return w
}
@ -33,25 +35,27 @@ func (w Weights) String() string {
type Table struct {
t *table
w []weights
}
func GetTable(c *Collator) *Table {
return &Table{c.t, nil}
return &Table{c.t}
}
func convertToWeights(ws []weights) []Weights {
func convertToWeights(ws []colElem) []Weights {
out := make([]Weights, len(ws))
for i, w := range ws {
out[i] = Weights{int(w.primary), int(w.secondary), int(w.tertiary), int(w.quaternary)}
out[i] = Weights{int(w.primary()), int(w.secondary()), int(w.tertiary()), int(w.quaternary())}
}
return out
}
func convertFromWeights(ws []Weights) []weights {
out := make([]weights, len(ws))
func convertFromWeights(ws []Weights) []colElem {
out := make([]colElem, len(ws))
for i, w := range ws {
out[i] = weights{uint32(w.Primary), uint16(w.Secondary), uint8(w.Tertiary), uint32(w.Quaternary)}
out[i] = makeCE([]int{w.Primary, w.Secondary, w.Tertiary})
if out[i] == ceIgnore && w.Quaternary > 0 {
out[i] = makeQuaternary(w.Quaternary)
}
}
return out
}
@ -68,10 +72,9 @@ func SetTop(c *Collator, top int) {
c.t.variableTop = uint32(top)
}
func GetColElems(c *Collator, buf *Buffer, str []byte) []Weights {
buf.ResetKeys()
c.getColElems(buf, str)
return convertToWeights(buf.ce)
func GetColElems(c *Collator, str []byte) []Weights {
ce := c.getColElems(str)
return convertToWeights(ce)
}
func ProcessWeights(h AlternateHandling, top int, w []Weights) []Weights {

10
libgo/go/exp/locale/collate/maketables.go
View file

@ -38,7 +38,7 @@ var (
`URL of the Default Unicode Collation Element Table (DUCET). This can be a zip
file containing the file allkeys_CLDR.txt or an allkeys.txt file.`)
cldr = flag.String("cldr",
"http://www.unicode.org/Public/cldr/2.0.1/core.zip",
"http://www.unicode.org/Public/cldr/22/core.zip",
"URL of CLDR archive.")
test = flag.Bool("test", false,
"test existing tables; can be used to compare web data with package data.")
@ -180,7 +180,7 @@ func skipAlt(a string) bool {
func failOnError(e error) {
if e != nil {
log.Fatal(e)
log.Panic(e)
}
}
@ -607,7 +607,7 @@ func insertTailoring(t *build.Tailoring, r RuleElem, context, extend string) {
if *test {
testInput.add(str)
}
err := t.Insert(lmap[l[0]], str, extend)
err := t.Insert(lmap[l[0]], str, context+extend)
failOnError(err)
}
case "pc", "sc", "tc", "ic":
@ -617,7 +617,7 @@ func insertTailoring(t *build.Tailoring, r RuleElem, context, extend string) {
if *test {
testInput.add(str)
}
err := t.Insert(level, str, extend)
err := t.Insert(level, str, context+extend)
failOnError(err)
}
default:
@ -677,7 +677,7 @@ func testCollator(c *collate.Collator) {
if bytes.Compare(k0, k) != 0 {
failOnError(fmt.Errorf("test:%U: keys differ (%x vs %x)", []rune(str), k0, k))
}
buf.ResetKeys()
buf.Reset()
}
fmt.Println("PASS")
}

6
libgo/go/exp/locale/collate/regtest.go
View file

@ -236,9 +236,9 @@ func doTest(t Test) {
if strings.Contains(t.name, "NON_IGNOR") {
c.Alternate = collate.AltNonIgnorable
}
prev := t.str[0]
for i := 1; i < len(t.str); i++ {
b.Reset()
s := t.str[i]
ka := c.Key(b, prev)
kb := c.Key(b, s)
@ -247,10 +247,10 @@ func doTest(t Test) {
prev = s
continue
}
if r := c.Compare(b, prev, s); r == 1 {
if r := c.Compare(prev, s); r == 1 {
fail(t, "%d: Compare(%.4X, %.4X) == %d; want -1 or 0", i, runes(prev), runes(s), r)
}
if r := c.Compare(b, s, prev); r == -1 {
if r := c.Compare(s, prev); r == -1 {
fail(t, "%d: Compare(%.4X, %.4X) == %d; want 1 or 0", i, runes(s), runes(prev), r)
}
prev = s

38
libgo/go/exp/locale/collate/table.go
View file

@ -37,17 +37,21 @@ func (t *table) indexedTable(idx tableIndex) *table {
return &nt
}
// appendNext appends the weights corresponding to the next rune or
// appendNext appends the weights corresponding to the next rune or
// contraction in s. If a contraction is matched to a discontinuous
// sequence of runes, the weights for the interstitial runes are
// sequence of runes, the weights for the interstitial runes are
// appended as well. It returns a new slice that includes the appended
// weights and the number of bytes consumed from s.
func (t *table) appendNext(w []weights, s []byte) ([]weights, int) {
func (t *table) appendNext(w []colElem, s []byte) ([]colElem, int) {
v, sz := t.index.lookup(s)
ce := colElem(v)
tp := ce.ctype()
if tp == ceNormal {
w = append(w, getWeights(ce, s))
if ce == 0 {
r, _ := utf8.DecodeRune(s)
ce = makeImplicitCE(implicitPrimary(r))
}
w = append(w, ce)
} else if tp == ceExpansionIndex {
w = t.appendExpansion(w, ce)
} else if tp == ceContractionIndex {
@ -62,40 +66,28 @@ func (t *table) appendNext(w []weights, s []byte) ([]weights, int) {
for p := 0; len(nfkd) > 0; nfkd = nfkd[p:] {
w, p = t.appendNext(w, nfkd)
}
w[i].tertiary = t1
w[i] = w[i].updateTertiary(t1)
if i++; i < len(w) {
w[i].tertiary = t2
w[i] = w[i].updateTertiary(t2)
for i++; i < len(w); i++ {
w[i].tertiary = maxTertiary
w[i] = w[i].updateTertiary(maxTertiary)
}
}
}
return w, sz
}
func getWeights(ce colElem, s []byte) weights {
if ce == 0 { // implicit
r, _ := utf8.DecodeRune(s)
return weights{
primary: uint32(implicitPrimary(r)),
secondary: defaultSecondary,
tertiary: defaultTertiary,
}
}
return splitCE(ce)
}
func (t *table) appendExpansion(w []weights, ce colElem) []weights {
func (t *table) appendExpansion(w []colElem, ce colElem) []colElem {
i := splitExpandIndex(ce)
n := int(t.expandElem[i])
i++
for _, ce := range t.expandElem[i : i+n] {
w = append(w, splitCE(colElem(ce)))
w = append(w, colElem(ce))
}
return w
}
func (t *table) matchContraction(w []weights, ce colElem, suffix []byte) ([]weights, int) {
func (t *table) matchContraction(w []colElem, ce colElem, suffix []byte) ([]colElem, int) {
index, n, offset := splitContractIndex(ce)
scan := t.contractTries.scanner(index, n, suffix)
@ -138,7 +130,7 @@ func (t *table) matchContraction(w []weights, ce colElem, suffix []byte) ([]weig
i, n := scan.result()
ce = colElem(t.contractElem[i+offset])
if ce.ctype() == ceNormal {
w = append(w, splitCE(ce))
w = append(w, ce)
} else {
w = t.appendExpansion(w, ce)
}

53716
libgo/go/exp/locale/collate/tables.go
View file

File diff suppressed because it is too large Load diff

2
libgo/go/exp/locale/collate/tools/colcmp/col.go
View file

@ -91,5 +91,5 @@ func (c *goCollator) Key(b Input) []byte {
}
func (c *goCollator) Compare(a, b Input) int {
return c.c.Compare(&c.buf, a.UTF8, b.UTF8)
return c.c.Compare(a.UTF8, b.UTF8)
}

8
libgo/go/exp/locale/collate/tools/colcmp/colcmp.go
View file

@ -399,7 +399,7 @@ var cmdRegress = &Command{
}
const failedKeyCompare = `
%d: incorrect comparison result for input:
%s:%d: incorrect comparison result for input:
a: %q (%.4X)
key: %s
b: %q (%.4X)
@ -412,7 +412,7 @@ const failedKeyCompare = `
`
const failedCompare = `
%d: incorrect comparison result for input:
%s:%d: incorrect comparison result for input:
a: %q (%.4X)
b: %q (%.4X)
Compare(a, b) = %d; want %d.
@ -453,12 +453,12 @@ func runRegress(ctxt *Context, args []string) {
count++
a := string(ia.UTF8)
b := string(ib.UTF8)
fmt.Printf(failedKeyCompare, i-1, a, []rune(a), keyStr(ia.key), b, []rune(b), keyStr(ib.key), cmp, goldCmp, keyStr(gold.Key(ia)), keyStr(gold.Key(ib)))
fmt.Printf(failedKeyCompare, t.Locale, i-1, a, []rune(a), keyStr(ia.key), b, []rune(b), keyStr(ib.key), cmp, goldCmp, keyStr(gold.Key(ia)), keyStr(gold.Key(ib)))
} else if cmp := t.Col.Compare(ia, ib); cmp != goldCmp {
count++
a := string(ia.UTF8)
b := string(ib.UTF8)
fmt.Printf(failedKeyCompare, i-1, a, []rune(a), b, []rune(b), cmp, goldCmp)
fmt.Printf(failedCompare, t.Locale, i-1, a, []rune(a), b, []rune(b), cmp, goldCmp)
}
}
if count > 0 {

2
libgo/go/exp/locale/collate/trie.go
View file

@ -4,7 +4,7 @@
// The trie in this file is used to associate the first full character
// in an UTF-8 string to a collation element.
// All but the last byte in a UTF-8 byte sequence are
// All but the last byte in a UTF-8 byte sequence are
// used to lookup offsets in the index table to be used for the next byte.
// The last byte is used to index into a table of collation elements.
// For a full description, see exp/locale/collate/build/trie.go.

2
libgo/go/exp/locale/collate/trie_test.go
View file

@ -8,7 +8,7 @@ import (
"testing"
)
// We take the smallest, largest and an arbitrary value for each
// We take the smallest, largest and an arbitrary value for each
// of the UTF-8 sequence lengths.
var testRunes = []rune{
0x01, 0x0C, 0x7F, // 1-byte sequences

4
libgo/go/exp/norm/forminfo.go
View file

@ -76,7 +76,7 @@ func init() {
// We do not distinguish between boundaries for NFC, NFD, etc. to avoid
// unexpected behavior for the user. For example, in NFD, there is a boundary
// after 'a'. However, 'a' might combine with modifiers, so from the application's
// perspective it is not a good boundary. We will therefore always use the
// perspective it is not a good boundary. We will therefore always use the
// boundaries for the combining variants.
// BoundaryBefore returns true if this rune starts a new segment and
@ -101,7 +101,7 @@ func (p Properties) BoundaryAfter() bool {
// 0: NFD_QC Yes (0) or No (1). No also means there is a decomposition.
// 1..2: NFC_QC Yes(00), No (10), or Maybe (11)
// 3: Combines forward (0 == false, 1 == true)
//
//
// When all 4 bits are zero, the character is inert, meaning it is never
// influenced by normalization.
type qcInfo uint8

4
libgo/go/exp/norm/iter.go
View file

@ -64,9 +64,9 @@ func (i *Iter) Done() bool {
}
// Next writes f(i.input[i.Pos():n]...) to buffer buf, where n is the
// largest boundary of i.input such that the result fits in buf.
// largest boundary of i.input such that the result fits in buf.
// It returns the number of bytes written to buf.
// len(buf) should be at least MaxSegmentSize.
// len(buf) should be at least MaxSegmentSize.
// Done must be false before calling Next.
func (i *Iter) Next(buf []byte) int {
return i.next(i, buf)

2
libgo/go/exp/norm/maketables.go
View file

@ -86,7 +86,7 @@ const (
// Quick Check properties of runes allow us to quickly
// determine whether a rune may occur in a normal form.
// For a given normal form, a rune may be guaranteed to occur
// verbatim (QC=Yes), may or may not combine with another
// verbatim (QC=Yes), may or may not combine with another
// rune (QC=Maybe), or may not occur (QC=No).
type QCResult int

2
libgo/go/exp/norm/maketesttables.go
View file

@ -16,7 +16,7 @@ func main() {
printTestTables()
}
// We take the smallest, largest and an arbitrary value for each
// We take the smallest, largest and an arbitrary value for each
// of the UTF-8 sequence lengths.
var testRunes = []rune{
0x01, 0x0C, 0x7F, // 1-byte sequences

2
libgo/go/exp/norm/normregtest.go
View file

@ -57,7 +57,7 @@ var logger = log.New(os.Stderr, "", log.Lshortfile)
// 1E0A;1E0A;0044 0307;1E0A;0044 0307; # (Ḋ; Ḋ; D◌̇; Ḋ; D◌̇; ) LATIN CAPITAL LETTER D WITH DOT ABOVE
// 1E0C;1E0C;0044 0323;1E0C;0044 0323; # (Ḍ; Ḍ; D◌̣; Ḍ; D◌̣; ) LATIN CAPITAL LETTER D WITH DOT BELOW
//
// Each test has 5 columns (c1, c2, c3, c4, c5), where
// Each test has 5 columns (c1, c2, c3, c4, c5), where
// (c1, c2, c3, c4, c5) == (c1, NFC(c1), NFD(c1), NFKC(c1), NFKD(c1))
//
// CONFORMANCE:

8648
libgo/go/exp/norm/tables.go
View file

File diff suppressed because it is too large Load diff

22
libgo/go/exp/types/staging/builtins.go → libgo/go/exp/types/builtins.go
View file

@ -44,7 +44,7 @@ func (check *checker) builtin(x *operand, call *ast.CallExpr, bin *builtin, iota
switch id {
case _Make, _New:
// argument must be a type
typ0 = underlying(check.typ(arg0, false))
typ0 = check.typ(arg0, false)
if typ0 == Typ[Invalid] {
goto Error
}
@ -191,7 +191,7 @@ func (check *checker) builtin(x *operand, call *ast.CallExpr, bin *builtin, iota
case _Make:
var min int // minimum number of arguments
switch typ0.(type) {
switch underlying(typ0).(type) {
case *Slice:
min = 2
case *Map, *Chan:
@ -204,13 +204,27 @@ func (check *checker) builtin(x *operand, call *ast.CallExpr, bin *builtin, iota
check.errorf(call.Pos(), "%s expects %d or %d arguments; found %d", call, min, min+1, n)
goto Error
}
var sizes []interface{} // constant integer arguments, if any
for _, arg := range args[1:] {
check.expr(x, arg, nil, iota)
if !x.isInteger() {
if x.isInteger() {
if x.mode == constant {
if isNegConst(x.val) {
check.invalidArg(x.pos(), "%s must not be negative", x)
// safe to continue
} else {
sizes = append(sizes, x.val) // x.val >= 0
}
}
} else {
check.invalidArg(x.pos(), "%s must be an integer", x)
// safe to continue
}
}
if len(sizes) == 2 && compareConst(sizes[0], sizes[1], token.GTR) {
check.invalidArg(args[1].Pos(), "length and capacity swapped")
// safe to continue
}
x.mode = variable
x.typ = typ0
@ -287,7 +301,7 @@ func (check *checker) builtin(x *operand, call *ast.CallExpr, bin *builtin, iota
var t operand
x1 := x
for _, arg := range args {
check.exprOrType(x1, arg, nil, iota, true) // permit trace for types, e.g.: new(trace(T))
check.rawExpr(x1, arg, nil, iota, true) // permit trace for types, e.g.: new(trace(T))
check.dump("%s: %s", x1.pos(), x1)
x1 = &t // use incoming x only for first argument
}

532
libgo/go/exp/types/check.go
View file

@ -9,244 +9,366 @@ package types
import (
"fmt"
"go/ast"
"go/scanner"
"go/token"
"strconv"
"sort"
)
const debug = false
// enable for debugging
const trace = false
type checker struct {
fset *token.FileSet
errors scanner.ErrorList
types map[ast.Expr]Type
fset *token.FileSet
pkg *ast.Package
errh func(token.Pos, string)
mapf func(ast.Expr, Type)
// lazily initialized
firsterr error
filenames []string // sorted list of package file names for reproducible iteration order
initexprs map[*ast.ValueSpec][]ast.Expr // "inherited" initialization expressions for constant declarations
functypes []*Signature // stack of function signatures; actively typechecked function on top
pos []token.Pos // stack of expr positions; debugging support, used if trace is set
}
func (c *checker) errorf(pos token.Pos, format string, args ...interface{}) string {
msg := fmt.Sprintf(format, args...)
c.errors.Add(c.fset.Position(pos), msg)
return msg
}
// collectFields collects struct fields tok = token.STRUCT), interface methods
// (tok = token.INTERFACE), and function arguments/results (tok = token.FUNC).
// declare declares an object of the given kind and name (ident) in scope;
// decl is the corresponding declaration in the AST. An error is reported
// if the object was declared before.
//
func (c *checker) collectFields(tok token.Token, list *ast.FieldList, cycleOk bool) (fields ObjList, tags []string, isVariadic bool) {
if list != nil {
for _, field := range list.List {
ftype := field.Type
if t, ok := ftype.(*ast.Ellipsis); ok {
ftype = t.Elt
isVariadic = true
}
typ := c.makeType(ftype, cycleOk)
tag := ""
if field.Tag != nil {
assert(field.Tag.Kind == token.STRING)
tag, _ = strconv.Unquote(field.Tag.Value)
}
if len(field.Names) > 0 {
// named fields
for _, name := range field.Names {
obj := name.Obj
obj.Type = typ
fields = append(fields, obj)
if tok == token.STRUCT {
tags = append(tags, tag)
}
}
} else {
// anonymous field
switch tok {
case token.STRUCT:
tags = append(tags, tag)
fallthrough
case token.FUNC:
obj := ast.NewObj(ast.Var, "")
obj.Type = typ
fields = append(fields, obj)
case token.INTERFACE:
utyp := Underlying(typ)
if typ, ok := utyp.(*Interface); ok {
// TODO(gri) This is not good enough. Check for double declarations!
fields = append(fields, typ.Methods...)
} else if _, ok := utyp.(*Bad); !ok {
// if utyp is Bad, don't complain (the root cause was reported before)
c.errorf(ftype.Pos(), "interface contains embedded non-interface type")
}
default:
panic("unreachable")
}
}
}
}
return
}
// makeType makes a new type for an AST type specification x or returns
// the type referred to by a type name x. If cycleOk is set, a type may
// refer to itself directly or indirectly; otherwise cycles are errors.
// TODO(gri) This is very similar to the declare function in go/parser; it
// is only used to associate methods with their respective receiver base types.
// In a future version, it might be simpler and cleaner do to all the resolution
// in the type-checking phase. It would simplify the parser, AST, and also
// reduce some amount of code duplication.
//
func (c *checker) makeType(x ast.Expr, cycleOk bool) (typ Type) {
if debug {
fmt.Printf("makeType (cycleOk = %v)\n", cycleOk)
ast.Print(c.fset, x)
defer func() {
fmt.Printf("-> %T %v\n\n", typ, typ)
}()
}
switch t := x.(type) {
case *ast.BadExpr:
return &Bad{}
case *ast.Ident:
// type name
obj := t.Obj
if obj == nil {
// unresolved identifier (error has been reported before)
return &Bad{Msg: fmt.Sprintf("%s is unresolved", t.Name)}
}
if obj.Kind != ast.Typ {
msg := c.errorf(t.Pos(), "%s is not a type", t.Name)
return &Bad{Msg: msg}
}
c.checkObj(obj, cycleOk)
if !cycleOk && obj.Type.(*Name).Underlying == nil {
msg := c.errorf(obj.Pos(), "illegal cycle in declaration of %s", obj.Name)
return &Bad{Msg: msg}
}
return obj.Type.(Type)
case *ast.ParenExpr:
return c.makeType(t.X, cycleOk)
case *ast.SelectorExpr:
// qualified identifier
// TODO (gri) eventually, this code belongs to expression
// type checking - here for the time being
if ident, ok := t.X.(*ast.Ident); ok {
if obj := ident.Obj; obj != nil {
if obj.Kind != ast.Pkg {
msg := c.errorf(ident.Pos(), "%s is not a package", obj.Name)
return &Bad{Msg: msg}
}
// TODO(gri) we have a package name but don't
// have the mapping from package name to package
// scope anymore (created in ast.NewPackage).
return &Bad{} // for now
func (check *checker) declare(scope *ast.Scope, kind ast.ObjKind, ident *ast.Ident, decl ast.Decl) {
assert(ident.Obj == nil) // identifier already declared or resolved
obj := ast.NewObj(kind, ident.Name)
obj.Decl = decl
ident.Obj = obj
if ident.Name != "_" {
if alt := scope.Insert(obj); alt != nil {
prevDecl := ""
if pos := alt.Pos(); pos.IsValid() {
prevDecl = fmt.Sprintf("\n\tprevious declaration at %s", check.fset.Position(pos))
}
check.errorf(ident.Pos(), fmt.Sprintf("%s redeclared in this block%s", ident.Name, prevDecl))
}
// TODO(gri) can this really happen (the parser should have excluded this)?
msg := c.errorf(t.Pos(), "expected qualified identifier")
return &Bad{Msg: msg}
case *ast.StarExpr:
return &Pointer{Base: c.makeType(t.X, true)}
case *ast.ArrayType:
if t.Len != nil {
// TODO(gri) compute length
return &Array{Elt: c.makeType(t.Elt, cycleOk)}
}
return &Slice{Elt: c.makeType(t.Elt, true)}
case *ast.StructType:
fields, tags, _ := c.collectFields(token.STRUCT, t.Fields, cycleOk)
return &Struct{Fields: fields, Tags: tags}
case *ast.FuncType:
params, _, isVariadic := c.collectFields(token.FUNC, t.Params, true)
results, _, _ := c.collectFields(token.FUNC, t.Results, true)
return &Func{Recv: nil, Params: params, Results: results, IsVariadic: isVariadic}
case *ast.InterfaceType:
methods, _, _ := c.collectFields(token.INTERFACE, t.Methods, cycleOk)
methods.Sort()
return &Interface{Methods: methods}
case *ast.MapType:
return &Map{Key: c.makeType(t.Key, true), Elt: c.makeType(t.Value, true)}
case *ast.ChanType:
return &Chan{Dir: t.Dir, Elt: c.makeType(t.Value, true)}
}
panic(fmt.Sprintf("unreachable (%T)", x))
}
// checkObj type checks an object.
func (c *checker) checkObj(obj *ast.Object, ref bool) {
if obj.Type != nil {
// object has already been type checked
func (check *checker) valueSpec(pos token.Pos, obj *ast.Object, lhs []*ast.Ident, typ ast.Expr, rhs []ast.Expr, iota int) {
if len(lhs) == 0 {
check.invalidAST(pos, "missing lhs in declaration")
return
}
switch obj.Kind {
case ast.Bad:
// ignore
// determine type for all of lhs, if any
// (but only set it for the object we typecheck!)
var t Type
if typ != nil {
t = check.typ(typ, false)
}
case ast.Con:
// TODO(gri) complete this
// len(lhs) > 0
if len(lhs) == len(rhs) {
// check only lhs and rhs corresponding to obj
var l, r ast.Expr
for i, name := range lhs {
if name.Obj == obj {
l = lhs[i]
r = rhs[i]
break
}
}
assert(l != nil)
obj.Type = t
check.assign1to1(l, r, nil, true, iota)
return
}
// there must be a type or initialization expressions
if t == nil && len(rhs) == 0 {
check.invalidAST(pos, "missing type or initialization expression")
t = Typ[Invalid]
}
// if we have a type, mark all of lhs
if t != nil {
for _, name := range lhs {
name.Obj.Type = t
}
}
// check initial values, if any
if len(rhs) > 0 {
// TODO(gri) should try to avoid this conversion
lhx := make([]ast.Expr, len(lhs))
for i, e := range lhs {
lhx[i] = e
}
check.assignNtoM(lhx, rhs, true, iota)
}
}
func (check *checker) function(typ *Signature, body *ast.BlockStmt) {
check.functypes = append(check.functypes, typ)
check.stmt(body)
check.functypes = check.functypes[0 : len(check.functypes)-1]
}
// object typechecks an object by assigning it a type; obj.Type must be nil.
// Callers must check obj.Type before calling object; this eliminates a call
// for each identifier that has been typechecked already, a common scenario.
//
func (check *checker) object(obj *ast.Object, cycleOk bool) {
assert(obj.Type == nil)
switch obj.Kind {
case ast.Bad, ast.Pkg:
// nothing to do
case ast.Con, ast.Var:
// The obj.Data field for constants and variables is initialized
// to the respective (hypothetical, for variables) iota value by
// the parser. The object's fields can be in one of the following
// states:
// Type != nil => the constant value is Data
// Type == nil => the object is not typechecked yet, and Data can be:
// Data is int => Data is the value of iota for this declaration
// Data == nil => the object's expression is being evaluated
if obj.Data == nil {
check.errorf(obj.Pos(), "illegal cycle in initialization of %s", obj.Name)
obj.Type = Typ[Invalid]
return
}
spec := obj.Decl.(*ast.ValueSpec)
iota := obj.Data.(int)
obj.Data = nil
// determine initialization expressions
values := spec.Values
if len(values) == 0 && obj.Kind == ast.Con {
values = check.initexprs[spec]
}
check.valueSpec(spec.Pos(), obj, spec.Names, spec.Type, values, iota)
case ast.Typ:
typ := &Name{Obj: obj}
typ := &NamedType{Obj: obj}
obj.Type = typ // "mark" object so recursion terminates
typ.Underlying = Underlying(c.makeType(obj.Decl.(*ast.TypeSpec).Type, ref))
case ast.Var:
// TODO(gri) complete this
typ.Underlying = underlying(check.typ(obj.Decl.(*ast.TypeSpec).Type, cycleOk))
// typecheck associated method signatures
if obj.Data != nil {
scope := obj.Data.(*ast.Scope)
switch t := typ.Underlying.(type) {
case *Struct:
// struct fields must not conflict with methods
for _, f := range t.Fields {
if m := scope.Lookup(f.Name); m != nil {
check.errorf(m.Pos(), "type %s has both field and method named %s", obj.Name, f.Name)
}
}
// ok to continue
case *Interface:
// methods cannot be associated with an interface type
for _, m := range scope.Objects {
recv := m.Decl.(*ast.FuncDecl).Recv.List[0].Type
check.errorf(recv.Pos(), "invalid receiver type %s (%s is an interface type)", obj.Name, obj.Name)
}
// ok to continue
}
// typecheck method signatures
for _, m := range scope.Objects {
mdecl := m.Decl.(*ast.FuncDecl)
// TODO(gri) At the moment, the receiver is type-checked when checking
// the method body. Also, we don't properly track if the receiver is
// a pointer (i.e., currently, method sets are too large). FIX THIS.
mtyp := check.typ(mdecl.Type, cycleOk).(*Signature)
m.Type = mtyp
}
}
case ast.Fun:
fdecl := obj.Decl.(*ast.FuncDecl)
ftyp := c.makeType(fdecl.Type, ref).(*Func)
obj.Type = ftyp
if fdecl.Recv != nil {
recvField := fdecl.Recv.List[0]
if len(recvField.Names) > 0 {
ftyp.Recv = recvField.Names[0].Obj
} else {
ftyp.Recv = ast.NewObj(ast.Var, "_")
ftyp.Recv.Decl = recvField
}
c.checkObj(ftyp.Recv, ref)
// TODO(axw) add method to a list in the receiver type.
// This will ensure that the method base type is
// type-checked
check.collectFields(token.FUNC, fdecl.Recv, true)
}
// TODO(axw) check function body, if non-nil.
ftyp := check.typ(fdecl.Type, cycleOk).(*Signature)
obj.Type = ftyp
check.function(ftyp, fdecl.Body)
default:
panic("unreachable")
}
}
// Check typechecks a package.
// It augments the AST by assigning types to all ast.Objects and returns a map
// of types for all expression nodes in statements, and a scanner.ErrorList if
// there are errors.
// assocInitvals associates "inherited" initialization expressions
// with the corresponding *ast.ValueSpec in the check.initexprs map
// for constant declarations without explicit initialization expressions.
//
func Check(fset *token.FileSet, pkg *ast.Package) (types map[ast.Expr]Type, err error) {
// Sort objects so that we get reproducible error
// positions (this is only needed for testing).
// TODO(gri): Consider ast.Scope implementation that
// provides both a list and a map for fast lookup.
// Would permit the use of scopes instead of ObjMaps
// elsewhere.
list := make(ObjList, len(pkg.Scope.Objects))
i := 0
for _, obj := range pkg.Scope.Objects {
list[i] = obj
i++
func (check *checker) assocInitvals(decl *ast.GenDecl) {
var values []ast.Expr
for _, s := range decl.Specs {
if s, ok := s.(*ast.ValueSpec); ok {
if len(s.Values) > 0 {
values = s.Values
} else {
check.initexprs[s] = values
}
}
}
list.Sort()
var c checker
c.fset = fset
c.types = make(map[ast.Expr]Type)
for _, obj := range list {
c.checkObj(obj, false)
if len(values) == 0 {
check.invalidAST(decl.Pos(), "no initialization values provided")
}
c.errors.RemoveMultiples()
return c.types, c.errors.Err()
}
// assocMethod associates a method declaration with the respective
// receiver base type. meth.Recv must exist.
//
func (check *checker) assocMethod(meth *ast.FuncDecl) {
// The receiver type is one of the following (enforced by parser):
// - *ast.Ident
// - *ast.StarExpr{*ast.Ident}
// - *ast.BadExpr (parser error)
typ := meth.Recv.List[0].Type
if ptr, ok := typ.(*ast.StarExpr); ok {
typ = ptr.X
}
// determine receiver base type object (or nil if error)
var obj *ast.Object
if ident, ok := typ.(*ast.Ident); ok && ident.Obj != nil {
obj = ident.Obj
if obj.Kind != ast.Typ {
check.errorf(ident.Pos(), "%s is not a type", ident.Name)
obj = nil
}
// TODO(gri) determine if obj was defined in this package
/*
if check.notLocal(obj) {
check.errorf(ident.Pos(), "cannot define methods on non-local type %s", ident.Name)
obj = nil
}
*/
} else {
// If it's not an identifier or the identifier wasn't declared/resolved,
// the parser/resolver already reported an error. Nothing to do here.
}
// determine base type scope (or nil if error)
var scope *ast.Scope
if obj != nil {
if obj.Data != nil {
scope = obj.Data.(*ast.Scope)
} else {
scope = ast.NewScope(nil)
obj.Data = scope
}
} else {
// use a dummy scope so that meth can be declared in
// presence of an error and get an associated object
// (always use a new scope so that we don't get double
// declaration errors)
scope = ast.NewScope(nil)
}
check.declare(scope, ast.Fun, meth.Name, meth)
}
func (check *checker) assocInitvalsOrMethod(decl ast.Decl) {
switch d := decl.(type) {
case *ast.GenDecl:
if d.Tok == token.CONST {
check.assocInitvals(d)
}
case *ast.FuncDecl:
if d.Recv != nil {
check.assocMethod(d)
}
}
}
func (check *checker) decl(decl ast.Decl) {
switch d := decl.(type) {
case *ast.BadDecl:
// ignore
case *ast.GenDecl:
for _, spec := range d.Specs {
switch s := spec.(type) {
case *ast.ImportSpec:
// nothing to do (handled by ast.NewPackage)
case *ast.ValueSpec:
for _, name := range s.Names {
if obj := name.Obj; obj.Type == nil {
check.object(obj, false)
}
}
case *ast.TypeSpec:
if obj := s.Name.Obj; obj.Type == nil {
check.object(obj, false)
}
default:
check.invalidAST(s.Pos(), "unknown ast.Spec node %T", s)
}
}
case *ast.FuncDecl:
if d.Name.Name == "init" {
// initialization function
// TODO(gri) ignore for now (has no object associated with it)
// (should probably collect in a first phase and properly initialize)
return
}
if obj := d.Name.Obj; obj.Type == nil {
check.object(obj, false)
}
default:
check.invalidAST(d.Pos(), "unknown ast.Decl node %T", d)
}
}
// iterate calls f for each package-level declaration.
func (check *checker) iterate(f func(*checker, ast.Decl)) {
list := check.filenames
if list == nil {
// initialize lazily
for filename := range check.pkg.Files {
list = append(list, filename)
}
sort.Strings(list)
check.filenames = list
}
for _, filename := range list {
for _, decl := range check.pkg.Files[filename].Decls {
f(check, decl)
}
}
}
// A bailout panic is raised to indicate early termination.
type bailout struct{}
func check(fset *token.FileSet, pkg *ast.Package, errh func(token.Pos, string), f func(ast.Expr, Type)) (err error) {
// initialize checker
var check checker
check.fset = fset
check.pkg = pkg
check.errh = errh
check.mapf = f
check.initexprs = make(map[*ast.ValueSpec][]ast.Expr)
// handle bailouts
defer func() {
if p := recover(); p != nil {
_ = p.(bailout) // re-panic if not a bailout
}
err = check.firsterr
}()
// determine missing constant initialization expressions
// and associate methods with types
check.iterate((*checker).assocInitvalsOrMethod)
// typecheck all declarations
check.iterate((*checker).decl)
return
}

Some files were not shown because too many files have changed in this diff Show more