procyberian/gcc
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

libgo: Update to go1.6rc1.

Browse source 
Reviewed-on: https://go-review.googlesource.com/19200

From-SVN: r233110
This commit is contained in:
Ian Lance Taylor 2016-02-03 21:58:02 +00:00 • committed by Ian Lance Taylor
parent b081ed4efc
commit f98dd1a338
728 changed files with 43810 additions and 11912 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
gotools/ChangeLog
View file

@ -1,3 +1,9 @@
2016-02-03 Ian Lance Taylor <iant@google.com>
* Makefile.am (go_cmd_gofmt_files): Update to Go 1.6rc1 by adding
internal.go.
* Makefile.in: Rebuild.
2015-12-02 Ian Lance Taylor <iant@google.com>
PR go/66147

1
gotools/Makefile.am
View file

@ -76,6 +76,7 @@ go_cmd_go_files = \
go_cmd_gofmt_files = \
$(cmdsrcdir)/gofmt/doc.go \
$(cmdsrcdir)/gofmt/gofmt.go \
$(cmdsrcdir)/gofmt/internal.go \
$(cmdsrcdir)/gofmt/rewrite.go \
$(cmdsrcdir)/gofmt/simplify.go

3
gotools/Makefile.in
View file

@ -294,6 +294,7 @@ go_cmd_go_files = \
go_cmd_gofmt_files = \
$(cmdsrcdir)/gofmt/doc.go \
$(cmdsrcdir)/gofmt/gofmt.go \
$(cmdsrcdir)/gofmt/internal.go \
$(cmdsrcdir)/gofmt/rewrite.go \
$(cmdsrcdir)/gofmt/simplify.go
@ -563,8 +564,8 @@ distclean-generic:
maintainer-clean-generic:
@echo "This command is intended for maintainers to use"
@echo "it deletes files that may require special tools to rebuild."
@NATIVE_FALSE@install-exec-local:
@NATIVE_FALSE@uninstall-local:
@NATIVE_FALSE@install-exec-local:
clean: clean-am
clean-am: clean-binPROGRAMS clean-generic clean-noinstPROGRAMS \

2
libgo/MERGE
View file

@ -1,4 +1,4 @@
f2e4c8b5fb3660d793b2c545ef207153db0a34b1
036b8fd40b60830ca1d152f17148e52b96d8aa6c
The first line of this file holds the git revision number of the
last merge done from the master library sources.

95
libgo/Makefile.am
View file

@ -846,9 +846,7 @@ go_net_common_files = \
go/net/parse.go \
go/net/pipe.go \
go/net/fd_poll_runtime.go \
go/net/port.go \
go/net/port_unix.go \
go/net/race0.go \
$(go_net_sendfile_file) \
go/net/sock_posix.go \
$(go_net_sock_file) \
@ -1018,7 +1016,7 @@ go_os_files = \
$(go_os_sys_file) \
$(go_os_cloexec_file) \
go/os/types.go \
go/os/types_notwin.go
go/os/types_unix.go
go_path_files = \
go/path/match.go \
@ -1100,7 +1098,8 @@ go_strings_files = \
go/strings/replace.go \
go/strings/search.go \
go/strings/strings.go \
go/strings/strings_decl.go
go/strings/strings_decl.go \
go/strings/strings_generic.go
go_strings_c_files = \
go/strings/indexbyte.c
@ -1109,7 +1108,6 @@ go_sync_files = \
go/sync/mutex.go \
go/sync/once.go \
go/sync/pool.go \
go/sync/race0.go \
go/sync/runtime.go \
go/sync/rwmutex.go \
go/sync/waitgroup.go
@ -1196,7 +1194,6 @@ go_compress_bzip2_files = \
go_compress_flate_files = \
go/compress/flate/copy.go \
go/compress/flate/deflate.go \
go/compress/flate/fixedhuff.go \
go/compress/flate/huffman_bit_writer.go \
go/compress/flate/huffman_code.go \
go/compress/flate/inflate.go \
@ -1367,7 +1364,8 @@ go_debug_dwarf_files = \
go/debug/dwarf/unit.go
go_debug_elf_files = \
go/debug/elf/elf.go \
go/debug/elf/file.go
go/debug/elf/file.go \
go/debug/elf/reader.go
go_debug_gosym_files = \
go/debug/gosym/pclntab.go \
go/debug/gosym/symtab.go
@ -1450,7 +1448,6 @@ go_go_build_files = \
go/go/build/read.go \
go/go/build/syslist.go
go_go_constant_files = \
go/go/constant/go14.go \
go/go/constant/value.go
go_go_doc_files = \
go/go/doc/comment.go \
@ -1461,7 +1458,8 @@ go_go_doc_files = \
go/go/doc/reader.go \
go/go/doc/synopsis.go
go_go_format_files = \
go/go/format/format.go
go/go/format/format.go \
go/go/format/internal.go
go_go_importer_files = \
go/go/importer/importer.go
go_go_parser_files = \
@ -1489,7 +1487,6 @@ go_go_types_files = \
go/go/types/eval.go \
go/go/types/expr.go \
go/go/types/exprstring.go \
go/go/types/go12.go \
go/go/types/initorder.go \
go/go/types/labels.go \
go/go/types/lookup.go \
@ -1512,6 +1509,7 @@ go_go_types_files = \
go/go/types/universe.go
go_go_internal_gcimporter_files = \
go/go/internal/gcimporter/bimport.go \
go/go/internal/gcimporter/exportdata.go \
go/go/internal/gcimporter/gcimporter.go
go_go_internal_gccgoimporter_files = \
@ -1578,20 +1576,46 @@ go_index_suffixarray_files = \
go/index/suffixarray/qsufsort.go \
go/index/suffixarray/suffixarray.go
go_internal_format_files = \
go/internal/format/format.go
go_internal_golang_org_x_net_http2_hpack_files = \
go/internal/golang.org/x/net/http2/hpack/encode.go \
go/internal/golang.org/x/net/http2/hpack/hpack.go \
go/internal/golang.org/x/net/http2/hpack/huffman.go \
go/internal/golang.org/x/net/http2/hpack/tables.go
go_internal_race_files = \
go/internal/race/doc.go \
go/internal/race/norace.go
go_internal_singleflight_files = \
go/internal/singleflight/singleflight.go
if LIBGO_IS_LINUX
internal_syscall_unix_getrandom_file = go/internal/syscall/unix/getrandom_linux.go
if LIBGO_IS_386
internal_syscall_unix_getrandom_files = go/internal/syscall/unix/getrandom_linux.go go/internal/syscall/unix/getrandom_linux_386.go
else
internal_syscall_unix_getrandom_file =
if LIBGO_IS_X86_64
internal_syscall_unix_getrandom_files = go/internal/syscall/unix/getrandom_linux.go go/internal/syscall/unix/getrandom_linux_amd64.go
else
if LIBGO_IS_ARM
internal_syscall_unix_getrandom_files = go/internal/syscall/unix/getrandom_linux.go go/internal/syscall/unix/getrandom_linux_arm.go
else
if LIBGO_IS_PPC64
internal_syscall_unix_getrandom_files = go/internal/syscall/unix/getrandom_linux.go go/internal/syscall/unix/getrandom_linux_ppc64x.go
else
if LIBGO_IS_MIPS64
internal_syscall_unix_getrandom_files = go/internal/syscall/unix/getrandom_linux.go go/internal/syscall/unix/getrandom_linux_mips64x.go
else
internal_syscall_unix_getrandom_files = go/internal/syscall/unix/getrandom_linux.go go/internal/syscall/unix/getrandom_linux_generic.go
endif
endif
endif
endif
endif
else
internal_syscall_unix_getrandom_files =
endif
go_internal_syscall_unix_files = \
go/internal/syscall/unix/dummy.go \
$(internal_syscall_unix_getrandom_file)
$(internal_syscall_unix_getrandom_files)
go_internal_testenv_files = \
go/internal/testenv/testenv.go
@ -1608,15 +1632,19 @@ go_math_big_files = \
go/math/big/arith.go \
go/math/big/arith_decl_pure.go \
go/math/big/decimal.go \
go/math/big/doc.go \
go/math/big/float.go \
go/math/big/floatconv.go \
go/math/big/floatmarsh.go \
go/math/big/ftoa.go \
go/math/big/int.go \
go/math/big/intmarsh.go \
go/math/big/intconv.go \
go/math/big/nat.go \
go/math/big/natconv.go \
go/math/big/rat.go \
go/math/big/ratconv.go \
go/math/big/ratmarsh.go \
go/math/big/roundingmode_string.go
go_math_cmplx_files = \
go/math/cmplx/abs.go \
@ -1654,9 +1682,11 @@ go_net_http_files = \
go/net/http/cookie.go \
go/net/http/filetransport.go \
go/net/http/fs.go \
go/net/http/h2_bundle.go \
go/net/http/header.go \
go/net/http/jar.go \
go/net/http/lex.go \
go/net/http/method.go \
go/net/http/request.go \
go/net/http/response.go \
go/net/http/server.go \
@ -1698,7 +1728,8 @@ go_net_http_httputil_files = \
go/net/http/httputil/persist.go \
go/net/http/httputil/reverseproxy.go
go_net_http_internal_files = \
go/net/http/internal/chunked.go
go/net/http/internal/chunked.go \
go/net/http/internal/testcert.go
if LIBGO_IS_LINUX
go_net_internal_socktest_sys = go/net/internal/socktest/sys_cloexec.go
@ -1731,6 +1762,7 @@ go_os_exec_files = \
go/os/exec/lp_unix.go
go_os_signal_files = \
go/os/signal/doc.go \
go/os/signal/signal.go \
go/os/signal/signal_unix.go
@ -1999,7 +2031,7 @@ go_base_syscall_files = \
go/syscall/syscall_errno.go \
go/syscall/libcall_support.go \
go/syscall/libcall_posix.go \
go/syscall/race0.go \
go/syscall/msan0.go \
go/syscall/socket.go \
go/syscall/sockcmsg_unix.go \
go/syscall/str.go \
@ -2229,7 +2261,8 @@ libgo_go_objs = \
image/jpeg.lo \
image/png.lo \
index/suffixarray.lo \
internal/format.lo \
internal/golang.org/x/net/http2/hpack.lo \
internal/race.lo \
internal/singleflight.lo \
internal/syscall/unix.lo \
internal/testenv.lo \
@ -2358,7 +2391,7 @@ CHECK = \
elif test "$(GOBENCH)" != ""; then \
$(SHELL) $(srcdir)/testsuite/gotest --goarch=$(GOARCH) --goos=$(GOOS) --basedir=$(srcdir) --srcdir=$(srcdir)/go/$(@D) --pkgpath="$(@D)" --pkgfiles="$(go_$(subst /,_,$(@D))_files)" --bench="$(GOBENCH)" $(GOTESTFLAGS) $(go_$(subst /,_,$(@D))_test_files); \
else \
if $(SHELL) $(srcdir)/testsuite/gotest --goarch=$(GOARCH) --goos=$(GOOS) --basedir=$(srcdir) --srcdir=$(srcdir)/go/$(@D) --pkgpath="$(@D)" --pkgfiles="$(go_$(subst /,_,$(@D))_files)" $(GOTESTFLAGS) $(go_$(subst /,_,$(@D))_test_files) >>$@-testlog 2>&1; then \
if $(SHELL) $(srcdir)/testsuite/gotest --goarch=$(GOARCH) --goos=$(GOOS) --basedir=$(srcdir) --srcdir=$(srcdir)/go/$(@D) --pkgpath="$(@D)" --pkgfiles="$(go_$(subst .,_,$(subst /,_,$(@D)))_files)" $(GOTESTFLAGS) $(go_$(subst /,_,$(@D))_test_files) >>$@-testlog 2>&1; then \
echo "PASS: $(@D)" >> $@-testlog; \
echo "PASS: $(@D)"; \
echo "PASS: $(@D)" > $@-testsum; \
@ -3350,14 +3383,23 @@ index/suffixarray/check: $(CHECK_DEPS)
@$(CHECK)
.PHONY: index/suffixarray/check
@go_include@ internal/format.lo.dep
internal/format.lo.dep: $(go_internal_format_files)
@go_include@ internal/golang.org/x/net/http2/hpack.lo.dep
internal/golang.org/x/net/http2/hpack.lo.dep: $(go_internal_golang_org_x_net_http2_hpack_files)
$(BUILDDEPS)
internal/format.lo: $(go_internal_format_files)
internal/golang.org/x/net/http2/hpack.lo: $(go_internal_golang_org_x_net_http2_hpack_files)
$(BUILDPACKAGE)
internal/format/check: $(CHECK_DEPS)
internal/golang.org/x/net/http2/hpack/check: $(CHECK_DEPS)
@$(CHECK)
.PHONY: internal/format/check
.PHONY: internal/golang.org/x/net/http2/hpack/check
@go_include@ internal/race.lo.dep
internal/race.lo.dep: $(go_internal_race_files)
$(BUILDDEPS)
internal/race.lo: $(go_internal_race_files)
$(BUILDPACKAGE)
internal/race/check: $(CHECK_DEPS)
@$(CHECK)
.PHONY: internal/race/check
@go_include@ internal/singleflight.lo.dep
internal/singleflight.lo.dep: $(go_internal_singleflight_files)
@ -4013,7 +4055,9 @@ image/color/palette.gox: image/color/palette.lo
index/suffixarray.gox: index/suffixarray.lo
$(BUILDGOX)
internal/format.gox: internal/format.lo
internal/golang.org/x/net/http2/hpack.gox: internal/golang.org/x/net/http2/hpack.lo
$(BUILDGOX)
internal/race.gox: internal/race.lo
$(BUILDGOX)
internal/singleflight.gox: internal/singleflight.lo
$(BUILDGOX)
@ -4217,6 +4261,7 @@ TEST_PACKAGES = \
image/jpeg/check \
image/png/check \
index/suffixarray/check \
internal/golang.org/x/net/http2/hpack/check \
internal/singleflight/check \
internal/trace/check \
io/ioutil/check \

84
libgo/Makefile.in
View file

@ -205,7 +205,8 @@ am__DEPENDENCIES_2 = bufio.lo bytes.lo bytes/index.lo crypto.lo \
net/http/httputil.lo net/http/internal.lo net/http/pprof.lo \
image/color.lo image/color/palette.lo image/draw.lo \
image/gif.lo image/internal/imageutil.lo image/jpeg.lo \
image/png.lo index/suffixarray.lo internal/format.lo \
image/png.lo index/suffixarray.lo \
internal/golang.org/x/net/http2/hpack.lo internal/race.lo \
internal/singleflight.lo internal/syscall/unix.lo \
internal/testenv.lo internal/trace.lo io/ioutil.lo \
log/syslog.lo log/syslog/syslog_c.lo math/big.lo math/cmplx.lo \
@ -1129,9 +1130,7 @@ go_net_common_files = \
go/net/parse.go \
go/net/pipe.go \
go/net/fd_poll_runtime.go \
go/net/port.go \
go/net/port_unix.go \
go/net/race0.go \
$(go_net_sendfile_file) \
go/net/sock_posix.go \
$(go_net_sock_file) \
@ -1216,7 +1215,7 @@ go_os_files = \
$(go_os_sys_file) \
$(go_os_cloexec_file) \
go/os/types.go \
go/os/types_notwin.go
go/os/types_unix.go
go_path_files = \
go/path/match.go \
@ -1274,7 +1273,8 @@ go_strings_files = \
go/strings/replace.go \
go/strings/search.go \
go/strings/strings.go \
go/strings/strings_decl.go
go/strings/strings_decl.go \
go/strings/strings_generic.go
go_strings_c_files = \
go/strings/indexbyte.c
@ -1284,7 +1284,6 @@ go_sync_files = \
go/sync/mutex.go \
go/sync/once.go \
go/sync/pool.go \
go/sync/race0.go \
go/sync/runtime.go \
go/sync/rwmutex.go \
go/sync/waitgroup.go
@ -1352,7 +1351,6 @@ go_compress_bzip2_files = \
go_compress_flate_files = \
go/compress/flate/copy.go \
go/compress/flate/deflate.go \
go/compress/flate/fixedhuff.go \
go/compress/flate/huffman_bit_writer.go \
go/compress/flate/huffman_code.go \
go/compress/flate/inflate.go \
@ -1511,7 +1509,8 @@ go_debug_dwarf_files = \
go_debug_elf_files = \
go/debug/elf/elf.go \
go/debug/elf/file.go
go/debug/elf/file.go \
go/debug/elf/reader.go
go_debug_gosym_files = \
go/debug/gosym/pclntab.go \
@ -1611,7 +1610,6 @@ go_go_build_files = \
go/go/build/syslist.go
go_go_constant_files = \
go/go/constant/go14.go \
go/go/constant/value.go
go_go_doc_files = \
@ -1624,7 +1622,8 @@ go_go_doc_files = \
go/go/doc/synopsis.go
go_go_format_files = \
go/go/format/format.go
go/go/format/format.go \
go/go/format/internal.go
go_go_importer_files = \
go/go/importer/importer.go
@ -1658,7 +1657,6 @@ go_go_types_files = \
go/go/types/eval.go \
go/go/types/expr.go \
go/go/types/exprstring.go \
go/go/types/go12.go \
go/go/types/initorder.go \
go/go/types/labels.go \
go/go/types/lookup.go \
@ -1681,6 +1679,7 @@ go_go_types_files = \
go/go/types/universe.go
go_go_internal_gcimporter_files = \
go/go/internal/gcimporter/bimport.go \
go/go/internal/gcimporter/exportdata.go \
go/go/internal/gcimporter/gcimporter.go
@ -1751,17 +1750,29 @@ go_index_suffixarray_files = \
go/index/suffixarray/qsufsort.go \
go/index/suffixarray/suffixarray.go
go_internal_format_files = \
go/internal/format/format.go
go_internal_golang_org_x_net_http2_hpack_files = \
go/internal/golang.org/x/net/http2/hpack/encode.go \
go/internal/golang.org/x/net/http2/hpack/hpack.go \
go/internal/golang.org/x/net/http2/hpack/huffman.go \
go/internal/golang.org/x/net/http2/hpack/tables.go
go_internal_race_files = \
go/internal/race/doc.go \
go/internal/race/norace.go
go_internal_singleflight_files = \
go/internal/singleflight/singleflight.go
@LIBGO_IS_LINUX_FALSE@internal_syscall_unix_getrandom_file =
@LIBGO_IS_LINUX_TRUE@internal_syscall_unix_getrandom_file = go/internal/syscall/unix/getrandom_linux.go
@LIBGO_IS_386_FALSE@@LIBGO_IS_ARM_FALSE@@LIBGO_IS_LINUX_TRUE@@LIBGO_IS_MIPS64_FALSE@@LIBGO_IS_PPC64_FALSE@@LIBGO_IS_X86_64_FALSE@internal_syscall_unix_getrandom_files = go/internal/syscall/unix/getrandom_linux.go go/internal/syscall/unix/getrandom_linux_generic.go
@LIBGO_IS_386_FALSE@@LIBGO_IS_ARM_FALSE@@LIBGO_IS_LINUX_TRUE@@LIBGO_IS_MIPS64_TRUE@@LIBGO_IS_PPC64_FALSE@@LIBGO_IS_X86_64_FALSE@internal_syscall_unix_getrandom_files = go/internal/syscall/unix/getrandom_linux.go go/internal/syscall/unix/getrandom_linux_mips64x.go
@LIBGO_IS_386_FALSE@@LIBGO_IS_ARM_FALSE@@LIBGO_IS_LINUX_TRUE@@LIBGO_IS_PPC64_TRUE@@LIBGO_IS_X86_64_FALSE@internal_syscall_unix_getrandom_files = go/internal/syscall/unix/getrandom_linux.go go/internal/syscall/unix/getrandom_linux_ppc64x.go
@LIBGO_IS_386_FALSE@@LIBGO_IS_ARM_TRUE@@LIBGO_IS_LINUX_TRUE@@LIBGO_IS_X86_64_FALSE@internal_syscall_unix_getrandom_files = go/internal/syscall/unix/getrandom_linux.go go/internal/syscall/unix/getrandom_linux_arm.go
@LIBGO_IS_386_FALSE@@LIBGO_IS_LINUX_TRUE@@LIBGO_IS_X86_64_TRUE@internal_syscall_unix_getrandom_files = go/internal/syscall/unix/getrandom_linux.go go/internal/syscall/unix/getrandom_linux_amd64.go
@LIBGO_IS_386_TRUE@@LIBGO_IS_LINUX_TRUE@internal_syscall_unix_getrandom_files = go/internal/syscall/unix/getrandom_linux.go go/internal/syscall/unix/getrandom_linux_386.go
@LIBGO_IS_LINUX_FALSE@internal_syscall_unix_getrandom_files =
go_internal_syscall_unix_files = \
go/internal/syscall/unix/dummy.go \
$(internal_syscall_unix_getrandom_file)
$(internal_syscall_unix_getrandom_files)
go_internal_testenv_files = \
go/internal/testenv/testenv.go
@ -1779,15 +1790,19 @@ go_math_big_files = \
go/math/big/arith.go \
go/math/big/arith_decl_pure.go \
go/math/big/decimal.go \
go/math/big/doc.go \
go/math/big/float.go \
go/math/big/floatconv.go \
go/math/big/floatmarsh.go \
go/math/big/ftoa.go \
go/math/big/int.go \
go/math/big/intmarsh.go \
go/math/big/intconv.go \
go/math/big/nat.go \
go/math/big/natconv.go \
go/math/big/rat.go \
go/math/big/ratconv.go \
go/math/big/ratmarsh.go \
go/math/big/roundingmode_string.go
go_math_cmplx_files = \
@ -1827,9 +1842,11 @@ go_net_http_files = \
go/net/http/cookie.go \
go/net/http/filetransport.go \
go/net/http/fs.go \
go/net/http/h2_bundle.go \
go/net/http/header.go \
go/net/http/jar.go \
go/net/http/lex.go \
go/net/http/method.go \
go/net/http/request.go \
go/net/http/response.go \
go/net/http/server.go \
@ -1881,7 +1898,8 @@ go_net_http_httputil_files = \
go/net/http/httputil/reverseproxy.go
go_net_http_internal_files = \
go/net/http/internal/chunked.go
go/net/http/internal/chunked.go \
go/net/http/internal/testcert.go
@LIBGO_IS_FREEBSD_FALSE@@LIBGO_IS_LINUX_FALSE@go_net_internal_socktest_sys =
@LIBGO_IS_FREEBSD_TRUE@@LIBGO_IS_LINUX_FALSE@go_net_internal_socktest_sys = go/net/internal/socktest/sys_cloexec.go
@ -1908,6 +1926,7 @@ go_os_exec_files = \
go/os/exec/lp_unix.go
go_os_signal_files = \
go/os/signal/doc.go \
go/os/signal/signal.go \
go/os/signal/signal_unix.go
@ -2083,7 +2102,7 @@ go_base_syscall_files = \
go/syscall/syscall_errno.go \
go/syscall/libcall_support.go \
go/syscall/libcall_posix.go \
go/syscall/race0.go \
go/syscall/msan0.go \
go/syscall/socket.go \
go/syscall/sockcmsg_unix.go \
go/syscall/str.go \
@ -2253,7 +2272,8 @@ libgo_go_objs = \
image/jpeg.lo \
image/png.lo \
index/suffixarray.lo \
internal/format.lo \
internal/golang.org/x/net/http2/hpack.lo \
internal/race.lo \
internal/singleflight.lo \
internal/syscall/unix.lo \
internal/testenv.lo \
@ -2378,7 +2398,7 @@ CHECK = \
elif test "$(GOBENCH)" != ""; then \
$(SHELL) $(srcdir)/testsuite/gotest --goarch=$(GOARCH) --goos=$(GOOS) --basedir=$(srcdir) --srcdir=$(srcdir)/go/$(@D) --pkgpath="$(@D)" --pkgfiles="$(go_$(subst /,_,$(@D))_files)" --bench="$(GOBENCH)" $(GOTESTFLAGS) $(go_$(subst /,_,$(@D))_test_files); \
else \
if $(SHELL) $(srcdir)/testsuite/gotest --goarch=$(GOARCH) --goos=$(GOOS) --basedir=$(srcdir) --srcdir=$(srcdir)/go/$(@D) --pkgpath="$(@D)" --pkgfiles="$(go_$(subst /,_,$(@D))_files)" $(GOTESTFLAGS) $(go_$(subst /,_,$(@D))_test_files) >>$@-testlog 2>&1; then \
if $(SHELL) $(srcdir)/testsuite/gotest --goarch=$(GOARCH) --goos=$(GOOS) --basedir=$(srcdir) --srcdir=$(srcdir)/go/$(@D) --pkgpath="$(@D)" --pkgfiles="$(go_$(subst .,_,$(subst /,_,$(@D)))_files)" $(GOTESTFLAGS) $(go_$(subst /,_,$(@D))_test_files) >>$@-testlog 2>&1; then \
echo "PASS: $(@D)" >> $@-testlog; \
echo "PASS: $(@D)"; \
echo "PASS: $(@D)" > $@-testsum; \
@ -2511,6 +2531,7 @@ TEST_PACKAGES = \
image/jpeg/check \
image/png/check \
index/suffixarray/check \
internal/golang.org/x/net/http2/hpack/check \
internal/singleflight/check \
internal/trace/check \
io/ioutil/check \
@ -5743,14 +5764,23 @@ index/suffixarray/check: $(CHECK_DEPS)
@$(CHECK)
.PHONY: index/suffixarray/check
@go_include@ internal/format.lo.dep
internal/format.lo.dep: $(go_internal_format_files)
@go_include@ internal/golang.org/x/net/http2/hpack.lo.dep
internal/golang.org/x/net/http2/hpack.lo.dep: $(go_internal_golang_org_x_net_http2_hpack_files)
$(BUILDDEPS)
internal/format.lo: $(go_internal_format_files)
internal/golang.org/x/net/http2/hpack.lo: $(go_internal_golang_org_x_net_http2_hpack_files)
$(BUILDPACKAGE)
internal/format/check: $(CHECK_DEPS)
internal/golang.org/x/net/http2/hpack/check: $(CHECK_DEPS)
@$(CHECK)
.PHONY: internal/format/check
.PHONY: internal/golang.org/x/net/http2/hpack/check
@go_include@ internal/race.lo.dep
internal/race.lo.dep: $(go_internal_race_files)
$(BUILDDEPS)
internal/race.lo: $(go_internal_race_files)
$(BUILDPACKAGE)
internal/race/check: $(CHECK_DEPS)
@$(CHECK)
.PHONY: internal/race/check
@go_include@ internal/singleflight.lo.dep
internal/singleflight.lo.dep: $(go_internal_singleflight_files)
@ -6397,7 +6427,9 @@ image/color/palette.gox: image/color/palette.lo
index/suffixarray.gox: index/suffixarray.lo
$(BUILDGOX)
internal/format.gox: internal/format.lo
internal/golang.org/x/net/http2/hpack.gox: internal/golang.org/x/net/http2/hpack.lo
$(BUILDGOX)
internal/race.gox: internal/race.lo
$(BUILDGOX)
internal/singleflight.gox: internal/singleflight.lo
$(BUILDGOX)

2
libgo/VERSION
View file

@ -1 +1 @@
go1.5.1
go1.6rc1

18
libgo/configure vendored
View file

@ -646,6 +646,8 @@ LIBGO_IS_PPC64_FALSE
LIBGO_IS_PPC64_TRUE
LIBGO_IS_PPC_FALSE
LIBGO_IS_PPC_TRUE
LIBGO_IS_MIPS64_FALSE
LIBGO_IS_MIPS64_TRUE
LIBGO_IS_MIPSO64_FALSE
LIBGO_IS_MIPSO64_TRUE
LIBGO_IS_MIPSN64_FALSE
@ -11124,7 +11126,7 @@ else
lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
lt_status=$lt_dlunknown
cat > conftest.$ac_ext <<_LT_EOF
#line 11127 "configure"
#line 11129 "configure"
#include "confdefs.h"
#if HAVE_DLFCN_H
@ -11230,7 +11232,7 @@ else
lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
lt_status=$lt_dlunknown
cat > conftest.$ac_ext <<_LT_EOF
#line 11233 "configure"
#line 11235 "configure"
#include "confdefs.h"
#if HAVE_DLFCN_H
@ -13884,6 +13886,14 @@ else
LIBGO_IS_MIPSO64_FALSE=
fi
if test $mips_abi = n64 -o $mips_abi = o64; then
LIBGO_IS_MIPS64_TRUE=
LIBGO_IS_MIPS64_FALSE='#'
else
LIBGO_IS_MIPS64_TRUE='#'
LIBGO_IS_MIPS64_FALSE=
fi
if test $is_ppc = yes; then
LIBGO_IS_PPC_TRUE=
LIBGO_IS_PPC_FALSE='#'
@ -15799,6 +15809,10 @@ if test -z "${LIBGO_IS_MIPSO64_TRUE}" && test -z "${LIBGO_IS_MIPSO64_FALSE}"; th
as_fn_error "conditional \"LIBGO_IS_MIPSO64\" was never defined.
Usually this means the macro was only invoked conditionally." "$LINENO" 5
fi
if test -z "${LIBGO_IS_MIPS64_TRUE}" && test -z "${LIBGO_IS_MIPS64_FALSE}"; then
as_fn_error "conditional \"LIBGO_IS_MIPS64\" was never defined.
Usually this means the macro was only invoked conditionally." "$LINENO" 5
fi
if test -z "${LIBGO_IS_PPC_TRUE}" && test -z "${LIBGO_IS_PPC_FALSE}"; then
as_fn_error "conditional \"LIBGO_IS_PPC\" was never defined.
Usually this means the macro was only invoked conditionally." "$LINENO" 5

1
libgo/configure.ac
View file

@ -316,6 +316,7 @@ AM_CONDITIONAL(LIBGO_IS_MIPSO32, test $mips_abi = o32)
AM_CONDITIONAL(LIBGO_IS_MIPSN32, test $mips_abi = n32)
AM_CONDITIONAL(LIBGO_IS_MIPSN64, test $mips_abi = n64)
AM_CONDITIONAL(LIBGO_IS_MIPSO64, test $mips_abi = o64)
AM_CONDITIONAL(LIBGO_IS_MIPS64, test $mips_abi = n64 -o $mips_abi = o64)
AM_CONDITIONAL(LIBGO_IS_PPC, test $is_ppc = yes)
AM_CONDITIONAL(LIBGO_IS_PPC64, test $is_ppc64 = yes)
AM_CONDITIONAL(LIBGO_IS_PPC64LE, test $is_ppc64le = yes)

11
libgo/go/archive/tar/common.go
View file

@ -327,3 +327,14 @@ func toASCII(s string) string {
}
return buf.String()
}
// isHeaderOnlyType checks if the given type flag is of the type that has no
// data section even if a size is specified.
func isHeaderOnlyType(flag byte) bool {
switch flag {
case TypeLink, TypeSymlink, TypeChar, TypeBlock, TypeDir, TypeFifo:
return true
default:
return false
}
}

672
libgo/go/archive/tar/reader.go
View file

@ -12,6 +12,7 @@ import (
"errors"
"io"
"io/ioutil"
"math"
"os"
"strconv"
"strings"
@ -36,6 +37,10 @@ type Reader struct {
hdrBuff [blockSize]byte // buffer to use in readHeader
}
type parser struct {
err error // Last error seen
}
// A numBytesReader is an io.Reader with a numBytes method, returning the number
// of bytes remaining in the underlying encoded data.
type numBytesReader interface {
@ -49,12 +54,36 @@ type regFileReader struct {
nb int64 // number of unread bytes for current file entry
}
// A sparseFileReader is a numBytesReader for reading sparse file data from a tar archive.
// A sparseFileReader is a numBytesReader for reading sparse file data from a
// tar archive.
type sparseFileReader struct {
rfr *regFileReader // reads the sparse-encoded file data
sp []sparseEntry // the sparse map for the file
pos int64 // keeps track of file position
tot int64 // total size of the file
rfr numBytesReader // Reads the sparse-encoded file data
sp []sparseEntry // The sparse map for the file
pos int64 // Keeps track of file position
total int64 // Total size of the file
}
// A sparseEntry holds a single entry in a sparse file's sparse map.
//
// Sparse files are represented using a series of sparseEntrys.
// Despite the name, a sparseEntry represents an actual data fragment that
// references data found in the underlying archive stream. All regions not
// covered by a sparseEntry are logically filled with zeros.
//
// For example, if the underlying raw file contains the 10-byte data:
// var compactData = "abcdefgh"
//
// And the sparse map has the following entries:
// var sp = []sparseEntry{
// {offset: 2, numBytes: 5} // Data fragment for [2..7]
// {offset: 18, numBytes: 3} // Data fragment for [18..21]
// }
//
// Then the content of the resulting sparse file with a "real" size of 25 is:
// var sparseData = "\x00"*2 + "abcde" + "\x00"*11 + "fgh" + "\x00"*4
type sparseEntry struct {
offset int64 // Starting position of the fragment
numBytes int64 // Length of the fragment
}
// Keywords for GNU sparse files in a PAX extended header
@ -88,69 +117,82 @@ func NewReader(r io.Reader) *Reader { return &Reader{r: r} }
//
// io.EOF is returned at the end of the input.
func (tr *Reader) Next() (*Header, error) {
var hdr *Header
if tr.err == nil {
tr.skipUnread()
}
if tr.err != nil {
return hdr, tr.err
return nil, tr.err
}
hdr = tr.readHeader()
if hdr == nil {
return hdr, tr.err
}
// Check for PAX/GNU header.
switch hdr.Typeflag {
case TypeXHeader:
// PAX extended header
headers, err := parsePAX(tr)
if err != nil {
return nil, err
}
// We actually read the whole file,
// but this skips alignment padding
tr.skipUnread()
var hdr *Header
var extHdrs map[string]string
// Externally, Next iterates through the tar archive as if it is a series of
// files. Internally, the tar format often uses fake "files" to add meta
// data that describes the next file. These meta data "files" should not
// normally be visible to the outside. As such, this loop iterates through
// one or more "header files" until it finds a "normal file".
loop:
for {
tr.err = tr.skipUnread()
if tr.err != nil {
return nil, tr.err
}
hdr = tr.readHeader()
if hdr == nil {
if tr.err != nil {
return nil, tr.err
}
mergePAX(hdr, headers)
// Check for a PAX format sparse file
sp, err := tr.checkForGNUSparsePAXHeaders(hdr, headers)
if err != nil {
tr.err = err
return nil, err
// Check for PAX/GNU special headers and files.
switch hdr.Typeflag {
case TypeXHeader:
extHdrs, tr.err = parsePAX(tr)
if tr.err != nil {
return nil, tr.err
}
continue loop // This is a meta header affecting the next header
case TypeGNULongName, TypeGNULongLink:
var realname []byte
realname, tr.err = ioutil.ReadAll(tr)
if tr.err != nil {
return nil, tr.err
}
// Convert GNU extensions to use PAX headers.
if extHdrs == nil {
extHdrs = make(map[string]string)
}
var p parser
switch hdr.Typeflag {
case TypeGNULongName:
extHdrs[paxPath] = p.parseString(realname)
case TypeGNULongLink:
extHdrs[paxLinkpath] = p.parseString(realname)
}
if p.err != nil {
tr.err = p.err
return nil, tr.err
}
continue loop // This is a meta header affecting the next header
default:
mergePAX(hdr, extHdrs)
// Check for a PAX format sparse file
sp, err := tr.checkForGNUSparsePAXHeaders(hdr, extHdrs)
if err != nil {
tr.err = err
return nil, err
}
if sp != nil {
// Current file is a PAX format GNU sparse file.
// Set the current file reader to a sparse file reader.
tr.curr, tr.err = newSparseFileReader(tr.curr, sp, hdr.Size)
if tr.err != nil {
return nil, tr.err
}
}
break loop // This is a file, so stop
}
if sp != nil {
// Current file is a PAX format GNU sparse file.
// Set the current file reader to a sparse file reader.
tr.curr = &sparseFileReader{rfr: tr.curr.(*regFileReader), sp: sp, tot: hdr.Size}
}
return hdr, nil
case TypeGNULongName:
// We have a GNU long name header. Its contents are the real file name.
realname, err := ioutil.ReadAll(tr)
if err != nil {
return nil, err
}
hdr, err := tr.Next()
hdr.Name = cString(realname)
return hdr, err
case TypeGNULongLink:
// We have a GNU long link header.
realname, err := ioutil.ReadAll(tr)
if err != nil {
return nil, err
}
hdr, err := tr.Next()
hdr.Linkname = cString(realname)
return hdr, err
}
return hdr, tr.err
return hdr, nil
}
// checkForGNUSparsePAXHeaders checks the PAX headers for GNU sparse headers. If they are found, then
@ -321,6 +363,7 @@ func parsePAX(r io.Reader) (map[string]string, error) {
if err != nil {
return nil, err
}
sbuf := string(buf)
// For GNU PAX sparse format 0.0 support.
// This function transforms the sparse format 0.0 headers into sparse format 0.1 headers.
@ -329,35 +372,17 @@ func parsePAX(r io.Reader) (map[string]string, error) {
headers := make(map[string]string)
// Each record is constructed as
// "%d %s=%s\n", length, keyword, value
for len(buf) > 0 {
// or the header was empty to start with.
var sp int
// The size field ends at the first space.
sp = bytes.IndexByte(buf, ' ')
if sp == -1 {
for len(sbuf) > 0 {
key, value, residual, err := parsePAXRecord(sbuf)
if err != nil {
return nil, ErrHeader
}
// Parse the first token as a decimal integer.
n, err := strconv.ParseInt(string(buf[:sp]), 10, 0)
if err != nil || n < 5 || int64(len(buf)) < n {
return nil, ErrHeader
}
// Extract everything between the decimal and the n -1 on the
// beginning to eat the ' ', -1 on the end to skip the newline.
var record []byte
record, buf = buf[sp+1:n-1], buf[n:]
// The first equals is guaranteed to mark the end of the key.
// Everything else is value.
eq := bytes.IndexByte(record, '=')
if eq == -1 {
return nil, ErrHeader
}
key, value := record[:eq], record[eq+1:]
sbuf = residual
keyStr := string(key)
if keyStr == paxGNUSparseOffset || keyStr == paxGNUSparseNumBytes {
// GNU sparse format 0.0 special key. Write to sparseMap instead of using the headers map.
sparseMap.Write(value)
sparseMap.WriteString(value)
sparseMap.Write([]byte{','})
} else {
// Normal key. Set the value in the headers map.
@ -372,9 +397,42 @@ func parsePAX(r io.Reader) (map[string]string, error) {
return headers, nil
}
// cString parses bytes as a NUL-terminated C-style string.
// parsePAXRecord parses the input PAX record string into a key-value pair.
// If parsing is successful, it will slice off the currently read record and
// return the remainder as r.
//
// A PAX record is of the following form:
// "%d %s=%s\n" % (size, key, value)
func parsePAXRecord(s string) (k, v, r string, err error) {
// The size field ends at the first space.
sp := strings.IndexByte(s, ' ')
if sp == -1 {
return "", "", s, ErrHeader
}
// Parse the first token as a decimal integer.
n, perr := strconv.ParseInt(s[:sp], 10, 0) // Intentionally parse as native int
if perr != nil || n < 5 || int64(len(s)) < n {
return "", "", s, ErrHeader
}
// Extract everything between the space and the final newline.
rec, nl, rem := s[sp+1:n-1], s[n-1:n], s[n:]
if nl != "\n" {
return "", "", s, ErrHeader
}
// The first equals separates the key from the value.
eq := strings.IndexByte(rec, '=')
if eq == -1 {
return "", "", s, ErrHeader
}
return rec[:eq], rec[eq+1:], rem, nil
}
// parseString parses bytes as a NUL-terminated C-style string.
// If a NUL byte is not found then the whole slice is returned as a string.
func cString(b []byte) string {
func (*parser) parseString(b []byte) string {
n := 0
for n < len(b) && b[n] != 0 {
n++
@ -382,19 +440,51 @@ func cString(b []byte) string {
return string(b[0:n])
}
func (tr *Reader) octal(b []byte) int64 {
// Check for binary format first.
// parseNumeric parses the input as being encoded in either base-256 or octal.
// This function may return negative numbers.
// If parsing fails or an integer overflow occurs, err will be set.
func (p *parser) parseNumeric(b []byte) int64 {
// Check for base-256 (binary) format first.
// If the first bit is set, then all following bits constitute a two's
// complement encoded number in big-endian byte order.
if len(b) > 0 && b[0]&0x80 != 0 {
var x int64
for i, c := range b {
if i == 0 {
c &= 0x7f // ignore signal bit in first byte
}
x = x<<8 | int64(c)
// Handling negative numbers relies on the following identity:
// -a-1 == ^a
//
// If the number is negative, we use an inversion mask to invert the
// data bytes and treat the value as an unsigned number.
var inv byte // 0x00 if positive or zero, 0xff if negative
if b[0]&0x40 != 0 {
inv = 0xff
}
return x
var x uint64
for i, c := range b {
c ^= inv // Inverts c only if inv is 0xff, otherwise does nothing
if i == 0 {
c &= 0x7f // Ignore signal bit in first byte
}
if (x >> 56) > 0 {
p.err = ErrHeader // Integer overflow
return 0
}
x = x<<8 | uint64(c)
}
if (x >> 63) > 0 {
p.err = ErrHeader // Integer overflow
return 0
}
if inv == 0xff {
return ^int64(x)
}
return int64(x)
}
// Normal case is base-8 (octal) format.
return p.parseOctal(b)
}
func (p *parser) parseOctal(b []byte) int64 {
// Because unused fields are filled with NULs, we need
// to skip leading NULs. Fields may also be padded with
// spaces or NULs.
@ -405,23 +495,52 @@ func (tr *Reader) octal(b []byte) int64 {
if len(b) == 0 {
return 0
}
x, err := strconv.ParseUint(cString(b), 8, 64)
if err != nil {
tr.err = err
x, perr := strconv.ParseUint(p.parseString(b), 8, 64)
if perr != nil {
p.err = ErrHeader
}
return int64(x)
}
// skipUnread skips any unread bytes in the existing file entry, as well as any alignment padding.
func (tr *Reader) skipUnread() {
nr := tr.numBytes() + tr.pad // number of bytes to skip
// skipUnread skips any unread bytes in the existing file entry, as well as any
// alignment padding. It returns io.ErrUnexpectedEOF if any io.EOF is
// encountered in the data portion; it is okay to hit io.EOF in the padding.
//
// Note that this function still works properly even when sparse files are being
// used since numBytes returns the bytes remaining in the underlying io.Reader.
func (tr *Reader) skipUnread() error {
dataSkip := tr.numBytes() // Number of data bytes to skip
totalSkip := dataSkip + tr.pad // Total number of bytes to skip
tr.curr, tr.pad = nil, 0
if sr, ok := tr.r.(io.Seeker); ok {
if _, err := sr.Seek(nr, os.SEEK_CUR); err == nil {
return
// If possible, Seek to the last byte before the end of the data section.
// Do this because Seek is often lazy about reporting errors; this will mask
// the fact that the tar stream may be truncated. We can rely on the
// io.CopyN done shortly afterwards to trigger any IO errors.
var seekSkipped int64 // Number of bytes skipped via Seek
if sr, ok := tr.r.(io.Seeker); ok && dataSkip > 1 {
// Not all io.Seeker can actually Seek. For example, os.Stdin implements
// io.Seeker, but calling Seek always returns an error and performs
// no action. Thus, we try an innocent seek to the current position
// to see if Seek is really supported.
pos1, err := sr.Seek(0, os.SEEK_CUR)
if err == nil {
// Seek seems supported, so perform the real Seek.
pos2, err := sr.Seek(dataSkip-1, os.SEEK_CUR)
if err != nil {
tr.err = err
return tr.err
}
seekSkipped = pos2 - pos1
}
}
_, tr.err = io.CopyN(ioutil.Discard, tr.r, nr)
var copySkipped int64 // Number of bytes skipped via CopyN
copySkipped, tr.err = io.CopyN(ioutil.Discard, tr.r, totalSkip-seekSkipped)
if tr.err == io.EOF && seekSkipped+copySkipped < dataSkip {
tr.err = io.ErrUnexpectedEOF
}
return tr.err
}
func (tr *Reader) verifyChecksum(header []byte) bool {
@ -429,23 +548,31 @@ func (tr *Reader) verifyChecksum(header []byte) bool {
return false
}
given := tr.octal(header[148:156])
var p parser
given := p.parseOctal(header[148:156])
unsigned, signed := checksum(header)
return given == unsigned || given == signed
return p.err == nil && (given == unsigned || given == signed)
}
// readHeader reads the next block header and assumes that the underlying reader
// is already aligned to a block boundary.
//
// The err will be set to io.EOF only when one of the following occurs:
// * Exactly 0 bytes are read and EOF is hit.
// * Exactly 1 block of zeros is read and EOF is hit.
// * At least 2 blocks of zeros are read.
func (tr *Reader) readHeader() *Header {
header := tr.hdrBuff[:]
copy(header, zeroBlock)
if _, tr.err = io.ReadFull(tr.r, header); tr.err != nil {
return nil
return nil // io.EOF is okay here
}
// Two blocks of zero bytes marks the end of the archive.
if bytes.Equal(header, zeroBlock[0:blockSize]) {
if _, tr.err = io.ReadFull(tr.r, header); tr.err != nil {
return nil
return nil // io.EOF is okay here
}
if bytes.Equal(header, zeroBlock[0:blockSize]) {
tr.err = io.EOF
@ -461,22 +588,19 @@ func (tr *Reader) readHeader() *Header {
}
// Unpack
var p parser
hdr := new(Header)
s := slicer(header)
hdr.Name = cString(s.next(100))
hdr.Mode = tr.octal(s.next(8))
hdr.Uid = int(tr.octal(s.next(8)))
hdr.Gid = int(tr.octal(s.next(8)))
hdr.Size = tr.octal(s.next(12))
if hdr.Size < 0 {
tr.err = ErrHeader
return nil
}
hdr.ModTime = time.Unix(tr.octal(s.next(12)), 0)
hdr.Name = p.parseString(s.next(100))
hdr.Mode = p.parseNumeric(s.next(8))
hdr.Uid = int(p.parseNumeric(s.next(8)))
hdr.Gid = int(p.parseNumeric(s.next(8)))
hdr.Size = p.parseNumeric(s.next(12))
hdr.ModTime = time.Unix(p.parseNumeric(s.next(12)), 0)
s.next(8) // chksum
hdr.Typeflag = s.next(1)[0]
hdr.Linkname = cString(s.next(100))
hdr.Linkname = p.parseString(s.next(100))
// The remainder of the header depends on the value of magic.
// The original (v7) version of tar had no explicit magic field,
@ -496,70 +620,76 @@ func (tr *Reader) readHeader() *Header {
switch format {
case "posix", "gnu", "star":
hdr.Uname = cString(s.next(32))
hdr.Gname = cString(s.next(32))
hdr.Uname = p.parseString(s.next(32))
hdr.Gname = p.parseString(s.next(32))
devmajor := s.next(8)
devminor := s.next(8)
if hdr.Typeflag == TypeChar || hdr.Typeflag == TypeBlock {
hdr.Devmajor = tr.octal(devmajor)
hdr.Devminor = tr.octal(devminor)
hdr.Devmajor = p.parseNumeric(devmajor)
hdr.Devminor = p.parseNumeric(devminor)
}
var prefix string
switch format {
case "posix", "gnu":
prefix = cString(s.next(155))
prefix = p.parseString(s.next(155))
case "star":
prefix = cString(s.next(131))
hdr.AccessTime = time.Unix(tr.octal(s.next(12)), 0)
hdr.ChangeTime = time.Unix(tr.octal(s.next(12)), 0)
prefix = p.parseString(s.next(131))
hdr.AccessTime = time.Unix(p.parseNumeric(s.next(12)), 0)
hdr.ChangeTime = time.Unix(p.parseNumeric(s.next(12)), 0)
}
if len(prefix) > 0 {
hdr.Name = prefix + "/" + hdr.Name
}
}
if tr.err != nil {
if p.err != nil {
tr.err = p.err
return nil
}
nb := hdr.Size
if isHeaderOnlyType(hdr.Typeflag) {
nb = 0
}
if nb < 0 {
tr.err = ErrHeader
return nil
}
// Maximum value of hdr.Size is 64 GB (12 octal digits),
// so there's no risk of int64 overflowing.
nb := int64(hdr.Size)
tr.pad = -nb & (blockSize - 1) // blockSize is a power of two
// Set the current file reader.
tr.pad = -nb & (blockSize - 1) // blockSize is a power of two
tr.curr = &regFileReader{r: tr.r, nb: nb}
// Check for old GNU sparse format entry.
if hdr.Typeflag == TypeGNUSparse {
// Get the real size of the file.
hdr.Size = tr.octal(header[483:495])
hdr.Size = p.parseNumeric(header[483:495])
if p.err != nil {
tr.err = p.err
return nil
}
// Read the sparse map.
sp := tr.readOldGNUSparseMap(header)
if tr.err != nil {
return nil
}
// Current file is a GNU sparse file. Update the current file reader.
tr.curr = &sparseFileReader{rfr: tr.curr.(*regFileReader), sp: sp, tot: hdr.Size}
tr.curr, tr.err = newSparseFileReader(tr.curr, sp, hdr.Size)
if tr.err != nil {
return nil
}
}
return hdr
}
// A sparseEntry holds a single entry in a sparse file's sparse map.
// A sparse entry indicates the offset and size in a sparse file of a
// block of data.
type sparseEntry struct {
offset int64
numBytes int64
}
// readOldGNUSparseMap reads the sparse map as stored in the old GNU sparse format.
// The sparse map is stored in the tar header if it's small enough. If it's larger than four entries,
// then one or more extension headers are used to store the rest of the sparse map.
func (tr *Reader) readOldGNUSparseMap(header []byte) []sparseEntry {
var p parser
isExtended := header[oldGNUSparseMainHeaderIsExtendedOffset] != 0
spCap := oldGNUSparseMainHeaderNumEntries
if isExtended {
@ -570,10 +700,10 @@ func (tr *Reader) readOldGNUSparseMap(header []byte) []sparseEntry {
// Read the four entries from the main tar header
for i := 0; i < oldGNUSparseMainHeaderNumEntries; i++ {
offset := tr.octal(s.next(oldGNUSparseOffsetSize))
numBytes := tr.octal(s.next(oldGNUSparseNumBytesSize))
if tr.err != nil {
tr.err = ErrHeader
offset := p.parseNumeric(s.next(oldGNUSparseOffsetSize))
numBytes := p.parseNumeric(s.next(oldGNUSparseNumBytesSize))
if p.err != nil {
tr.err = p.err
return nil
}
if offset == 0 && numBytes == 0 {
@ -591,10 +721,10 @@ func (tr *Reader) readOldGNUSparseMap(header []byte) []sparseEntry {
isExtended = sparseHeader[oldGNUSparseExtendedHeaderIsExtendedOffset] != 0
s = slicer(sparseHeader)
for i := 0; i < oldGNUSparseExtendedHeaderNumEntries; i++ {
offset := tr.octal(s.next(oldGNUSparseOffsetSize))
numBytes := tr.octal(s.next(oldGNUSparseNumBytesSize))
if tr.err != nil {
tr.err = ErrHeader
offset := p.parseNumeric(s.next(oldGNUSparseOffsetSize))
numBytes := p.parseNumeric(s.next(oldGNUSparseNumBytesSize))
if p.err != nil {
tr.err = p.err
return nil
}
if offset == 0 && numBytes == 0 {
@ -606,122 +736,111 @@ func (tr *Reader) readOldGNUSparseMap(header []byte) []sparseEntry {
return sp
}
// readGNUSparseMap1x0 reads the sparse map as stored in GNU's PAX sparse format version 1.0.
// The sparse map is stored just before the file data and padded out to the nearest block boundary.
// readGNUSparseMap1x0 reads the sparse map as stored in GNU's PAX sparse format
// version 1.0. The format of the sparse map consists of a series of
// newline-terminated numeric fields. The first field is the number of entries
// and is always present. Following this are the entries, consisting of two
// fields (offset, numBytes). This function must stop reading at the end
// boundary of the block containing the last newline.
//
// Note that the GNU manual says that numeric values should be encoded in octal
// format. However, the GNU tar utility itself outputs these values in decimal.
// As such, this library treats values as being encoded in decimal.
func readGNUSparseMap1x0(r io.Reader) ([]sparseEntry, error) {
buf := make([]byte, 2*blockSize)
sparseHeader := buf[:blockSize]
var cntNewline int64
var buf bytes.Buffer
var blk = make([]byte, blockSize)
// readDecimal is a helper function to read a decimal integer from the sparse map
// while making sure to read from the file in blocks of size blockSize
readDecimal := func() (int64, error) {
// Look for newline
nl := bytes.IndexByte(sparseHeader, '\n')
if nl == -1 {
if len(sparseHeader) >= blockSize {
// This is an error
return 0, ErrHeader
// feedTokens copies data in numBlock chunks from r into buf until there are
// at least cnt newlines in buf. It will not read more blocks than needed.
var feedTokens = func(cnt int64) error {
for cntNewline < cnt {
if _, err := io.ReadFull(r, blk); err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
return err
}
oldLen := len(sparseHeader)
newLen := oldLen + blockSize
if cap(sparseHeader) < newLen {
// There's more header, but we need to make room for the next block
copy(buf, sparseHeader)
sparseHeader = buf[:newLen]
} else {
// There's more header, and we can just reslice
sparseHeader = sparseHeader[:newLen]
buf.Write(blk)
for _, c := range blk {
if c == '\n' {
cntNewline++
}
}
// Now that sparseHeader is large enough, read next block
if _, err := io.ReadFull(r, sparseHeader[oldLen:newLen]); err != nil {
return 0, err
}
// Look for a newline in the new data
nl = bytes.IndexByte(sparseHeader[oldLen:newLen], '\n')
if nl == -1 {
// This is an error
return 0, ErrHeader
}
nl += oldLen // We want the position from the beginning
}
// Now that we've found a newline, read a number
n, err := strconv.ParseInt(string(sparseHeader[:nl]), 10, 0)
if err != nil {
return 0, ErrHeader
}
// Update sparseHeader to consume this number
sparseHeader = sparseHeader[nl+1:]
return n, nil
return nil
}
// Read the first block
if _, err := io.ReadFull(r, sparseHeader); err != nil {
// nextToken gets the next token delimited by a newline. This assumes that
// at least one newline exists in the buffer.
var nextToken = func() string {
cntNewline--
tok, _ := buf.ReadString('\n')
return tok[:len(tok)-1] // Cut off newline
}
// Parse for the number of entries.
// Use integer overflow resistant math to check this.
if err := feedTokens(1); err != nil {
return nil, err
}
// The first line contains the number of entries
numEntries, err := readDecimal()
if err != nil {
return nil, err
numEntries, err := strconv.ParseInt(nextToken(), 10, 0) // Intentionally parse as native int
if err != nil || numEntries < 0 || int(2*numEntries) < int(numEntries) {
return nil, ErrHeader
}
// Read all the entries
// Parse for all member entries.
// numEntries is trusted after this since a potential attacker must have
// committed resources proportional to what this library used.
if err := feedTokens(2 * numEntries); err != nil {
return nil, err
}
sp := make([]sparseEntry, 0, numEntries)
for i := int64(0); i < numEntries; i++ {
// Read the offset
offset, err := readDecimal()
offset, err := strconv.ParseInt(nextToken(), 10, 64)
if err != nil {
return nil, err
return nil, ErrHeader
}
// Read numBytes
numBytes, err := readDecimal()
numBytes, err := strconv.ParseInt(nextToken(), 10, 64)
if err != nil {
return nil, err
return nil, ErrHeader
}
sp = append(sp, sparseEntry{offset: offset, numBytes: numBytes})
}
return sp, nil
}
// readGNUSparseMap0x1 reads the sparse map as stored in GNU's PAX sparse format version 0.1.
// The sparse map is stored in the PAX headers.
func readGNUSparseMap0x1(headers map[string]string) ([]sparseEntry, error) {
// Get number of entries
numEntriesStr, ok := headers[paxGNUSparseNumBlocks]
if !ok {
return nil, ErrHeader
}
numEntries, err := strconv.ParseInt(numEntriesStr, 10, 0)
if err != nil {
// readGNUSparseMap0x1 reads the sparse map as stored in GNU's PAX sparse format
// version 0.1. The sparse map is stored in the PAX headers.
func readGNUSparseMap0x1(extHdrs map[string]string) ([]sparseEntry, error) {
// Get number of entries.
// Use integer overflow resistant math to check this.
numEntriesStr := extHdrs[paxGNUSparseNumBlocks]
numEntries, err := strconv.ParseInt(numEntriesStr, 10, 0) // Intentionally parse as native int
if err != nil || numEntries < 0 || int(2*numEntries) < int(numEntries) {
return nil, ErrHeader
}
sparseMap := strings.Split(headers[paxGNUSparseMap], ",")
// There should be two numbers in sparseMap for each entry
// There should be two numbers in sparseMap for each entry.
sparseMap := strings.Split(extHdrs[paxGNUSparseMap], ",")
if int64(len(sparseMap)) != 2*numEntries {
return nil, ErrHeader
}
// Loop through the entries in the sparse map
// Loop through the entries in the sparse map.
// numEntries is trusted now.
sp := make([]sparseEntry, 0, numEntries)
for i := int64(0); i < numEntries; i++ {
offset, err := strconv.ParseInt(sparseMap[2*i], 10, 0)
offset, err := strconv.ParseInt(sparseMap[2*i], 10, 64)
if err != nil {
return nil, ErrHeader
}
numBytes, err := strconv.ParseInt(sparseMap[2*i+1], 10, 0)
numBytes, err := strconv.ParseInt(sparseMap[2*i+1], 10, 64)
if err != nil {
return nil, ErrHeader
}
sp = append(sp, sparseEntry{offset: offset, numBytes: numBytes})
}
return sp, nil
}
@ -738,10 +857,18 @@ func (tr *Reader) numBytes() int64 {
// Read reads from the current entry in the tar archive.
// It returns 0, io.EOF when it reaches the end of that entry,
// until Next is called to advance to the next entry.
//
// Calling Read on special types like TypeLink, TypeSymLink, TypeChar,
// TypeBlock, TypeDir, and TypeFifo returns 0, io.EOF regardless of what
// the Header.Size claims.
func (tr *Reader) Read(b []byte) (n int, err error) {
if tr.err != nil {
return 0, tr.err
}
if tr.curr == nil {
return 0, io.EOF
}
n, err = tr.curr.Read(b)
if err != nil && err != io.EOF {
tr.err = err
@ -771,9 +898,33 @@ func (rfr *regFileReader) numBytes() int64 {
return rfr.nb
}
// readHole reads a sparse file hole ending at offset toOffset
func (sfr *sparseFileReader) readHole(b []byte, toOffset int64) int {
n64 := toOffset - sfr.pos
// newSparseFileReader creates a new sparseFileReader, but validates all of the
// sparse entries before doing so.
func newSparseFileReader(rfr numBytesReader, sp []sparseEntry, total int64) (*sparseFileReader, error) {
if total < 0 {
return nil, ErrHeader // Total size cannot be negative
}
// Validate all sparse entries. These are the same checks as performed by
// the BSD tar utility.
for i, s := range sp {
switch {
case s.offset < 0 || s.numBytes < 0:
return nil, ErrHeader // Negative values are never okay
case s.offset > math.MaxInt64-s.numBytes:
return nil, ErrHeader // Integer overflow with large length
case s.offset+s.numBytes > total:
return nil, ErrHeader // Region extends beyond the "real" size
case i > 0 && sp[i-1].offset+sp[i-1].numBytes > s.offset:
return nil, ErrHeader // Regions can't overlap and must be in order
}
}
return &sparseFileReader{rfr: rfr, sp: sp, total: total}, nil
}
// readHole reads a sparse hole ending at endOffset.
func (sfr *sparseFileReader) readHole(b []byte, endOffset int64) int {
n64 := endOffset - sfr.pos
if n64 > int64(len(b)) {
n64 = int64(len(b))
}
@ -787,49 +938,54 @@ func (sfr *sparseFileReader) readHole(b []byte, toOffset int64) int {
// Read reads the sparse file data in expanded form.
func (sfr *sparseFileReader) Read(b []byte) (n int, err error) {
if len(sfr.sp) == 0 {
// No more data fragments to read from.
if sfr.pos < sfr.tot {
// We're in the last hole
n = sfr.readHole(b, sfr.tot)
return
}
// Otherwise, we're at the end of the file
return 0, io.EOF
}
if sfr.tot < sfr.sp[0].offset {
return 0, io.ErrUnexpectedEOF
}
if sfr.pos < sfr.sp[0].offset {
// We're in a hole
n = sfr.readHole(b, sfr.sp[0].offset)
return
// Skip past all empty fragments.
for len(sfr.sp) > 0 && sfr.sp[0].numBytes == 0 {
sfr.sp = sfr.sp[1:]
}
// We're not in a hole, so we'll read from the next data fragment
posInFragment := sfr.pos - sfr.sp[0].offset
bytesLeft := sfr.sp[0].numBytes - posInFragment
// If there are no more fragments, then it is possible that there
// is one last sparse hole.
if len(sfr.sp) == 0 {
// This behavior matches the BSD tar utility.
// However, GNU tar stops returning data even if sfr.total is unmet.
if sfr.pos < sfr.total {
return sfr.readHole(b, sfr.total), nil
}
return 0, io.EOF
}
// In front of a data fragment, so read a hole.
if sfr.pos < sfr.sp[0].offset {
return sfr.readHole(b, sfr.sp[0].offset), nil
}
// In a data fragment, so read from it.
// This math is overflow free since we verify that offset and numBytes can
// be safely added when creating the sparseFileReader.
endPos := sfr.sp[0].offset + sfr.sp[0].numBytes // End offset of fragment
bytesLeft := endPos - sfr.pos // Bytes left in fragment
if int64(len(b)) > bytesLeft {
b = b[0:bytesLeft]
b = b[:bytesLeft]
}
n, err = sfr.rfr.Read(b)
sfr.pos += int64(n)
if int64(n) == bytesLeft {
// We're done with this fragment
sfr.sp = sfr.sp[1:]
if err == io.EOF {
if sfr.pos < endPos {
err = io.ErrUnexpectedEOF // There was supposed to be more data
} else if sfr.pos < sfr.total {
err = nil // There is still an implicit sparse hole at the end
}
}
if err == io.EOF && sfr.pos < sfr.tot {
// We reached the end of the last fragment's data, but there's a final hole
err = nil
if sfr.pos == endPos {
sfr.sp = sfr.sp[1:] // We are done with this fragment, so pop it
}
return
return n, err
}
// numBytes returns the number of bytes left to read in the sparse file's
// sparse-encoded data in the tar archive.
func (sfr *sparseFileReader) numBytes() int64 {
return sfr.rfr.nb
return sfr.rfr.numBytes()
}

1005
libgo/go/archive/tar/reader_test.go
View file

File diff suppressed because it is too large Load diff

BIN
libgo/go/archive/tar/testdata/gnu-multi-hdrs.tar vendored Normal file
View file

Binary file not shown.

BIN
libgo/go/archive/tar/testdata/hdr-only.tar vendored Normal file
View file

Binary file not shown.

BIN
libgo/go/archive/tar/testdata/issue12435.tar vendored Normal file
View file

Binary file not shown.

BIN
libgo/go/archive/tar/testdata/neg-size.tar vendored
View file

Binary file not shown.

BIN
libgo/go/archive/tar/testdata/pax-multi-hdrs.tar vendored Normal file
View file

Binary file not shown.

BIN
libgo/go/archive/tar/testdata/pax-path-hdr.tar vendored Normal file
View file

Binary file not shown.

BIN
libgo/go/archive/tar/testdata/ustar-file-reg.tar vendored Normal file
View file

Binary file not shown.

222
libgo/go/archive/tar/writer.go
View file

@ -12,8 +12,8 @@ import (
"errors"
"fmt"
"io"
"os"
"path"
"sort"
"strconv"
"strings"
"time"
@ -23,7 +23,6 @@ var (
ErrWriteTooLong = errors.New("archive/tar: write too long")
ErrFieldTooLong = errors.New("archive/tar: header field too long")
ErrWriteAfterClose = errors.New("archive/tar: write after close")
errNameTooLong = errors.New("archive/tar: name too long")
errInvalidHeader = errors.New("archive/tar: header field too long or contains invalid values")
)
@ -43,6 +42,10 @@ type Writer struct {
paxHdrBuff [blockSize]byte // buffer to use in writeHeader when writing a pax header
}
type formatter struct {
err error // Last error seen
}
// NewWriter creates a new Writer writing to w.
func NewWriter(w io.Writer) *Writer { return &Writer{w: w} }
@ -69,17 +72,9 @@ func (tw *Writer) Flush() error {
}
// Write s into b, terminating it with a NUL if there is room.
// If the value is too long for the field and allowPax is true add a paxheader record instead
func (tw *Writer) cString(b []byte, s string, allowPax bool, paxKeyword string, paxHeaders map[string]string) {
needsPaxHeader := allowPax && len(s) > len(b) || !isASCII(s)
if needsPaxHeader {
paxHeaders[paxKeyword] = s
return
}
func (f *formatter) formatString(b []byte, s string) {
if len(s) > len(b) {
if tw.err == nil {
tw.err = ErrFieldTooLong
}
f.err = ErrFieldTooLong
return
}
ascii := toASCII(s)
@ -90,40 +85,40 @@ func (tw *Writer) cString(b []byte, s string, allowPax bool, paxKeyword string,
}
// Encode x as an octal ASCII string and write it into b with leading zeros.
func (tw *Writer) octal(b []byte, x int64) {
func (f *formatter) formatOctal(b []byte, x int64) {
s := strconv.FormatInt(x, 8)
// leading zeros, but leave room for a NUL.
for len(s)+1 < len(b) {
s = "0" + s
}
tw.cString(b, s, false, paxNone, nil)
f.formatString(b, s)
}
// Write x into b, either as octal or as binary (GNUtar/star extension).
// If the value is too long for the field and writingPax is enabled both for the field and the add a paxheader record instead
func (tw *Writer) numeric(b []byte, x int64, allowPax bool, paxKeyword string, paxHeaders map[string]string) {
// Try octal first.
s := strconv.FormatInt(x, 8)
if len(s) < len(b) {
tw.octal(b, x)
// fitsInBase256 reports whether x can be encoded into n bytes using base-256
// encoding. Unlike octal encoding, base-256 encoding does not require that the
// string ends with a NUL character. Thus, all n bytes are available for output.
//
// If operating in binary mode, this assumes strict GNU binary mode; which means
// that the first byte can only be either 0x80 or 0xff. Thus, the first byte is
// equivalent to the sign bit in two's complement form.
func fitsInBase256(n int, x int64) bool {
var binBits = uint(n-1) * 8
return n >= 9 || (x >= -1<<binBits && x < 1<<binBits)
}
// Write x into b, as binary (GNUtar/star extension).
func (f *formatter) formatNumeric(b []byte, x int64) {
if fitsInBase256(len(b), x) {
for i := len(b) - 1; i >= 0; i-- {
b[i] = byte(x)
x >>= 8
}
b[0] |= 0x80 // Highest bit indicates binary format
return
}
// If it is too long for octal, and pax is preferred, use a pax header
if allowPax && tw.preferPax {
tw.octal(b, 0)
s := strconv.FormatInt(x, 10)
paxHeaders[paxKeyword] = s
return
}
// Too big: use binary (big-endian).
tw.usedBinary = true
for i := len(b) - 1; x > 0 && i >= 0; i-- {
b[i] = byte(x)
x >>= 8
}
b[0] |= 0x80 // highest bit indicates binary format
f.formatOctal(b, 0) // Last resort, just write zero
f.err = ErrFieldTooLong
}
var (
@ -162,6 +157,7 @@ func (tw *Writer) writeHeader(hdr *Header, allowPax bool) error {
// subsecond time resolution, but for now let's just capture
// too long fields or non ascii characters
var f formatter
var header []byte
// We need to select which scratch buffer to use carefully,
@ -176,10 +172,40 @@ func (tw *Writer) writeHeader(hdr *Header, allowPax bool) error {
copy(header, zeroBlock)
s := slicer(header)
// Wrappers around formatter that automatically sets paxHeaders if the
// argument extends beyond the capacity of the input byte slice.
var formatString = func(b []byte, s string, paxKeyword string) {
needsPaxHeader := paxKeyword != paxNone && len(s) > len(b) || !isASCII(s)
if needsPaxHeader {
paxHeaders[paxKeyword] = s
return
}
f.formatString(b, s)
}
var formatNumeric = func(b []byte, x int64, paxKeyword string) {
// Try octal first.
s := strconv.FormatInt(x, 8)
if len(s) < len(b) {
f.formatOctal(b, x)
return
}
// If it is too long for octal, and PAX is preferred, use a PAX header.
if paxKeyword != paxNone && tw.preferPax {
f.formatOctal(b, 0)
s := strconv.FormatInt(x, 10)
paxHeaders[paxKeyword] = s
return
}
tw.usedBinary = true
f.formatNumeric(b, x)
}
// keep a reference to the filename to allow to overwrite it later if we detect that we can use ustar longnames instead of pax
pathHeaderBytes := s.next(fileNameSize)
tw.cString(pathHeaderBytes, hdr.Name, true, paxPath, paxHeaders)
formatString(pathHeaderBytes, hdr.Name, paxPath)
// Handle out of range ModTime carefully.
var modTime int64
@ -187,25 +213,25 @@ func (tw *Writer) writeHeader(hdr *Header, allowPax bool) error {
modTime = hdr.ModTime.Unix()
}
tw.octal(s.next(8), hdr.Mode) // 100:108
tw.numeric(s.next(8), int64(hdr.Uid), true, paxUid, paxHeaders) // 108:116
tw.numeric(s.next(8), int64(hdr.Gid), true, paxGid, paxHeaders) // 116:124
tw.numeric(s.next(12), hdr.Size, true, paxSize, paxHeaders) // 124:136
tw.numeric(s.next(12), modTime, false, paxNone, nil) // 136:148 --- consider using pax for finer granularity
s.next(8) // chksum (148:156)
s.next(1)[0] = hdr.Typeflag // 156:157
f.formatOctal(s.next(8), hdr.Mode) // 100:108
formatNumeric(s.next(8), int64(hdr.Uid), paxUid) // 108:116
formatNumeric(s.next(8), int64(hdr.Gid), paxGid) // 116:124
formatNumeric(s.next(12), hdr.Size, paxSize) // 124:136
formatNumeric(s.next(12), modTime, paxNone) // 136:148 --- consider using pax for finer granularity
s.next(8) // chksum (148:156)
s.next(1)[0] = hdr.Typeflag // 156:157
tw.cString(s.next(100), hdr.Linkname, true, paxLinkpath, paxHeaders)
formatString(s.next(100), hdr.Linkname, paxLinkpath)
copy(s.next(8), []byte("ustar\x0000")) // 257:265
tw.cString(s.next(32), hdr.Uname, true, paxUname, paxHeaders) // 265:297
tw.cString(s.next(32), hdr.Gname, true, paxGname, paxHeaders) // 297:329
tw.numeric(s.next(8), hdr.Devmajor, false, paxNone, nil) // 329:337
tw.numeric(s.next(8), hdr.Devminor, false, paxNone, nil) // 337:345
copy(s.next(8), []byte("ustar\x0000")) // 257:265
formatString(s.next(32), hdr.Uname, paxUname) // 265:297
formatString(s.next(32), hdr.Gname, paxGname) // 297:329
formatNumeric(s.next(8), hdr.Devmajor, paxNone) // 329:337
formatNumeric(s.next(8), hdr.Devminor, paxNone) // 337:345
// keep a reference to the prefix to allow to overwrite it later if we detect that we can use ustar longnames instead of pax
prefixHeaderBytes := s.next(155)
tw.cString(prefixHeaderBytes, "", false, paxNone, nil) // 345:500 prefix
formatString(prefixHeaderBytes, "", paxNone) // 345:500 prefix
// Use the GNU magic instead of POSIX magic if we used any GNU extensions.
if tw.usedBinary {
@ -215,37 +241,26 @@ func (tw *Writer) writeHeader(hdr *Header, allowPax bool) error {
_, paxPathUsed := paxHeaders[paxPath]
// try to use a ustar header when only the name is too long
if !tw.preferPax && len(paxHeaders) == 1 && paxPathUsed {
suffix := hdr.Name
prefix := ""
if len(hdr.Name) > fileNameSize && isASCII(hdr.Name) {
var err error
prefix, suffix, err = tw.splitUSTARLongName(hdr.Name)
if err == nil {
// ok we can use a ustar long name instead of pax, now correct the fields
prefix, suffix, ok := splitUSTARPath(hdr.Name)
if ok {
// Since we can encode in USTAR format, disable PAX header.
delete(paxHeaders, paxPath)
// remove the path field from the pax header. this will suppress the pax header
delete(paxHeaders, paxPath)
// update the path fields
tw.cString(pathHeaderBytes, suffix, false, paxNone, nil)
tw.cString(prefixHeaderBytes, prefix, false, paxNone, nil)
// Use the ustar magic if we used ustar long names.
if len(prefix) > 0 && !tw.usedBinary {
copy(header[257:265], []byte("ustar\x00"))
}
}
// Update the path fields
formatString(pathHeaderBytes, suffix, paxNone)
formatString(prefixHeaderBytes, prefix, paxNone)
}
}
// The chksum field is terminated by a NUL and a space.
// This is different from the other octal fields.
chksum, _ := checksum(header)
tw.octal(header[148:155], chksum)
f.formatOctal(header[148:155], chksum) // Never fails
header[155] = ' '
if tw.err != nil {
// problem with header; probably integer too big for a field.
// Check if there were any formatting errors.
if f.err != nil {
tw.err = f.err
return tw.err
}
@ -270,28 +285,25 @@ func (tw *Writer) writeHeader(hdr *Header, allowPax bool) error {
return tw.err
}
// writeUSTARLongName splits a USTAR long name hdr.Name.
// name must be < 256 characters. errNameTooLong is returned
// if hdr.Name can't be split. The splitting heuristic
// is compatible with gnu tar.
func (tw *Writer) splitUSTARLongName(name string) (prefix, suffix string, err error) {
// splitUSTARPath splits a path according to USTAR prefix and suffix rules.
// If the path is not splittable, then it will return ("", "", false).
func splitUSTARPath(name string) (prefix, suffix string, ok bool) {
length := len(name)
if length > fileNamePrefixSize+1 {
if length <= fileNameSize || !isASCII(name) {
return "", "", false
} else if length > fileNamePrefixSize+1 {
length = fileNamePrefixSize + 1
} else if name[length-1] == '/' {
length--
}
i := strings.LastIndex(name[:length], "/")
// nlen contains the resulting length in the name field.
// plen contains the resulting length in the prefix field.
nlen := len(name) - i - 1
plen := i
nlen := len(name) - i - 1 // nlen is length of suffix
plen := i // plen is length of prefix
if i <= 0 || nlen > fileNameSize || nlen == 0 || plen > fileNamePrefixSize {
err = errNameTooLong
return
return "", "", false
}
prefix, suffix = name[:i], name[i+1:]
return
return name[:i], name[i+1:], true
}
// writePaxHeader writes an extended pax header to the
@ -304,11 +316,11 @@ func (tw *Writer) writePAXHeader(hdr *Header, paxHeaders map[string]string) erro
// succeed, and seems harmless enough.
ext.ModTime = hdr.ModTime
// The spec asks that we namespace our pseudo files
// with the current pid.
pid := os.Getpid()
// with the current pid. However, this results in differing outputs
// for identical inputs. As such, the constant 0 is now used instead.
// golang.org/issue/12358
dir, file := path.Split(hdr.Name)
fullName := path.Join(dir,
fmt.Sprintf("PaxHeaders.%d", pid), file)
fullName := path.Join(dir, "PaxHeaders.0", file)
ascii := toASCII(fullName)
if len(ascii) > 100 {
@ -318,8 +330,15 @@ func (tw *Writer) writePAXHeader(hdr *Header, paxHeaders map[string]string) erro
// Construct the body
var buf bytes.Buffer
for k, v := range paxHeaders {
fmt.Fprint(&buf, paxHeader(k+"="+v))
// Keys are sorted before writing to body to allow deterministic output.
var keys []string
for k := range paxHeaders {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
fmt.Fprint(&buf, formatPAXRecord(k, paxHeaders[k]))
}
ext.Size = int64(len(buf.Bytes()))
@ -335,17 +354,18 @@ func (tw *Writer) writePAXHeader(hdr *Header, paxHeaders map[string]string) erro
return nil
}
// paxHeader formats a single pax record, prefixing it with the appropriate length
func paxHeader(msg string) string {
const padding = 2 // Extra padding for space and newline
size := len(msg) + padding
// formatPAXRecord formats a single PAX record, prefixing it with the
// appropriate length.
func formatPAXRecord(k, v string) string {
const padding = 3 // Extra padding for ' ', '=', and '\n'
size := len(k) + len(v) + padding
size += len(strconv.Itoa(size))
record := fmt.Sprintf("%d %s\n", size, msg)
record := fmt.Sprintf("%d %s=%s\n", size, k, v)
// Final adjustment if adding size field increased the record size.
if len(record) != size {
// Final adjustment if adding size increased
// the number of digits in size
size = len(record)
record = fmt.Sprintf("%d %s\n", size, msg)
record = fmt.Sprintf("%d %s=%s\n", size, k, v)
}
return record
}

210
libgo/go/archive/tar/writer_test.go
View file

@ -9,8 +9,10 @@ import (
"fmt"
"io"
"io/ioutil"
"math"
"os"
"reflect"
"sort"
"strings"
"testing"
"testing/iotest"
@ -291,7 +293,7 @@ func TestPax(t *testing.T) {
t.Fatal(err)
}
// Simple test to make sure PAX extensions are in effect
if !bytes.Contains(buf.Bytes(), []byte("PaxHeaders.")) {
if !bytes.Contains(buf.Bytes(), []byte("PaxHeaders.0")) {
t.Fatal("Expected at least one PAX header to be written.")
}
// Test that we can get a long name back out of the archive.
@ -330,7 +332,7 @@ func TestPaxSymlink(t *testing.T) {
t.Fatal(err)
}
// Simple test to make sure PAX extensions are in effect
if !bytes.Contains(buf.Bytes(), []byte("PaxHeaders.")) {
if !bytes.Contains(buf.Bytes(), []byte("PaxHeaders.0")) {
t.Fatal("Expected at least one PAX header to be written.")
}
// Test that we can get a long name back out of the archive.
@ -380,7 +382,7 @@ func TestPaxNonAscii(t *testing.T) {
t.Fatal(err)
}
// Simple test to make sure PAX extensions are in effect
if !bytes.Contains(buf.Bytes(), []byte("PaxHeaders.")) {
if !bytes.Contains(buf.Bytes(), []byte("PaxHeaders.0")) {
t.Fatal("Expected at least one PAX header to be written.")
}
// Test that we can get a long name back out of the archive.
@ -439,21 +441,49 @@ func TestPaxXattrs(t *testing.T) {
}
}
func TestPAXHeader(t *testing.T) {
medName := strings.Repeat("CD", 50)
longName := strings.Repeat("AB", 100)
paxTests := [][2]string{
{paxPath + "=/etc/hosts", "19 path=/etc/hosts\n"},
{"a=b", "6 a=b\n"}, // Single digit length
{"a=names", "11 a=names\n"}, // Test case involving carries
{paxPath + "=" + longName, fmt.Sprintf("210 path=%s\n", longName)},
{paxPath + "=" + medName, fmt.Sprintf("110 path=%s\n", medName)}}
func TestPaxHeadersSorted(t *testing.T) {
fileinfo, err := os.Stat("testdata/small.txt")
if err != nil {
t.Fatal(err)
}
hdr, err := FileInfoHeader(fileinfo, "")
if err != nil {
t.Fatalf("os.Stat: %v", err)
}
contents := strings.Repeat(" ", int(hdr.Size))
for _, test := range paxTests {
key, expected := test[0], test[1]
if result := paxHeader(key); result != expected {
t.Fatalf("paxHeader: got %s, expected %s", result, expected)
}
hdr.Xattrs = map[string]string{
"foo": "foo",
"bar": "bar",
"baz": "baz",
"qux": "qux",
}
var buf bytes.Buffer
writer := NewWriter(&buf)
if err := writer.WriteHeader(hdr); err != nil {
t.Fatal(err)
}
if _, err = writer.Write([]byte(contents)); err != nil {
t.Fatal(err)
}
if err := writer.Close(); err != nil {
t.Fatal(err)
}
// Simple test to make sure PAX extensions are in effect
if !bytes.Contains(buf.Bytes(), []byte("PaxHeaders.0")) {
t.Fatal("Expected at least one PAX header to be written.")
}
// xattr bar should always appear before others
indices := []int{
bytes.Index(buf.Bytes(), []byte("bar=bar")),
bytes.Index(buf.Bytes(), []byte("baz=baz")),
bytes.Index(buf.Bytes(), []byte("foo=foo")),
bytes.Index(buf.Bytes(), []byte("qux=qux")),
}
if !sort.IntsAreSorted(indices) {
t.Fatal("PAX headers are not sorted")
}
}
@ -544,3 +574,149 @@ func TestWriteAfterClose(t *testing.T) {
t.Fatalf("Write: got %v; want ErrWriteAfterClose", err)
}
}
func TestSplitUSTARPath(t *testing.T) {
var sr = strings.Repeat
var vectors = []struct {
input string // Input path
prefix string // Expected output prefix
suffix string // Expected output suffix
ok bool // Split success?
}{
{"", "", "", false},
{"abc", "", "", false},
{"用戶名", "", "", false},
{sr("a", fileNameSize), "", "", false},
{sr("a", fileNameSize) + "/", "", "", false},
{sr("a", fileNameSize) + "/a", sr("a", fileNameSize), "a", true},
{sr("a", fileNamePrefixSize) + "/", "", "", false},
{sr("a", fileNamePrefixSize) + "/a", sr("a", fileNamePrefixSize), "a", true},
{sr("a", fileNameSize+1), "", "", false},
{sr("/", fileNameSize+1), sr("/", fileNameSize-1), "/", true},
{sr("a", fileNamePrefixSize) + "/" + sr("b", fileNameSize),
sr("a", fileNamePrefixSize), sr("b", fileNameSize), true},
{sr("a", fileNamePrefixSize) + "//" + sr("b", fileNameSize), "", "", false},
{sr("a/", fileNameSize), sr("a/", 77) + "a", sr("a/", 22), true},
}
for _, v := range vectors {
prefix, suffix, ok := splitUSTARPath(v.input)
if prefix != v.prefix || suffix != v.suffix || ok != v.ok {
t.Errorf("splitUSTARPath(%q):\ngot (%q, %q, %v)\nwant (%q, %q, %v)",
v.input, prefix, suffix, ok, v.prefix, v.suffix, v.ok)
}
}
}
func TestFormatPAXRecord(t *testing.T) {
var medName = strings.Repeat("CD", 50)
var longName = strings.Repeat("AB", 100)
var vectors = []struct {
inputKey string
inputVal string
output string
}{
{"k", "v", "6 k=v\n"},
{"path", "/etc/hosts", "19 path=/etc/hosts\n"},
{"path", longName, "210 path=" + longName + "\n"},
{"path", medName, "110 path=" + medName + "\n"},
{"foo", "ba", "9 foo=ba\n"},
{"foo", "bar", "11 foo=bar\n"},
{"foo", "b=\nar=\n==\x00", "18 foo=b=\nar=\n==\x00\n"},
{"foo", "hello9 foo=ba\nworld", "27 foo=hello9 foo=ba\nworld\n"},
{"☺☻☹", "日a本b語ç", "27 ☺☻☹=日a本b語ç\n"},
{"\x00hello", "\x00world", "17 \x00hello=\x00world\n"},
}
for _, v := range vectors {
output := formatPAXRecord(v.inputKey, v.inputVal)
if output != v.output {
t.Errorf("formatPAXRecord(%q, %q): got %q, want %q",
v.inputKey, v.inputVal, output, v.output)
}
}
}
func TestFitsInBase256(t *testing.T) {
var vectors = []struct {
input int64
width int
ok bool
}{
{+1, 8, true},
{0, 8, true},
{-1, 8, true},
{1 << 56, 8, false},
{(1 << 56) - 1, 8, true},
{-1 << 56, 8, true},
{(-1 << 56) - 1, 8, false},
{121654, 8, true},
{-9849849, 8, true},
{math.MaxInt64, 9, true},
{0, 9, true},
{math.MinInt64, 9, true},
{math.MaxInt64, 12, true},
{0, 12, true},
{math.MinInt64, 12, true},
}
for _, v := range vectors {
ok := fitsInBase256(v.width, v.input)
if ok != v.ok {
t.Errorf("checkNumeric(%d, %d): got %v, want %v", v.input, v.width, ok, v.ok)
}
}
}
func TestFormatNumeric(t *testing.T) {
var vectors = []struct {
input int64
output string
ok bool
}{
// Test base-256 (binary) encoded values.
{-1, "\xff", true},
{-1, "\xff\xff", true},
{-1, "\xff\xff\xff", true},
{(1 << 0), "0", false},
{(1 << 8) - 1, "\x80\xff", true},
{(1 << 8), "0\x00", false},
{(1 << 16) - 1, "\x80\xff\xff", true},
{(1 << 16), "00\x00", false},
{-1 * (1 << 0), "\xff", true},
{-1*(1<<0) - 1, "0", false},
{-1 * (1 << 8), "\xff\x00", true},
{-1*(1<<8) - 1, "0\x00", false},
{-1 * (1 << 16), "\xff\x00\x00", true},
{-1*(1<<16) - 1, "00\x00", false},
{537795476381659745, "0000000\x00", false},
{537795476381659745, "\x80\x00\x00\x00\x07\x76\xa2\x22\xeb\x8a\x72\x61", true},
{-615126028225187231, "0000000\x00", false},
{-615126028225187231, "\xff\xff\xff\xff\xf7\x76\xa2\x22\xeb\x8a\x72\x61", true},
{math.MaxInt64, "0000000\x00", false},
{math.MaxInt64, "\x80\x00\x00\x00\x7f\xff\xff\xff\xff\xff\xff\xff", true},
{math.MinInt64, "0000000\x00", false},
{math.MinInt64, "\xff\xff\xff\xff\x80\x00\x00\x00\x00\x00\x00\x00", true},
{math.MaxInt64, "\x80\x7f\xff\xff\xff\xff\xff\xff\xff", true},
{math.MinInt64, "\xff\x80\x00\x00\x00\x00\x00\x00\x00", true},
}
for _, v := range vectors {
var f formatter
output := make([]byte, len(v.output))
f.formatNumeric(output, v.input)
ok := (f.err == nil)
if ok != v.ok {
if v.ok {
t.Errorf("formatNumeric(%d): got formatting failure, want success", v.input)
} else {
t.Errorf("formatNumeric(%d): got formatting success, want failure", v.input)
}
}
if string(output) != v.output {
t.Errorf("formatNumeric(%d): got %q, want %q", v.input, output, v.output)
}
}
}

103
libgo/go/archive/zip/reader.go
View file

@ -22,9 +22,10 @@ var (
)
type Reader struct {
r io.ReaderAt
File []*File
Comment string
r io.ReaderAt
File []*File
Comment string
decompressors map[uint16]Decompressor
}
type ReadCloser struct {
@ -34,6 +35,7 @@ type ReadCloser struct {
type File struct {
FileHeader
zip *Reader
zipr io.ReaderAt
zipsize int64
headerOffset int64
@ -95,7 +97,7 @@ func (z *Reader) init(r io.ReaderAt, size int64) error {
// a bad one, and then only report a ErrFormat or UnexpectedEOF if
// the file count modulo 65536 is incorrect.
for {
f := &File{zipr: r, zipsize: size}
f := &File{zip: z, zipr: r, zipsize: size}
err = readDirectoryHeader(f, buf)
if err == ErrFormat || err == io.ErrUnexpectedEOF {
break
@ -113,6 +115,24 @@ func (z *Reader) init(r io.ReaderAt, size int64) error {
return nil
}
// RegisterDecompressor registers or overrides a custom decompressor for a
// specific method ID. If a decompressor for a given method is not found,
// Reader will default to looking up the decompressor at the package level.
func (z *Reader) RegisterDecompressor(method uint16, dcomp Decompressor) {
if z.decompressors == nil {
z.decompressors = make(map[uint16]Decompressor)
}
z.decompressors[method] = dcomp
}
func (z *Reader) decompressor(method uint16) Decompressor {
dcomp := z.decompressors[method]
if dcomp == nil {
dcomp = decompressor(method)
}
return dcomp
}
// Close closes the Zip file, rendering it unusable for I/O.
func (rc *ReadCloser) Close() error {
return rc.f.Close()
@ -140,7 +160,7 @@ func (f *File) Open() (rc io.ReadCloser, err error) {
}
size := int64(f.CompressedSize64)
r := io.NewSectionReader(f.zipr, f.headerOffset+bodyOffset, size)
dcomp := decompressor(f.Method)
dcomp := f.zip.decompressor(f.Method)
if dcomp == nil {
err = ErrAlgorithm
return
@ -261,39 +281,59 @@ func readDirectoryHeader(f *File, r io.Reader) error {
f.Extra = d[filenameLen : filenameLen+extraLen]
f.Comment = string(d[filenameLen+extraLen:])
needUSize := f.UncompressedSize == ^uint32(0)
needCSize := f.CompressedSize == ^uint32(0)
needHeaderOffset := f.headerOffset == int64(^uint32(0))
if len(f.Extra) > 0 {
// Best effort to find what we need.
// Other zip authors might not even follow the basic format,
// and we'll just ignore the Extra content in that case.
b := readBuf(f.Extra)
for len(b) >= 4 { // need at least tag and size
tag := b.uint16()
size := b.uint16()
if int(size) > len(b) {
return ErrFormat
break
}
if tag == zip64ExtraId {
// update directory values from the zip64 extra block
// update directory values from the zip64 extra block.
// They should only be consulted if the sizes read earlier
// are maxed out.
// See golang.org/issue/13367.
eb := readBuf(b[:size])
if len(eb) >= 8 {
if needUSize {
needUSize = false
if len(eb) < 8 {
return ErrFormat
}
f.UncompressedSize64 = eb.uint64()
}
if len(eb) >= 8 {
if needCSize {
needCSize = false
if len(eb) < 8 {
return ErrFormat
}
f.CompressedSize64 = eb.uint64()
}
if len(eb) >= 8 {
if needHeaderOffset {
needHeaderOffset = false
if len(eb) < 8 {
return ErrFormat
}
f.headerOffset = int64(eb.uint64())
}
break
}
b = b[size:]
}
// Should have consumed the whole header.
// But popular zip & JAR creation tools are broken and
// may pad extra zeros at the end, so accept those
// too. See golang.org/issue/8186.
for _, v := range b {
if v != 0 {
return ErrFormat
}
}
}
if needUSize || needCSize || needHeaderOffset {
return ErrFormat
}
return nil
}
@ -376,14 +416,16 @@ func readDirectoryEnd(r io.ReaderAt, size int64) (dir *directoryEnd, err error)
}
d.comment = string(b[:l])
p, err := findDirectory64End(r, directoryEndOffset)
if err == nil && p >= 0 {
err = readDirectory64End(r, p, d)
// These values mean that the file can be a zip64 file
if d.directoryRecords == 0xffff || d.directorySize == 0xffff || d.directoryOffset == 0xffffffff {
p, err := findDirectory64End(r, directoryEndOffset)
if err == nil && p >= 0 {
err = readDirectory64End(r, p, d)
}
if err != nil {
return nil, err
}
}
if err != nil {
return nil, err
}
// Make sure directoryOffset points to somewhere in our file.
if o := int64(d.directoryOffset); o < 0 || o >= size {
return nil, ErrFormat
@ -407,8 +449,13 @@ func findDirectory64End(r io.ReaderAt, directoryEndOffset int64) (int64, error)
if sig := b.uint32(); sig != directory64LocSignature {
return -1, nil
}
b = b[4:] // skip number of the disk with the start of the zip64 end of central directory
p := b.uint64() // relative offset of the zip64 end of central directory record
if b.uint32() != 0 { // number of the disk with the start of the zip64 end of central directory
return -1, nil // the file is not a valid zip64-file
}
p := b.uint64() // relative offset of the zip64 end of central directory record
if b.uint32() != 1 { // total number of disks
return -1, nil // the file is not a valid zip64-file
}
return int64(p), nil
}

37
libgo/go/archive/zip/reader_test.go
View file

@ -605,3 +605,40 @@ func TestIssue11146(t *testing.T) {
}
r.Close()
}
// Verify we do not treat non-zip64 archives as zip64
func TestIssue12449(t *testing.T) {
data := []byte{
0x50, 0x4b, 0x03, 0x04, 0x14, 0x00, 0x08, 0x00,
0x00, 0x00, 0x6b, 0xb4, 0xba, 0x46, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x03, 0x00, 0x18, 0x00, 0xca, 0x64,
0x55, 0x75, 0x78, 0x0b, 0x00, 0x50, 0x4b, 0x05,
0x06, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01,
0x00, 0x49, 0x00, 0x00, 0x00, 0x44, 0x00, 0x00,
0x00, 0x31, 0x31, 0x31, 0x32, 0x32, 0x32, 0x0a,
0x50, 0x4b, 0x07, 0x08, 0x1d, 0x88, 0x77, 0xb0,
0x07, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00,
0x50, 0x4b, 0x01, 0x02, 0x14, 0x03, 0x14, 0x00,
0x08, 0x00, 0x00, 0x00, 0x6b, 0xb4, 0xba, 0x46,
0x1d, 0x88, 0x77, 0xb0, 0x07, 0x00, 0x00, 0x00,
0x07, 0x00, 0x00, 0x00, 0x03, 0x00, 0x18, 0x00,
0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0xa0, 0x81, 0x00, 0x00, 0x00, 0x00, 0xca, 0x64,
0x55, 0x75, 0x78, 0x0b, 0x00, 0x50, 0x4b, 0x05,
0x06, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01,
0x00, 0x49, 0x00, 0x00, 0x00, 0x44, 0x00, 0x00,
0x00, 0x97, 0x2b, 0x49, 0x23, 0x05, 0xc5, 0x0b,
0xa7, 0xd1, 0x52, 0xa2, 0x9c, 0x50, 0x4b, 0x06,
0x07, 0xc8, 0x19, 0xc1, 0xaf, 0x94, 0x9c, 0x61,
0x44, 0xbe, 0x94, 0x19, 0x42, 0x58, 0x12, 0xc6,
0x5b, 0x50, 0x4b, 0x05, 0x06, 0x00, 0x00, 0x00,
0x00, 0x01, 0x00, 0x01, 0x00, 0x69, 0x00, 0x00,
0x00, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00,
}
// Read in the archive.
_, err := NewReader(bytes.NewReader([]byte(data)), int64(len(data)))
if err != nil {
t.Errorf("Error reading the archive: %v", err)
}
}

25
libgo/go/archive/zip/register.go
View file

@ -12,15 +12,19 @@ import (
"sync"
)
// A Compressor returns a compressing writer, writing to the
// provided writer. On Close, any pending data should be flushed.
type Compressor func(io.Writer) (io.WriteCloser, error)
// A Compressor returns a new compressing writer, writing to w.
// The WriteCloser's Close method must be used to flush pending data to w.
// The Compressor itself must be safe to invoke from multiple goroutines
// simultaneously, but each returned writer will be used only by
// one goroutine at a time.
type Compressor func(w io.Writer) (io.WriteCloser, error)
// Decompressor is a function that wraps a Reader with a decompressing Reader.
// The decompressed ReadCloser is returned to callers who open files from
// within the archive. These callers are responsible for closing this reader
// when they're finished reading.
type Decompressor func(io.Reader) io.ReadCloser
// A Decompressor returns a new decompressing reader, reading from r.
// The ReadCloser's Close method must be used to release associated resources.
// The Decompressor itself must be safe to invoke from multiple goroutines
// simultaneously, but each returned reader will be used only by
// one goroutine at a time.
type Decompressor func(r io.Reader) io.ReadCloser
var flateWriterPool sync.Pool
@ -75,14 +79,15 @@ var (
)
// RegisterDecompressor allows custom decompressors for a specified method ID.
func RegisterDecompressor(method uint16, d Decompressor) {
// The common methods Store and Deflate are built in.
func RegisterDecompressor(method uint16, dcomp Decompressor) {
mu.Lock()
defer mu.Unlock()
if _, ok := decompressors[method]; ok {
panic("decompressor already registered")
}
decompressors[method] = d
decompressors[method] = dcomp
}
// RegisterCompressor registers custom compressors for a specified method ID.

2
libgo/go/archive/zip/struct.go
View file

@ -235,7 +235,7 @@ func (h *FileHeader) SetMode(mode os.FileMode) {
// isZip64 reports whether the file size exceeds the 32 bit limit
func (fh *FileHeader) isZip64() bool {
return fh.CompressedSize64 > uint32max || fh.UncompressedSize64 > uint32max
return fh.CompressedSize64 >= uint32max || fh.UncompressedSize64 >= uint32max
}
func msdosModeToFileMode(m uint32) (mode os.FileMode) {

32
libgo/go/archive/zip/writer.go
View file

@ -14,14 +14,14 @@ import (
)
// TODO(adg): support zip file comments
// TODO(adg): support specifying deflate level
// Writer implements a zip file writer.
type Writer struct {
cw *countWriter
dir []*header
last *fileWriter
closed bool
cw *countWriter
dir []*header
last *fileWriter
closed bool
compressors map[uint16]Compressor
}
type header struct {
@ -78,7 +78,7 @@ func (w *Writer) Close() error {
b.uint16(h.ModifiedTime)
b.uint16(h.ModifiedDate)
b.uint32(h.CRC32)
if h.isZip64() || h.offset > uint32max {
if h.isZip64() || h.offset >= uint32max {
// the file needs a zip64 header. store maxint in both
// 32 bit size fields (and offset later) to signal that the
// zip64 extra header should be used.
@ -220,7 +220,7 @@ func (w *Writer) CreateHeader(fh *FileHeader) (io.Writer, error) {
compCount: &countWriter{w: w.cw},
crc32: crc32.NewIEEE(),
}
comp := compressor(fh.Method)
comp := w.compressor(fh.Method)
if comp == nil {
return nil, ErrAlgorithm
}
@ -270,6 +270,24 @@ func writeHeader(w io.Writer, h *FileHeader) error {
return err
}
// RegisterCompressor registers or overrides a custom compressor for a specific
// method ID. If a compressor for a given method is not found, Writer will
// default to looking up the compressor at the package level.
func (w *Writer) RegisterCompressor(method uint16, comp Compressor) {
if w.compressors == nil {
w.compressors = make(map[uint16]Compressor)
}
w.compressors[method] = comp
}
func (w *Writer) compressor(method uint16) Compressor {
comp := w.compressors[method]
if comp == nil {
comp = compressor(method)
}
return comp
}
type fileWriter struct {
*header
zipw io.Writer

62
libgo/go/archive/zip/zip_test.go
View file

@ -10,6 +10,7 @@ import (
"bytes"
"fmt"
"hash"
"internal/testenv"
"io"
"io/ioutil"
"sort"
@ -19,6 +20,9 @@ import (
)
func TestOver65kFiles(t *testing.T) {
if testing.Short() && testenv.Builder() == "" {
t.Skip("skipping in short mode")
}
buf := new(bytes.Buffer)
w := NewWriter(buf)
const nFiles = (1 << 16) + 42
@ -233,10 +237,24 @@ func TestZip64(t *testing.T) {
testZip64DirectoryRecordLength(buf, t)
}
func TestZip64EdgeCase(t *testing.T) {
if testing.Short() {
t.Skip("slow test; skipping")
}
// Test a zip file with uncompressed size 0xFFFFFFFF.
// That's the magic marker for a 64-bit file, so even though
// it fits in a 32-bit field we must use the 64-bit field.
// Go 1.5 and earlier got this wrong,
// writing an invalid zip file.
const size = 1<<32 - 1 - int64(len("END\n")) // before the "END\n" part
buf := testZip64(t, size)
testZip64DirectoryRecordLength(buf, t)
}
func testZip64(t testing.TB, size int64) *rleBuffer {
const chunkSize = 1024
chunks := int(size / chunkSize)
// write 2^32 bytes plus "END\n" to a zip file
// write size bytes plus "END\n" to a zip file
buf := new(rleBuffer)
w := NewWriter(buf)
f, err := w.CreateHeader(&FileHeader{
@ -257,6 +275,12 @@ func testZip64(t testing.TB, size int64) *rleBuffer {
t.Fatal("write chunk:", err)
}
}
if frag := int(size % chunkSize); frag > 0 {
_, err := f.Write(chunk[:frag])
if err != nil {
t.Fatal("write chunk:", err)
}
}
end := []byte("END\n")
_, err = f.Write(end)
if err != nil {
@ -283,6 +307,12 @@ func testZip64(t testing.TB, size int64) *rleBuffer {
t.Fatal("read:", err)
}
}
if frag := int(size % chunkSize); frag > 0 {
_, err := io.ReadFull(rc, chunk[:frag])
if err != nil {
t.Fatal("read:", err)
}
}
gotEnd, err := ioutil.ReadAll(rc)
if err != nil {
t.Fatal("read end:", err)
@ -294,14 +324,14 @@ func testZip64(t testing.TB, size int64) *rleBuffer {
if err != nil {
t.Fatal("closing:", err)
}
if size == 1<<32 {
if size+int64(len("END\n")) >= 1<<32-1 {
if got, want := f0.UncompressedSize, uint32(uint32max); got != want {
t.Errorf("UncompressedSize %d, want %d", got, want)
t.Errorf("UncompressedSize %#x, want %#x", got, want)
}
}
if got, want := f0.UncompressedSize64, uint64(size)+uint64(len(end)); got != want {
t.Errorf("UncompressedSize64 %d, want %d", got, want)
t.Errorf("UncompressedSize64 %#x, want %#x", got, want)
}
return buf
@ -373,9 +403,14 @@ func testValidHeader(h *FileHeader, t *testing.T) {
}
b := buf.Bytes()
if _, err = NewReader(bytes.NewReader(b), int64(len(b))); err != nil {
zf, err := NewReader(bytes.NewReader(b), int64(len(b)))
if err != nil {
t.Fatalf("got %v, expected nil", err)
}
zh := zf.File[0].FileHeader
if zh.Name != h.Name || zh.Method != h.Method || zh.UncompressedSize64 != uint64(len("hi")) {
t.Fatalf("got %q/%d/%d expected %q/%d/%d", zh.Name, zh.Method, zh.UncompressedSize64, h.Name, h.Method, len("hi"))
}
}
// Issue 4302.
@ -388,20 +423,29 @@ func TestHeaderInvalidTagAndSize(t *testing.T) {
h := FileHeader{
Name: filename,
Method: Deflate,
Extra: []byte(ts.Format(time.RFC3339Nano)), // missing tag and len
Extra: []byte(ts.Format(time.RFC3339Nano)), // missing tag and len, but Extra is best-effort parsing
}
h.SetModTime(ts)
testInvalidHeader(&h, t)
testValidHeader(&h, t)
}
func TestHeaderTooShort(t *testing.T) {
h := FileHeader{
Name: "foo.txt",
Method: Deflate,
Extra: []byte{zip64ExtraId}, // missing size
Extra: []byte{zip64ExtraId}, // missing size and second half of tag, but Extra is best-effort parsing
}
testInvalidHeader(&h, t)
testValidHeader(&h, t)
}
func TestHeaderIgnoredSize(t *testing.T) {
h := FileHeader{
Name: "foo.txt",
Method: Deflate,
Extra: []byte{zip64ExtraId & 0xFF, zip64ExtraId >> 8, 24, 0, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8}, // bad size but shouldn't be consulted
}
testValidHeader(&h, t)
}
// Issue 4393. It is valid to have an extra data header

2
libgo/go/bufio/bufio.go
View file

@ -179,7 +179,7 @@ func (b *Reader) Discard(n int) (discarded int, err error) {
// Read reads data into p.
// It returns the number of bytes read into p.
// It calls Read at most once on the underlying Reader,
// The bytes are taken from at most one Read on the underlying Reader,
// hence n may be less than len(p).
// At EOF, the count will be zero and err will be io.EOF.
func (b *Reader) Read(p []byte) (n int, err error) {

29
libgo/go/bufio/example_test.go
View file

@ -80,3 +80,32 @@ func ExampleScanner_custom() {
// 5678
// Invalid input: strconv.ParseInt: parsing "1234567901234567890": value out of range
}
// Use a Scanner with a custom split function to parse a comma-separated
// list with an empty final value.
func ExampleScanner_emptyFinalToken() {
// Comma-separated list; last entry is empty.
const input = "1,2,3,4,"
scanner := bufio.NewScanner(strings.NewReader(input))
// Define a split function that separates on commas.
onComma := func(data []byte, atEOF bool) (advance int, token []byte, err error) {
for i := 0; i < len(data); i++ {
if data[i] == ',' {
return i + 1, data[:i], nil
}
}
// There is one final token to be delivered, which may be the empty string.
// Returning bufio.ErrFinalToken here tells Scan there are no more tokens after this
// but does not trigger an error to be returned from Scan itself.
return 0, data, bufio.ErrFinalToken
}
scanner.Split(onComma)
// Scan.
for scanner.Scan() {
fmt.Printf("%q ", scanner.Text())
}
if err := scanner.Err(); err != nil {
fmt.Fprintln(os.Stderr, "reading input:", err)
}
// Output: "1" "2" "3" "4" ""
}

60
libgo/go/bufio/scan.go
View file

@ -37,6 +37,8 @@ type Scanner struct {
end int // End of data in buf.
err error // Sticky error.
empties int // Count of successive empty tokens.
scanCalled bool // Scan has been called; buffer is in use.
done bool // Scan has finished.
}
// SplitFunc is the signature of the split function used to tokenize the
@ -65,10 +67,13 @@ var (
)
const (
// MaxScanTokenSize is the maximum size used to buffer a token.
// MaxScanTokenSize is the maximum size used to buffer a token
// unless the user provides an explicit buffer with Scan.Buffer.
// The actual maximum token size may be smaller as the buffer
// may need to include, for instance, a newline.
MaxScanTokenSize = 64 * 1024
startBufSize = 4096 // Size of initial allocation for buffer.
)
// NewScanner returns a new Scanner to read from r.
@ -78,7 +83,6 @@ func NewScanner(r io.Reader) *Scanner {
r: r,
split: ScanLines,
maxTokenSize: MaxScanTokenSize,
buf: make([]byte, 4096), // Plausible starting size; needn't be large.
}
}
@ -103,6 +107,16 @@ func (s *Scanner) Text() string {
return string(s.token)
}
// ErrFinalToken is a special sentinel error value. It is intended to be
// returned by a Split function to indicate that the token being delivered
// with the error is the last token and scanning should stop after this one.
// After ErrFinalToken is received by Scan, scanning stops with no error.
// The value is useful to stop processing early or when it is necessary to
// deliver a final empty token. One could achieve the same behavior
// with a custom error value but providing one here is tidier.
// See the emptyFinalToken example for a use of this value.
var ErrFinalToken = errors.New("final token")
// Scan advances the Scanner to the next token, which will then be
// available through the Bytes or Text method. It returns false when the
// scan stops, either by reaching the end of the input or an error.
@ -112,6 +126,10 @@ func (s *Scanner) Text() string {
// Scan panics if the split function returns 100 empty tokens without
// advancing the input. This is a common error mode for scanners.
func (s *Scanner) Scan() bool {
if s.done {
return false
}
s.scanCalled = true
// Loop until we have a token.
for {
// See if we can get a token with what we already have.
@ -120,6 +138,11 @@ func (s *Scanner) Scan() bool {
if s.end > s.start || s.err != nil {
advance, token, err := s.split(s.buf[s.start:s.end], s.err != nil)
if err != nil {
if err == ErrFinalToken {
s.token = token
s.done = true
return true
}
s.setErr(err)
return false
}
@ -158,11 +181,16 @@ func (s *Scanner) Scan() bool {
}
// Is the buffer full? If so, resize.
if s.end == len(s.buf) {
if len(s.buf) >= s.maxTokenSize {
// Guarantee no overflow in the multiplication below.
const maxInt = int(^uint(0) >> 1)
if len(s.buf) >= s.maxTokenSize || len(s.buf) > maxInt/2 {
s.setErr(ErrTooLong)
return false
}
newSize := len(s.buf) * 2
if newSize == 0 {
newSize = startBufSize
}
if newSize > s.maxTokenSize {
newSize = s.maxTokenSize
}
@ -217,9 +245,31 @@ func (s *Scanner) setErr(err error) {
}
}
// Split sets the split function for the Scanner. If called, it must be
// called before Scan. The default split function is ScanLines.
// Buffer sets the initial buffer to use when scanning and the maximum
// size of buffer that may be allocated during scanning. The maximum
// token size is the larger of max and cap(buf). If max <= cap(buf),
// Scan will use this buffer only and do no allocation.
//
// By default, Scan uses an internal buffer and sets the
// maximum token size to MaxScanTokenSize.
//
// Buffer panics if it is called after scanning has started.
func (s *Scanner) Buffer(buf []byte, max int) {
if s.scanCalled {
panic("Buffer called after Scan")
}
s.buf = buf[0:cap(buf)]
s.maxTokenSize = max
}
// Split sets the split function for the Scanner.
// The default split function is ScanLines.
//
// Split panics if it is called after scanning has started.
func (s *Scanner) Split(split SplitFunc) {
if s.scanCalled {
panic("Split called after Scan")
}
s.split = split
}

42
libgo/go/bufio/scan_test.go
View file

@ -429,33 +429,37 @@ func commaSplit(data []byte, atEOF bool) (advance int, token []byte, err error)
return i + 1, data[:i], nil
}
}
if !atEOF {
return 0, nil, nil
}
return 0, data, nil
return 0, data, ErrFinalToken
}
func TestEmptyTokens(t *testing.T) {
s := NewScanner(strings.NewReader("1,2,3,"))
values := []string{"1", "2", "3", ""}
func testEmptyTokens(t *testing.T, text string, values []string) {
s := NewScanner(strings.NewReader(text))
s.Split(commaSplit)
var i int
for i = 0; i < len(values); i++ {
if !s.Scan() {
break
for i = 0; s.Scan(); i++ {
if i >= len(values) {
t.Fatalf("got %d fields, expected %d", i+1, len(values))
}
if s.Text() != values[i] {
t.Errorf("%d: expected %q got %q", i, values[i], s.Text())
}
}
if i != len(values) {
t.Errorf("got %d fields, expected %d", i, len(values))
t.Fatalf("got %d fields, expected %d", i, len(values))
}
if err := s.Err(); err != nil {
t.Fatal(err)
}
}
func TestEmptyTokens(t *testing.T) {
testEmptyTokens(t, "1,2,3,", []string{"1", "2", "3", ""})
}
func TestWithNoEmptyTokens(t *testing.T) {
testEmptyTokens(t, "1,2,3", []string{"1", "2", "3"})
}
func loopAtEOFSplit(data []byte, atEOF bool) (advance int, token []byte, err error) {
if len(data) > 0 {
return 1, data[:1], nil
@ -522,3 +526,19 @@ func TestEmptyLinesOK(t *testing.T) {
t.Fatalf("stopped with %d left to process", c)
}
}
// Make sure we can read a huge token if a big enough buffer is provided.
func TestHugeBuffer(t *testing.T) {
text := strings.Repeat("x", 2*MaxScanTokenSize)
s := NewScanner(strings.NewReader(text + "\n"))
s.Buffer(make([]byte, 100), 3*MaxScanTokenSize)
for s.Scan() {
token := s.Text()
if token != text {
t.Errorf("scan got incorrect token of length %d", len(token))
}
}
if s.Err() != nil {
t.Fatal("after scan:", s.Err())
}
}

17
libgo/go/bytes/buffer.go
View file

@ -36,10 +36,11 @@ const (
// ErrTooLarge is passed to panic if memory cannot be allocated to store data in a buffer.
var ErrTooLarge = errors.New("bytes.Buffer: too large")
// Bytes returns a slice of the contents of the unread portion of the buffer;
// len(b.Bytes()) == b.Len(). If the caller changes the contents of the
// returned slice, the contents of the buffer will change provided there
// are no intervening method calls on the Buffer.
// Bytes returns a slice of length b.Len() holding the unread portion of the buffer.
// The slice is valid for use only until the next buffer modification (that is,
// only until the next call to a method like Read, Write, Reset, or Truncate).
// The slice aliases the buffer content at least until the next buffer modification,
// so immediate changes to the slice will affect the result of future reads.
func (b *Buffer) Bytes() []byte { return b.buf[b.off:] }
// String returns the contents of the unread portion of the buffer
@ -60,7 +61,8 @@ func (b *Buffer) Len() int { return len(b.buf) - b.off }
// total space allocated for the buffer's data.
func (b *Buffer) Cap() int { return cap(b.buf) }
// Truncate discards all but the first n unread bytes from the buffer.
// Truncate discards all but the first n unread bytes from the buffer
// but continues to use the same allocated storage.
// It panics if n is negative or greater than the length of the buffer.
func (b *Buffer) Truncate(n int) {
b.lastRead = opInvalid
@ -74,8 +76,9 @@ func (b *Buffer) Truncate(n int) {
b.buf = b.buf[0 : b.off+n]
}
// Reset resets the buffer so it has no content.
// b.Reset() is the same as b.Truncate(0).
// Reset resets the buffer to be empty,
// but it retains the underlying storage for use by future writes.
// Reset is the same as Truncate(0).
func (b *Buffer) Reset() { b.Truncate(0) }
// grow grows the buffer to guarantee space for n more bytes.

31
libgo/go/bytes/bytes_test.go
View file

@ -1255,3 +1255,34 @@ func BenchmarkRepeat(b *testing.B) {
Repeat([]byte("-"), 80)
}
}
func benchmarkBytesCompare(b *testing.B, n int) {
var x = make([]byte, n)
var y = make([]byte, n)
for i := 0; i < n; i++ {
x[i] = 'a'
}
for i := 0; i < n; i++ {
y[i] = 'a'
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
Compare(x, y)
}
}
func BenchmarkBytesCompare1(b *testing.B) { benchmarkBytesCompare(b, 1) }
func BenchmarkBytesCompare2(b *testing.B) { benchmarkBytesCompare(b, 2) }
func BenchmarkBytesCompare4(b *testing.B) { benchmarkBytesCompare(b, 4) }
func BenchmarkBytesCompare8(b *testing.B) { benchmarkBytesCompare(b, 8) }
func BenchmarkBytesCompare16(b *testing.B) { benchmarkBytesCompare(b, 16) }
func BenchmarkBytesCompare32(b *testing.B) { benchmarkBytesCompare(b, 32) }
func BenchmarkBytesCompare64(b *testing.B) { benchmarkBytesCompare(b, 64) }
func BenchmarkBytesCompare128(b *testing.B) { benchmarkBytesCompare(b, 128) }
func BenchmarkBytesCompare256(b *testing.B) { benchmarkBytesCompare(b, 256) }
func BenchmarkBytesCompare512(b *testing.B) { benchmarkBytesCompare(b, 512) }
func BenchmarkBytesCompare1024(b *testing.B) { benchmarkBytesCompare(b, 1024) }
func BenchmarkBytesCompare2048(b *testing.B) { benchmarkBytesCompare(b, 2048) }

104
libgo/go/cmd/cgo/ast.go
View file

@ -124,7 +124,7 @@ func (f *File) ReadGo(name string) {
if f.Ref == nil {
f.Ref = make([]*Ref, 0, 8)
}
f.walk(ast2, "prog", (*File).saveRef)
f.walk(ast2, "prog", (*File).saveExprs)
// Accumulate exported functions.
// The comments are only on ast1 but we need to
@ -163,52 +163,72 @@ func commentText(g *ast.CommentGroup) string {
return strings.Join(pieces, "")
}
// Save various references we are going to need later.
func (f *File) saveExprs(x interface{}, context string) {
switch x := x.(type) {
case *ast.Expr:
switch (*x).(type) {
case *ast.SelectorExpr:
f.saveRef(x, context)
}
case *ast.CallExpr:
f.saveCall(x)
}
}
// Save references to C.xxx for later processing.
func (f *File) saveRef(x interface{}, context string) {
n, ok := x.(*ast.Expr)
func (f *File) saveRef(n *ast.Expr, context string) {
sel := (*n).(*ast.SelectorExpr)
// For now, assume that the only instance of capital C is when
// used as the imported package identifier.
// The parser should take care of scoping in the future, so
// that we will be able to distinguish a "top-level C" from a
// local C.
if l, ok := sel.X.(*ast.Ident); !ok || l.Name != "C" {
return
}
if context == "as2" {
context = "expr"
}
if context == "embed-type" {
error_(sel.Pos(), "cannot embed C type")
}
goname := sel.Sel.Name
if goname == "errno" {
error_(sel.Pos(), "cannot refer to errno directly; see documentation")
return
}
if goname == "_CMalloc" {
error_(sel.Pos(), "cannot refer to C._CMalloc; use C.malloc")
return
}
if goname == "malloc" {
goname = "_CMalloc"
}
name := f.Name[goname]
if name == nil {
name = &Name{
Go: goname,
}
f.Name[goname] = name
}
f.Ref = append(f.Ref, &Ref{
Name: name,
Expr: n,
Context: context,
})
}
// Save calls to C.xxx for later processing.
func (f *File) saveCall(call *ast.CallExpr) {
sel, ok := call.Fun.(*ast.SelectorExpr)
if !ok {
return
}
if sel, ok := (*n).(*ast.SelectorExpr); ok {
// For now, assume that the only instance of capital C is
// when used as the imported package identifier.
// The parser should take care of scoping in the future,
// so that we will be able to distinguish a "top-level C"
// from a local C.
if l, ok := sel.X.(*ast.Ident); ok && l.Name == "C" {
if context == "as2" {
context = "expr"
}
if context == "embed-type" {
error_(sel.Pos(), "cannot embed C type")
}
goname := sel.Sel.Name
if goname == "errno" {
error_(sel.Pos(), "cannot refer to errno directly; see documentation")
return
}
if goname == "_CMalloc" {
error_(sel.Pos(), "cannot refer to C._CMalloc; use C.malloc")
return
}
if goname == "malloc" {
goname = "_CMalloc"
}
name := f.Name[goname]
if name == nil {
name = &Name{
Go: goname,
}
f.Name[goname] = name
}
f.Ref = append(f.Ref, &Ref{
Name: name,
Expr: n,
Context: context,
})
return
}
if l, ok := sel.X.(*ast.Ident); !ok || l.Name != "C" {
return
}
f.Calls = append(f.Calls, call)
}
// If a function should be exported add it to ExpFunc.

149
libgo/go/cmd/cgo/doc.go
View file

@ -117,17 +117,27 @@ The standard C numeric types are available under the names
C.char, C.schar (signed char), C.uchar (unsigned char),
C.short, C.ushort (unsigned short), C.int, C.uint (unsigned int),
C.long, C.ulong (unsigned long), C.longlong (long long),
C.ulonglong (unsigned long long), C.float, C.double.
C.ulonglong (unsigned long long), C.float, C.double,
C.complexfloat (complex float), and C.complexdouble (complex double).
The C type void* is represented by Go's unsafe.Pointer.
The C types __int128_t and __uint128_t are represented by [16]byte.
To access a struct, union, or enum type directly, prefix it with
struct_, union_, or enum_, as in C.struct_stat.
The size of any C type T is available as C.sizeof_T, as in
C.sizeof_struct_stat.
As Go doesn't have support for C's union type in the general case,
C's union types are represented as a Go byte array with the same length.
Go structs cannot embed fields with C types.
Go code can not refer to zero-sized fields that occur at the end of
non-empty C structs. To get the address of such a field (which is the
only operation you can do with a zero-sized field) you must take the
address of the struct and add the size of the struct.
Cgo translates C types into equivalent unexported Go types.
Because the translations are unexported, a Go package should not
expose C types in its exported API: a C type used in one Go package
@ -188,10 +198,10 @@ by making copies of the data. In pseudo-Go definitions:
// C string to Go string
func C.GoString(*C.char) string
// C string, length to Go string
// C data with explicit length to Go string
func C.GoStringN(*C.char, C.int) string
// C pointer, length to Go []byte
// C data with explicit length to Go []byte
func C.GoBytes(unsafe.Pointer, C.int) []byte
C references to Go
@ -221,6 +231,55 @@ definitions and declarations, then the two output files will produce
duplicate symbols and the linker will fail. To avoid this, definitions
must be placed in preambles in other files, or in C source files.
Passing pointers
Go is a garbage collected language, and the garbage collector needs to
know the location of every pointer to Go memory. Because of this,
there are restrictions on passing pointers between Go and C.
In this section the term Go pointer means a pointer to memory
allocated by Go (such as by using the & operator or calling the
predefined new function) and the term C pointer means a pointer to
memory allocated by C (such as by a call to C.malloc). Whether a
pointer is a Go pointer or a C pointer is a dynamic property
determined by how the memory was allocated; it has nothing to do with
the type of the pointer.
Go code may pass a Go pointer to C provided the Go memory to which it
points does not contain any Go pointers. The C code must preserve
this property: it must not store any Go pointers in Go memory, even
temporarily. When passing a pointer to a field in a struct, the Go
memory in question is the memory occupied by the field, not the entire
struct. When passing a pointer to an element in an array or slice,
the Go memory in question is the entire array or the entire backing
array of the slice.
C code may not keep a copy of a Go pointer after the call returns.
A Go function called by C code may not return a Go pointer. A Go
function called by C code may take C pointers as arguments, and it may
store non-pointer or C pointer data through those pointers, but it may
not store a Go pointer in memory pointed to by a C pointer. A Go
function called by C code may take a Go pointer as an argument, but it
must preserve the property that the Go memory to which it points does
not contain any Go pointers.
Go code may not store a Go pointer in C memory. C code may store Go
pointers in C memory, subject to the rule above: it must stop storing
the Go pointer when the C function returns.
These rules are checked dynamically at runtime. The checking is
controlled by the cgocheck setting of the GODEBUG environment
variable. The default setting is GODEBUG=cgocheck=1, which implements
reasonably cheap dynamic checks. These checks may be disabled
entirely using GODEBUG=cgocheck=0. Complete checking of pointer
handling, at some cost in run time, is available via GODEBUG=cgocheck=2.
It is possible to defeat this enforcement by using the unsafe package,
and of course there is nothing stopping the C code from doing anything
it likes. However, programs that break these rules are likely to fail
in unexpected and unpredictable ways.
Using cgo directly
Usage:
@ -391,17 +450,13 @@ the translation process.
Translating Go
[The rest of this comment refers to 6g, the Go compiler that is part
of the amd64 port of the gc Go toolchain. Everything here applies to
another architecture's compilers as well.]
Given the input Go files x.go and y.go, cgo generates these source
files:
x.cgo1.go # for 6g
y.cgo1.go # for 6g
_cgo_gotypes.go # for 6g
_cgo_import.go # for 6g (if -dynout _cgo_import.go)
x.cgo1.go # for gc (cmd/compile)
y.cgo1.go # for gc
_cgo_gotypes.go # for gc
_cgo_import.go # for gc (if -dynout _cgo_import.go)
x.cgo2.c # for gcc
y.cgo2.c # for gcc
_cgo_defun.c # for gcc (if -gccgo)
@ -464,7 +519,7 @@ Linking
Once the _cgo_export.c and *.cgo2.c files have been compiled with gcc,
they need to be linked into the final binary, along with the libraries
they might depend on (in the case of puts, stdio). 6l has been
they might depend on (in the case of puts, stdio). cmd/link has been
extended to understand basic ELF files, but it does not understand ELF
in the full complexity that modern C libraries embrace, so it cannot
in general generate direct references to the system libraries.
@ -495,23 +550,23 @@ _cgo_import.go, which looks like:
//go:cgo_import_dynamic _ _ "libc.so.6"
In the end, the compiled Go package, which will eventually be
presented to 6l as part of a larger program, contains:
presented to cmd/link as part of a larger program, contains:
_go_.6 # 6g-compiled object for _cgo_gotypes.go, _cgo_import.go, *.cgo1.go
_go_.o # gc-compiled object for _cgo_gotypes.go, _cgo_import.go, *.cgo1.go
_all.o # gcc-compiled object for _cgo_export.c, *.cgo2.c
The final program will be a dynamic executable, so that 6l can avoid
The final program will be a dynamic executable, so that cmd/link can avoid
needing to process arbitrary .o files. It only needs to process the .o
files generated from C files that cgo writes, and those are much more
limited in the ELF or other features that they use.
In essence, the _cgo_import.6 file includes the extra linking
directives that 6l is not sophisticated enough to derive from _all.o
In essence, the _cgo_import.o file includes the extra linking
directives that cmd/link is not sophisticated enough to derive from _all.o
on its own. Similarly, the _all.o uses dynamic references to real
system object code because 6l is not sophisticated enough to process
system object code because cmd/link is not sophisticated enough to process
the real code.
The main benefits of this system are that 6l remains relatively simple
The main benefits of this system are that cmd/link remains relatively simple
(it does not need to implement a complete ELF and Mach-O linker) and
that gcc is not needed after the package is compiled. For example,
package net uses cgo for access to name resolution functions provided
@ -540,17 +595,17 @@ system calls.
Internal and External Linking
The text above describes "internal" linking, in which 6l parses and
The text above describes "internal" linking, in which cmd/link parses and
links host object files (ELF, Mach-O, PE, and so on) into the final
executable itself. Keeping 6l simple means we cannot possibly
executable itself. Keeping cmd/link simple means we cannot possibly
implement the full semantics of the host linker, so the kinds of
objects that can be linked directly into the binary is limited (other
code can only be used as a dynamic library). On the other hand, when
using internal linking, 6l can generate Go binaries by itself.
using internal linking, cmd/link can generate Go binaries by itself.
In order to allow linking arbitrary object files without requiring
dynamic libraries, cgo supports an "external" linking mode too. In
external linking mode, 6l does not process any host object files.
external linking mode, cmd/link does not process any host object files.
Instead, it collects all the Go code and writes a single go.o object
file containing it. Then it invokes the host linker (usually gcc) to
combine the go.o object file and any supporting non-Go code into a
@ -582,8 +637,8 @@ to be made when linking the final binary.
Linking Directives
In either linking mode, package-specific directives must be passed
through to 6l. These are communicated by writing //go: directives in a
Go source file compiled by 6g. The directives are copied into the .6
through to cmd/link. These are communicated by writing //go: directives in a
Go source file compiled by gc. The directives are copied into the .o
object file and then processed by the linker.
The directives are:
@ -672,7 +727,7 @@ Example
As a simple example, consider a package that uses cgo to call C.sin.
The following code will be generated by cgo:
// compiled by 6g
// compiled by gc
//go:cgo_ldflag "-lm"
@ -708,7 +763,7 @@ Otherwise the link will be an internal one.
The linking directives are used according to the kind of final link
used.
In internal mode, 6l itself processes all the host object files, in
In internal mode, cmd/link itself processes all the host object files, in
particular foo.cgo2.o. To do so, it uses the cgo_import_dynamic and
cgo_dynamic_linker directives to learn that the otherwise undefined
reference to sin in foo.cgo2.o should be rewritten to refer to the
@ -716,56 +771,56 @@ symbol sin with version GLIBC_2.2.5 from the dynamic library
"libm.so.6", and the binary should request "/lib/ld-linux.so.2" as its
runtime dynamic linker.
In external mode, 6l does not process any host object files, in
particular foo.cgo2.o. It links together the 6g-generated object
In external mode, cmd/link does not process any host object files, in
particular foo.cgo2.o. It links together the gc-generated object
files, along with any other Go code, into a go.o file. While doing
that, 6l will discover that there is no definition for
_cgo_gcc_Cfunc_sin, referred to by the 6g-compiled source file. This
is okay, because 6l also processes the cgo_import_static directive and
that, cmd/link will discover that there is no definition for
_cgo_gcc_Cfunc_sin, referred to by the gc-compiled source file. This
is okay, because cmd/link also processes the cgo_import_static directive and
knows that _cgo_gcc_Cfunc_sin is expected to be supplied by a host
object file, so 6l does not treat the missing symbol as an error when
object file, so cmd/link does not treat the missing symbol as an error when
creating go.o. Indeed, the definition for _cgo_gcc_Cfunc_sin will be
provided to the host linker by foo2.cgo.o, which in turn will need the
symbol 'sin'. 6l also processes the cgo_ldflag directives, so that it
symbol 'sin'. cmd/link also processes the cgo_ldflag directives, so that it
knows that the eventual host link command must include the -lm
argument, so that the host linker will be able to find 'sin' in the
math library.
6l Command Line Interface
cmd/link Command Line Interface
The go command and any other Go-aware build systems invoke 6l
to link a collection of packages into a single binary. By default, 6l will
The go command and any other Go-aware build systems invoke cmd/link
to link a collection of packages into a single binary. By default, cmd/link will
present the same interface it does today:
6l main.a
cmd/link main.a
produces a file named 6.out, even if 6l does so by invoking the host
produces a file named a.out, even if cmd/link does so by invoking the host
linker in external linking mode.
By default, 6l will decide the linking mode as follows: if the only
By default, cmd/link will decide the linking mode as follows: if the only
packages using cgo are those on a whitelist of standard library
packages (net, os/user, runtime/cgo), 6l will use internal linking
mode. Otherwise, there are non-standard cgo packages involved, and 6l
packages (net, os/user, runtime/cgo), cmd/link will use internal linking
mode. Otherwise, there are non-standard cgo packages involved, and cmd/link
will use external linking mode. The first rule means that a build of
the godoc binary, which uses net but no other cgo, can run without
needing gcc available. The second rule means that a build of a
cgo-wrapped library like sqlite3 can generate a standalone executable
instead of needing to refer to a dynamic library. The specific choice
can be overridden using a command line flag: 6l -linkmode=internal or
6l -linkmode=external.
can be overridden using a command line flag: cmd/link -linkmode=internal or
cmd/link -linkmode=external.
In an external link, 6l will create a temporary directory, write any
In an external link, cmd/link will create a temporary directory, write any
host object files found in package archives to that directory (renamed
to avoid conflicts), write the go.o file to that directory, and invoke
the host linker. The default value for the host linker is $CC, split
into fields, or else "gcc". The specific host linker command line can
be overridden using command line flags: 6l -extld=clang
be overridden using command line flags: cmd/link -extld=clang
-extldflags='-ggdb -O3'. If any package in a build includes a .cc or
other file compiled by the C++ compiler, the go tool will use the
-extld option to set the host linker to the C++ compiler.
These defaults mean that Go-aware build systems can ignore the linking
changes and keep running plain '6l' and get reasonable results, but
changes and keep running plain 'cmd/link' and get reasonable results, but
they can also control the linking details if desired.
*/

354
libgo/go/cmd/cgo/gcc.go
View file

@ -38,8 +38,8 @@ var nameToC = map[string]string{
"ulong": "unsigned long",
"longlong": "long long",
"ulonglong": "unsigned long long",
"complexfloat": "float complex",
"complexdouble": "double complex",
"complexfloat": "float _Complex",
"complexdouble": "double _Complex",
}
// cname returns the C name to use for C.s.
@ -167,6 +167,7 @@ func (p *Package) Translate(f *File) {
if len(needType) > 0 {
p.loadDWARF(f, needType)
}
p.rewriteCalls(f)
p.rewriteRef(f)
}
@ -575,6 +576,331 @@ func (p *Package) mangleName(n *Name) {
n.Mangle = prefix + n.Kind + "_" + n.Go
}
// rewriteCalls rewrites all calls that pass pointers to check that
// they follow the rules for passing pointers between Go and C.
func (p *Package) rewriteCalls(f *File) {
for _, call := range f.Calls {
// This is a call to C.xxx; set goname to "xxx".
goname := call.Fun.(*ast.SelectorExpr).Sel.Name
if goname == "malloc" {
continue
}
name := f.Name[goname]
if name.Kind != "func" {
// Probably a type conversion.
continue
}
p.rewriteCall(f, call, name)
}
}
// rewriteCall rewrites one call to add pointer checks. We replace
// each pointer argument x with _cgoCheckPointer(x).(T).
func (p *Package) rewriteCall(f *File, call *ast.CallExpr, name *Name) {
for i, param := range name.FuncType.Params {
if len(call.Args) <= i {
// Avoid a crash; this will be caught when the
// generated file is compiled.
return
}
// An untyped nil does not need a pointer check, and
// when _cgoCheckPointer returns the untyped nil the
// type assertion we are going to insert will fail.
// Easier to just skip nil arguments.
// TODO: Note that this fails if nil is shadowed.
if id, ok := call.Args[i].(*ast.Ident); ok && id.Name == "nil" {
continue
}
if !p.needsPointerCheck(f, param.Go) {
continue
}
c := &ast.CallExpr{
Fun: ast.NewIdent("_cgoCheckPointer"),
Args: []ast.Expr{
call.Args[i],
},
}
// Add optional additional arguments for an address
// expression.
c.Args = p.checkAddrArgs(f, c.Args, call.Args[i])
// _cgoCheckPointer returns interface{}.
// We need to type assert that to the type we want.
// If the Go version of this C type uses
// unsafe.Pointer, we can't use a type assertion,
// because the Go file might not import unsafe.
// Instead we use a local variant of _cgoCheckPointer.
var arg ast.Expr
if n := p.unsafeCheckPointerName(param.Go); n != "" {
c.Fun = ast.NewIdent(n)
arg = c
} else {
// In order for the type assertion to succeed,
// we need it to match the actual type of the
// argument. The only type we have is the
// type of the function parameter. We know
// that the argument type must be assignable
// to the function parameter type, or the code
// would not compile, but there is nothing
// requiring that the types be exactly the
// same. Add a type conversion to the
// argument so that the type assertion will
// succeed.
c.Args[0] = &ast.CallExpr{
Fun: param.Go,
Args: []ast.Expr{
c.Args[0],
},
}
arg = &ast.TypeAssertExpr{
X: c,
Type: param.Go,
}
}
call.Args[i] = arg
}
}
// needsPointerCheck returns whether the type t needs a pointer check.
// This is true if t is a pointer and if the value to which it points
// might contain a pointer.
func (p *Package) needsPointerCheck(f *File, t ast.Expr) bool {
return p.hasPointer(f, t, true)
}
// hasPointer is used by needsPointerCheck. If top is true it returns
// whether t is or contains a pointer that might point to a pointer.
// If top is false it returns whether t is or contains a pointer.
// f may be nil.
func (p *Package) hasPointer(f *File, t ast.Expr, top bool) bool {
switch t := t.(type) {
case *ast.ArrayType:
if t.Len == nil {
if !top {
return true
}
return p.hasPointer(f, t.Elt, false)
}
return p.hasPointer(f, t.Elt, top)
case *ast.StructType:
for _, field := range t.Fields.List {
if p.hasPointer(f, field.Type, top) {
return true
}
}
return false
case *ast.StarExpr: // Pointer type.
if !top {
return true
}
return p.hasPointer(f, t.X, false)
case *ast.FuncType, *ast.InterfaceType, *ast.MapType, *ast.ChanType:
return true
case *ast.Ident:
// TODO: Handle types defined within function.
for _, d := range p.Decl {
gd, ok := d.(*ast.GenDecl)
if !ok || gd.Tok != token.TYPE {
continue
}
for _, spec := range gd.Specs {
ts, ok := spec.(*ast.TypeSpec)
if !ok {
continue
}
if ts.Name.Name == t.Name {
return p.hasPointer(f, ts.Type, top)
}
}
}
if def := typedef[t.Name]; def != nil {
return p.hasPointer(f, def.Go, top)
}
if t.Name == "string" {
return !top
}
if t.Name == "error" {
return true
}
if goTypes[t.Name] != nil {
return false
}
// We can't figure out the type. Conservative
// approach is to assume it has a pointer.
return true
case *ast.SelectorExpr:
if l, ok := t.X.(*ast.Ident); !ok || l.Name != "C" {
// Type defined in a different package.
// Conservative approach is to assume it has a
// pointer.
return true
}
if f == nil {
// Conservative approach: assume pointer.
return true
}
name := f.Name[t.Sel.Name]
if name != nil && name.Kind == "type" && name.Type != nil && name.Type.Go != nil {
return p.hasPointer(f, name.Type.Go, top)
}
// We can't figure out the type. Conservative
// approach is to assume it has a pointer.
return true
default:
error_(t.Pos(), "could not understand type %s", gofmt(t))
return true
}
}
// checkAddrArgs tries to add arguments to the call of
// _cgoCheckPointer when the argument is an address expression. We
// pass true to mean that the argument is an address operation of
// something other than a slice index, which means that it's only
// necessary to check the specific element pointed to, not the entire
// object. This is for &s.f, where f is a field in a struct. We can
// pass a slice or array, meaning that we should check the entire
// slice or array but need not check any other part of the object.
// This is for &s.a[i], where we need to check all of a. However, we
// only pass the slice or array if we can refer to it without side
// effects.
func (p *Package) checkAddrArgs(f *File, args []ast.Expr, x ast.Expr) []ast.Expr {
// Strip type conversions.
for {
c, ok := x.(*ast.CallExpr)
if !ok || len(c.Args) != 1 || !p.isType(c.Fun) {
break
}
x = c.Args[0]
}
u, ok := x.(*ast.UnaryExpr)
if !ok || u.Op != token.AND {
return args
}
index, ok := u.X.(*ast.IndexExpr)
if !ok {
// This is the address of something that is not an
// index expression. We only need to examine the
// single value to which it points.
// TODO: what if true is shadowed?
return append(args, ast.NewIdent("true"))
}
if !p.hasSideEffects(f, index.X) {
// Examine the entire slice.
return append(args, index.X)
}
// Treat the pointer as unknown.
return args
}
// hasSideEffects returns whether the expression x has any side
// effects. x is an expression, not a statement, so the only side
// effect is a function call.
func (p *Package) hasSideEffects(f *File, x ast.Expr) bool {
found := false
f.walk(x, "expr",
func(f *File, x interface{}, context string) {
switch x.(type) {
case *ast.CallExpr:
found = true
}
})
return found
}
// isType returns whether the expression is definitely a type.
// This is conservative--it returns false for an unknown identifier.
func (p *Package) isType(t ast.Expr) bool {
switch t := t.(type) {
case *ast.SelectorExpr:
if t.Sel.Name != "Pointer" {
return false
}
id, ok := t.X.(*ast.Ident)
if !ok {
return false
}
return id.Name == "unsafe"
case *ast.Ident:
// TODO: This ignores shadowing.
switch t.Name {
case "unsafe.Pointer", "bool", "byte",
"complex64", "complex128",
"error",
"float32", "float64",
"int", "int8", "int16", "int32", "int64",
"rune", "string",
"uint", "uint8", "uint16", "uint32", "uint64", "uintptr":
return true
}
case *ast.StarExpr:
return p.isType(t.X)
case *ast.ArrayType, *ast.StructType, *ast.FuncType, *ast.InterfaceType,
*ast.MapType, *ast.ChanType:
return true
}
return false
}
// unsafeCheckPointerName is given the Go version of a C type. If the
// type uses unsafe.Pointer, we arrange to build a version of
// _cgoCheckPointer that returns that type. This avoids using a type
// assertion to unsafe.Pointer in our copy of user code. We return
// the name of the _cgoCheckPointer function we are going to build, or
// the empty string if the type does not use unsafe.Pointer.
func (p *Package) unsafeCheckPointerName(t ast.Expr) string {
if !p.hasUnsafePointer(t) {
return ""
}
var buf bytes.Buffer
conf.Fprint(&buf, fset, t)
s := buf.String()
for i, t := range p.CgoChecks {
if s == t {
return p.unsafeCheckPointerNameIndex(i)
}
}
p.CgoChecks = append(p.CgoChecks, s)
return p.unsafeCheckPointerNameIndex(len(p.CgoChecks) - 1)
}
// hasUnsafePointer returns whether the Go type t uses unsafe.Pointer.
// t is the Go version of a C type, so we don't need to handle every case.
// We only care about direct references, not references via typedefs.
func (p *Package) hasUnsafePointer(t ast.Expr) bool {
switch t := t.(type) {
case *ast.Ident:
// We don't see a SelectorExpr for unsafe.Pointer;
// this is created by code in this file.
return t.Name == "unsafe.Pointer"
case *ast.ArrayType:
return p.hasUnsafePointer(t.Elt)
case *ast.StructType:
for _, f := range t.Fields.List {
if p.hasUnsafePointer(f.Type) {
return true
}
}
case *ast.StarExpr: // Pointer type.
return p.hasUnsafePointer(t.X)
}
return false
}
// unsafeCheckPointerNameIndex returns the name to use for a
// _cgoCheckPointer variant based on the index in the CgoChecks slice.
func (p *Package) unsafeCheckPointerNameIndex(i int) string {
return fmt.Sprintf("_cgoCheckPointer%d", i)
}
// rewriteRef rewrites all the C.xxx references in f.AST to refer to the
// Go equivalents, now that we have figured out the meaning of all
// the xxx. In *godefs mode, rewriteRef replaces the names
@ -612,6 +938,10 @@ func (p *Package) rewriteRef(f *File) {
if r.Name.Kind != "func" {
if r.Name.Kind == "type" {
r.Context = "type"
if r.Name.Type == nil {
error_(r.Pos(), "invalid conversion to C.%s: undefined C type '%s'", fixGo(r.Name.Go), r.Name.C)
break
}
expr = r.Name.Type.Go
break
}
@ -663,6 +993,10 @@ func (p *Package) rewriteRef(f *File) {
}
} else if r.Name.Kind == "type" {
// Okay - might be new(T)
if r.Name.Type == nil {
error_(r.Pos(), "expression C.%s: undefined C type '%s'", fixGo(r.Name.Go), r.Name.C)
break
}
expr = r.Name.Type.Go
} else if r.Name.Kind == "var" {
expr = &ast.StarExpr{Star: (*r.Expr).Pos(), X: expr}
@ -1028,12 +1362,12 @@ var dwarfToName = map[string]string{
"long unsigned int": "ulong",
"unsigned int": "uint",
"short unsigned int": "ushort",
"unsigned short": "ushort", // Used by Clang; issue 13129.
"short int": "short",
"long long int": "longlong",
"long long unsigned int": "ulonglong",
"signed char": "schar",
"float complex": "complexfloat",
"double complex": "complexdouble",
"unsigned char": "uchar",
}
const signedDelta = 64
@ -1224,6 +1558,11 @@ func (c *typeConv) Type(dtype dwarf.Type, pos token.Pos) *Type {
t.Go = c.int32
case 8:
t.Go = c.int64
case 16:
t.Go = &ast.ArrayType{
Len: c.intExpr(t.Size),
Elt: c.uint8,
}
}
if t.Align = t.Size; t.Align >= c.ptrSize {
t.Align = c.ptrSize
@ -1381,6 +1720,11 @@ func (c *typeConv) Type(dtype dwarf.Type, pos token.Pos) *Type {
t.Go = c.uint32
case 8:
t.Go = c.uint64
case 16:
t.Go = &ast.ArrayType{
Len: c.intExpr(t.Size),
Elt: c.uint8,
}
}
if t.Align = t.Size; t.Align >= c.ptrSize {
t.Align = c.ptrSize
@ -1393,7 +1737,7 @@ func (c *typeConv) Type(dtype dwarf.Type, pos token.Pos) *Type {
}
switch dtype.(type) {
case *dwarf.AddrType, *dwarf.BoolType, *dwarf.CharType, *dwarf.IntType, *dwarf.FloatType, *dwarf.UcharType, *dwarf.UintType:
case *dwarf.AddrType, *dwarf.BoolType, *dwarf.CharType, *dwarf.ComplexType, *dwarf.IntType, *dwarf.FloatType, *dwarf.UcharType, *dwarf.UintType:
s := dtype.Common().Name
if s != "" {
if ss, ok := dwarfToName[s]; ok {

3
libgo/go/cmd/cgo/godefs.go
View file

@ -11,6 +11,7 @@ import (
"go/printer"
"go/token"
"os"
"path/filepath"
"strings"
)
@ -19,7 +20,7 @@ func (p *Package) godefs(f *File, srcfile string) string {
var buf bytes.Buffer
fmt.Fprintf(&buf, "// Created by cgo -godefs - DO NOT EDIT\n")
fmt.Fprintf(&buf, "// %s\n", strings.Join(os.Args, " "))
fmt.Fprintf(&buf, "// %s %s\n", filepath.Base(os.Args[0]), strings.Join(os.Args[1:], " "))
fmt.Fprintf(&buf, "\n")
override := make(map[string]string)

80
libgo/go/cmd/cgo/main.go
View file

@ -42,6 +42,7 @@ type Package struct {
GoFiles []string // list of Go files
GccFiles []string // list of gcc output files
Preamble string // collected preamble for _cgo_export.h
CgoChecks []string // see unsafeCheckPointerName
}
// A File collects information about a single Go input file.
@ -51,6 +52,7 @@ type File struct {
Package string // Package name
Preamble string // C preamble (doc comment on import "C")
Ref []*Ref // all references to C.xxx in AST
Calls []*ast.CallExpr // all calls to C.xxx in AST
ExpFunc []*ExpFunc // exported functions for this file
Name map[string]*Name // map from Go name to Name
}
@ -132,43 +134,47 @@ func usage() {
}
var ptrSizeMap = map[string]int64{
"386": 4,
"alpha": 8,
"amd64": 8,
"arm": 4,
"arm64": 8,
"m68k": 4,
"mipso32": 4,
"mipsn32": 4,
"mipso64": 8,
"mipsn64": 8,
"ppc": 4,
"ppc64": 8,
"ppc64le": 8,
"s390": 4,
"s390x": 8,
"sparc": 4,
"sparc64": 8,
"386": 4,
"alpha": 8,
"amd64": 8,
"arm": 4,
"arm64": 8,
"m68k": 4,
"mipso32": 4,
"mipsn32": 4,
"mipso64": 8,
"mipsn64": 8,
"mips64": 8,
"mips64le": 8,
"ppc": 4,
"ppc64": 8,
"ppc64le": 8,
"s390": 4,
"s390x": 8,
"sparc": 4,
"sparc64": 8,
}
var intSizeMap = map[string]int64{
"386": 4,
"alpha": 8,
"amd64": 8,
"arm": 4,
"arm64": 8,
"m68k": 4,
"mipso32": 4,
"mipsn32": 4,
"mipso64": 8,
"mipsn64": 8,
"ppc": 4,
"ppc64": 8,
"ppc64le": 8,
"s390": 4,
"s390x": 8,
"sparc": 4,
"sparc64": 8,
"386": 4,
"alpha": 8,
"amd64": 8,
"arm": 4,
"arm64": 8,
"m68k": 4,
"mipso32": 4,
"mipsn32": 4,
"mipso64": 8,
"mipsn64": 8,
"mips64": 8,
"mips64le": 8,
"ppc": 4,
"ppc64": 8,
"ppc64le": 8,
"s390": 4,
"s390x": 8,
"sparc": 4,
"sparc64": 8,
}
var cPrefix string
@ -297,11 +303,7 @@ func main() {
if nerrors > 0 {
os.Exit(2)
}
pkg := f.Package
if dir := os.Getenv("CGOPKGPATH"); dir != "" {
pkg = filepath.Join(dir, pkg)
}
p.PackagePath = pkg
p.PackagePath = f.Package
p.Record(f)
if *godefs {
os.Stdout.WriteString(p.godefs(f, input))

248
libgo/go/cmd/cgo/out.go
View file

@ -103,11 +103,19 @@ func (p *Package) writeDefs() {
}
if *gccgo {
fmt.Fprint(fgo2, gccgoGoProlog)
fmt.Fprint(fc, p.cPrologGccgo())
} else {
fmt.Fprint(fgo2, goProlog)
}
for i, t := range p.CgoChecks {
n := p.unsafeCheckPointerNameIndex(i)
fmt.Fprintf(fgo2, "\nfunc %s(p interface{}, args ...interface{}) %s {\n", n, t)
fmt.Fprintf(fgo2, "\treturn _cgoCheckPointer(p, args...).(%s)\n", t)
fmt.Fprintf(fgo2, "}\n")
}
gccgoSymbolPrefix := p.gccgoSymbolPrefix()
cVars := make(map[string]bool)
@ -693,7 +701,7 @@ func (p *Package) writeExports(fgo2, fm, fgcc, fgcch io.Writer) {
}
fntype := fn.Type
forFieldList(fntype.Params,
func(i int, atype ast.Expr) {
func(i int, aname string, atype ast.Expr) {
t := p.cgoType(atype)
if off%t.Align != 0 {
pad := t.Align - off%t.Align
@ -711,7 +719,7 @@ func (p *Package) writeExports(fgo2, fm, fgcc, fgcch io.Writer) {
npad++
}
forFieldList(fntype.Results,
func(i int, atype ast.Expr) {
func(i int, aname string, atype ast.Expr) {
t := p.cgoType(atype)
if off%t.Align != 0 {
pad := t.Align - off%t.Align
@ -744,8 +752,12 @@ func (p *Package) writeExports(fgo2, fm, fgcc, fgcch io.Writer) {
fmt.Fprintf(fgcch, "\n/* Return type for %s */\n", exp.ExpName)
fmt.Fprintf(fgcch, "struct %s_return {\n", exp.ExpName)
forFieldList(fntype.Results,
func(i int, atype ast.Expr) {
fmt.Fprintf(fgcch, "\t%s r%d;\n", p.cgoType(atype).C, i)
func(i int, aname string, atype ast.Expr) {
fmt.Fprintf(fgcch, "\t%s r%d;", p.cgoType(atype).C, i)
if len(aname) > 0 {
fmt.Fprintf(fgcch, " /* %s */", aname)
}
fmt.Fprint(fgcch, "\n")
})
fmt.Fprintf(fgcch, "};\n")
gccResult = "struct " + exp.ExpName + "_return"
@ -758,7 +770,7 @@ func (p *Package) writeExports(fgo2, fm, fgcc, fgcch io.Writer) {
s += " recv"
}
forFieldList(fntype.Params,
func(i int, atype ast.Expr) {
func(i int, aname string, atype ast.Expr) {
if i > 0 || fn.Recv != nil {
s += ", "
}
@ -783,7 +795,7 @@ func (p *Package) writeExports(fgo2, fm, fgcc, fgcch io.Writer) {
fmt.Fprintf(fgcc, "\ta.recv = recv;\n")
}
forFieldList(fntype.Params,
func(i int, atype ast.Expr) {
func(i int, aname string, atype ast.Expr) {
fmt.Fprintf(fgcc, "\ta.p%d = p%d;\n", i, i)
})
fmt.Fprintf(fgcc, "\tcrosscall2(_cgoexp%s_%s, &a, %d);\n", cPrefix, exp.ExpName, off)
@ -792,7 +804,7 @@ func (p *Package) writeExports(fgo2, fm, fgcc, fgcch io.Writer) {
fmt.Fprintf(fgcc, "\treturn a.r0;\n")
} else {
forFieldList(fntype.Results,
func(i int, atype ast.Expr) {
func(i int, aname string, atype ast.Expr) {
fmt.Fprintf(fgcc, "\tr.r%d = a.r%d;\n", i, i)
})
fmt.Fprintf(fgcc, "\treturn r;\n")
@ -800,17 +812,18 @@ func (p *Package) writeExports(fgo2, fm, fgcc, fgcch io.Writer) {
}
fmt.Fprintf(fgcc, "}\n")
// Build the wrapper function compiled by gc.
goname := exp.Func.Name.Name
// Build the wrapper function compiled by cmd/compile.
goname := "_cgoexpwrap" + cPrefix + "_"
if fn.Recv != nil {
goname = "_cgoexpwrap" + cPrefix + "_" + fn.Recv.List[0].Names[0].Name + "_" + goname
goname += fn.Recv.List[0].Names[0].Name + "_"
}
fmt.Fprintf(fgo2, "//go:cgo_export_dynamic %s\n", goname)
goname += exp.Func.Name.Name
fmt.Fprintf(fgo2, "//go:cgo_export_dynamic %s\n", exp.ExpName)
fmt.Fprintf(fgo2, "//go:linkname _cgoexp%s_%s _cgoexp%s_%s\n", cPrefix, exp.ExpName, cPrefix, exp.ExpName)
fmt.Fprintf(fgo2, "//go:cgo_export_static _cgoexp%s_%s\n", cPrefix, exp.ExpName)
fmt.Fprintf(fgo2, "//go:nosplit\n") // no split stack, so no use of m or g
fmt.Fprintf(fgo2, "//go:norace\n") // must not have race detector calls inserted
fmt.Fprintf(fgo2, "func _cgoexp%s_%s(a unsafe.Pointer, n int32) {", cPrefix, exp.ExpName)
fmt.Fprintf(fgo2, "func _cgoexp%s_%s(a unsafe.Pointer, n int32) {\n", cPrefix, exp.ExpName)
fmt.Fprintf(fgo2, "\tfn := %s\n", goname)
// The indirect here is converting from a Go function pointer to a C function pointer.
fmt.Fprintf(fgo2, "\t_cgo_runtime_cgocallback(**(**unsafe.Pointer)(unsafe.Pointer(&fn)), a, uintptr(n));\n")
@ -818,44 +831,75 @@ func (p *Package) writeExports(fgo2, fm, fgcc, fgcch io.Writer) {
fmt.Fprintf(fm, "int _cgoexp%s_%s;\n", cPrefix, exp.ExpName)
// Calling a function with a receiver from C requires
// a Go wrapper function.
// This code uses printer.Fprint, not conf.Fprint,
// because we don't want //line comments in the middle
// of the function types.
fmt.Fprintf(fgo2, "\n")
fmt.Fprintf(fgo2, "func %s(", goname)
comma := false
if fn.Recv != nil {
fmt.Fprintf(fgo2, "func %s(recv ", goname)
conf.Fprint(fgo2, fset, fn.Recv.List[0].Type)
forFieldList(fntype.Params,
func(i int, atype ast.Expr) {
fmt.Fprintf(fgo2, ", p%d ", i)
conf.Fprint(fgo2, fset, atype)
})
fmt.Fprintf(fgo2, ")")
if gccResult != "void" {
fmt.Fprint(fgo2, " (")
forFieldList(fntype.Results,
func(i int, atype ast.Expr) {
if i > 0 {
fmt.Fprint(fgo2, ", ")
}
conf.Fprint(fgo2, fset, atype)
})
fmt.Fprint(fgo2, ")")
}
fmt.Fprint(fgo2, " {\n")
fmt.Fprint(fgo2, "\t")
if gccResult != "void" {
fmt.Fprint(fgo2, "return ")
}
fmt.Fprintf(fgo2, "recv.%s(", exp.Func.Name)
forFieldList(fntype.Params,
func(i int, atype ast.Expr) {
fmt.Fprintf(fgo2, "recv ")
printer.Fprint(fgo2, fset, fn.Recv.List[0].Type)
comma = true
}
forFieldList(fntype.Params,
func(i int, aname string, atype ast.Expr) {
if comma {
fmt.Fprintf(fgo2, ", ")
}
fmt.Fprintf(fgo2, "p%d ", i)
printer.Fprint(fgo2, fset, atype)
comma = true
})
fmt.Fprintf(fgo2, ")")
if gccResult != "void" {
fmt.Fprint(fgo2, " (")
forFieldList(fntype.Results,
func(i int, aname string, atype ast.Expr) {
if i > 0 {
fmt.Fprint(fgo2, ", ")
}
fmt.Fprintf(fgo2, "p%d", i)
fmt.Fprintf(fgo2, "r%d ", i)
printer.Fprint(fgo2, fset, atype)
})
fmt.Fprint(fgo2, ")\n")
fmt.Fprint(fgo2, "}\n")
fmt.Fprint(fgo2, ")")
}
fmt.Fprint(fgo2, " {\n")
if gccResult == "void" {
fmt.Fprint(fgo2, "\t")
} else {
// Verify that any results don't contain any
// Go pointers.
addedDefer := false
forFieldList(fntype.Results,
func(i int, aname string, atype ast.Expr) {
if !p.hasPointer(nil, atype, false) {
return
}
if !addedDefer {
fmt.Fprint(fgo2, "\tdefer func() {\n")
addedDefer = true
}
fmt.Fprintf(fgo2, "\t\t_cgoCheckResult(r%d)\n", i)
})
if addedDefer {
fmt.Fprint(fgo2, "\t}()\n")
}
fmt.Fprint(fgo2, "\treturn ")
}
if fn.Recv != nil {
fmt.Fprintf(fgo2, "recv.")
}
fmt.Fprintf(fgo2, "%s(", exp.Func.Name)
forFieldList(fntype.Params,
func(i int, aname string, atype ast.Expr) {
if i > 0 {
fmt.Fprint(fgo2, ", ")
}
fmt.Fprintf(fgo2, "p%d", i)
})
fmt.Fprint(fgo2, ")\n")
fmt.Fprint(fgo2, "}\n")
}
fmt.Fprintf(fgcch, "%s", gccExportHeaderEpilog)
@ -879,13 +923,13 @@ func (p *Package) writeGccgoExports(fgo2, fm, fgcc, fgcch io.Writer) {
cdeclBuf := new(bytes.Buffer)
resultCount := 0
forFieldList(fntype.Results,
func(i int, atype ast.Expr) { resultCount++ })
func(i int, aname string, atype ast.Expr) { resultCount++ })
switch resultCount {
case 0:
fmt.Fprintf(cdeclBuf, "void")
case 1:
forFieldList(fntype.Results,
func(i int, atype ast.Expr) {
func(i int, aname string, atype ast.Expr) {
t := p.cgoType(atype)
fmt.Fprintf(cdeclBuf, "%s", t.C)
})
@ -894,9 +938,13 @@ func (p *Package) writeGccgoExports(fgo2, fm, fgcc, fgcch io.Writer) {
fmt.Fprintf(fgcch, "\n/* Return type for %s */\n", exp.ExpName)
fmt.Fprintf(fgcch, "struct %s_result {\n", exp.ExpName)
forFieldList(fntype.Results,
func(i int, atype ast.Expr) {
func(i int, aname string, atype ast.Expr) {
t := p.cgoType(atype)
fmt.Fprintf(fgcch, "\t%s r%d;\n", t.C, i)
fmt.Fprintf(fgcch, "\t%s r%d;", t.C, i)
if len(aname) > 0 {
fmt.Fprintf(fgcch, " /* %s */", aname)
}
fmt.Fprint(fgcch, "\n")
})
fmt.Fprintf(fgcch, "};\n")
fmt.Fprintf(cdeclBuf, "struct %s_result", exp.ExpName)
@ -911,7 +959,7 @@ func (p *Package) writeGccgoExports(fgo2, fm, fgcc, fgcch io.Writer) {
}
// Function parameters.
forFieldList(fntype.Params,
func(i int, atype ast.Expr) {
func(i int, aname string, atype ast.Expr) {
if i > 0 || fn.Recv != nil {
fmt.Fprintf(cdeclBuf, ", ")
}
@ -925,23 +973,15 @@ func (p *Package) writeGccgoExports(fgo2, fm, fgcc, fgcch io.Writer) {
fmt.Fprintf(fgcch, "\n%s", exp.Doc)
}
fmt.Fprintf(fgcch, "extern %s %s %s;\n", cRet, exp.ExpName, cParams)
// We need to use a name that will be exported by the
// Go code; otherwise gccgo will make it static and we
// will not be able to link against it from the C
// code.
goName := "Cgoexp_" + exp.ExpName
fmt.Fprintf(fgcch, `extern %s %s %s __asm__("%s.%s");`, cRet, goName, cParams, gccgoSymbolPrefix, goName)
fmt.Fprint(fgcch, "\n")
// Use a #define so that the C code that includes
// cgo_export.h will be able to refer to the Go
// function using the expected name.
fmt.Fprintf(fgcch, "#define %s %s\n", exp.ExpName, goName)
// Use a #undef in _cgo_export.c so that we ignore the
// #define from cgo_export.h, since here we are
// defining the real function.
fmt.Fprintf(fgcc, "#undef %s\n", exp.ExpName)
fmt.Fprintf(fgcc, `extern %s %s %s __asm__("%s.%s");`, cRet, goName, cParams, gccgoSymbolPrefix, goName)
fmt.Fprint(fgcc, "\n")
fmt.Fprint(fgcc, "\n")
fmt.Fprintf(fgcc, "%s %s %s {\n", cRet, exp.ExpName, cParams)
@ -956,7 +996,7 @@ func (p *Package) writeGccgoExports(fgo2, fm, fgcc, fgcch io.Writer) {
fmt.Fprint(fgcc, "recv")
}
forFieldList(fntype.Params,
func(i int, atype ast.Expr) {
func(i int, aname string, atype ast.Expr) {
if i > 0 || fn.Recv != nil {
fmt.Fprintf(fgcc, ", ")
}
@ -982,7 +1022,7 @@ func (p *Package) writeGccgoExports(fgo2, fm, fgcc, fgcch io.Writer) {
printer.Fprint(fgo2, fset, fn.Recv.List[0].Type)
}
forFieldList(fntype.Params,
func(i int, atype ast.Expr) {
func(i int, aname string, atype ast.Expr) {
if i > 0 || fn.Recv != nil {
fmt.Fprintf(fgo2, ", ")
}
@ -993,7 +1033,7 @@ func (p *Package) writeGccgoExports(fgo2, fm, fgcc, fgcch io.Writer) {
if resultCount > 0 {
fmt.Fprintf(fgo2, " (")
forFieldList(fntype.Results,
func(i int, atype ast.Expr) {
func(i int, aname string, atype ast.Expr) {
if i > 0 {
fmt.Fprint(fgo2, ", ")
}
@ -1013,7 +1053,7 @@ func (p *Package) writeGccgoExports(fgo2, fm, fgcc, fgcch io.Writer) {
}
fmt.Fprintf(fgo2, "%s(", exp.Func.Name)
forFieldList(fntype.Params,
func(i int, atype ast.Expr) {
func(i int, aname string, atype ast.Expr) {
if i > 0 {
fmt.Fprint(fgo2, ", ")
}
@ -1071,19 +1111,19 @@ func (p *Package) gccgoSymbolPrefix() string {
}
// Call a function for each entry in an ast.FieldList, passing the
// index into the list and the type.
func forFieldList(fl *ast.FieldList, fn func(int, ast.Expr)) {
// index into the list, the name if any, and the type.
func forFieldList(fl *ast.FieldList, fn func(int, string, ast.Expr)) {
if fl == nil {
return
}
i := 0
for _, r := range fl.List {
if r.Names == nil {
fn(i, r.Type)
fn(i, "", r.Type)
i++
} else {
for range r.Names {
fn(i, r.Type)
for _, n := range r.Names {
fn(i, n.Name, r.Type)
i++
}
}
@ -1193,9 +1233,11 @@ func (p *Package) cgoType(e ast.Expr) *Type {
}
const gccProlog = `
// Usual nonsense: if x and y are not equal, the type will be invalid
// (have a negative array count) and an inscrutable error will come
// out of the compiler and hopefully mention "name".
/*
If x and y are not equal, the type will be invalid
(have a negative array count) and an inscrutable error will come
out of the compiler and hopefully mention "name".
*/
#define __cgo_compile_assert_eq(x, y, name) typedef char name[(x-y)*(x-y)*-2+1];
// Check at compile time that the sizes we use match our expectations.
@ -1239,6 +1281,18 @@ func _cgo_runtime_cmalloc(uintptr) unsafe.Pointer
//go:linkname _cgo_runtime_cgocallback runtime.cgocallback
func _cgo_runtime_cgocallback(unsafe.Pointer, unsafe.Pointer, uintptr)
//go:linkname _cgoCheckPointer runtime.cgoCheckPointer
func _cgoCheckPointer(interface{}, ...interface{}) interface{}
//go:linkname _cgoCheckResult runtime.cgoCheckResult
func _cgoCheckResult(interface{})
`
const gccgoGoProlog = `
func _cgoCheckPointer(interface{}, ...interface{}) interface{}
func _cgoCheckResult(interface{})
`
const goStringDef = `
@ -1293,7 +1347,8 @@ var builtinDefs = map[string]string{
}
func (p *Package) cPrologGccgo() string {
return strings.Replace(cPrologGccgo, "PREFIX", cPrefix, -1)
return strings.Replace(strings.Replace(cPrologGccgo, "PREFIX", cPrefix, -1),
"GCCGOSYMBOLPREF", p.gccgoSymbolPrefix(), -1)
}
const cPrologGccgo = `
@ -1348,6 +1403,39 @@ void *_cgoPREFIX_Cfunc__CMalloc(size_t n) {
runtime_throw("runtime: C malloc failed");
return p;
}
struct __go_type_descriptor;
typedef struct __go_empty_interface {
const struct __go_type_descriptor *__type_descriptor;
void *__object;
} Eface;
extern Eface runtimeCgoCheckPointer(Eface, Slice)
__asm__("runtime.cgoCheckPointer")
__attribute__((weak));
extern Eface localCgoCheckPointer(Eface, Slice)
__asm__("GCCGOSYMBOLPREF._cgoCheckPointer");
Eface localCgoCheckPointer(Eface ptr, Slice args) {
if(runtimeCgoCheckPointer) {
return runtimeCgoCheckPointer(ptr, args);
}
return ptr;
}
extern void runtimeCgoCheckResult(Eface)
__asm__("runtime.cgoCheckResult")
__attribute__((weak));
extern void localCgoCheckResult(Eface)
__asm__("GCCGOSYMBOLPREF._cgoCheckResult");
void localCgoCheckResult(Eface val) {
if(runtimeCgoCheckResult) {
runtimeCgoCheckResult(val);
}
}
`
func (p *Package) gccExportHeaderProlog() string {
@ -1373,14 +1461,16 @@ typedef GoUintGOINTBITS GoUint;
typedef __SIZE_TYPE__ GoUintptr;
typedef float GoFloat32;
typedef double GoFloat64;
typedef __complex float GoComplex64;
typedef __complex double GoComplex128;
typedef float _Complex GoComplex64;
typedef double _Complex GoComplex128;
// static assertion to make sure the file is being used on architecture
// at least with matching size of GoInt.
/*
static assertion to make sure the file is being used on architecture
at least with matching size of GoInt.
*/
typedef char _check_for_GOINTBITS_bit_pointer_matching_GoInt[sizeof(void*)==GOINTBITS/8 ? 1:-1];
typedef struct { char *p; GoInt n; } GoString;
typedef struct { const char *p; GoInt n; } GoString;
typedef void *GoMap;
typedef void *GoChan;
typedef struct { void *t; void *v; } GoInterface;

195
libgo/go/cmd/go/alldocs.go
View file

@ -84,12 +84,16 @@ and test commands:
-n
print the commands but do not run them.
-p n
the number of builds that can be run in parallel.
the number of programs, such as build commands or
test binaries, that can be run in parallel.
The default is the number of CPUs available, except
on darwin/arm which defaults to 1.
-race
enable data race detection.
Supported only on linux/amd64, freebsd/amd64, darwin/amd64 and windows/amd64.
-msan
enable interoperation with memory sanitizer.
Supported only on linux/amd64.
-v
print the names of packages as they are compiled.
-work
@ -112,13 +116,14 @@ and test commands:
a suffix to use in the name of the package installation directory,
in order to keep output separate from default builds.
If using the -race flag, the install suffix is automatically set to race
or, if set explicitly, has _race appended to it. Using a -buildmode
option that requires non-default compile flags has a similar effect.
or, if set explicitly, has _race appended to it. Likewise for the -msan
flag. Using a -buildmode option that requires non-default compile flags
has a similar effect.
-ldflags 'flag list'
arguments to pass on each go tool link invocation.
-linkshared
link against shared libraries previously created with
-buildmode=shared
-buildmode=shared.
-pkgdir dir
install and load all packages from dir instead of the usual locations.
For example, when building with a non-standard configuration,
@ -225,12 +230,17 @@ which is schematically one of these:
go doc <pkg>
go doc <sym>[.<method>]
go doc [<pkg>].<sym>[.<method>]
go doc [<pkg>.]<sym>[.<method>]
go doc [<pkg>.][<sym>.]<method>
The first item in this list matched by the argument is the one whose
documentation is printed. (See the examples below.) For packages, the order of
scanning is determined lexically, but the GOROOT tree is always scanned before
GOPATH.
The first item in this list matched by the argument is the one whose documentation
is printed. (See the examples below.) However, if the argument starts with a capital
letter it is assumed to identify a symbol or method in the current directory.
For packages, the order of scanning is determined lexically in breadth-first order.
That is, the package presented is the one that matches the search and is nearest
the root and lexically first at its level of the hierarchy. The GOROOT tree is
always scanned in its entirety before GOPATH.
If there is no package specified or matched, the package in the current
directory is selected, so "go doc Foo" shows the documentation for symbol Foo in
@ -278,6 +288,14 @@ Examples:
go doc text/template new # Two arguments
Show documentation for text/template's New function.
At least in the current tree, these invocations all print the
documentation for json.Decoder's Decode method:
go doc json.Decoder.Decode
go doc json.decoder.decode
go doc json.decode
cd go/src/encoding/json; go doc decode
Flags:
-c
Respect case when matching symbols.
@ -344,7 +362,7 @@ Generate Go files by processing source
Usage:
go generate [-run regexp] [file.go... | packages]
go generate [-run regexp] [-n] [-v] [-x] [build flags] [file.go... | packages]
Generate runs commands described by directives within existing
files. Those commands can run any process but the intent is to
@ -436,12 +454,14 @@ Go generate accepts one specific flag:
any trailing spaces and final newline) matches the
expression.
It also accepts the standard build flags -v, -n, and -x.
It also accepts the standard build flags including -v, -n, and -x.
The -v flag prints the names of packages and files as they are
processed.
The -n flag prints commands that would be executed.
The -x flag prints commands as they are executed.
For more about build flags, see 'go help build'.
For more about specifying packages, see 'go help packages'.
@ -477,16 +497,22 @@ missing packages but does not use it to look for updates to existing packages.
Get also accepts build flags to control the installation. See 'go help build'.
When checking out a new package, get creates the target directory
GOPATH/src/<import-path>. If the GOPATH contains multiple entries,
get uses the first one. See 'go help gopath'.
When checking out or updating a package, get looks for a branch or tag
that matches the locally installed version of Go. The most important
rule is that if the local installation is running version "go1", get
searches for a branch or tag named "go1". If no such version exists it
retrieves the most recent version of the package.
If the vendoring experiment is enabled (see 'go help gopath'),
then when go get checks out or updates a Git repository,
Unless vendoring support is disabled (see 'go help gopath'),
when go get checks out or updates a Git repository,
it also updates any git submodules referenced by the repository.
Get never checks out or updates code stored in vendor directories.
For more about specifying packages, see 'go help packages'.
For more about how 'go get' finds source code to
@ -577,6 +603,14 @@ syntax of package template. The default output is equivalent to -f
XTestImports []string // imports from XTestGoFiles
}
The error information, if any, is
type PackageError struct {
ImportStack []string // shortest path from package named on command line to this one
Pos string // position of error (if present, file:line:col)
Err string // the error itself
}
The template function "join" calls strings.Join.
The template function "context" returns the build context, defined as:
@ -643,7 +677,7 @@ Test packages
Usage:
go test [-c] [-i] [build and test flags] [packages] [flags for test binary]
go test [build/test flags] [packages] [build/test flags & test binary flags]
'Go test' automates testing the packages named by the import paths.
It prints a summary of the test results in the format:
@ -673,10 +707,16 @@ non-test installation.
In addition to the build flags, the flags handled by 'go test' itself are:
-args
Pass the remainder of the command line (everything after -args)
to the test binary, uninterpreted and unchanged.
Because this flag consumes the remainder of the command line,
the package list (if present) must appear before this flag.
-c
Compile the test binary to pkg.test but do not run it
(where pkg is the last element of the package's import path).
The file name can be changed with the -o flag.
Compile the test binary to pkg.test but do not run it
(where pkg is the last element of the package's import path).
The file name can be changed with the -o flag.
-exec xprog
Run the test binary using xprog. The behavior is the same as
@ -687,17 +727,12 @@ In addition to the build flags, the flags handled by 'go test' itself are:
Do not run the test.
-o file
Compile the test binary to the named file.
The test still runs (unless -c or -i is specified).
Compile the test binary to the named file.
The test still runs (unless -c or -i is specified).
The test binary also accepts flags that control execution of the test; these
flags are also accessible by 'go test'. See 'go help testflag' for details.
If the test binary needs any other flags, they should be presented after the
package names. The go tool treats as a flag the first argument that begins with
a minus sign that it does not recognize itself; that argument and all subsequent
arguments are passed as arguments to the test binary.
For more about build flags, see 'go help build'.
For more about specifying packages, see 'go help packages'.
@ -804,6 +839,11 @@ are:
Build the listed main packages and everything they import into
executables. Packages not named main are ignored.
-buildmode=pie
Build the listed main packages and everything they import into
position independent executables (PIE). Packages not named
main are ignored.
File types
@ -875,7 +915,7 @@ DIR/bin/quux, not DIR/bin/foo/quux. The "foo/" prefix is stripped
so that you can add DIR/bin to your PATH to get at the
installed commands. If the GOBIN environment variable is
set, commands are installed to the directory it names instead
of DIR/bin.
of DIR/bin. GOBIN must be an absolute path.
Here's an example directory layout:
@ -933,13 +973,10 @@ See https://golang.org/s/go14internal for details.
Vendor Directories
Go 1.5 includes experimental support for using local copies
of external dependencies to satisfy imports of those dependencies,
often referred to as vendoring. Setting the environment variable
GO15VENDOREXPERIMENT=1 enables that experimental support.
Go 1.6 includes support for using local copies of external dependencies
to satisfy imports of those dependencies, often referred to as vendoring.
When the vendor experiment is enabled,
code below a directory named "vendor" is importable only
Code below a directory named "vendor" is importable only
by code in the directory tree rooted at the parent of "vendor",
and only using an import path that omits the prefix up to and
including the vendor element.
@ -977,12 +1014,18 @@ top-level "crash/bang".
Code in vendor directories is not subject to import path
checking (see 'go help importpath').
When the vendor experiment is enabled, 'go get' checks out
submodules when checking out or updating a git repository
(see 'go help get').
When 'go get' checks out or updates a git repository, it now also
updates submodules.
The vendoring semantics are an experiment, and they may change
in future releases. Once settled, they will be on by default.
Vendor directories do not affect the placement of new repositories
being checked out for the first time by 'go get': those are always
placed in the main GOPATH, never in a vendor subtree.
In Go 1.5, as an experiment, setting the environment variable
GO15VENDOREXPERIMENT=1 enabled these features.
As of Go 1.6 they are on by default. To turn them off, set
GO15VENDOREXPERIMENT=0. In Go 1.7, the environment
variable will stop having any effect.
See https://golang.org/s/go15vendor for details.
@ -1051,7 +1094,7 @@ Special-purpose environment variables:
File names in stack traces are rewritten from GOROOT to
GOROOT_FINAL.
GO15VENDOREXPERIMENT
Set to 1 to enable the Go 1.5 vendoring experiment.
Set to 0 to disable vendoring semantics.
GO_EXTLINK_ENABLED
Whether the linker should use external linking mode
when using -linkmode=auto with code that uses cgo.
@ -1227,10 +1270,10 @@ unless it is being referred to by that import path. In this way, import comments
let package authors make sure the custom import path is used and not a
direct path to the underlying code hosting site.
If the vendoring experiment is enabled (see 'go help gopath'),
then import path checking is disabled for code found within vendor trees.
This makes it possible to copy code into alternate locations in vendor trees
without needing to update import comments.
If vendoring is enabled (see 'go help gopath'), then import path checking is
disabled for code found within vendor trees. This makes it possible to copy
code into alternate locations in vendor trees without needing to update import
comments.
See https://golang.org/s/go14customimport for details.
@ -1286,6 +1329,14 @@ internally at Google all begin with 'google', and paths
denoting remote repositories begin with the path to the code,
such as 'github.com/user/repo'.
Packages in a program need not have unique package names,
but there are two reserved package names with special meaning.
The name main indicates a command, not a library.
Commands are built into binaries and cannot be imported.
The name documentation indicates documentation for
a non-Go program in the directory. Files in package documentation
are ignored by the go command.
As a special case, if the package list is a list of .go files from a
single directory, the command is applied to a single synthesized
package made up of exactly those files, ignoring any build constraints
@ -1391,6 +1442,10 @@ control the execution of any test:
Allow parallel execution of test functions that call t.Parallel.
The value of this flag is the maximum number of tests to run
simultaneously; by default, it is set to the value of GOMAXPROCS.
Note that -parallel only applies within a single test binary.
The 'go test' command may run tests for different packages
in parallel as well, according to the setting of the -p flag
(see 'go help build').
-run regexp
Run only those tests and examples matching the regular
@ -1414,25 +1469,63 @@ control the execution of any test:
Verbose output: log all tests as they are run. Also print all
text from Log and Logf calls even if the test succeeds.
The test binary, called pkg.test where pkg is the name of the
directory containing the package sources, can be invoked directly
after building it with 'go test -c'. When invoking the test binary
directly, each of the standard flag names must be prefixed with 'test.',
as in -test.run=TestMyFunc or -test.v.
Each of these flags is also recognized with an optional 'test.' prefix,
as in -test.v. When invoking the generated test binary (the result of
'go test -c') directly, however, the prefix is mandatory.
When running 'go test', flags not listed above are passed through
unaltered. For instance, the command
The 'go test' command rewrites or removes recognized flags,
as appropriate, both before and after the optional package list,
before invoking the test binary.
go test -x -v -cpuprofile=prof.out -dir=testdata -update
For instance, the command
go test -v -myflag testdata -cpuprofile=prof.out -x
will compile the test binary and then run it as
pkg.test -test.v -test.cpuprofile=prof.out -dir=testdata -update
pkg.test -test.v -myflag testdata -test.cpuprofile=prof.out
(The -x flag is removed because it applies only to the go command's
execution, not to the test itself.)
The test flags that generate profiles (other than for coverage) also
leave the test binary in pkg.test for use when analyzing the profiles.
Flags not recognized by 'go test' must be placed after any specified packages.
When 'go test' runs a test binary, it does so from within the
corresponding package's source code directory. Depending on the test,
it may be necessary to do the same when invoking a generated test
binary directly.
The command-line package list, if present, must appear before any
flag not known to the go test command. Continuing the example above,
the package list would have to appear before -myflag, but could appear
on either side of -v.
To keep an argument for a test binary from being interpreted as a
known flag or a package name, use -args (see 'go help test') which
passes the remainder of the command line through to the test binary
uninterpreted and unaltered.
For instance, the command
go test -v -args -x -v
will compile the test binary and then run it as
pkg.test -test.v -x -v
Similarly,
go test -args math
will compile the test binary and then run it as
pkg.test math
In the first example, the -x and the second -v are passed through to the
test binary unchanged and with no effect on the go command itself.
In the second example, the argument math is passed through to the test
binary, instead of being interpreted as the package list.
Description of testing functions

499
libgo/go/cmd/go/build.go
View file

@ -63,12 +63,16 @@ and test commands:
-n
print the commands but do not run them.
-p n
the number of builds that can be run in parallel.
the number of programs, such as build commands or
test binaries, that can be run in parallel.
The default is the number of CPUs available, except
on darwin/arm which defaults to 1.
-race
enable data race detection.
Supported only on linux/amd64, freebsd/amd64, darwin/amd64 and windows/amd64.
-msan
enable interoperation with memory sanitizer.
Supported only on linux/amd64.
-v
print the names of packages as they are compiled.
-work
@ -91,13 +95,14 @@ and test commands:
a suffix to use in the name of the package installation directory,
in order to keep output separate from default builds.
If using the -race flag, the install suffix is automatically set to race
or, if set explicitly, has _race appended to it. Using a -buildmode
option that requires non-default compile flags has a similar effect.
or, if set explicitly, has _race appended to it. Likewise for the -msan
flag. Using a -buildmode option that requires non-default compile flags
has a similar effect.
-ldflags 'flag list'
arguments to pass on each go tool link invocation.
-linkshared
link against shared libraries previously created with
-buildmode=shared
-buildmode=shared.
-pkgdir dir
install and load all packages from dir instead of the usual locations.
For example, when building with a non-standard configuration,
@ -166,6 +171,7 @@ var buildGcflags []string // -gcflags flag
var buildLdflags []string // -ldflags flag
var buildGccgoflags []string // -gccgoflags flag
var buildRace bool // -race flag
var buildMSan bool // -msan flag
var buildToolExec []string // -toolexec flag
var buildBuildmode string // -buildmode flag
var buildLinkshared bool // -linkshared flag
@ -227,6 +233,7 @@ func addBuildFlags(cmd *Command) {
cmd.Flag.BoolVar(&buildLinkshared, "linkshared", false, "")
cmd.Flag.StringVar(&buildPkgdir, "pkgdir", "", "")
cmd.Flag.BoolVar(&buildRace, "race", false, "")
cmd.Flag.BoolVar(&buildMSan, "msan", false, "")
cmd.Flag.Var((*stringsFlag)(&buildContext.BuildTags), "tags", "")
cmd.Flag.Var((*stringsFlag)(&buildToolExec), "toolexec", "")
cmd.Flag.BoolVar(&buildWork, "work", false, "")
@ -352,29 +359,46 @@ func buildModeInit() {
codegenArg = "-fPIC"
} else {
switch platform {
case "linux/amd64":
case "linux/amd64", "linux/arm", "linux/arm64", "linux/386",
"android/amd64", "android/arm", "android/arm64", "android/386":
codegenArg = "-shared"
case "linux/arm":
buildAsmflags = append(buildAsmflags, "-shared")
case "darwin/amd64":
case "android/arm":
case "darwin/amd64", "darwin/386":
default:
fatalf("-buildmode=c-shared not supported on %s\n", platform)
}
}
ldBuildmode = "c-shared"
case "default":
ldBuildmode = "exe"
switch platform {
case "android/arm", "android/arm64", "android/amd64", "android/386":
codegenArg = "-shared"
ldBuildmode = "pie"
default:
ldBuildmode = "exe"
}
case "exe":
pkgsFilter = pkgsMain
ldBuildmode = "exe"
case "pie":
if gccgo {
fatalf("-buildmode=pie not supported by gccgo")
} else {
switch platform {
case "linux/386", "linux/amd64", "linux/arm", "linux/arm64", "linux/ppc64le",
"android/amd64", "android/arm", "android/arm64", "android/386":
codegenArg = "-shared"
default:
fatalf("-buildmode=pie not supported on %s\n", platform)
}
}
ldBuildmode = "pie"
case "shared":
pkgsFilter = pkgsNotMain
if gccgo {
codegenArg = "-fPIC"
} else {
switch platform {
case "linux/amd64":
case "linux/386", "linux/amd64", "linux/arm", "linux/arm64", "linux/ppc64le":
default:
fatalf("-buildmode=shared not supported on %s\n", platform)
}
@ -391,9 +415,11 @@ func buildModeInit() {
if gccgo {
codegenArg = "-fPIC"
} else {
if platform != "linux/amd64" {
fmt.Fprintf(os.Stderr, "go %s: -linkshared is only supported on linux/amd64\n", flag.Args()[0])
os.Exit(2)
switch platform {
case "linux/386", "linux/amd64", "linux/arm", "linux/arm64", "linux/ppc64le":
buildAsmflags = append(buildAsmflags, "-D=GOBUILDMODE_shared=1")
default:
fatalf("-linkshared not supported on %s\n", platform)
}
codegenArg = "-dynlink"
// TODO(mwhudson): remove -w when that gets fixed in linker.
@ -415,7 +441,7 @@ func buildModeInit() {
}
func runBuild(cmd *Command, args []string) {
raceInit()
instrumentInit()
buildModeInit()
var b builder
b.init()
@ -463,7 +489,12 @@ func runBuild(cmd *Command, args []string) {
var a *action
if buildBuildmode == "shared" {
a = b.libaction(libname(args), pkgsFilter(packages(args)), modeBuild, depMode)
pkgs := pkgsFilter(packages(args))
if libName, err := libname(args, pkgs); err != nil {
fatalf("%s", err.Error())
} else {
a = b.libaction(libName, pkgs, modeBuild, depMode)
}
} else {
a = &action{}
for _, p := range pkgsFilter(packages(args)) {
@ -487,32 +518,73 @@ See also: go build, go get, go clean.
`,
}
// isMetaPackage checks if name is a reserved package name that expands to multiple packages
func isMetaPackage(name string) bool {
return name == "std" || name == "cmd" || name == "all"
}
// libname returns the filename to use for the shared library when using
// -buildmode=shared. The rules we use are:
// 1) Drop any trailing "/..."s if present
// 2) Change / to -
// 3) Join arguments with ,
// So std -> libstd.so
// a b/... -> liba,b.so
// gopkg.in/tomb.v2 -> libgopkg.in-tomb.v2.so
func libname(args []string) string {
// Use arguments for special 'meta' packages:
// std --> libstd.so
// std cmd --> libstd,cmd.so
// A single non-meta argument with trailing "/..." is special cased:
// foo/... --> libfoo.so
// (A relative path like "./..." expands the "." first)
// Use import paths for other cases, changing '/' to '-':
// somelib --> libsubdir-somelib.so
// ./ or ../ --> libsubdir-somelib.so
// gopkg.in/tomb.v2 -> libgopkg.in-tomb.v2.so
// a/... b/... ---> liba/c,b/d.so - all matching import paths
// Name parts are joined with ','.
func libname(args []string, pkgs []*Package) (string, error) {
var libname string
for _, arg := range args {
arg = strings.TrimSuffix(arg, "/...")
arg = strings.Replace(arg, "/", "-", -1)
appendName := func(arg string) {
if libname == "" {
libname = arg
} else {
libname += "," + arg
}
}
var haveNonMeta bool
for _, arg := range args {
if isMetaPackage(arg) {
appendName(arg)
} else {
haveNonMeta = true
}
}
if len(libname) == 0 { // non-meta packages only. use import paths
if len(args) == 1 && strings.HasSuffix(args[0], "/...") {
// Special case of "foo/..." as mentioned above.
arg := strings.TrimSuffix(args[0], "/...")
if build.IsLocalImport(arg) {
cwd, _ := os.Getwd()
bp, _ := buildContext.ImportDir(filepath.Join(cwd, arg), build.FindOnly)
if bp.ImportPath != "" && bp.ImportPath != "." {
arg = bp.ImportPath
}
}
appendName(strings.Replace(arg, "/", "-", -1))
} else {
for _, pkg := range pkgs {
appendName(strings.Replace(pkg.ImportPath, "/", "-", -1))
}
}
} else if haveNonMeta { // have both meta package and a non-meta one
return "", errors.New("mixing of meta and non-meta packages is not allowed")
}
// TODO(mwhudson): Needs to change for platforms that use different naming
// conventions...
return "lib" + libname + ".so"
return "lib" + libname + ".so", nil
}
func runInstall(cmd *Command, args []string) {
raceInit()
if gobin != "" && !filepath.IsAbs(gobin) {
fatalf("cannot install, GOBIN must be an absolute path")
}
instrumentInit()
buildModeInit()
pkgs := pkgsFilter(packagesForBuild(args))
@ -537,7 +609,11 @@ func runInstall(cmd *Command, args []string) {
b.init()
var a *action
if buildBuildmode == "shared" {
a = b.libaction(libname(args), pkgs, modeInstall, modeInstall)
if libName, err := libname(args, pkgs); err != nil {
fatalf("%s", err.Error())
} else {
a = b.libaction(libName, pkgs, modeInstall, modeInstall)
}
} else {
a = &action{}
var tools []*action
@ -754,7 +830,9 @@ func goFilesPackage(gofiles []string) *Package {
pkg := new(Package)
pkg.local = true
pkg.cmdline = true
stk.push("main")
pkg.load(&stk, bp, err)
stk.pop()
pkg.localPrefix = dirToImportPath(dir)
pkg.ImportPath = "command-line-arguments"
pkg.target = ""
@ -812,15 +890,17 @@ func readpkglist(shlibpath string) (pkgs []*Package) {
// action returns the action for applying the given operation (mode) to the package.
// depMode is the action to use when building dependencies.
// action never looks for p in a shared library.
// action never looks for p in a shared library, but may find p's dependencies in a
// shared library if buildLinkshared is true.
func (b *builder) action(mode buildMode, depMode buildMode, p *Package) *action {
return b.action1(mode, depMode, p, false)
return b.action1(mode, depMode, p, false, "")
}
// action1 returns the action for applying the given operation (mode) to the package.
// depMode is the action to use when building dependencies.
// action1 will look for p in a shared library if lookshared is true.
func (b *builder) action1(mode buildMode, depMode buildMode, p *Package, lookshared bool) *action {
// forShlib is the shared library that p will become part of, if any.
func (b *builder) action1(mode buildMode, depMode buildMode, p *Package, lookshared bool, forShlib string) *action {
shlib := ""
if lookshared {
shlib = p.Shlib
@ -852,13 +932,23 @@ func (b *builder) action1(mode buildMode, depMode buildMode, p *Package, looksha
b.actionCache[key] = a
for _, p1 := range p.imports {
ls := buildLinkshared
// If p1 is part of the same shared library as p, we need the action
// that builds p here, not the shared libary or we get action loops.
if p1.Shlib == p.Shlib {
ls = false
if forShlib != "" {
// p is part of a shared library.
if p1.Shlib != "" && p1.Shlib != forShlib {
// p1 is explicitly part of a different shared library.
// Put the action for that shared library into a.deps.
a.deps = append(a.deps, b.action1(depMode, depMode, p1, true, p1.Shlib))
} else {
// p1 is (implicitly or not) part of this shared library.
// Put the action for p1 into a.deps.
a.deps = append(a.deps, b.action1(depMode, depMode, p1, false, forShlib))
}
} else {
// p is not part of a shared library.
// If p1 is in a shared library, put the action for that into
// a.deps, otherwise put the action for p1 into a.deps.
a.deps = append(a.deps, b.action1(depMode, depMode, p1, buildLinkshared, p1.Shlib))
}
a.deps = append(a.deps, b.action1(depMode, depMode, p1, ls))
}
// If we are not doing a cross-build, then record the binary we'll
@ -866,7 +956,7 @@ func (b *builder) action1(mode buildMode, depMode buildMode, p *Package, looksha
// using cgo, to make sure we do not overwrite the binary while
// a package is using it. If this is a cross-build, then the cgo we
// are writing is not the cgo we need to use.
if goos == runtime.GOOS && goarch == runtime.GOARCH && !buildRace && reqStdPkgSrc {
if goos == runtime.GOOS && goarch == runtime.GOARCH && !buildRace && !buildMSan && reqStdPkgSrc {
if (len(p.CgoFiles) > 0 || p.Standard && p.ImportPath == "runtime/cgo") && !buildLinkshared && buildBuildmode != "shared" {
var stk importStack
p1 := loadPackage("cmd/cgo", &stk)
@ -914,18 +1004,27 @@ func (b *builder) action1(mode buildMode, depMode buildMode, p *Package, looksha
switch mode {
case modeInstall:
a.f = (*builder).install
a.deps = []*action{b.action1(modeBuild, depMode, p, lookshared)}
a.deps = []*action{b.action1(modeBuild, depMode, p, lookshared, forShlib)}
a.target = a.p.target
// Install header for cgo in c-archive and c-shared modes.
if p.usesCgo() && (buildBuildmode == "c-archive" || buildBuildmode == "c-shared") {
hdrTarget := a.target[:len(a.target)-len(filepath.Ext(a.target))] + ".h"
if buildContext.Compiler == "gccgo" {
// For the header file, remove the "lib"
// added by go/build, so we generate pkg.h
// rather than libpkg.h.
dir, file := filepath.Split(hdrTarget)
file = strings.TrimPrefix(file, "lib")
hdrTarget = filepath.Join(dir, file)
}
ah := &action{
p: a.p,
deps: []*action{a.deps[0]},
f: (*builder).installHeader,
pkgdir: a.pkgdir,
objdir: a.objdir,
target: a.target[:len(a.target)-len(filepath.Ext(a.target))] + ".h",
target: hdrTarget,
}
a.deps = append(a.deps, ah)
}
@ -961,7 +1060,11 @@ func (b *builder) action1(mode buildMode, depMode buildMode, p *Package, looksha
func (b *builder) libaction(libname string, pkgs []*Package, mode, depMode buildMode) *action {
a := &action{}
if mode == modeBuild {
switch mode {
default:
fatalf("unrecognized mode %v", mode)
case modeBuild:
a.f = (*builder).linkShared
a.target = filepath.Join(b.work, libname)
for _, p := range pkgs {
@ -970,14 +1073,15 @@ func (b *builder) libaction(libname string, pkgs []*Package, mode, depMode build
}
a.deps = append(a.deps, b.action(depMode, depMode, p))
}
} else if mode == modeInstall {
case modeInstall:
// Currently build mode shared forces external linking mode, and
// external linking mode forces an import of runtime/cgo. So if it
// was not passed on the command line and it is not present in
// another shared library, add it here.
seencgo := false
// external linking mode forces an import of runtime/cgo (and
// math on arm). So if it was not passed on the command line and
// it is not present in another shared library, add it here.
_, gccgo := buildToolchain.(gccgoToolchain)
if !gccgo {
seencgo := false
for _, p := range pkgs {
seencgo = seencgo || (p.Standard && p.ImportPath == "runtime/cgo")
}
@ -997,6 +1101,28 @@ func (b *builder) libaction(libname string, pkgs []*Package, mode, depMode build
pkgs = append(pkgs, p)
}
}
if goarch == "arm" {
seenmath := false
for _, p := range pkgs {
seenmath = seenmath || (p.Standard && p.ImportPath == "math")
}
if !seenmath {
var stk importStack
p := loadPackage("math", &stk)
if p.Error != nil {
fatalf("load math: %v", p.Error)
}
computeStale(p)
// If math is in another shared library, then that's
// also the shared library that contains runtime, so
// something will depend on it and so math's staleness
// will be checked when processing that library.
if p.Shlib == "" || p.Shlib == libname {
pkgs = append([]*Package{}, pkgs...)
pkgs = append(pkgs, p)
}
}
}
}
// Figure out where the library will go.
@ -1029,7 +1155,7 @@ func (b *builder) libaction(libname string, pkgs []*Package, mode, depMode build
if err != nil || lstat.ModTime().After(built) {
stale = true
}
a.deps = append(a.deps, b.action(depMode, depMode, p))
a.deps = append(a.deps, b.action1(depMode, depMode, p, false, a.target))
}
if stale {
@ -1047,8 +1173,6 @@ func (b *builder) libaction(libname string, pkgs []*Package, mode, depMode build
shlibnameaction.deps = append(shlibnameaction.deps, buildAction)
}
}
} else {
fatalf("unregonized mode %v", mode)
}
return a
}
@ -1239,17 +1363,11 @@ func (b *builder) build(a *action) (err error) {
// different sections of the bootstrap script have to
// be merged, the banners give patch something
// to use to find its context.
fmt.Printf("\n#\n# %s\n#\n\n", a.p.ImportPath)
b.print("\n#\n# " + a.p.ImportPath + "\n#\n\n")
}
if buildV {
fmt.Fprintf(os.Stderr, "%s\n", a.p.ImportPath)
}
if a.p.Standard && a.p.ImportPath == "runtime" && buildContext.Compiler == "gc" &&
(!hasString(a.p.GoFiles, "zgoos_"+buildContext.GOOS+".go") ||
!hasString(a.p.GoFiles, "zgoarch_"+buildContext.GOARCH+".go")) {
return fmt.Errorf("%s/%s must be bootstrapped using make%v", buildContext.GOOS, buildContext.GOARCH, defaultSuffix())
b.print(a.p.ImportPath + "\n")
}
// Make build directory.
@ -1393,17 +1511,17 @@ func (b *builder) build(a *action) (err error) {
switch {
case strings.HasSuffix(name, _goos_goarch):
targ := file[:len(name)-len(_goos_goarch)] + "_GOOS_GOARCH." + ext
if err := b.copyFile(a, obj+targ, filepath.Join(a.p.Dir, file), 0644, true); err != nil {
if err := b.copyFile(a, obj+targ, filepath.Join(a.p.Dir, file), 0666, true); err != nil {
return err
}
case strings.HasSuffix(name, _goarch):
targ := file[:len(name)-len(_goarch)] + "_GOARCH." + ext
if err := b.copyFile(a, obj+targ, filepath.Join(a.p.Dir, file), 0644, true); err != nil {
if err := b.copyFile(a, obj+targ, filepath.Join(a.p.Dir, file), 0666, true); err != nil {
return err
}
case strings.HasSuffix(name, _goos):
targ := file[:len(name)-len(_goos)] + "_GOOS." + ext
if err := b.copyFile(a, obj+targ, filepath.Join(a.p.Dir, file), 0644, true); err != nil {
if err := b.copyFile(a, obj+targ, filepath.Join(a.p.Dir, file), 0666, true); err != nil {
return err
}
}
@ -1492,7 +1610,7 @@ func (b *builder) getPkgConfigFlags(p *Package) (cflags, ldflags []string, err e
func (b *builder) installShlibname(a *action) error {
a1 := a.deps[0]
err := ioutil.WriteFile(a.target, []byte(filepath.Base(a1.target)+"\n"), 0644)
err := ioutil.WriteFile(a.target, []byte(filepath.Base(a1.target)+"\n"), 0666)
if err != nil {
return err
}
@ -1516,12 +1634,12 @@ func (b *builder) install(a *action) (err error) {
}
}()
a1 := a.deps[0]
perm := os.FileMode(0644)
perm := os.FileMode(0666)
if a1.link {
switch buildBuildmode {
case "c-archive", "c-shared":
default:
perm = 0755
perm = 0777
}
}
@ -1595,7 +1713,25 @@ func (b *builder) moveOrCopyFile(a *action, dst, src string, perm os.FileMode, f
// If we can update the mode and rename to the dst, do it.
// Otherwise fall back to standard copy.
if err := os.Chmod(src, perm); err == nil {
// The perm argument is meant to be adjusted according to umask,
// but we don't know what the umask is.
// Create a dummy file to find out.
// This avoids build tags and works even on systems like Plan 9
// where the file mask computation incorporates other information.
mode := perm
f, err := os.OpenFile(filepath.Clean(dst)+"-go-tmp-umask", os.O_WRONLY|os.O_CREATE|os.O_EXCL, perm)
if err == nil {
fi, err := f.Stat()
if err == nil {
mode = fi.Mode() & 0777
}
name := f.Name()
f.Close()
os.Remove(name)
}
if err := os.Chmod(src, mode); err == nil {
if err := os.Rename(src, dst); err == nil {
if buildX {
b.showcmd("", "mv %s %s", src, dst)
@ -1629,7 +1765,7 @@ func (b *builder) copyFile(a *action, dst, src string, perm os.FileMode, force b
if fi.IsDir() {
return fmt.Errorf("build output %q already exists and is a directory", dst)
}
if !force && !isObject(dst) {
if !force && fi.Mode().IsRegular() && !isObject(dst) {
return fmt.Errorf("build output %q already exists and is not an object file", dst)
}
}
@ -1641,7 +1777,7 @@ func (b *builder) copyFile(a *action, dst, src string, perm os.FileMode, force b
}
}
os.Remove(dst)
mayberemovefile(dst)
df, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm)
if err != nil && toolIsWindows {
// Windows does not allow deletion of a binary file
@ -1660,7 +1796,7 @@ func (b *builder) copyFile(a *action, dst, src string, perm os.FileMode, force b
_, err = io.Copy(df, sf)
df.Close()
if err != nil {
os.Remove(dst)
mayberemovefile(dst)
return fmt.Errorf("copying %s to %s: %v", src, dst, err)
}
return nil
@ -1682,7 +1818,7 @@ func (b *builder) installHeader(a *action) error {
}
}
return b.moveOrCopyFile(a, a.target, src, 0644, true)
return b.moveOrCopyFile(a, a.target, src, 0666, true)
}
// cover runs, in effect,
@ -1707,6 +1843,7 @@ var objectMagic = [][]byte{
{0x4d, 0x5a, 0x90, 0x00, 0x03, 0x00}, // PE (Windows) as generated by 6l/8l and gcc
{0x00, 0x00, 0x01, 0xEB}, // Plan 9 i386
{0x00, 0x00, 0x8a, 0x97}, // Plan 9 amd64
{0x00, 0x00, 0x06, 0x47}, // Plan 9 arm
}
func isObject(s string) bool {
@ -1725,6 +1862,16 @@ func isObject(s string) bool {
return false
}
// mayberemovefile removes a file only if it is a regular file
// When running as a user with sufficient privileges, we may delete
// even device files, for example, which is not intended.
func mayberemovefile(s string) {
if fi, err := os.Lstat(s); err == nil && !fi.Mode().IsRegular() {
return
}
os.Remove(s)
}
// fmtcmd formats a command in the manner of fmt.Sprintf but also:
//
// If dir is non-empty and the script is not in dir right now,
@ -2103,7 +2250,7 @@ func (gcToolchain) gc(b *builder, p *Package, archive, obj string, asmhdr bool,
if p.Name == "main" {
gcargs[1] = "main"
}
if p.Standard && p.ImportPath == "runtime" {
if p.Standard && (p.ImportPath == "runtime" || strings.HasPrefix(p.ImportPath, "runtime/internal")) {
// runtime compiles with a special gc flag to emit
// additional reflect type data.
gcargs = append(gcargs, "-+")
@ -2208,33 +2355,26 @@ func (gcToolchain) pack(b *builder, p *Package, objDir, afile string, ofiles []s
for _, f := range ofiles {
absOfiles = append(absOfiles, mkAbs(objDir, f))
}
cmd := "c"
absAfile := mkAbs(objDir, afile)
appending := false
if _, err := os.Stat(absAfile); err == nil {
appending = true
cmd = "r"
// The archive file should have been created by the compiler.
// Since it used to not work that way, verify.
if _, err := os.Stat(absAfile); err != nil {
fatalf("os.Stat of archive file failed: %v", err)
}
cmdline := stringList("pack", cmd, absAfile, absOfiles)
if appending {
if buildN || buildX {
b.showcmd(p.Dir, "%s # internal", joinUnambiguously(cmdline))
}
if buildN {
return nil
}
if err := packInternal(b, absAfile, absOfiles); err != nil {
b.showOutput(p.Dir, p.ImportPath, err.Error()+"\n")
return errPrintedOutput
}
if buildN || buildX {
cmdline := stringList("pack", "r", absAfile, absOfiles)
b.showcmd(p.Dir, "%s # internal", joinUnambiguously(cmdline))
}
if buildN {
return nil
}
// Need actual pack.
cmdline[0] = tool("pack")
return b.run(p.Dir, p.ImportPath, nil, buildToolExec, cmdline)
if err := packInternal(b, absAfile, absOfiles); err != nil {
b.showOutput(p.Dir, p.ImportPath, err.Error()+"\n")
return errPrintedOutput
}
return nil
}
func packInternal(b *builder, afile string, ofiles []string) error {
@ -2445,11 +2585,11 @@ func (tools gccgoToolchain) asm(b *builder, p *Package, obj, ofile, sfile string
sfile = mkAbs(p.Dir, sfile)
defs := []string{"-D", "GOOS_" + goos, "-D", "GOARCH_" + goarch}
if pkgpath := gccgoCleanPkgpath(p); pkgpath != "" {
defs = append(defs, `-D`, `GOPKGPATH="`+pkgpath+`"`)
defs = append(defs, `-D`, `GOPKGPATH=`+pkgpath)
}
defs = tools.maybePIC(defs)
defs = append(defs, b.gccArchArgs()...)
return b.run(p.Dir, p.ImportPath, nil, tools.compiler(), "-c", "-I", obj, "-o", ofile, defs, sfile)
return b.run(p.Dir, p.ImportPath, nil, tools.compiler(), "-xassembler-with-cpp", "-I", obj, "-c", "-o", ofile, defs, sfile)
}
func (gccgoToolchain) pkgpath(basedir string, p *Package) string {
@ -2464,7 +2604,7 @@ func (gccgoToolchain) pack(b *builder, p *Package, objDir, afile string, ofiles
for _, f := range ofiles {
absOfiles = append(absOfiles, mkAbs(objDir, f))
}
return b.run(p.Dir, p.ImportPath, nil, "ar", "cru", mkAbs(objDir, afile), absOfiles)
return b.run(p.Dir, p.ImportPath, nil, "ar", "rc", mkAbs(objDir, afile), absOfiles)
}
func (tools gccgoToolchain) ld(b *builder, root *action, out string, allactions []*action, mainpkg string, ofiles []string) error {
@ -2599,6 +2739,10 @@ func (tools gccgoToolchain) ld(b *builder, root *action, out string, allactions
// libffi.
ldflags = append(ldflags, "-Wl,-r", "-nostdlib", "-Wl,--whole-archive", "-lgolibbegin", "-Wl,--no-whole-archive")
if b.gccSupportsNoPie() {
ldflags = append(ldflags, "-no-pie")
}
// We are creating an object file, so we don't want a build ID.
ldflags = b.disableBuildID(ldflags)
@ -2606,7 +2750,7 @@ func (tools gccgoToolchain) ld(b *builder, root *action, out string, allactions
out = out + ".o"
case "c-shared":
ldflags = append(ldflags, "-shared", "-nostdlib", "-Wl,--whole-archive", "-lgolibbegin", "-Wl,--no-whole-archive", "-lgo", "-lgcc_s", "-lgcc")
ldflags = append(ldflags, "-shared", "-nostdlib", "-Wl,--whole-archive", "-lgolibbegin", "-Wl,--no-whole-archive", "-lgo", "-lgcc_s", "-lgcc", "-lc", "-lgcc")
default:
fatalf("-buildmode=%s not supported for gccgo", ldBuildmode)
@ -2704,43 +2848,6 @@ func gccgoCleanPkgpath(p *Package) string {
return strings.Map(clean, gccgoPkgpath(p))
}
// libgcc returns the filename for libgcc, as determined by invoking gcc with
// the -print-libgcc-file-name option.
func (b *builder) libgcc(p *Package) (string, error) {
var buf bytes.Buffer
gccCmd := b.gccCmd(p.Dir)
prev := b.print
if buildN {
// In -n mode we temporarily swap out the builder's
// print function to capture the command-line. This
// let's us assign it to $LIBGCC and produce a valid
// buildscript for cgo packages.
b.print = func(a ...interface{}) (int, error) {
return fmt.Fprint(&buf, a...)
}
}
f, err := b.runOut(p.Dir, p.ImportPath, nil, gccCmd, "-print-libgcc-file-name")
if err != nil {
return "", fmt.Errorf("gcc -print-libgcc-file-name: %v (%s)", err, f)
}
if buildN {
s := fmt.Sprintf("LIBGCC=$(%s)\n", buf.Next(buf.Len()-1))
b.print = prev
b.print(s)
return "$LIBGCC", nil
}
// The compiler might not be able to find libgcc, and in that case,
// it will simply return "libgcc.a", which is of no use to us.
if !filepath.IsAbs(string(f)) {
return "", nil
}
return strings.Trim(string(f), "\r\n"), nil
}
// gcc runs the gcc C compiler to create an object from a single C file.
func (b *builder) gcc(p *Package, out string, flags []string, cfile string) error {
return b.ccompile(p, out, flags, cfile, b.gccCmd(p.Dir))
@ -2827,6 +2934,36 @@ func (b *builder) ccompilerCmd(envvar, defcmd, objdir string) []string {
return a
}
// On systems with PIE (position independent executables) enabled by default,
// -no-pie must be passed when doing a partial link with -Wl,-r. But -no-pie is
// not supported by all compilers.
func (b *builder) gccSupportsNoPie() bool {
if goos != "linux" {
// On some BSD platforms, error messages from the
// compiler make it to the console despite cmd.Std*
// all being nil. As -no-pie is only required on linux
// systems so far, we only test there.
return false
}
src := filepath.Join(b.work, "trivial.c")
if err := ioutil.WriteFile(src, []byte{}, 0666); err != nil {
return false
}
cmdArgs := b.gccCmd(b.work)
cmdArgs = append(cmdArgs, "-no-pie", "-c", "trivial.c")
if buildN || buildX {
b.showcmd(b.work, "%s", joinUnambiguously(cmdArgs))
if buildN {
return false
}
}
cmd := exec.Command(cmdArgs[0], cmdArgs[1:]...)
cmd.Dir = b.work
cmd.Env = envForDir(cmd.Dir, os.Environ())
out, err := cmd.CombinedOutput()
return err == nil && !bytes.Contains(out, []byte("unrecognized"))
}
// gccArchArgs returns arguments to pass to gcc based on the architecture.
func (b *builder) gccArchArgs() []string {
switch goarch {
@ -2866,12 +3003,6 @@ func (b *builder) cflags(p *Package, def bool) (cppflags, cflags, cxxflags, ldfl
var cgoRe = regexp.MustCompile(`[/\\:]`)
var (
cgoLibGccFile string
cgoLibGccErr error
cgoLibGccFileOnce sync.Once
)
func (b *builder) cgo(p *Package, cgoExe, obj string, pcCFLAGS, pcLDFLAGS, cgofiles, gccfiles, gxxfiles, mfiles []string) (outGo, outObj []string, err error) {
cgoCPPFLAGS, cgoCFLAGS, cgoCXXFLAGS, cgoLDFLAGS := b.cflags(p, true)
_, cgoexeCFLAGS, _, _ := b.cflags(p, false)
@ -2882,11 +3013,16 @@ func (b *builder) cgo(p *Package, cgoExe, obj string, pcCFLAGS, pcLDFLAGS, cgofi
cgoLDFLAGS = append(cgoLDFLAGS, "-lobjc")
}
if buildMSan && p.ImportPath != "runtime/cgo" {
cgoCFLAGS = append([]string{"-fsanitize=memory"}, cgoCFLAGS...)
cgoLDFLAGS = append([]string{"-fsanitize=memory"}, cgoLDFLAGS...)
}
// Allows including _cgo_export.h from .[ch] files in the package.
cgoCPPFLAGS = append(cgoCPPFLAGS, "-I", obj)
// cgo
// TODO: CGOPKGPATH, CGO_FLAGS?
// TODO: CGO_FLAGS?
gofiles := []string{obj + "_cgo_gotypes.go"}
cfiles := []string{"_cgo_main.c", "_cgo_export.c"}
for _, fn := range cgofiles {
@ -2902,7 +3038,7 @@ func (b *builder) cgo(p *Package, cgoExe, obj string, pcCFLAGS, pcLDFLAGS, cgofi
if p.Standard && p.ImportPath == "runtime/cgo" {
cgoflags = append(cgoflags, "-import_runtime_cgo=false")
}
if p.Standard && (p.ImportPath == "runtime/race" || p.ImportPath == "runtime/cgo") {
if p.Standard && (p.ImportPath == "runtime/race" || p.ImportPath == "runtime/msan" || p.ImportPath == "runtime/cgo") {
cgoflags = append(cgoflags, "-import_syscall=false")
}
@ -2954,7 +3090,9 @@ func (b *builder) cgo(p *Package, cgoExe, obj string, pcCFLAGS, pcLDFLAGS, cgofi
var linkobj []string
var bareLDFLAGS []string
// filter out -lsomelib, -l somelib, *.{so,dll,dylib}, and (on Darwin) -framework X
// When linking relocatable objects, various flags need to be
// filtered out as they are inapplicable and can cause some linkers
// to fail.
for i := 0; i < len(cgoLDFLAGS); i++ {
f := cgoLDFLAGS[i]
switch {
@ -2970,7 +3108,6 @@ func (b *builder) cgo(p *Package, cgoExe, obj string, pcCFLAGS, pcLDFLAGS, cgofi
case strings.HasSuffix(f, ".dylib"),
strings.HasSuffix(f, ".so"),
strings.HasSuffix(f, ".dll"):
continue
// Remove any -fsanitize=foo flags.
// Otherwise the compiler driver thinks that we are doing final link
// and links sanitizer runtime into the object file. But we are not doing
@ -2979,27 +3116,27 @@ func (b *builder) cgo(p *Package, cgoExe, obj string, pcCFLAGS, pcLDFLAGS, cgofi
// See issue 8788 for details.
case strings.HasPrefix(f, "-fsanitize="):
continue
// runpath flags not applicable unless building a shared
// object or executable; see issue 12115 for details. This
// is necessary as Go currently does not offer a way to
// specify the set of LDFLAGS that only apply to shared
// objects.
case strings.HasPrefix(f, "-Wl,-rpath"):
if f == "-Wl,-rpath" || f == "-Wl,-rpath-link" {
// Skip following argument to -rpath* too.
i++
}
default:
bareLDFLAGS = append(bareLDFLAGS, f)
}
}
cgoLibGccFileOnce.Do(func() {
cgoLibGccFile, cgoLibGccErr = b.libgcc(p)
})
if cgoLibGccFile == "" && cgoLibGccErr != nil {
return nil, nil, err
}
var staticLibs []string
if goos == "windows" {
// libmingw32 and libmingwex might also use libgcc, so libgcc must come last,
// and they also have some inter-dependencies, so must use linker groups.
// libmingw32 and libmingwex have some inter-dependencies,
// so must use linker groups.
staticLibs = []string{"-Wl,--start-group", "-lmingwex", "-lmingw32", "-Wl,--end-group"}
}
if cgoLibGccFile != "" {
staticLibs = append(staticLibs, cgoLibGccFile)
}
cflags := stringList(cgoCPPFLAGS, cgoCFLAGS)
for _, cfile := range cfiles {
@ -3045,7 +3182,7 @@ func (b *builder) cgo(p *Package, cgoExe, obj string, pcCFLAGS, pcLDFLAGS, cgofi
linkobj = append(linkobj, p.SysoFiles...)
dynobj := obj + "_cgo_.o"
pie := goarch == "arm" && (goos == "linux" || goos == "android")
pie := (goarch == "arm" && goos == "linux") || goos == "android"
if pie { // we need to use -pie for Linux/ARM to get accurate imported sym
cgoLDFLAGS = append(cgoLDFLAGS, "-pie")
}
@ -3083,6 +3220,10 @@ func (b *builder) cgo(p *Package, cgoExe, obj string, pcCFLAGS, pcLDFLAGS, cgofi
}
ldflags := stringList(bareLDFLAGS, "-Wl,-r", "-nostdlib", staticLibs)
if b.gccSupportsNoPie() {
ldflags = append(ldflags, "-no-pie")
}
// We are creating an object file, so we don't want a build ID.
ldflags = b.disableBuildID(ldflags)
@ -3217,7 +3358,7 @@ func (b *builder) swigIntSize(obj string) (intsize string, err error) {
return "$INTBITS", nil
}
src := filepath.Join(b.work, "swig_intsize.go")
if err = ioutil.WriteFile(src, []byte(swigIntSizeCode), 0644); err != nil {
if err = ioutil.WriteFile(src, []byte(swigIntSizeCode), 0666); err != nil {
return
}
srcs := []string{src}
@ -3339,32 +3480,38 @@ func (q *actionQueue) pop() *action {
return heap.Pop(q).(*action)
}
func raceInit() {
if !buildRace {
func instrumentInit() {
if !buildRace && !buildMSan {
return
}
if goarch != "amd64" || goos != "linux" && goos != "freebsd" && goos != "darwin" && goos != "windows" {
fmt.Fprintf(os.Stderr, "go %s: -race is only supported on linux/amd64, freebsd/amd64, darwin/amd64 and windows/amd64\n", flag.Args()[0])
if buildRace && buildMSan {
fmt.Fprintf(os.Stderr, "go %s: may not use -race and -msan simultaneously", flag.Args()[0])
os.Exit(2)
}
buildGcflags = append(buildGcflags, "-race")
buildLdflags = append(buildLdflags, "-race")
if goarch != "amd64" || goos != "linux" && goos != "freebsd" && goos != "darwin" && goos != "windows" {
fmt.Fprintf(os.Stderr, "go %s: -race and -msan are only supported on linux/amd64, freebsd/amd64, darwin/amd64 and windows/amd64\n", flag.Args()[0])
os.Exit(2)
}
if !buildContext.CgoEnabled {
fmt.Fprintf(os.Stderr, "go %s: -race requires cgo; enable cgo by setting CGO_ENABLED=1\n", flag.Args()[0])
os.Exit(2)
}
if buildRace {
buildGcflags = append(buildGcflags, "-race")
buildLdflags = append(buildLdflags, "-race")
} else {
buildGcflags = append(buildGcflags, "-msan")
buildLdflags = append(buildLdflags, "-msan")
}
if buildContext.InstallSuffix != "" {
buildContext.InstallSuffix += "_"
}
buildContext.InstallSuffix += "race"
buildContext.BuildTags = append(buildContext.BuildTags, "race")
}
// defaultSuffix returns file extension used for command files in
// current os environment.
func defaultSuffix() string {
switch runtime.GOOS {
case "windows":
return ".bat"
case "plan9":
return ".rc"
default:
return ".bash"
if buildRace {
buildContext.InstallSuffix += "race"
buildContext.BuildTags = append(buildContext.BuildTags, "race")
} else {
buildContext.InstallSuffix += "msan"
buildContext.BuildTags = append(buildContext.BuildTags, "msan")
}
}

4
libgo/go/cmd/go/discovery.go
View file

@ -41,9 +41,9 @@ func parseMetaGoImports(r io.Reader) (imports []metaImport, err error) {
d.Strict = false
var t xml.Token
for {
t, err = d.Token()
t, err = d.RawToken()
if err != nil {
if err == io.EOF {
if err == io.EOF || len(imports) > 0 {
err = nil
}
return

23
libgo/go/cmd/go/doc.go
View file

@ -32,12 +32,17 @@ which is schematically one of these:
go doc <pkg>
go doc <sym>[.<method>]
go doc [<pkg>].<sym>[.<method>]
go doc [<pkg>.]<sym>[.<method>]
go doc [<pkg>.][<sym>.]<method>
The first item in this list matched by the argument is the one whose
documentation is printed. (See the examples below.) For packages, the order of
scanning is determined lexically, but the GOROOT tree is always scanned before
GOPATH.
The first item in this list matched by the argument is the one whose documentation
is printed. (See the examples below.) However, if the argument starts with a capital
letter it is assumed to identify a symbol or method in the current directory.
For packages, the order of scanning is determined lexically in breadth-first order.
That is, the package presented is the one that matches the search and is nearest
the root and lexically first at its level of the hierarchy. The GOROOT tree is
always scanned in its entirety before GOPATH.
If there is no package specified or matched, the package in the current
directory is selected, so "go doc Foo" shows the documentation for symbol Foo in
@ -85,6 +90,14 @@ Examples:
go doc text/template new # Two arguments
Show documentation for text/template's New function.
At least in the current tree, these invocations all print the
documentation for json.Decoder's Decode method:
go doc json.Decoder.Decode
go doc json.decoder.decode
go doc json.decode
cd go/src/encoding/json; go doc decode
Flags:
-c
Respect case when matching symbols.

7
libgo/go/cmd/go/env.go
View file

@ -33,6 +33,11 @@ func mkEnv() []envVar {
var b builder
b.init()
vendorExpValue := "0"
if go15VendorExperiment {
vendorExpValue = "1"
}
env := []envVar{
{"GOARCH", goarch},
{"GOBIN", gobin},
@ -44,7 +49,7 @@ func mkEnv() []envVar {
{"GORACE", os.Getenv("GORACE")},
{"GOROOT", goroot},
{"GOTOOLDIR", toolDir},
{"GO15VENDOREXPERIMENT", os.Getenv("GO15VENDOREXPERIMENT")},
{"GO15VENDOREXPERIMENT", vendorExpValue},
// disable escape codes in clang errors
{"TERM", "dumb"},

50
libgo/go/cmd/go/generate.go
View file

@ -22,7 +22,7 @@ import (
var cmdGenerate = &Command{
Run: runGenerate,
UsageLine: "generate [-run regexp] [file.go... | packages]",
UsageLine: "generate [-run regexp] [-n] [-v] [-x] [build flags] [file.go... | packages]",
Short: "generate Go files by processing source",
Long: `
Generate runs commands described by directives within existing
@ -115,12 +115,14 @@ Go generate accepts one specific flag:
any trailing spaces and final newline) matches the
expression.
It also accepts the standard build flags -v, -n, and -x.
It also accepts the standard build flags including -v, -n, and -x.
The -v flag prints the names of packages and files as they are
processed.
The -n flag prints commands that would be executed.
The -x flag prints commands as they are executed.
For more about build flags, see 'go help build'.
For more about specifying packages, see 'go help packages'.
`,
}
@ -179,6 +181,7 @@ type Generator struct {
pkg string
commands map[string][]string
lineNum int // current line number.
env []string
}
// run runs the generators in the current file.
@ -242,6 +245,7 @@ func (g *Generator) run() (ok bool) {
}
}
g.setEnv()
words := g.split(string(buf))
if len(words) == 0 {
g.errorf("no arguments to directive")
@ -269,6 +273,19 @@ func isGoGenerate(buf []byte) bool {
return bytes.HasPrefix(buf, []byte("//go:generate ")) || bytes.HasPrefix(buf, []byte("//go:generate\t"))
}
// setEnv sets the extra environment variables used when executing a
// single go:generate command.
func (g *Generator) setEnv() {
g.env = []string{
"GOARCH=" + runtime.GOARCH,
"GOOS=" + runtime.GOOS,
"GOFILE=" + g.file,
"GOLINE=" + strconv.Itoa(g.lineNum),
"GOPACKAGE=" + g.pkg,
"DOLLAR=" + "$",
}
}
// split breaks the line into words, evaluating quoted
// strings and evaluating environment variables.
// The initial //go:generate element is present in line.
@ -345,22 +362,13 @@ func (g *Generator) errorf(format string, args ...interface{}) {
// expandVar expands the $XXX invocation in word. It is called
// by os.Expand.
func (g *Generator) expandVar(word string) string {
switch word {
case "GOARCH":
return buildContext.GOARCH
case "GOOS":
return buildContext.GOOS
case "GOFILE":
return g.file
case "GOLINE":
return fmt.Sprint(g.lineNum)
case "GOPACKAGE":
return g.pkg
case "DOLLAR":
return "$"
default:
return os.Getenv(word)
w := word + "="
for _, e := range g.env {
if strings.HasPrefix(e, w) {
return e[len(w):]
}
}
return os.Getenv(word)
}
// identLength returns the length of the identifier beginning the string.
@ -396,13 +404,7 @@ func (g *Generator) exec(words []string) {
cmd.Stderr = os.Stderr
// Run the command in the package directory.
cmd.Dir = g.dir
env := []string{
"GOARCH=" + runtime.GOARCH,
"GOOS=" + runtime.GOOS,
"GOFILE=" + g.file,
"GOPACKAGE=" + g.pkg,
}
cmd.Env = mergeEnvLists(env, origEnv)
cmd.Env = mergeEnvLists(g.env, origEnv)
err := cmd.Run()
if err != nil {
g.errorf("running %q: %s", words[0], err)

1
libgo/go/cmd/go/generate_test.go
View file

@ -39,6 +39,7 @@ func TestGenerateCommandParse(t *testing.T) {
pkg: "sys",
commands: make(map[string][]string),
}
g.setEnv()
g.setShorthand([]string{"-command", "yacc", "go", "tool", "yacc"})
for _, test := range splitTests {
// First with newlines.

18
libgo/go/cmd/go/get.go
View file

@ -45,16 +45,22 @@ missing packages but does not use it to look for updates to existing packages.
Get also accepts build flags to control the installation. See 'go help build'.
When checking out a new package, get creates the target directory
GOPATH/src/<import-path>. If the GOPATH contains multiple entries,
get uses the first one. See 'go help gopath'.
When checking out or updating a package, get looks for a branch or tag
that matches the locally installed version of Go. The most important
rule is that if the local installation is running version "go1", get
searches for a branch or tag named "go1". If no such version exists it
retrieves the most recent version of the package.
If the vendoring experiment is enabled (see 'go help gopath'),
then when go get checks out or updates a Git repository,
Unless vendoring support is disabled (see 'go help gopath'),
when go get checks out or updates a Git repository,
it also updates any git submodules referenced by the repository.
Get never checks out or updates code stored in vendor directories.
For more about specifying packages, see 'go help packages'.
For more about how 'go get' finds source code to
@ -84,8 +90,12 @@ func runGet(cmd *Command, args []string) {
// Disable any prompting for passwords by Git.
// Only has an effect for 2.3.0 or later, but avoiding
// the prompt in earlier versions is just too hard.
// See golang.org/issue/9341.
os.Setenv("GIT_TERMINAL_PROMPT", "0")
// If user has explicitly set GIT_TERMINAL_PROMPT=1, keep
// prompting.
// See golang.org/issue/9341 and golang.org/issue/12706.
if os.Getenv("GIT_TERMINAL_PROMPT") == "" {
os.Setenv("GIT_TERMINAL_PROMPT", "0")
}
// Phase 1. Download/update.
var stk importStack

249
libgo/go/cmd/go/go_test.go
View file

@ -31,8 +31,7 @@ var (
exeSuffix string // ".exe" on Windows
builder = testenv.Builder()
skipExternalBuilder = false // skip external tests on this builder
skipExternal = false // skip external tests
)
func init() {
@ -44,14 +43,21 @@ func init() {
case "arm", "arm64":
canRun = false
}
}
if strings.HasPrefix(builder+"-", "freebsd-arm-") {
skipExternalBuilder = true
canRun = false
}
switch runtime.GOOS {
case "linux":
switch runtime.GOARCH {
case "arm":
// many linux/arm machines are too slow to run
// the full set of external tests.
skipExternal = true
}
case "freebsd":
switch runtime.GOARCH {
case "arm":
// many freebsd/arm machines are too slow to run
// the full set of external tests.
skipExternal = true
canRun = false
}
case "windows":
exeSuffix = ".exe"
}
@ -83,8 +89,6 @@ func TestMain(m *testing.M) {
case "linux", "darwin", "freebsd", "windows":
canRace = canCgo && runtime.GOARCH == "amd64"
}
measureTick("./testgo" + exeSuffix)
}
// Don't let these environment variables confuse the test.
@ -103,24 +107,8 @@ func TestMain(m *testing.M) {
// The length of an mtime tick on this system. This is an estimate of
// how long we need to sleep to ensure that the mtime of two files is
// different.
var mtimeTick time.Duration
// measureTick sets mtimeTick by looking at the rounding of the mtime
// of a file.
func measureTick(path string) {
st, err := os.Stat(path)
if err != nil {
// Default to one second, the most conservative value.
mtimeTick = time.Second
return
}
mtime := st.ModTime()
t := time.Microsecond
for mtime.Round(t).Equal(mtime) && t < time.Second {
t *= 10
}
mtimeTick = t
}
// We used to try to be clever but that didn't always work (see golang.org/issue/12205).
var mtimeTick time.Duration = 1 * time.Second
// Manage a single run of the testgo binary.
type testgoData struct {
@ -138,8 +126,8 @@ type testgoData struct {
func testgo(t *testing.T) *testgoData {
testenv.MustHaveGoBuild(t)
if skipExternalBuilder {
t.Skip("skipping external tests on %s builder", builder)
if skipExternal {
t.Skip("skipping external tests on %s/%s", runtime.GOOS, runtime.GOARCH)
}
return &testgoData{t: t}
@ -452,7 +440,7 @@ func (tg *testgoData) grepCountBoth(match string) int {
// removed if it exists.
func (tg *testgoData) creatingTemp(path string) {
if filepath.IsAbs(path) && !strings.HasPrefix(path, tg.tempdir) {
tg.t.Fatal("internal testsuite error: creatingTemp(%q) with absolute path not in temporary directory", path)
tg.t.Fatalf("internal testsuite error: creatingTemp(%q) with absolute path not in temporary directory", path)
}
// If we have changed the working directory, make sure we have
// an absolute path, because we are going to change directory
@ -671,17 +659,48 @@ func TestGoBuildDashAInDevBranch(t *testing.T) {
tg.grepStderr("runtime", "testgo build -a math in dev branch DID NOT build runtime, but should have")
}
func TestGoBuilDashAInReleaseBranch(t *testing.T) {
func TestGoBuildDashAInReleaseBranch(t *testing.T) {
if testing.Short() {
t.Skip("don't rebuild the standard library in short mode")
}
tg := testgo(t)
defer tg.cleanup()
tg.run("install", "math") // should be up to date already but just in case
tg.run("install", "math", "net/http") // should be up to date already but just in case
tg.setenv("TESTGO_IS_GO_RELEASE", "1")
tg.run("build", "-v", "-a", "math")
tg.grepStderr("runtime", "testgo build -a math in dev branch did not build runtime, but should have")
tg.run("install", "-v", "-a", "math")
tg.grepStderr("runtime", "testgo build -a math in release branch DID NOT build runtime, but should have")
// Now runtime.a is updated (newer mtime), so everything would look stale if not for being a release.
//
tg.run("build", "-v", "net/http")
tg.grepStderrNot("strconv", "testgo build -v net/http in release branch with newer runtime.a DID build strconv but should not have")
tg.grepStderrNot("golang.org/x/net/http2/hpack", "testgo build -v net/http in release branch with newer runtime.a DID build .../golang.org/x/net/http2/hpack but should not have")
tg.grepStderrNot("net/http", "testgo build -v net/http in release branch with newer runtime.a DID build net/http but should not have")
}
func TestGoListStandard(t *testing.T) {
tg := testgo(t)
defer tg.cleanup()
tg.cd(runtime.GOROOT() + "/src")
tg.run("list", "-f", "{{if not .Standard}}{{.ImportPath}}{{end}}", "./...")
stdout := tg.getStdout()
for _, line := range strings.Split(stdout, "\n") {
if strings.HasPrefix(line, "_/") && strings.HasSuffix(line, "/src") {
// $GOROOT/src shows up if there are any .go files there.
// We don't care.
continue
}
if line == "" {
continue
}
t.Errorf("package in GOROOT not listed as standard: %v", line)
}
// Similarly, expanding std should include some of our vendored code.
tg.run("list", "std", "cmd")
tg.grepStdout("golang.org/x/net/http2/hpack", "list std cmd did not mention vendored hpack")
tg.grepStdout("golang.org/x/arch/x86/x86asm", "list std cmd did not mention vendored x86asm")
}
func TestGoInstallCleansUpAfterGoBuild(t *testing.T) {
@ -775,6 +794,28 @@ func TestGoInstallDetectsRemovedFiles(t *testing.T) {
tg.wantStale("mypkg", "./testgo list mypkg claims mypkg is NOT stale after removing y.go; should be stale")
}
func TestWildcardMatchesSyntaxErrorDirs(t *testing.T) {
tg := testgo(t)
defer tg.cleanup()
tg.tempFile("src/mypkg/x.go", `package mypkg`)
tg.tempFile("src/mypkg/y.go", `pkg mypackage`)
tg.setenv("GOPATH", tg.path("."))
tg.cd(tg.path("src/mypkg"))
tg.runFail("list", "./...")
tg.runFail("build", "./...")
tg.runFail("install", "./...")
}
func TestGoListWithTags(t *testing.T) {
tg := testgo(t)
defer tg.cleanup()
tg.tempFile("src/mypkg/x.go", "// +build thetag\n\npackage mypkg\n")
tg.setenv("GOPATH", tg.path("."))
tg.cd(tg.path("./src"))
tg.run("list", "-tags=thetag", "./my...")
tg.grepStdout("mypkg", "did not find mypkg")
}
func TestGoInstallErrorOnCrossCompileToBin(t *testing.T) {
if testing.Short() {
t.Skip("don't install into GOROOT in short mode")
@ -951,6 +992,16 @@ func TestInternalPackagesOutsideGOROOTAreRespected(t *testing.T) {
tg.grepBoth("use of internal package not allowed", "wrote error message for testdata/testinternal2")
}
func TestRunInternal(t *testing.T) {
tg := testgo(t)
defer tg.cleanup()
dir := filepath.Join(tg.pwd(), "testdata")
tg.setenv("GOPATH", dir)
tg.run("run", filepath.Join(dir, "src/run/good.go"))
tg.runFail("run", filepath.Join(dir, "src/run/bad.go"))
tg.grepStderr("use of internal package not allowed", "unexpected error for run/bad.go")
}
func testMove(t *testing.T, vcs, url, base, config string) {
testenv.MustHaveExternalNetwork(t)
@ -1053,7 +1104,6 @@ func TestImportCommentConflict(t *testing.T) {
// cmd/go: custom import path checking should not apply to github.com/xxx/yyy.
func TestIssue10952(t *testing.T) {
testenv.MustHaveExternalNetwork(t)
if _, err := exec.LookPath("git"); err != nil {
t.Skip("skipping because git binary not found")
}
@ -1071,6 +1121,34 @@ func TestIssue10952(t *testing.T) {
tg.run("get", "-d", "-u", importPath)
}
func TestGetGitDefaultBranch(t *testing.T) {
testenv.MustHaveExternalNetwork(t)
if _, err := exec.LookPath("git"); err != nil {
t.Skip("skipping because git binary not found")
}
tg := testgo(t)
defer tg.cleanup()
tg.parallel()
tg.tempDir("src")
tg.setenv("GOPATH", tg.path("."))
// This repo has two branches, master and another-branch.
// The another-branch is the default that you get from 'git clone'.
// The go get command variants should not override this.
const importPath = "github.com/rsc/go-get-default-branch"
tg.run("get", "-d", importPath)
repoDir := tg.path("src/" + importPath)
defer tg.resetReadOnlyFlagAll(repoDir)
tg.runGit(repoDir, "branch", "--contains", "HEAD")
tg.grepStdout(`\* another-branch`, "not on correct default branch")
tg.run("get", "-d", "-u", importPath)
tg.runGit(repoDir, "branch", "--contains", "HEAD")
tg.grepStdout(`\* another-branch`, "not on correct default branch")
}
func TestDisallowedCSourceFiles(t *testing.T) {
tg := testgo(t)
defer tg.cleanup()
@ -1139,6 +1217,15 @@ func TestInstallFailsWithNoBuildableFiles(t *testing.T) {
tg.grepStderr("no buildable Go source files", "go install cgotest did not report 'no buildable Go Source files'")
}
func TestRelativeGOBINFail(t *testing.T) {
tg := testgo(t)
defer tg.cleanup()
tg.tempFile("triv.go", `package main; func main() {}`)
tg.setenv("GOBIN", ".")
tg.runFail("install")
tg.grepStderr("cannot install, GOBIN must be an absolute path", "go install must fail if $GOBIN is a relative path")
}
// Test that without $GOBIN set, binaries get installed
// into the GOPATH bin directory.
func TestInstallIntoGOPATH(t *testing.T) {
@ -1150,9 +1237,21 @@ func TestInstallIntoGOPATH(t *testing.T) {
tg.wantExecutable("testdata/bin/go-cmd-test"+exeSuffix, "go install go-cmd-test did not write to testdata/bin/go-cmd-test")
}
// Issue 12407
func TestBuildOutputToDevNull(t *testing.T) {
if runtime.GOOS == "plan9" {
t.Skip("skipping because /dev/null is a regular file on plan9")
}
tg := testgo(t)
defer tg.cleanup()
tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata"))
tg.run("build", "-o", os.DevNull, "go-cmd-test")
}
func TestPackageMainTestImportsArchiveNotBinary(t *testing.T) {
tg := testgo(t)
defer tg.cleanup()
tg.parallel()
gobin := filepath.Join(tg.pwd(), "testdata", "bin")
tg.creatingTemp(gobin)
tg.setenv("GOBIN", gobin)
@ -1165,6 +1264,17 @@ func TestPackageMainTestImportsArchiveNotBinary(t *testing.T) {
tg.run("test", "main_test")
}
// Issue 12690
func TestPackageNotStaleWithTrailingSlash(t *testing.T) {
tg := testgo(t)
defer tg.cleanup()
goroot := runtime.GOROOT()
tg.setenv("GOROOT", goroot+"/")
tg.wantNotStale("runtime", "with trailing slash in GOROOT, runtime listed as stale")
tg.wantNotStale("os", "with trailing slash in GOROOT, os listed as stale")
tg.wantNotStale("io", "with trailing slash in GOROOT, io listed as stale")
}
// With $GOBIN set, binaries get installed to $GOBIN.
func TestInstallIntoGOBIN(t *testing.T) {
tg := testgo(t)
@ -1357,6 +1467,18 @@ func TestGoListCmdOnlyShowsCommands(t *testing.T) {
}
}
func TestGoListDedupsPackages(t *testing.T) {
tg := testgo(t)
defer tg.cleanup()
tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata"))
tg.run("list", "xtestonly", "./testdata/src/xtestonly/...")
got := strings.TrimSpace(tg.getStdout())
const want = "xtestonly"
if got != want {
t.Errorf("got %q; want %q", got, want)
}
}
// Issue 4096. Validate the output of unsuccessful go install foo/quxx.
func TestUnsuccessfulGoInstallShouldMentionMissingPackage(t *testing.T) {
tg := testgo(t)
@ -1833,6 +1955,9 @@ func TestIssue6480(t *testing.T) {
// cmd/cgo: undefined reference when linking a C-library using gccgo
func TestIssue7573(t *testing.T) {
if !canCgo {
t.Skip("skipping because cgo not enabled")
}
if _, err := exec.LookPath("gccgo"); err != nil {
t.Skip("skipping because no gccgo compiler found")
}
@ -1931,6 +2056,13 @@ func TestGoTestFooTestWorks(t *testing.T) {
tg.run("test", "testdata/standalone_test.go")
}
func TestGoTestFlagsAfterPackage(t *testing.T) {
tg := testgo(t)
defer tg.cleanup()
tg.run("test", "testdata/flag_test.go", "-v", "-args", "-v=7") // Two distinct -v flags.
tg.run("test", "-v", "testdata/flag_test.go", "-args", "-v=7") // Two distinct -v flags.
}
func TestGoTestXtestonlyWorks(t *testing.T) {
tg := testgo(t)
defer tg.cleanup()
@ -1991,6 +2123,21 @@ func TestGoGenerateRunFlag(t *testing.T) {
tg.grepStdoutNot("no", "go generate -run yes ./testdata/generate/test4.go selected no")
}
func TestGoGenerateEnv(t *testing.T) {
switch runtime.GOOS {
case "plan9", "windows":
t.Skipf("skipping because %s does not have the env command", runtime.GOOS)
}
tg := testgo(t)
defer tg.cleanup()
tg.parallel()
tg.tempFile("env.go", "package main\n\n//go:generate env")
tg.run("generate", tg.path("env.go"))
for _, v := range []string{"GOARCH", "GOOS", "GOFILE", "GOLINE", "GOPACKAGE", "DOLLAR"} {
tg.grepStdout("^"+v+"=", "go generate environment missing "+v)
}
}
func TestGoGetCustomDomainWildcard(t *testing.T) {
testenv.MustHaveExternalNetwork(t)
@ -2051,6 +2198,17 @@ func TestGoGetRscIoToolstash(t *testing.T) {
tg.run("get", "./toolstash")
}
// Issue 13037: Was not parsing <meta> tags in 404 served over HTTPS
func TestGoGetHTTPS404(t *testing.T) {
testenv.MustHaveExternalNetwork(t)
tg := testgo(t)
defer tg.cleanup()
tg.tempDir("src")
tg.setenv("GOPATH", tg.path("."))
tg.run("get", "bazil.org/fuse/fs/fstestutil")
}
// Test that you can not import a main package.
func TestIssue4210(t *testing.T) {
tg := testgo(t)
@ -2128,7 +2286,11 @@ func TestGoGetInsecureCustomDomain(t *testing.T) {
}
func TestIssue10193(t *testing.T) {
t.Skip("depends on code.google.com")
testenv.MustHaveExternalNetwork(t)
if _, err := exec.LookPath("hg"); err != nil {
t.Skip("skipping because hg binary not found")
}
tg := testgo(t)
defer tg.cleanup()
@ -2199,7 +2361,7 @@ func TestGoTestImportErrorStack(t *testing.T) {
tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata"))
tg.runFail("test", "testdep/p1")
if !strings.Contains(tg.stderr.String(), out) {
t.Fatal("did not give full import stack:\n\n%s", tg.stderr.String())
t.Fatalf("did not give full import stack:\n\n%s", tg.stderr.String())
}
}
@ -2387,3 +2549,12 @@ func TestGoBuildARM(t *testing.T) {
tg.run("build", "hello.go")
tg.grepStderrNot("unable to find math.a", "did not build math.a correctly")
}
func TestIssue13655(t *testing.T) {
tg := testgo(t)
defer tg.cleanup()
for _, pkg := range []string{"runtime", "runtime/internal/atomic"} {
tg.run("list", "-f", "{{.Deps}}", pkg)
tg.grepStdout("runtime/internal/sys", "did not find required dependency of "+pkg+" on runtime/internal/sys")
}
}

31
libgo/go/cmd/go/go_unix_test.go Normal file
View file

@ -0,0 +1,31 @@
// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build darwin dragonfly freebsd linux netbsd openbsd solaris
package main_test
import (
"os"
"syscall"
"testing"
)
func TestGoBuildUmask(t *testing.T) {
// Do not use tg.parallel; avoid other tests seeing umask manipulation.
mask := syscall.Umask(0077) // prohibit low bits
defer syscall.Umask(mask)
tg := testgo(t)
defer tg.cleanup()
tg.tempFile("x.go", `package main; func main() {}`)
tg.creatingTemp("x")
tg.run("build", tg.path("x.go"))
fi, err := os.Stat("x")
if err != nil {
t.Fatal(err)
}
if mode := fi.Mode(); mode&0077 != 0 {
t.Fatalf("wrote x with mode=%v, wanted no 0077 bits", mode)
}
}

50
libgo/go/cmd/go/help.go
View file

@ -79,6 +79,14 @@ internally at Google all begin with 'google', and paths
denoting remote repositories begin with the path to the code,
such as 'github.com/user/repo'.
Packages in a program need not have unique package names,
but there are two reserved package names with special meaning.
The name main indicates a command, not a library.
Commands are built into binaries and cannot be imported.
The name documentation indicates documentation for
a non-Go program in the directory. Files in package documentation
are ignored by the go command.
As a special case, if the package list is a list of .go files from a
single directory, the command is applied to a single synthesized
package made up of exactly those files, ignoring any build constraints
@ -261,10 +269,10 @@ unless it is being referred to by that import path. In this way, import comments
let package authors make sure the custom import path is used and not a
direct path to the underlying code hosting site.
If the vendoring experiment is enabled (see 'go help gopath'),
then import path checking is disabled for code found within vendor trees.
This makes it possible to copy code into alternate locations in vendor trees
without needing to update import comments.
If vendoring is enabled (see 'go help gopath'), then import path checking is
disabled for code found within vendor trees. This makes it possible to copy
code into alternate locations in vendor trees without needing to update import
comments.
See https://golang.org/s/go14customimport for details.
`,
@ -307,7 +315,7 @@ DIR/bin/quux, not DIR/bin/foo/quux. The "foo/" prefix is stripped
so that you can add DIR/bin to your PATH to get at the
installed commands. If the GOBIN environment variable is
set, commands are installed to the directory it names instead
of DIR/bin.
of DIR/bin. GOBIN must be an absolute path.
Here's an example directory layout:
@ -365,13 +373,10 @@ See https://golang.org/s/go14internal for details.
Vendor Directories
Go 1.5 includes experimental support for using local copies
of external dependencies to satisfy imports of those dependencies,
often referred to as vendoring. Setting the environment variable
GO15VENDOREXPERIMENT=1 enables that experimental support.
Go 1.6 includes support for using local copies of external dependencies
to satisfy imports of those dependencies, often referred to as vendoring.
When the vendor experiment is enabled,
code below a directory named "vendor" is importable only
Code below a directory named "vendor" is importable only
by code in the directory tree rooted at the parent of "vendor",
and only using an import path that omits the prefix up to and
including the vendor element.
@ -409,12 +414,18 @@ top-level "crash/bang".
Code in vendor directories is not subject to import path
checking (see 'go help importpath').
When the vendor experiment is enabled, 'go get' checks out
submodules when checking out or updating a git repository
(see 'go help get').
When 'go get' checks out or updates a git repository, it now also
updates submodules.
The vendoring semantics are an experiment, and they may change
in future releases. Once settled, they will be on by default.
Vendor directories do not affect the placement of new repositories
being checked out for the first time by 'go get': those are always
placed in the main GOPATH, never in a vendor subtree.
In Go 1.5, as an experiment, setting the environment variable
GO15VENDOREXPERIMENT=1 enabled these features.
As of Go 1.6 they are on by default. To turn them off, set
GO15VENDOREXPERIMENT=0. In Go 1.7, the environment
variable will stop having any effect.
See https://golang.org/s/go15vendor for details.
`,
@ -487,7 +498,7 @@ Special-purpose environment variables:
File names in stack traces are rewritten from GOROOT to
GOROOT_FINAL.
GO15VENDOREXPERIMENT
Set to 1 to enable the Go 1.5 vendoring experiment.
Set to 0 to disable vendoring semantics.
GO_EXTLINK_ENABLED
Whether the linker should use external linking mode
when using -linkmode=auto with code that uses cgo.
@ -570,5 +581,10 @@ are:
-buildmode=exe
Build the listed main packages and everything they import into
executables. Packages not named main are ignored.
-buildmode=pie
Build the listed main packages and everything they import into
position independent executables (PIE). Packages not named
main are ignored.
`,
}

24
libgo/go/cmd/go/http.go
View file

@ -12,6 +12,7 @@
package main
import (
"crypto/tls"
"fmt"
"io"
"io/ioutil"
@ -24,8 +25,17 @@ import (
// httpClient is the default HTTP client, but a variable so it can be
// changed by tests, without modifying http.DefaultClient.
var httpClient = http.DefaultClient
var impatientHTTPClient = &http.Client{
// impatientInsecureHTTPClient is used in -insecure mode,
// when we're connecting to https servers that might not be there
// or might be using self-signed certificates.
var impatientInsecureHTTPClient = &http.Client{
Timeout: time.Duration(5 * time.Second),
Transport: &http.Transport{
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
},
},
}
type httpError struct {
@ -71,7 +81,7 @@ func httpsOrHTTP(importPath string, security securityMode) (urlStr string, body
log.Printf("Fetching %s", urlStr)
}
if security == insecure && scheme == "https" { // fail earlier
res, err = impatientHTTPClient.Get(urlStr)
res, err = impatientInsecureHTTPClient.Get(urlStr)
} else {
res, err = httpClient.Get(urlStr)
}
@ -83,16 +93,12 @@ func httpsOrHTTP(importPath string, security securityMode) (urlStr string, body
}
}
urlStr, res, err := fetch("https")
if err != nil || res.StatusCode != 200 {
if err != nil {
if buildV {
if err != nil {
log.Printf("https fetch failed.")
} else {
log.Printf("ignoring https fetch with status code %d", res.StatusCode)
}
log.Printf("https fetch failed: %v", err)
}
closeBody(res)
if security == insecure {
closeBody(res)
urlStr, res, err = fetch("http")
}
}

8
libgo/go/cmd/go/list.go
View file

@ -78,6 +78,14 @@ syntax of package template. The default output is equivalent to -f
XTestImports []string // imports from XTestGoFiles
}
The error information, if any, is
type PackageError struct {
ImportStack []string // shortest path from package named on command line to this one
Pos string // position of error (if present, file:line:col)
Err string // the error itself
}
The template function "join" calls strings.Join.
The template function "context" returns the build context, defined as:

22
libgo/go/cmd/go/main.go
View file

@ -285,8 +285,8 @@ func printUsage(w io.Writer) {
func usage() {
// special case "go test -h"
if len(os.Args) > 1 && os.Args[1] == "test" {
os.Stdout.WriteString(testUsage + "\n\n" +
strings.TrimSpace(testFlag1) + "\n\n" +
os.Stderr.WriteString(testUsage + "\n\n" +
strings.TrimSpace(testFlag1) + "\n\n\t" +
strings.TrimSpace(testFlag2) + "\n")
os.Exit(2)
}
@ -353,7 +353,7 @@ func importPathsNoDotExpansion(args []string) []string {
} else {
a = path.Clean(a)
}
if a == "all" || a == "std" || a == "cmd" {
if isMetaPackage(a) {
out = append(out, allPackages(a)...)
continue
}
@ -554,7 +554,7 @@ func allPackages(pattern string) []string {
func matchPackages(pattern string) []string {
match := func(string) bool { return true }
treeCanMatch := func(string) bool { return true }
if pattern != "all" && pattern != "std" && pattern != "cmd" {
if !isMetaPackage(pattern) {
match = matchPattern(pattern)
treeCanMatch = treeCanMatchPattern(pattern)
}
@ -588,10 +588,9 @@ func matchPackages(pattern string) []string {
}
name := filepath.ToSlash(path[len(src):])
if pattern == "std" && (strings.Contains(name, ".") || name == "cmd") {
if pattern == "std" && (!isStandardImportPath(name) || name == "cmd") {
// The name "std" is only the standard library.
// If the name has a dot, assume it's a domain name for go get,
// and if the name is cmd, it's the root of the command tree.
// If the name is cmd, it's the root of the command tree.
return filepath.SkipDir
}
if !treeCanMatch(name) {
@ -674,7 +673,14 @@ func matchPackagesInFS(pattern string) []string {
if !match(name) {
return nil
}
if _, err = build.ImportDir(path, 0); err != nil {
// We keep the directory if we can import it, or if we can't import it
// due to invalid Go source files. This means that directories containing
// parse errors will be built (and fail) instead of being silently skipped
// as not matching the pattern. Go 1.5 and earlier skipped, but that
// behavior means people miss serious mistakes.
// See golang.org/issue/11407.
if p, err := buildContext.ImportDir(path, 0); err != nil && (p == nil || len(p.InvalidGoFiles) == 0) {
if _, noGo := err.(*build.NoGoError); !noGo {
log.Print(err)
}

97
libgo/go/cmd/go/note.go
View file

@ -7,6 +7,7 @@ package main
import (
"bytes"
"debug/elf"
"debug/macho"
"encoding/binary"
"fmt"
"io"
@ -69,11 +70,11 @@ func readELFNote(filename, name string, typ int32) ([]byte, error) {
var elfGoNote = []byte("Go\x00\x00")
// readELFGoBuildID the Go build ID string from an ELF binary.
// The Go build ID is stored in a note described by an ELF PT_NOTE prog header.
// The caller has already opened filename, to get f, and read the first 4 kB out, in data.
// The Go build ID is stored in a note described by an ELF PT_NOTE prog
// header. The caller has already opened filename, to get f, and read
// at least 4 kB out, in data.
func readELFGoBuildID(filename string, f *os.File, data []byte) (buildid string, err error) {
// Assume the note content is in the first 4 kB, already read.
// Assume the note content is in the data, already read.
// Rewrite the ELF header to set shnum to 0, so that we can pass
// the data to elf.NewFile and it will decode the Prog list but not
// try to read the section headers and the string table from disk.
@ -95,22 +96,92 @@ func readELFGoBuildID(filename string, f *os.File, data []byte) (buildid string,
return "", &os.PathError{Path: filename, Op: "parse", Err: err}
}
for _, p := range ef.Progs {
if p.Type != elf.PT_NOTE || p.Off >= uint64(len(data)) || p.Off+p.Filesz >= uint64(len(data)) || p.Filesz < 16 {
if p.Type != elf.PT_NOTE || p.Filesz < 16 {
continue
}
note := data[p.Off : p.Off+p.Filesz]
nameSize := ef.ByteOrder.Uint32(note)
valSize := ef.ByteOrder.Uint32(note[4:])
tag := ef.ByteOrder.Uint32(note[8:])
name := note[12:16]
if nameSize != 4 || 16+valSize > uint32(len(note)) || tag != elfGoBuildIDTag || !bytes.Equal(name, elfGoNote) {
continue
var note []byte
if p.Off+p.Filesz < uint64(len(data)) {
note = data[p.Off : p.Off+p.Filesz]
} else {
// For some linkers, such as the Solaris linker,
// the buildid may not be found in data (which
// likely contains the first 16kB of the file)
// or even the first few megabytes of the file
// due to differences in note segment placement;
// in that case, extract the note data manually.
_, err = f.Seek(int64(p.Off), 0)
if err != nil {
return "", err
}
note = make([]byte, p.Filesz)
_, err = io.ReadFull(f, note)
if err != nil {
return "", err
}
}
return string(note[16 : 16+valSize]), nil
filesz := p.Filesz
for filesz >= 16 {
nameSize := ef.ByteOrder.Uint32(note)
valSize := ef.ByteOrder.Uint32(note[4:])
tag := ef.ByteOrder.Uint32(note[8:])
name := note[12:16]
if nameSize == 4 && 16+valSize <= uint32(len(note)) && tag == elfGoBuildIDTag && bytes.Equal(name, elfGoNote) {
return string(note[16 : 16+valSize]), nil
}
nameSize = (nameSize + 3) &^ 3
valSize = (valSize + 3) &^ 3
notesz := uint64(12 + nameSize + valSize)
if filesz <= notesz {
break
}
filesz -= notesz
note = note[notesz:]
}
}
// No note. Treat as successful but build ID empty.
return "", nil
}
// The Go build ID is stored at the beginning of the Mach-O __text segment.
// The caller has already opened filename, to get f, and read a few kB out, in data.
// Sadly, that's not guaranteed to hold the note, because there is an arbitrary amount
// of other junk placed in the file ahead of the main text.
func readMachoGoBuildID(filename string, f *os.File, data []byte) (buildid string, err error) {
// If the data we want has already been read, don't worry about Mach-O parsing.
// This is both an optimization and a hedge against the Mach-O parsing failing
// in the future due to, for example, the name of the __text section changing.
if b, err := readRawGoBuildID(filename, data); b != "" && err == nil {
return b, err
}
mf, err := macho.NewFile(f)
if err != nil {
return "", &os.PathError{Path: filename, Op: "parse", Err: err}
}
sect := mf.Section("__text")
if sect == nil {
// Every binary has a __text section. Something is wrong.
return "", &os.PathError{Path: filename, Op: "parse", Err: fmt.Errorf("cannot find __text section")}
}
// It should be in the first few bytes, but read a lot just in case,
// especially given our past problems on OS X with the build ID moving.
// There shouldn't be much difference between reading 4kB and 32kB:
// the hard part is getting to the data, not transferring it.
n := sect.Size
if n > uint64(BuildIDReadSize) {
n = uint64(BuildIDReadSize)
}
buf := make([]byte, n)
if _, err := f.ReadAt(buf, int64(sect.Offset)); err != nil {
return "", err
}
return readRawGoBuildID(filename, buf)
}

55
libgo/go/cmd/go/note_test.go
View file

@ -6,11 +6,29 @@ package main_test
import (
main "cmd/go"
"go/build"
"runtime"
"testing"
)
func TestNoteReading(t *testing.T) {
testNoteReading(t)
}
func TestNoteReading2K(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skipf("2kB is not enough on %s", runtime.GOOS)
}
// Set BuildIDReadSize to 2kB to exercise Mach-O parsing more strictly.
defer func(old int) {
main.BuildIDReadSize = old
}(main.BuildIDReadSize)
main.BuildIDReadSize = 2 * 1024
testNoteReading(t)
}
func testNoteReading(t *testing.T) {
tg := testgo(t)
defer tg.cleanup()
tg.tempFile("hello.go", `package main; func main() { print("hello, world\n") }`)
@ -24,26 +42,25 @@ func TestNoteReading(t *testing.T) {
t.Fatalf("buildID in hello binary = %q, want %q", id, buildID)
}
if runtime.GOOS == "linux" && runtime.GOARCH == "ppc64le" {
t.Skipf("skipping - golang.org/issue/11184")
switch {
case !build.Default.CgoEnabled:
t.Skipf("skipping - no cgo, so assuming external linking not available")
case runtime.GOOS == "linux" && (runtime.GOARCH == "ppc64le" || runtime.GOARCH == "ppc64"):
t.Skipf("skipping - external linking not supported, golang.org/issue/11184")
case runtime.GOOS == "linux" && (runtime.GOARCH == "mips64le" || runtime.GOARCH == "mips64"):
t.Skipf("skipping - external linking not supported, golang.org/issue/12560")
case runtime.GOOS == "openbsd" && runtime.GOARCH == "arm":
t.Skipf("skipping - external linking not supported, golang.org/issue/10619")
case runtime.GOOS == "plan9":
t.Skipf("skipping - external linking not supported")
}
switch runtime.GOOS {
case "plan9":
// no external linking
t.Logf("no external linking - skipping linkmode=external test")
case "solaris":
t.Logf("skipping - golang.org/issue/12178")
default:
tg.run("build", "-ldflags", "-buildid="+buildID+" -linkmode=external", "-o", tg.path("hello.exe"), tg.path("hello.go"))
id, err := main.ReadBuildIDFromBinary(tg.path("hello.exe"))
if err != nil {
t.Fatalf("reading build ID from hello binary (linkmode=external): %v", err)
}
if id != buildID {
t.Fatalf("buildID in hello binary = %q, want %q (linkmode=external)", id, buildID)
}
tg.run("build", "-ldflags", "-buildid="+buildID+" -linkmode=external", "-o", tg.path("hello.exe"), tg.path("hello.go"))
id, err = main.ReadBuildIDFromBinary(tg.path("hello.exe"))
if err != nil {
t.Fatalf("reading build ID from hello binary (linkmode=external): %v", err)
}
if id != buildID {
t.Fatalf("buildID in hello binary = %q, want %q (linkmode=external)", id, buildID)
}
}

179
libgo/go/cmd/go/pkg.go
View file

@ -118,7 +118,7 @@ func (p *Package) vendored(imports []string) []string {
seen := make(map[string]bool)
var all []string
for _, path := range imports {
path, _ = vendoredImportPath(p, path)
path = vendoredImportPath(p, path)
if !seen[path] {
seen[path] = true
all = append(all, path)
@ -156,7 +156,7 @@ func (p *Package) copyBuild(pp *build.Package) {
if buildContext.Compiler == "gccgo" {
p.Standard = stdpkg[p.ImportPath]
} else {
p.Standard = p.Goroot && p.ImportPath != "" && !strings.Contains(p.ImportPath, ".")
p.Standard = p.Goroot && p.ImportPath != "" && isStandardImportPath(p.ImportPath)
}
p.GoFiles = pp.GoFiles
p.CgoFiles = pp.CgoFiles
@ -181,6 +181,19 @@ func (p *Package) copyBuild(pp *build.Package) {
p.XTestImports = pp.XTestImports
}
// isStandardImportPath reports whether $GOROOT/src/path should be considered
// part of the standard distribution. For historical reasons we allow people to add
// their own code to $GOROOT instead of using $GOPATH, but we assume that
// code will start with a domain name (dot in the first element).
func isStandardImportPath(path string) bool {
i := strings.Index(path, "/")
if i < 0 {
i = len(path)
}
elem := path[:i]
return !strings.Contains(elem, ".")
}
// A PackageError describes an error loading information about a package.
type PackageError struct {
ImportStack []string // shortest path from package named on command line to this one
@ -254,11 +267,14 @@ func reloadPackage(arg string, stk *importStack) *Package {
return loadPackage(arg, stk)
}
// The Go 1.5 vendoring experiment is enabled by setting GO15VENDOREXPERIMENT=1.
// The Go 1.5 vendoring experiment was enabled by setting GO15VENDOREXPERIMENT=1.
// In Go 1.6 this is on by default and is disabled by setting GO15VENDOREXPERIMENT=0.
// In Go 1.7 the variable will stop having any effect.
// The variable is obnoxiously long so that years from now when people find it in
// their profiles and wonder what it does, there is some chance that a web search
// might answer the question.
var go15VendorExperiment = os.Getenv("GO15VENDOREXPERIMENT") == "1"
// There is a copy of this variable in src/go/build/build.go. Delete that one when this one goes away.
var go15VendorExperiment = os.Getenv("GO15VENDOREXPERIMENT") != "0"
// dirToImportPath returns the pseudo-import path we use for a package
// outside the Go path. It begins with _/ and then contains the full path
@ -314,11 +330,14 @@ func loadImport(path, srcDir string, parent *Package, stk *importStack, importPo
importPath := path
origPath := path
isLocal := build.IsLocalImport(path)
var vendorSearch []string
if isLocal {
importPath = dirToImportPath(filepath.Join(srcDir, path))
} else if mode&useVendor != 0 {
path, vendorSearch = vendoredImportPath(parent, path)
// We do our own vendor resolution, because we want to
// find out the key to use in packageCache without the
// overhead of repeated calls to buildContext.Import.
// The code is also needed in a few other places anyway.
path = vendoredImportPath(parent, path)
importPath = path
}
@ -345,29 +364,12 @@ func loadImport(path, srcDir string, parent *Package, stk *importStack, importPo
//
// TODO: After Go 1, decide when to pass build.AllowBinary here.
// See issue 3268 for mistakes to avoid.
bp, err := buildContext.Import(path, srcDir, build.ImportComment)
// If we got an error from go/build about package not found,
// it contains the directories from $GOROOT and $GOPATH that
// were searched. Add to that message the vendor directories
// that were searched.
if err != nil && len(vendorSearch) > 0 {
// NOTE(rsc): The direct text manipulation here is fairly awful,
// but it avoids defining new go/build API (an exported error type)
// late in the Go 1.5 release cycle. If this turns out to be a more general
// problem we could define a real error type when the decision can be
// considered more carefully.
text := err.Error()
if strings.Contains(text, "cannot find package \"") && strings.Contains(text, "\" in any of:\n\t") {
old := strings.SplitAfter(text, "\n")
lines := []string{old[0]}
for _, dir := range vendorSearch {
lines = append(lines, "\t"+dir+" (vendor tree)\n")
}
lines = append(lines, old[1:]...)
err = errors.New(strings.Join(lines, ""))
}
buildMode := build.ImportComment
if !go15VendorExperiment || mode&useVendor == 0 || path != origPath {
// Not vendoring, or we already found the vendored path.
buildMode |= build.IgnoreVendor
}
bp, err := buildContext.Import(path, srcDir, buildMode)
bp.ImportPath = importPath
if gobin != "" {
bp.BinDir = gobin
@ -377,7 +379,7 @@ func loadImport(path, srcDir string, parent *Package, stk *importStack, importPo
err = fmt.Errorf("code in directory %s expects import %q", bp.Dir, bp.ImportComment)
}
p.load(stk, bp, err)
if p.Error != nil && len(importPos) > 0 {
if p.Error != nil && p.Error.Pos == "" && len(importPos) > 0 {
pos := importPos[0]
pos.Filename = shortPath(pos.Filename)
p.Error.Pos = pos.String()
@ -411,14 +413,11 @@ func isDir(path string) bool {
// vendoredImportPath returns the expansion of path when it appears in parent.
// If parent is x/y/z, then path might expand to x/y/z/vendor/path, x/y/vendor/path,
// x/vendor/path, vendor/path, or else stay x/y/z if none of those exist.
// x/vendor/path, vendor/path, or else stay path if none of those exist.
// vendoredImportPath returns the expanded path or, if no expansion is found, the original.
// If no expansion is found, vendoredImportPath also returns a list of vendor directories
// it searched along the way, to help prepare a useful error message should path turn
// out not to exist.
func vendoredImportPath(parent *Package, path string) (found string, searched []string) {
func vendoredImportPath(parent *Package, path string) (found string) {
if parent == nil || parent.Root == "" || !go15VendorExperiment {
return path, nil
return path
}
dir := filepath.Clean(parent.Dir)
root := filepath.Join(parent.Root, "src")
@ -438,7 +437,7 @@ func vendoredImportPath(parent *Package, path string) (found string, searched []
continue
}
targ := filepath.Join(dir[:i], vpath)
if isDir(targ) {
if isDir(targ) && hasGoFiles(targ) {
// We started with parent's dir c:\gopath\src\foo\bar\baz\quux\xyzzy.
// We know the import path for parent's dir.
// We chopped off some number of path elements and
@ -453,14 +452,26 @@ func vendoredImportPath(parent *Package, path string) (found string, searched []
// and found c:\gopath\src\vendor\path.
// We chopped \foo\bar (length 8) but the import path is "foo/bar" (length 7).
// Use "vendor/path" without any prefix.
return vpath, nil
return vpath
}
return parent.ImportPath[:len(parent.ImportPath)-chopped] + "/" + vpath, nil
return parent.ImportPath[:len(parent.ImportPath)-chopped] + "/" + vpath
}
// Note the existence of a vendor directory in case path is not found anywhere.
searched = append(searched, targ)
}
return path, searched
return path
}
// hasGoFiles reports whether dir contains any files with names ending in .go.
// For a vendor check we must exclude directories that contain no .go files.
// Otherwise it is not possible to vendor just a/b/c and still import the
// non-vendored a/b. See golang.org/issue/13832.
func hasGoFiles(dir string) bool {
fis, _ := ioutil.ReadDir(dir)
for _, fi := range fis {
if !fi.IsDir() && strings.HasSuffix(fi.Name(), ".go") {
return true
}
}
return false
}
// reusePackage reuses package p to satisfy the import at the top
@ -522,7 +533,7 @@ func disallowInternal(srcDir string, p *Package, stk *importStack) *Package {
i-- // rewind over slash in ".../internal"
}
parent := p.Dir[:i+len(p.Dir)-len(p.ImportPath)]
if hasPathPrefix(filepath.ToSlash(srcDir), filepath.ToSlash(parent)) {
if hasFilePathPrefix(filepath.Clean(srcDir), filepath.Clean(parent)) {
return p
}
@ -619,7 +630,7 @@ func disallowVendorVisibility(srcDir string, p *Package, stk *importStack) *Pack
return p
}
parent := p.Dir[:truncateTo]
if hasPathPrefix(filepath.ToSlash(srcDir), filepath.ToSlash(parent)) {
if hasFilePathPrefix(filepath.Clean(srcDir), filepath.Clean(parent)) {
return p
}
@ -712,6 +723,7 @@ func expandScanner(err error) error {
var raceExclude = map[string]bool{
"runtime/race": true,
"runtime/msan": true,
"runtime/cgo": true,
"cmd/cgo": true,
"syscall": true,
@ -725,6 +737,7 @@ var cgoExclude = map[string]bool{
var cgoSyscallExclude = map[string]bool{
"runtime/cgo": true,
"runtime/race": true,
"runtime/msan": true,
}
// load populates p using information from bp, err, which should
@ -829,27 +842,40 @@ func (p *Package) load(stk *importStack, bp *build.Package, err error) *Package
importPaths = append(importPaths, "syscall")
}
// Currently build mode c-shared, or -linkshared, forces
// Currently build modes c-shared, pie, and -linkshared force
// external linking mode, and external linking mode forces an
// import of runtime/cgo.
if p.Name == "main" && !p.Goroot && (buildBuildmode == "c-shared" || buildLinkshared) {
if p.Name == "main" && !p.Goroot && (buildBuildmode == "c-shared" || buildBuildmode == "pie" || buildLinkshared) {
importPaths = append(importPaths, "runtime/cgo")
}
// Everything depends on runtime, except runtime and unsafe.
if !p.Standard || (p.ImportPath != "runtime" && p.ImportPath != "unsafe") {
// Everything depends on runtime, except runtime, its internal
// subpackages, and unsafe.
if !p.Standard || (p.ImportPath != "runtime" && !strings.HasPrefix(p.ImportPath, "runtime/internal/") && p.ImportPath != "unsafe") {
importPaths = append(importPaths, "runtime")
// When race detection enabled everything depends on runtime/race.
// Exclude certain packages to avoid circular dependencies.
if buildRace && (!p.Standard || !raceExclude[p.ImportPath]) {
importPaths = append(importPaths, "runtime/race")
}
// MSan uses runtime/msan.
if buildMSan && (!p.Standard || !raceExclude[p.ImportPath]) {
importPaths = append(importPaths, "runtime/msan")
}
// On ARM with GOARM=5, everything depends on math for the link.
if p.Name == "main" && goarch == "arm" {
importPaths = append(importPaths, "math")
}
}
// Runtime and its internal packages depend on runtime/internal/sys,
// so that they pick up the generated zversion.go file.
// This can be an issue particularly for runtime/internal/atomic;
// see issue 13655.
if p.Standard && (p.ImportPath == "runtime" || strings.HasPrefix(p.ImportPath, "runtime/internal/")) && p.ImportPath != "runtime/internal/sys" {
importPaths = append(importPaths, "runtime/internal/sys")
}
// Build list of full paths to all Go files in the package,
// for use by commands like go fmt.
p.gofiles = stringList(p.GoFiles, p.CgoFiles, p.TestGoFiles, p.XTestGoFiles)
@ -931,6 +957,17 @@ func (p *Package) load(stk *importStack, bp *build.Package, err error) *Package
}
}
}
if p.Standard && !p1.Standard && p.Error == nil {
p.Error = &PackageError{
ImportStack: stk.copy(),
Err: fmt.Sprintf("non-standard import %q in standard package %q", path, p.ImportPath),
}
pos := p.build.ImportPos[path]
if len(pos) > 0 {
p.Error.Pos = pos[0].String()
}
}
path = p1.ImportPath
importPaths[i] = path
if i < len(p.Imports) {
@ -1613,15 +1650,24 @@ func packagesAndErrors(args []string) []*Package {
}
args = importPaths(args)
var pkgs []*Package
var stk importStack
var set = make(map[string]bool)
var (
pkgs []*Package
stk importStack
seenArg = make(map[string]bool)
seenPkg = make(map[*Package]bool)
)
for _, arg := range args {
if !set[arg] {
pkgs = append(pkgs, loadPackage(arg, &stk))
set[arg] = true
if seenArg[arg] {
continue
}
seenArg[arg] = true
pkg := loadPackage(arg, &stk)
if seenPkg[pkg] {
continue
}
seenPkg[pkg] = true
pkgs = append(pkgs, pkg)
}
computeStale(pkgs...)
@ -1792,8 +1838,17 @@ var (
goBuildEnd = []byte("\"\n \xff")
elfPrefix = []byte("\x7fELF")
machoPrefixes = [][]byte{
{0xfe, 0xed, 0xfa, 0xce},
{0xfe, 0xed, 0xfa, 0xcf},
{0xce, 0xfa, 0xed, 0xfe},
{0xcf, 0xfa, 0xed, 0xfe},
}
)
var BuildIDReadSize = 32 * 1024 // changed for testing
// ReadBuildIDFromBinary reads the build ID from a binary.
//
// ELF binaries store the build ID in a proper PT_NOTE section.
@ -1808,10 +1863,11 @@ func ReadBuildIDFromBinary(filename string) (id string, err error) {
return "", &os.PathError{Op: "parse", Path: filename, Err: errBuildIDUnknown}
}
// Read the first 16 kB of the binary file.
// Read the first 32 kB of the binary file.
// That should be enough to find the build ID.
// In ELF files, the build ID is in the leading headers,
// which are typically less than 4 kB, not to mention 16 kB.
// which are typically less than 4 kB, not to mention 32 kB.
// In Mach-O files, there's no limit, so we have to parse the file.
// On other systems, we're trying to read enough that
// we get the beginning of the text segment in the read.
// The offset where the text segment begins in a hello
@ -1819,7 +1875,6 @@ func ReadBuildIDFromBinary(filename string) (id string, err error) {
//
// Plan 9: 0x20
// Windows: 0x600
// Mach-O: 0x2000
//
f, err := os.Open(filename)
if err != nil {
@ -1827,7 +1882,7 @@ func ReadBuildIDFromBinary(filename string) (id string, err error) {
}
defer f.Close()
data := make([]byte, 16*1024)
data := make([]byte, BuildIDReadSize)
_, err = io.ReadFull(f, data)
if err == io.ErrUnexpectedEOF {
err = nil
@ -1839,7 +1894,17 @@ func ReadBuildIDFromBinary(filename string) (id string, err error) {
if bytes.HasPrefix(data, elfPrefix) {
return readELFGoBuildID(filename, f, data)
}
for _, m := range machoPrefixes {
if bytes.HasPrefix(data, m) {
return readMachoGoBuildID(filename, f, data)
}
}
return readRawGoBuildID(filename, data)
}
// readRawGoBuildID finds the raw build ID stored in text segment data.
func readRawGoBuildID(filename string, data []byte) (id string, err error) {
i := bytes.Index(data, goBuildPrefix)
if i < 0 {
// Missing. Treat as successful but build ID empty.

118
libgo/go/cmd/go/pkg_test.go
View file

@ -5,6 +5,9 @@
package main
import (
"io/ioutil"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
@ -57,6 +60,15 @@ var parseMetaGoImportsTests = []struct {
<body>`,
[]metaImport{{"foo/bar", "git", "https://github.com/rsc/foo/bar"}},
},
{
`<!doctype html><meta name="go-import" content="foo/bar git https://github.com/rsc/foo/bar">`,
[]metaImport{{"foo/bar", "git", "https://github.com/rsc/foo/bar"}},
},
{
// XML doesn't like <div style=position:relative>.
`<!doctype html><title>Page Not Found</title><meta name=go-import content="chitin.io/chitin git https://github.com/chitin-io/chitin"><div style=position:relative>DRAFT</div>`,
[]metaImport{{"chitin.io/chitin", "git", "https://github.com/chitin-io/chitin"}},
},
}
func TestParseMetaGoImports(t *testing.T) {
@ -71,3 +83,109 @@ func TestParseMetaGoImports(t *testing.T) {
}
}
}
func TestSharedLibName(t *testing.T) {
// TODO(avdva) - make these values platform-specific
prefix := "lib"
suffix := ".so"
testData := []struct {
args []string
pkgs []*Package
expected string
expectErr bool
rootedAt string
}{
{
args: []string{"std"},
pkgs: []*Package{},
expected: "std",
},
{
args: []string{"std", "cmd"},
pkgs: []*Package{},
expected: "std,cmd",
},
{
args: []string{},
pkgs: []*Package{&Package{ImportPath: "gopkg.in/somelib"}},
expected: "gopkg.in-somelib",
},
{
args: []string{"./..."},
pkgs: []*Package{&Package{ImportPath: "somelib"}},
expected: "somelib",
rootedAt: "somelib",
},
{
args: []string{"../somelib", "../somelib"},
pkgs: []*Package{&Package{ImportPath: "somelib"}},
expected: "somelib",
},
{
args: []string{"../lib1", "../lib2"},
pkgs: []*Package{&Package{ImportPath: "gopkg.in/lib1"}, &Package{ImportPath: "gopkg.in/lib2"}},
expected: "gopkg.in-lib1,gopkg.in-lib2",
},
{
args: []string{"./..."},
pkgs: []*Package{
&Package{ImportPath: "gopkg.in/dir/lib1"},
&Package{ImportPath: "gopkg.in/lib2"},
&Package{ImportPath: "gopkg.in/lib3"},
},
expected: "gopkg.in",
rootedAt: "gopkg.in",
},
{
args: []string{"std", "../lib2"},
pkgs: []*Package{},
expectErr: true,
},
{
args: []string{"all", "./"},
pkgs: []*Package{},
expectErr: true,
},
{
args: []string{"cmd", "fmt"},
pkgs: []*Package{},
expectErr: true,
},
}
for _, data := range testData {
func() {
if data.rootedAt != "" {
tmpGopath, err := ioutil.TempDir("", "gopath")
if err != nil {
t.Fatal(err)
}
oldGopath := buildContext.GOPATH
defer func() {
os.RemoveAll(tmpGopath)
buildContext.GOPATH = oldGopath
os.Chdir(cwd)
}()
root := filepath.Join(tmpGopath, "src", data.rootedAt)
err = os.MkdirAll(root, 0755)
if err != nil {
t.Fatal(err)
}
buildContext.GOPATH = tmpGopath
os.Chdir(root)
}
computed, err := libname(data.args, data.pkgs)
if err != nil {
if !data.expectErr {
t.Errorf("libname returned an error %q, expected a name", err.Error())
}
} else if data.expectErr {
t.Errorf("libname returned %q, expected an error", computed)
} else {
expected := prefix + data.expected + suffix
if expected != computed {
t.Errorf("libname returned %q, expected %q", computed, expected)
}
}
}()
}
}

16
libgo/go/cmd/go/run.go
View file

@ -64,7 +64,7 @@ func printStderr(args ...interface{}) (int, error) {
}
func runRun(cmd *Command, args []string) {
raceInit()
instrumentInit()
buildModeInit()
var b builder
b.init()
@ -89,8 +89,18 @@ func runRun(cmd *Command, args []string) {
fatalf("%s", p.Error)
}
p.omitDWARF = true
for _, err := range p.DepsErrors {
errorf("%s", err)
if len(p.DepsErrors) > 0 {
// Since these are errors in dependencies,
// the same error might show up multiple times,
// once in each package that depends on it.
// Only print each once.
printed := map[*PackageError]bool{}
for _, err := range p.DepsErrors {
if !printed[err] {
printed[err] = true
errorf("%s", err)
}
}
}
exitIfErrors()
if p.Name != "main" {

105
libgo/go/cmd/go/test.go
View file

@ -13,7 +13,6 @@ import (
"go/doc"
"go/parser"
"go/token"
"log"
"os"
"os/exec"
"path"
@ -33,7 +32,7 @@ func init() {
cmdTest.Run = runTest
}
const testUsage = "test [-c] [-i] [build and test flags] [packages] [flags for test binary]"
const testUsage = "test [build/test flags] [packages] [build/test flags & test binary flags]"
var cmdTest = &Command{
CustomFlags: true,
@ -68,11 +67,6 @@ non-test installation.
` + strings.TrimSpace(testFlag1) + ` See 'go help testflag' for details.
If the test binary needs any other flags, they should be presented after the
package names. The go tool treats as a flag the first argument that begins with
a minus sign that it does not recognize itself; that argument and all subsequent
arguments are passed as arguments to the test binary.
For more about build flags, see 'go help build'.
For more about specifying packages, see 'go help packages'.
@ -83,10 +77,16 @@ See also: go build, go vet.
const testFlag1 = `
In addition to the build flags, the flags handled by 'go test' itself are:
-args
Pass the remainder of the command line (everything after -args)
to the test binary, uninterpreted and unchanged.
Because this flag consumes the remainder of the command line,
the package list (if present) must appear before this flag.
-c
Compile the test binary to pkg.test but do not run it
(where pkg is the last element of the package's import path).
The file name can be changed with the -o flag.
Compile the test binary to pkg.test but do not run it
(where pkg is the last element of the package's import path).
The file name can be changed with the -o flag.
-exec xprog
Run the test binary using xprog. The behavior is the same as
@ -97,8 +97,8 @@ In addition to the build flags, the flags handled by 'go test' itself are:
Do not run the test.
-o file
Compile the test binary to the named file.
The test still runs (unless -c or -i is specified).
Compile the test binary to the named file.
The test still runs (unless -c or -i is specified).
The test binary also accepts flags that control execution of the test; these
flags are also accessible by 'go test'.
@ -207,6 +207,10 @@ const testFlag2 = `
Allow parallel execution of test functions that call t.Parallel.
The value of this flag is the maximum number of tests to run
simultaneously; by default, it is set to the value of GOMAXPROCS.
Note that -parallel only applies within a single test binary.
The 'go test' command may run tests for different packages
in parallel as well, according to the setting of the -p flag
(see 'go help build').
-run regexp
Run only those tests and examples matching the regular
@ -230,25 +234,63 @@ const testFlag2 = `
Verbose output: log all tests as they are run. Also print all
text from Log and Logf calls even if the test succeeds.
The test binary, called pkg.test where pkg is the name of the
directory containing the package sources, can be invoked directly
after building it with 'go test -c'. When invoking the test binary
directly, each of the standard flag names must be prefixed with 'test.',
as in -test.run=TestMyFunc or -test.v.
Each of these flags is also recognized with an optional 'test.' prefix,
as in -test.v. When invoking the generated test binary (the result of
'go test -c') directly, however, the prefix is mandatory.
When running 'go test', flags not listed above are passed through
unaltered. For instance, the command
The 'go test' command rewrites or removes recognized flags,
as appropriate, both before and after the optional package list,
before invoking the test binary.
go test -x -v -cpuprofile=prof.out -dir=testdata -update
For instance, the command
go test -v -myflag testdata -cpuprofile=prof.out -x
will compile the test binary and then run it as
pkg.test -test.v -test.cpuprofile=prof.out -dir=testdata -update
pkg.test -test.v -myflag testdata -test.cpuprofile=prof.out
(The -x flag is removed because it applies only to the go command's
execution, not to the test itself.)
The test flags that generate profiles (other than for coverage) also
leave the test binary in pkg.test for use when analyzing the profiles.
Flags not recognized by 'go test' must be placed after any specified packages.
When 'go test' runs a test binary, it does so from within the
corresponding package's source code directory. Depending on the test,
it may be necessary to do the same when invoking a generated test
binary directly.
The command-line package list, if present, must appear before any
flag not known to the go test command. Continuing the example above,
the package list would have to appear before -myflag, but could appear
on either side of -v.
To keep an argument for a test binary from being interpreted as a
known flag or a package name, use -args (see 'go help test') which
passes the remainder of the command line through to the test binary
uninterpreted and unaltered.
For instance, the command
go test -v -args -x -v
will compile the test binary and then run it as
pkg.test -test.v -x -v
Similarly,
go test -args math
will compile the test binary and then run it as
pkg.test math
In the first example, the -x and the second -v are passed through to the
test binary unchanged and with no effect on the go command itself.
In the second example, the argument math is passed through to the test
binary, instead of being interpreted as the package list.
`
var helpTestfunc = &Command{
@ -328,7 +370,7 @@ func runTest(cmd *Command, args []string) {
findExecCmd() // initialize cached result
raceInit()
instrumentInit()
buildModeInit()
pkgs := packagesForBuild(pkgArgs)
if len(pkgs) == 0 {
@ -396,7 +438,7 @@ func runTest(cmd *Command, args []string) {
if deps["C"] {
delete(deps, "C")
deps["runtime/cgo"] = true
if goos == runtime.GOOS && goarch == runtime.GOARCH && !buildRace {
if goos == runtime.GOOS && goarch == runtime.GOARCH && !buildRace && !buildMSan {
deps["cmd/cgo"] = true
}
}
@ -442,7 +484,7 @@ func runTest(cmd *Command, args []string) {
}
for _, p := range testCoverPkgs {
if !used[p.ImportPath] {
log.Printf("warning: no packages being tested depend on %s", p.ImportPath)
fmt.Fprintf(os.Stderr, "warning: no packages being tested depend on %s\n", p.ImportPath)
}
}
@ -547,6 +589,9 @@ func runTest(cmd *Command, args []string) {
if buildRace {
extraOpts = "-race "
}
if buildMSan {
extraOpts = "-msan "
}
fmt.Fprintf(os.Stderr, "installing these packages with 'go test %s-i%s' will speed future tests.\n\n", extraOpts, args)
}
@ -829,10 +874,12 @@ func (b *builder) test(p *Package) (buildAction, runAction, printAction *action,
}
}
// writeTestmain writes _testmain.go. This must happen after recompileForTest,
// because recompileForTest modifies XXX.
if err := writeTestmain(filepath.Join(testDir, "_testmain.go"), t); err != nil {
return nil, nil, nil, err
if !buildN {
// writeTestmain writes _testmain.go. This must happen after recompileForTest,
// because recompileForTest modifies XXX.
if err := writeTestmain(filepath.Join(testDir, "_testmain.go"), t); err != nil {
return nil, nil, nil, err
}
}
computeStale(pmain)

16
libgo/go/cmd/go/testdata/flag_test.go vendored Normal file
View file

@ -0,0 +1,16 @@
package flag_test
import (
"flag"
"log"
"testing"
)
var v = flag.Int("v", 0, "v flag")
// Run this as go test pkg -v=7
func TestVFlagIsSet(t *testing.T) {
if *v != 7 {
log.Fatal("v flag not set")
}
}

5
libgo/go/cmd/go/testdata/src/run/bad.go vendored Normal file
View file

@ -0,0 +1,5 @@
package main
import _ "run/subdir/internal/private"
func main() {}

5
libgo/go/cmd/go/testdata/src/run/good.go vendored Normal file
View file

@ -0,0 +1,5 @@
package main
import _ "run/internal"
func main() {}

1
libgo/go/cmd/go/testdata/src/run/internal/internal.go vendored Normal file
View file

@ -0,0 +1 @@
package internal

1
libgo/go/cmd/go/testdata/src/run/subdir/internal/private/private.go vendored Normal file
View file

@ -0,0 +1 @@
package private

1
libgo/go/cmd/go/testdata/src/vend/dir1/dir1.go vendored Normal file
View file

@ -0,0 +1 @@
package dir1

2
libgo/go/cmd/go/testdata/src/vend/hello/hello_test.go vendored
View file

@ -7,6 +7,6 @@ import (
func TestMsgInternal(t *testing.T) {
if strings.Msg != "hello, world" {
t.Fatal("unexpected msg: %v", strings.Msg)
t.Fatalf("unexpected msg: %v", strings.Msg)
}
}

2
libgo/go/cmd/go/testdata/src/vend/hello/hellox_test.go vendored
View file

@ -7,6 +7,6 @@ import (
func TestMsgExternal(t *testing.T) {
if strings.Msg != "hello, world" {
t.Fatal("unexpected msg: %v", strings.Msg)
t.Fatalf("unexpected msg: %v", strings.Msg)
}
}

1
libgo/go/cmd/go/testdata/src/vend/vendor/vend/dir1/dir2/dir2.go vendored Normal file
View file

@ -0,0 +1 @@
package dir2

2
libgo/go/cmd/go/testdata/src/vend/x/x.go vendored
View file

@ -3,3 +3,5 @@ package x
import _ "p"
import _ "q"
import _ "r"
import _ "vend/dir1" // not vendored
import _ "vend/dir1/dir2" // vendored

9
libgo/go/cmd/go/testflag.go
View file

@ -87,6 +87,7 @@ func init() {
func testFlags(args []string) (packageNames, passToTest []string) {
inPkg := false
outputDir := ""
var explicitArgs []string
for i := 0; i < len(args); i++ {
if !strings.HasPrefix(args[i], "-") {
if !inPkg && packageNames == nil {
@ -114,6 +115,12 @@ func testFlags(args []string) (packageNames, passToTest []string) {
// make non-nil: we have seen the empty package list
packageNames = []string{}
}
if args[i] == "-args" || args[i] == "--args" {
// -args or --args signals that everything that follows
// should be passed to the test.
explicitArgs = args[i+1:]
break
}
passToTest = append(passToTest, args[i])
continue
}
@ -191,6 +198,8 @@ func testFlags(args []string) (packageNames, passToTest []string) {
}
passToTest = append(passToTest, "-test.outputdir", dir)
}
passToTest = append(passToTest, explicitArgs...)
return
}

1
libgo/go/cmd/go/tool.go
View file

@ -104,6 +104,7 @@ func runTool(cmd *Command, args []string) {
fmt.Printf("%s\n", cmd)
return
}
args[0] = toolPath // in case the tool wants to re-exec itself, e.g. cmd/dist
toolCmd := &exec.Cmd{
Path: toolPath,
Args: args,

44
libgo/go/cmd/go/vcs.go
View file

@ -122,7 +122,7 @@ var vcsGit = &vcsCmd{
name: "Git",
cmd: "git",
createCmd: []string{"clone {repo} {dir}", "--git-dir={dir}/.git submodule update --init --recursive"},
createCmd: []string{"clone {repo} {dir}", "-C {dir} submodule update --init --recursive"},
downloadCmd: []string{"pull --ff-only", "submodule update --init --recursive"},
tagCmd: []tagCmd{
@ -137,8 +137,9 @@ var vcsGit = &vcsCmd{
// both createCmd and downloadCmd update the working dir.
// No need to do more here. We used to 'checkout master'
// but that doesn't work if the default branch is not named master.
// DO NOT add 'checkout master' here.
// See golang.org/issue/9032.
tagSyncDefault: []string{"checkout master", "submodule update --init --recursive"},
tagSyncDefault: []string{"submodule update --init --recursive"},
scheme: []string{"git", "https", "http", "git+ssh", "ssh"},
pingCmd: "ls-remote {scheme}://{repo}",
@ -385,9 +386,6 @@ func (v *vcsCmd) create(dir, repo string) error {
// download downloads any new changes for the repo in dir.
func (v *vcsCmd) download(dir string) error {
if err := v.fixDetachedHead(dir); err != nil {
return err
}
for _, cmd := range v.downloadCmd {
if !go15VendorExperiment && strings.Contains(cmd, "submodule") {
continue
@ -399,30 +397,6 @@ func (v *vcsCmd) download(dir string) error {
return nil
}
// fixDetachedHead switches a Git repository in dir from a detached head to the master branch.
// Go versions before 1.2 downloaded Git repositories in an unfortunate way
// that resulted in the working tree state being on a detached head.
// That meant the repository was not usable for normal Git operations.
// Go 1.2 fixed that, but we can't pull into a detached head, so if this is
// a Git repository we check for being on a detached head and switch to the
// real branch, almost always called "master".
// TODO(dsymonds): Consider removing this for Go 1.3.
func (v *vcsCmd) fixDetachedHead(dir string) error {
if v != vcsGit {
return nil
}
// "git symbolic-ref HEAD" succeeds iff we are not on a detached head.
if err := v.runVerboseOnly(dir, "symbolic-ref HEAD"); err == nil {
// not on a detached head
return nil
}
if buildV {
log.Printf("%s on detached head; repairing", dir)
}
return v.run(dir, "checkout master")
}
// tags returns the list of available tags for the repo in dir.
func (v *vcsCmd) tags(dir string) ([]string, error) {
var tags []string
@ -567,16 +541,8 @@ func repoRootForImportPath(importPath string, security securityMode) (*repoRoot,
lookup = lookup[:i]
}
rr, err = repoRootForImportDynamic(lookup, security)
// repoRootForImportDynamic returns error detail
// that is irrelevant if the user didn't intend to use a
// dynamic import in the first place.
// Squelch it.
if err != nil {
if buildV {
log.Printf("import %q: %v", importPath, err)
}
err = fmt.Errorf("unrecognized import path %q", importPath)
err = fmt.Errorf("unrecognized import path %q (%v)", importPath, err)
}
}
if err != nil {
@ -891,7 +857,7 @@ var vcsPaths = []*vcsPath{
// General syntax for any server.
// Must be last.
{
re: `^(?P<root>(?P<repo>([a-z0-9.\-]+\.)+[a-z0-9.\-]+(:[0-9]+)?/[A-Za-z0-9_.\-/]*?)\.(?P<vcs>bzr|git|hg|svn))(/[A-Za-z0-9_.\-]+)*$`,
re: `^(?P<root>(?P<repo>([a-z0-9.\-]+\.)+[a-z0-9.\-]+(:[0-9]+)?(/~?[A-Za-z0-9_.\-]+)+?)\.(?P<vcs>bzr|git|hg|svn))(/~?[A-Za-z0-9_.\-]+)*$`,
ping: true,
},
}

4
libgo/go/cmd/go/vcs_test.go
View file

@ -18,14 +18,14 @@ func TestRepoRootForImportPath(t *testing.T) {
path string
want *repoRoot
}{
{
/*{
"code.google.com/p/go",
&repoRoot{
vcs: vcsHg,
repo: "https://code.google.com/p/go",
},
},
/*{
{
"code.google.com/r/go",
&repoRoot{
vcs: vcsHg,

26
libgo/go/cmd/go/vendor_test.go
View file

@ -24,12 +24,14 @@ func TestVendorImports(t *testing.T) {
tg.run("list", "-f", "{{.ImportPath}} {{.Imports}}", "vend/...")
want := `
vend [vend/vendor/p r]
vend/dir1 []
vend/hello [fmt vend/vendor/strings]
vend/subdir [vend/vendor/p r]
vend/vendor/p []
vend/vendor/q []
vend/vendor/strings []
vend/x [vend/x/vendor/p vend/vendor/q vend/x/vendor/r]
vend/vendor/vend/dir1/dir2 []
vend/x [vend/x/vendor/p vend/vendor/q vend/x/vendor/r vend/dir1 vend/vendor/vend/dir1/dir2]
vend/x/invalid [vend/x/invalid/vendor/foo]
vend/x/vendor/p []
vend/x/vendor/p/p [notfound]
@ -45,6 +47,14 @@ func TestVendorImports(t *testing.T) {
}
}
func TestVendorBuild(t *testing.T) {
tg := testgo(t)
defer tg.cleanup()
tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata"))
tg.setenv("GO15VENDOREXPERIMENT", "1")
tg.run("build", "vend/x")
}
func TestVendorRun(t *testing.T) {
tg := testgo(t)
defer tg.cleanup()
@ -102,7 +112,7 @@ func TestVendorImportError(t *testing.T) {
re := regexp.MustCompile(`cannot find package "notfound" in any of:
.*[\\/]testdata[\\/]src[\\/]vend[\\/]x[\\/]vendor[\\/]notfound \(vendor tree\)
.*[\\/]testdata[\\/]src[\\/]vend[\\/]vendor[\\/]notfound \(vendor tree\)
.*[\\/]testdata[\\/]src[\\/]vend[\\/]vendor[\\/]notfound
.*[\\/]src[\\/]notfound \(from \$GOROOT\)
.*[\\/]testdata[\\/]src[\\/]notfound \(from \$GOPATH\)`)
@ -187,6 +197,18 @@ func TestVendorGetUpdate(t *testing.T) {
tg.run("get", "-u", "github.com/rsc/go-get-issue-11864")
}
func TestGetSubmodules(t *testing.T) {
testenv.MustHaveExternalNetwork(t)
tg := testgo(t)
defer tg.cleanup()
tg.makeTempdir()
tg.setenv("GOPATH", tg.path("."))
tg.setenv("GO15VENDOREXPERIMENT", "1")
tg.run("get", "-d", "github.com/rsc/go-get-issue-12612")
tg.run("get", "-u", "-d", "github.com/rsc/go-get-issue-12612")
}
func TestVendorCache(t *testing.T) {
tg := testgo(t)
defer tg.cleanup()

5
libgo/go/cmd/gofmt/gofmt.go
View file

@ -13,7 +13,6 @@ import (
"go/printer"
"go/scanner"
"go/token"
"internal/format"
"io"
"io/ioutil"
"os"
@ -88,7 +87,7 @@ func processFile(filename string, in io.Reader, out io.Writer, stdin bool) error
return err
}
file, sourceAdj, indentAdj, err := format.Parse(fileSet, filename, src, stdin)
file, sourceAdj, indentAdj, err := parse(fileSet, filename, src, stdin)
if err != nil {
return err
}
@ -107,7 +106,7 @@ func processFile(filename string, in io.Reader, out io.Writer, stdin bool) error
simplify(file)
}
res, err := format.Format(fileSet, file, sourceAdj, indentAdj, src, printer.Config{Mode: printerMode, Tabwidth: tabWidth})
res, err := format(fileSet, file, sourceAdj, indentAdj, src, printer.Config{Mode: printerMode, Tabwidth: tabWidth})
if err != nil {
return err
}

176
libgo/go/cmd/gofmt/internal.go Normal file
View file

@ -0,0 +1,176 @@
// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// TODO(gri): This file and the file src/go/format/internal.go are
// the same (but for this comment and the package name). Do not modify
// one without the other. Determine if we can factor out functionality
// in a public API. See also #11844 for context.
package main
import (
"bytes"
"go/ast"
"go/parser"
"go/printer"
"go/token"
"strings"
)
// parse parses src, which was read from the named file,
// as a Go source file, declaration, or statement list.
func parse(fset *token.FileSet, filename string, src []byte, fragmentOk bool) (
file *ast.File,
sourceAdj func(src []byte, indent int) []byte,
indentAdj int,
err error,
) {
// Try as whole source file.
file, err = parser.ParseFile(fset, filename, src, parserMode)
// If there's no error, return. If the error is that the source file didn't begin with a
// package line and source fragments are ok, fall through to
// try as a source fragment. Stop and return on any other error.
if err == nil || !fragmentOk || !strings.Contains(err.Error(), "expected 'package'") {
return
}
// If this is a declaration list, make it a source file
// by inserting a package clause.
// Insert using a ;, not a newline, so that the line numbers
// in psrc match the ones in src.
psrc := append([]byte("package p;"), src...)
file, err = parser.ParseFile(fset, filename, psrc, parserMode)
if err == nil {
sourceAdj = func(src []byte, indent int) []byte {
// Remove the package clause.
// Gofmt has turned the ; into a \n.
src = src[indent+len("package p\n"):]
return bytes.TrimSpace(src)
}
return
}
// If the error is that the source file didn't begin with a
// declaration, fall through to try as a statement list.
// Stop and return on any other error.
if !strings.Contains(err.Error(), "expected declaration") {
return
}
// If this is a statement list, make it a source file
// by inserting a package clause and turning the list
// into a function body. This handles expressions too.
// Insert using a ;, not a newline, so that the line numbers
// in fsrc match the ones in src. Add an extra '\n' before the '}'
// to make sure comments are flushed before the '}'.
fsrc := append(append([]byte("package p; func _() {"), src...), '\n', '\n', '}')
file, err = parser.ParseFile(fset, filename, fsrc, parserMode)
if err == nil {
sourceAdj = func(src []byte, indent int) []byte {
// Cap adjusted indent to zero.
if indent < 0 {
indent = 0
}
// Remove the wrapping.
// Gofmt has turned the ; into a \n\n.
// There will be two non-blank lines with indent, hence 2*indent.
src = src[2*indent+len("package p\n\nfunc _() {"):]
// Remove only the "}\n" suffix: remaining whitespaces will be trimmed anyway
src = src[:len(src)-len("}\n")]
return bytes.TrimSpace(src)
}
// Gofmt has also indented the function body one level.
// Adjust that with indentAdj.
indentAdj = -1
}
// Succeeded, or out of options.
return
}
// format formats the given package file originally obtained from src
// and adjusts the result based on the original source via sourceAdj
// and indentAdj.
func format(
fset *token.FileSet,
file *ast.File,
sourceAdj func(src []byte, indent int) []byte,
indentAdj int,
src []byte,
cfg printer.Config,
) ([]byte, error) {
if sourceAdj == nil {
// Complete source file.
var buf bytes.Buffer
err := cfg.Fprint(&buf, fset, file)
if err != nil {
return nil, err
}
return buf.Bytes(), nil
}
// Partial source file.
// Determine and prepend leading space.
i, j := 0, 0
for j < len(src) && isSpace(src[j]) {
if src[j] == '\n' {
i = j + 1 // byte offset of last line in leading space
}
j++
}
var res []byte
res = append(res, src[:i]...)
// Determine and prepend indentation of first code line.
// Spaces are ignored unless there are no tabs,
// in which case spaces count as one tab.
indent := 0
hasSpace := false
for _, b := range src[i:j] {
switch b {
case ' ':
hasSpace = true
case '\t':
indent++
}
}
if indent == 0 && hasSpace {
indent = 1
}
for i := 0; i < indent; i++ {
res = append(res, '\t')
}
// Format the source.
// Write it without any leading and trailing space.
cfg.Indent = indent + indentAdj
var buf bytes.Buffer
err := cfg.Fprint(&buf, fset, file)
if err != nil {
return nil, err
}
out := sourceAdj(buf.Bytes(), cfg.Indent)
// If the adjusted output is empty, the source
// was empty but (possibly) for white space.
// The result is the incoming source.
if len(out) == 0 {
return src, nil
}
// Otherwise, append output to leading space.
res = append(res, out...)
// Determine and append trailing space.
i = len(src)
for i > 0 && isSpace(src[i-1]) {
i--
}
return append(res, src[i:]...), nil
}
// isSpace reports whether the byte is a space character.
// isSpace defines a space as being among the following bytes: ' ', '\t', '\n' and '\r'.
func isSpace(b byte) bool {
return b == ' ' || b == '\t' || b == '\n' || b == '\r'
}

5
libgo/go/cmd/gofmt/long_test.go
View file

@ -15,7 +15,6 @@ import (
"go/ast"
"go/printer"
"go/token"
"internal/format"
"io"
"os"
"path/filepath"
@ -33,7 +32,7 @@ var (
)
func gofmt(fset *token.FileSet, filename string, src *bytes.Buffer) error {
f, _, _, err := format.Parse(fset, filename, src.Bytes(), false)
f, _, _, err := parse(fset, filename, src.Bytes(), false)
if err != nil {
return err
}
@ -61,7 +60,7 @@ func testFile(t *testing.T, b1, b2 *bytes.Buffer, filename string) {
// exclude files w/ syntax errors (typically test cases)
fset := token.NewFileSet()
if _, _, _, err = format.Parse(fset, filename, b1.Bytes(), false); err != nil {
if _, _, _, err = parse(fset, filename, b1.Bytes(), false); err != nil {
if *verbose {
fmt.Fprintf(os.Stderr, "ignoring %s\n", err)
}

9
libgo/go/cmd/gofmt/testdata/old.golden vendored
View file

@ -1,9 +0,0 @@
package P
func f() {
if x {
y
} else {
z
}
}

8
libgo/go/cmd/gofmt/testdata/old.input vendored
View file

@ -1,8 +0,0 @@
package P
func f() {
if x {
y
} else
z
}

8
libgo/go/compress/bzip2/bit_reader.go
View file

@ -77,14 +77,6 @@ func (br *bitReader) ReadBit() bool {
return n != 0
}
func (br *bitReader) TryReadBit() (bit byte, ok bool) {
if br.bits > 0 {
br.bits--
return byte(br.n>>br.bits) & 1, true
}
return 0, false
}
func (br *bitReader) Err() error {
return br.err
}

6
libgo/go/compress/bzip2/bzip2_test.go
View file

@ -173,6 +173,7 @@ const rand3Hex = "1744b384d68c042371244e13500d4bfb98c6244e3d71a5b700224420b59c59
const (
digits = iota
twain
random
)
var testfiles = []string{
@ -180,8 +181,10 @@ var testfiles = []string{
// does not repeat, but there are only 10 possible digits, so it should be
// reasonably compressible.
digits: "testdata/e.txt.bz2",
// Twain is Project Gutenberg's edition of Mark Twain's classic English novel.
// Twain is Mark Twain's classic English novel.
twain: "testdata/Mark.Twain-Tom.Sawyer.txt.bz2",
// 16KB of random data from /dev/urandom
random: "testdata/random.data.bz2",
}
func benchmarkDecode(b *testing.B, testfile int) {
@ -198,6 +201,7 @@ func benchmarkDecode(b *testing.B, testfile int) {
func BenchmarkDecodeDigits(b *testing.B) { benchmarkDecode(b, digits) }
func BenchmarkDecodeTwain(b *testing.B) { benchmarkDecode(b, twain) }
func BenchmarkDecodeRand(b *testing.B) { benchmarkDecode(b, random) }
func TestBufferOverrun(t *testing.T) {
// Tests https://golang.org/issue/5747.

44
libgo/go/compress/bzip2/huffman.go
View file

@ -38,23 +38,35 @@ func (t *huffmanTree) Decode(br *bitReader) (v uint16) {
for {
node := &t.nodes[nodeIndex]
bit, ok := br.TryReadBit()
if !ok && br.ReadBit() {
bit = 1
}
// bzip2 encodes left as a true bit.
if bit != 0 {
// left
if node.left == invalidNodeValue {
return node.leftValue
}
nodeIndex = node.left
var bit uint16
if br.bits > 0 {
// Get next bit - fast path.
br.bits--
bit = 0 - (uint16(br.n>>br.bits) & 1)
} else {
// right
if node.right == invalidNodeValue {
return node.rightValue
}
nodeIndex = node.right
// Get next bit - slow path.
// Use ReadBits to retrieve a single bit
// from the underling io.ByteReader.
bit = 0 - uint16(br.ReadBits(1))
}
// now
// bit = 0xffff if the next bit was 1
// bit = 0x0000 if the next bit was 0
// 1 means left, 0 means right.
//
// if bit == 0xffff {
// nodeIndex = node.left
// } else {
// nodeIndex = node.right
// }
nodeIndex = (bit & node.left) | (^bit & node.right)
if nodeIndex == invalidNodeValue {
// We found a leaf. Use the value of bit to decide
// whether is a left or a right value.
return (bit & node.leftValue) | (^bit & node.rightValue)
}
}
}

10
libgo/go/compress/flate/deflate_test.go
View file

@ -7,6 +7,7 @@ package flate
import (
"bytes"
"fmt"
"internal/testenv"
"io"
"io/ioutil"
"reflect"
@ -343,6 +344,9 @@ var deflateInflateStringTests = []deflateInflateStringTest{
}
func TestDeflateInflateString(t *testing.T) {
if testing.Short() && testenv.Builder() == "" {
t.Skip("skipping in short mode")
}
for _, test := range deflateInflateStringTests {
gold, err := ioutil.ReadFile(test.filename)
if err != nil {
@ -436,7 +440,11 @@ func TestWriterReset(t *testing.T) {
t.Fatalf("NewWriter: %v", err)
}
buf := []byte("hello world")
for i := 0; i < 1024; i++ {
n := 1024
if testing.Short() {
n = 10
}
for i := 0; i < n; i++ {
w.Write(buf)
}
w.Reset(ioutil.Discard)

78
libgo/go/compress/flate/fixedhuff.go
View file

@ -1,78 +0,0 @@
// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package flate
// autogenerated by go run gen.go -output fixedhuff.go, DO NOT EDIT
var fixedHuffmanDecoder = huffmanDecoder{
7,
[huffmanNumChunks]uint32{
0x1007, 0x0508, 0x0108, 0x1188, 0x1107, 0x0708, 0x0308, 0x0c09,
0x1087, 0x0608, 0x0208, 0x0a09, 0x0008, 0x0808, 0x0408, 0x0e09,
0x1047, 0x0588, 0x0188, 0x0909, 0x1147, 0x0788, 0x0388, 0x0d09,
0x10c7, 0x0688, 0x0288, 0x0b09, 0x0088, 0x0888, 0x0488, 0x0f09,
0x1027, 0x0548, 0x0148, 0x11c8, 0x1127, 0x0748, 0x0348, 0x0c89,
0x10a7, 0x0648, 0x0248, 0x0a89, 0x0048, 0x0848, 0x0448, 0x0e89,
0x1067, 0x05c8, 0x01c8, 0x0989, 0x1167, 0x07c8, 0x03c8, 0x0d89,
0x10e7, 0x06c8, 0x02c8, 0x0b89, 0x00c8, 0x08c8, 0x04c8, 0x0f89,
0x1017, 0x0528, 0x0128, 0x11a8, 0x1117, 0x0728, 0x0328, 0x0c49,
0x1097, 0x0628, 0x0228, 0x0a49, 0x0028, 0x0828, 0x0428, 0x0e49,
0x1057, 0x05a8, 0x01a8, 0x0949, 0x1157, 0x07a8, 0x03a8, 0x0d49,
0x10d7, 0x06a8, 0x02a8, 0x0b49, 0x00a8, 0x08a8, 0x04a8, 0x0f49,
0x1037, 0x0568, 0x0168, 0x11e8, 0x1137, 0x0768, 0x0368, 0x0cc9,
0x10b7, 0x0668, 0x0268, 0x0ac9, 0x0068, 0x0868, 0x0468, 0x0ec9,
0x1077, 0x05e8, 0x01e8, 0x09c9, 0x1177, 0x07e8, 0x03e8, 0x0dc9,
0x10f7, 0x06e8, 0x02e8, 0x0bc9, 0x00e8, 0x08e8, 0x04e8, 0x0fc9,
0x1007, 0x0518, 0x0118, 0x1198, 0x1107, 0x0718, 0x0318, 0x0c29,
0x1087, 0x0618, 0x0218, 0x0a29, 0x0018, 0x0818, 0x0418, 0x0e29,
0x1047, 0x0598, 0x0198, 0x0929, 0x1147, 0x0798, 0x0398, 0x0d29,
0x10c7, 0x0698, 0x0298, 0x0b29, 0x0098, 0x0898, 0x0498, 0x0f29,
0x1027, 0x0558, 0x0158, 0x11d8, 0x1127, 0x0758, 0x0358, 0x0ca9,
0x10a7, 0x0658, 0x0258, 0x0aa9, 0x0058, 0x0858, 0x0458, 0x0ea9,
0x1067, 0x05d8, 0x01d8, 0x09a9, 0x1167, 0x07d8, 0x03d8, 0x0da9,
0x10e7, 0x06d8, 0x02d8, 0x0ba9, 0x00d8, 0x08d8, 0x04d8, 0x0fa9,
0x1017, 0x0538, 0x0138, 0x11b8, 0x1117, 0x0738, 0x0338, 0x0c69,
0x1097, 0x0638, 0x0238, 0x0a69, 0x0038, 0x0838, 0x0438, 0x0e69,
0x1057, 0x05b8, 0x01b8, 0x0969, 0x1157, 0x07b8, 0x03b8, 0x0d69,
0x10d7, 0x06b8, 0x02b8, 0x0b69, 0x00b8, 0x08b8, 0x04b8, 0x0f69,
0x1037, 0x0578, 0x0178, 0x11f8, 0x1137, 0x0778, 0x0378, 0x0ce9,
0x10b7, 0x0678, 0x0278, 0x0ae9, 0x0078, 0x0878, 0x0478, 0x0ee9,
0x1077, 0x05f8, 0x01f8, 0x09e9, 0x1177, 0x07f8, 0x03f8, 0x0de9,
0x10f7, 0x06f8, 0x02f8, 0x0be9, 0x00f8, 0x08f8, 0x04f8, 0x0fe9,
0x1007, 0x0508, 0x0108, 0x1188, 0x1107, 0x0708, 0x0308, 0x0c19,
0x1087, 0x0608, 0x0208, 0x0a19, 0x0008, 0x0808, 0x0408, 0x0e19,
0x1047, 0x0588, 0x0188, 0x0919, 0x1147, 0x0788, 0x0388, 0x0d19,
0x10c7, 0x0688, 0x0288, 0x0b19, 0x0088, 0x0888, 0x0488, 0x0f19,
0x1027, 0x0548, 0x0148, 0x11c8, 0x1127, 0x0748, 0x0348, 0x0c99,
0x10a7, 0x0648, 0x0248, 0x0a99, 0x0048, 0x0848, 0x0448, 0x0e99,
0x1067, 0x05c8, 0x01c8, 0x0999, 0x1167, 0x07c8, 0x03c8, 0x0d99,
0x10e7, 0x06c8, 0x02c8, 0x0b99, 0x00c8, 0x08c8, 0x04c8, 0x0f99,
0x1017, 0x0528, 0x0128, 0x11a8, 0x1117, 0x0728, 0x0328, 0x0c59,
0x1097, 0x0628, 0x0228, 0x0a59, 0x0028, 0x0828, 0x0428, 0x0e59,
0x1057, 0x05a8, 0x01a8, 0x0959, 0x1157, 0x07a8, 0x03a8, 0x0d59,
0x10d7, 0x06a8, 0x02a8, 0x0b59, 0x00a8, 0x08a8, 0x04a8, 0x0f59,
0x1037, 0x0568, 0x0168, 0x11e8, 0x1137, 0x0768, 0x0368, 0x0cd9,
0x10b7, 0x0668, 0x0268, 0x0ad9, 0x0068, 0x0868, 0x0468, 0x0ed9,
0x1077, 0x05e8, 0x01e8, 0x09d9, 0x1177, 0x07e8, 0x03e8, 0x0dd9,
0x10f7, 0x06e8, 0x02e8, 0x0bd9, 0x00e8, 0x08e8, 0x04e8, 0x0fd9,
0x1007, 0x0518, 0x0118, 0x1198, 0x1107, 0x0718, 0x0318, 0x0c39,
0x1087, 0x0618, 0x0218, 0x0a39, 0x0018, 0x0818, 0x0418, 0x0e39,
0x1047, 0x0598, 0x0198, 0x0939, 0x1147, 0x0798, 0x0398, 0x0d39,
0x10c7, 0x0698, 0x0298, 0x0b39, 0x0098, 0x0898, 0x0498, 0x0f39,
0x1027, 0x0558, 0x0158, 0x11d8, 0x1127, 0x0758, 0x0358, 0x0cb9,
0x10a7, 0x0658, 0x0258, 0x0ab9, 0x0058, 0x0858, 0x0458, 0x0eb9,
0x1067, 0x05d8, 0x01d8, 0x09b9, 0x1167, 0x07d8, 0x03d8, 0x0db9,
0x10e7, 0x06d8, 0x02d8, 0x0bb9, 0x00d8, 0x08d8, 0x04d8, 0x0fb9,
0x1017, 0x0538, 0x0138, 0x11b8, 0x1117, 0x0738, 0x0338, 0x0c79,
0x1097, 0x0638, 0x0238, 0x0a79, 0x0038, 0x0838, 0x0438, 0x0e79,
0x1057, 0x05b8, 0x01b8, 0x0979, 0x1157, 0x07b8, 0x03b8, 0x0d79,
0x10d7, 0x06b8, 0x02b8, 0x0b79, 0x00b8, 0x08b8, 0x04b8, 0x0f79,
0x1037, 0x0578, 0x0178, 0x11f8, 0x1137, 0x0778, 0x0378, 0x0cf9,
0x10b7, 0x0678, 0x0278, 0x0af9, 0x0078, 0x0878, 0x0478, 0x0ef9,
0x1077, 0x05f8, 0x01f8, 0x09f9, 0x1177, 0x07f8, 0x03f8, 0x0df9,
0x10f7, 0x06f8, 0x02f8, 0x0bf9, 0x00f8, 0x08f8, 0x04f8, 0x0ff9,
},
nil, 0,
}

14
libgo/go/compress/flate/flate_test.go
View file

@ -11,7 +11,9 @@ package flate
import (
"bytes"
"encoding/hex"
"io"
"io/ioutil"
"strings"
"testing"
)
@ -258,3 +260,15 @@ func TestStreams(t *testing.T) {
}
}
}
func TestTruncatedStreams(t *testing.T) {
const data = "\x00\f\x00\xf3\xffhello, world\x01\x00\x00\xff\xff"
for i := 0; i < len(data)-1; i++ {
r := NewReader(strings.NewReader(data[:i]))
_, err := io.Copy(ioutil.Discard, r)
if err != io.ErrUnexpectedEOF {
t.Errorf("io.Copy(%d) on truncated stream: got %v, want %v", i, err, io.ErrUnexpectedEOF)
}
}
}

265
libgo/go/compress/flate/gen.go
View file

@ -1,265 +0,0 @@
// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build ignore
// This program generates fixedhuff.go
// Invoke as
//
// go run gen.go -output fixedhuff.go
package main
import (
"bytes"
"flag"
"fmt"
"go/format"
"io/ioutil"
"log"
)
var filename = flag.String("output", "fixedhuff.go", "output file name")
const maxCodeLen = 16
// Note: the definition of the huffmanDecoder struct is copied from
// inflate.go, as it is private to the implementation.
// chunk & 15 is number of bits
// chunk >> 4 is value, including table link
const (
huffmanChunkBits = 9
huffmanNumChunks = 1 << huffmanChunkBits
huffmanCountMask = 15
huffmanValueShift = 4
)
type huffmanDecoder struct {
min int // the minimum code length
chunks [huffmanNumChunks]uint32 // chunks as described above
links [][]uint32 // overflow links
linkMask uint32 // mask the width of the link table
}
// Initialize Huffman decoding tables from array of code lengths.
// Following this function, h is guaranteed to be initialized into a complete
// tree (i.e., neither over-subscribed nor under-subscribed). The exception is a
// degenerate case where the tree has only a single symbol with length 1. Empty
// trees are permitted.
func (h *huffmanDecoder) init(bits []int) bool {
// Sanity enables additional runtime tests during Huffman
// table construction. It's intended to be used during
// development to supplement the currently ad-hoc unit tests.
const sanity = false
if h.min != 0 {
*h = huffmanDecoder{}
}
// Count number of codes of each length,
// compute min and max length.
var count [maxCodeLen]int
var min, max int
for _, n := range bits {
if n == 0 {
continue
}
if min == 0 || n < min {
min = n
}
if n > max {
max = n
}
count[n]++
}
// Empty tree. The decompressor.huffSym function will fail later if the tree
// is used. Technically, an empty tree is only valid for the HDIST tree and
// not the HCLEN and HLIT tree. However, a stream with an empty HCLEN tree
// is guaranteed to fail since it will attempt to use the tree to decode the
// codes for the HLIT and HDIST trees. Similarly, an empty HLIT tree is
// guaranteed to fail later since the compressed data section must be
// composed of at least one symbol (the end-of-block marker).
if max == 0 {
return true
}
code := 0
var nextcode [maxCodeLen]int
for i := min; i <= max; i++ {
code <<= 1
nextcode[i] = code
code += count[i]
}
// Check that the coding is complete (i.e., that we've
// assigned all 2-to-the-max possible bit sequences).
// Exception: To be compatible with zlib, we also need to
// accept degenerate single-code codings. See also
// TestDegenerateHuffmanCoding.
if code != 1<<uint(max) && !(code == 1 && max == 1) {
return false
}
h.min = min
if max > huffmanChunkBits {
numLinks := 1 << (uint(max) - huffmanChunkBits)
h.linkMask = uint32(numLinks - 1)
// create link tables
link := nextcode[huffmanChunkBits+1] >> 1
h.links = make([][]uint32, huffmanNumChunks-link)
for j := uint(link); j < huffmanNumChunks; j++ {
reverse := int(reverseByte[j>>8]) | int(reverseByte[j&0xff])<<8
reverse >>= uint(16 - huffmanChunkBits)
off := j - uint(link)
if sanity && h.chunks[reverse] != 0 {
panic("impossible: overwriting existing chunk")
}
h.chunks[reverse] = uint32(off<<huffmanValueShift | (huffmanChunkBits + 1))
h.links[off] = make([]uint32, numLinks)
}
}
for i, n := range bits {
if n == 0 {
continue
}
code := nextcode[n]
nextcode[n]++
chunk := uint32(i<<huffmanValueShift | n)
reverse := int(reverseByte[code>>8]) | int(reverseByte[code&0xff])<<8
reverse >>= uint(16 - n)
if n <= huffmanChunkBits {
for off := reverse; off < len(h.chunks); off += 1 << uint(n) {
// We should never need to overwrite
// an existing chunk. Also, 0 is
// never a valid chunk, because the
// lower 4 "count" bits should be
// between 1 and 15.
if sanity && h.chunks[off] != 0 {
panic("impossible: overwriting existing chunk")
}
h.chunks[off] = chunk
}
} else {
j := reverse & (huffmanNumChunks - 1)
if sanity && h.chunks[j]&huffmanCountMask != huffmanChunkBits+1 {
// Longer codes should have been
// associated with a link table above.
panic("impossible: not an indirect chunk")
}
value := h.chunks[j] >> huffmanValueShift
linktab := h.links[value]
reverse >>= huffmanChunkBits
for off := reverse; off < len(linktab); off += 1 << uint(n-huffmanChunkBits) {
if sanity && linktab[off] != 0 {
panic("impossible: overwriting existing chunk")
}
linktab[off] = chunk
}
}
}
if sanity {
// Above we've sanity checked that we never overwrote
// an existing entry. Here we additionally check that
// we filled the tables completely.
for i, chunk := range h.chunks {
if chunk == 0 {
// As an exception, in the degenerate
// single-code case, we allow odd
// chunks to be missing.
if code == 1 && i%2 == 1 {
continue
}
panic("impossible: missing chunk")
}
}
for _, linktab := range h.links {
for _, chunk := range linktab {
if chunk == 0 {
panic("impossible: missing chunk")
}
}
}
}
return true
}
func main() {
flag.Parse()
var h huffmanDecoder
var bits [288]int
initReverseByte()
for i := 0; i < 144; i++ {
bits[i] = 8
}
for i := 144; i < 256; i++ {
bits[i] = 9
}
for i := 256; i < 280; i++ {
bits[i] = 7
}
for i := 280; i < 288; i++ {
bits[i] = 8
}
h.init(bits[:])
if h.links != nil {
log.Fatal("Unexpected links table in fixed Huffman decoder")
}
var buf bytes.Buffer
fmt.Fprintf(&buf, `// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.`+"\n\n")
fmt.Fprintln(&buf, "package flate")
fmt.Fprintln(&buf)
fmt.Fprintln(&buf, "// autogenerated by go run gen.go -output fixedhuff.go, DO NOT EDIT")
fmt.Fprintln(&buf)
fmt.Fprintln(&buf, "var fixedHuffmanDecoder = huffmanDecoder{")
fmt.Fprintf(&buf, "\t%d,\n", h.min)
fmt.Fprintln(&buf, "\t[huffmanNumChunks]uint32{")
for i := 0; i < huffmanNumChunks; i++ {
if i&7 == 0 {
fmt.Fprintf(&buf, "\t\t")
} else {
fmt.Fprintf(&buf, " ")
}
fmt.Fprintf(&buf, "0x%04x,", h.chunks[i])
if i&7 == 7 {
fmt.Fprintln(&buf)
}
}
fmt.Fprintln(&buf, "\t},")
fmt.Fprintln(&buf, "\tnil, 0,")
fmt.Fprintln(&buf, "}")
data, err := format.Source(buf.Bytes())
if err != nil {
log.Fatal(err)
}
err = ioutil.WriteFile(*filename, data, 0644)
if err != nil {
log.Fatal(err)
}
}
var reverseByte [256]byte
func initReverseByte() {
for x := 0; x < 256; x++ {
var result byte
for i := uint(0); i < 8; i++ {
result |= byte(((x >> i) & 1) << (7 - i))
}
reverseByte[x] = result
}
}

66
libgo/go/compress/flate/inflate.go
View file

@ -2,8 +2,6 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:generate go run gen.go -output fixedhuff.go
// Package flate implements the DEFLATE compressed data format, described in
// RFC 1951. The gzip and zlib packages implement access to DEFLATE-based file
// formats.
@ -13,6 +11,7 @@ import (
"bufio"
"io"
"strconv"
"sync"
)
const (
@ -26,6 +25,10 @@ const (
numCodes = 19 // number of codes in Huffman meta-code
)
// Initialize the fixedHuffmanDecoder only once upon first use.
var fixedOnce sync.Once
var fixedHuffmanDecoder huffmanDecoder
// A CorruptInputError reports the presence of corrupt input at a given offset.
type CorruptInputError int64
@ -39,6 +42,8 @@ type InternalError string
func (e InternalError) Error() string { return "flate: internal error: " + string(e) }
// A ReadError reports an error encountered while reading input.
//
// Deprecated: No longer returned.
type ReadError struct {
Offset int64 // byte offset where error occurred
Err error // error returned by underlying Read
@ -49,6 +54,8 @@ func (e *ReadError) Error() string {
}
// A WriteError reports an error encountered while writing output.
//
// Deprecated: No longer returned.
type WriteError struct {
Offset int64 // byte offset where error occurred
Err error // error returned by underlying Write
@ -67,10 +74,6 @@ type Resetter interface {
Reset(r io.Reader, dict []byte) error
}
// Note that much of the implementation of huffmanDecoder is also copied
// into gen.go (in package main) for the purpose of precomputing the
// fixed huffman tables so they can be included statically.
// The data structure for decoding Huffman tables is based on that of
// zlib. There is a lookup table of a fixed bit width (huffmanChunkBits),
// For codes smaller than the table width, there are multiple entries
@ -78,12 +81,15 @@ type Resetter interface {
// larger than the table width, the table contains a link to an overflow
// table. The width of each entry in the link table is the maximum code
// size minus the chunk width.
//
// Note that you can do a lookup in the table even without all bits
// filled. Since the extra bits are zero, and the DEFLATE Huffman codes
// have the property that shorter codes come before longer ones, the
// bit length estimate in the result is a lower bound on the actual
// number of bits.
//
// See the following:
// http://www.gzip.org/algorithm.txt
// chunk & 15 is number of bits
// chunk >> 4 is value, including table link
@ -459,6 +465,14 @@ func (f *decompressor) readHuffman() error {
return CorruptInputError(f.roffset)
}
// As an optimization, we can initialize the min bits to read at a time
// for the HLIT tree to the length of the EOB marker since we know that
// every block must terminate with one. This preserves the property that
// we never read any extra bytes after the end of the DEFLATE stream.
if f.h1.min < f.bits[endBlockMarker] {
f.h1.min = f.bits[endBlockMarker]
}
return nil
}
@ -635,7 +649,10 @@ func (f *decompressor) dataBlock() {
nr, err := io.ReadFull(f.r, f.buf[0:4])
f.roffset += int64(nr)
if err != nil {
f.err = &ReadError{f.roffset, err}
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
f.err = err
return
}
n := int(f.buf[0]) | int(f.buf[1])<<8
@ -667,7 +684,10 @@ func (f *decompressor) copyData() {
m, err := io.ReadFull(f.r, f.hist[f.hp:f.hp+m])
f.roffset += int64(m)
if err != nil {
f.err = &ReadError{f.roffset, err}
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
f.err = err
return
}
n -= m
@ -760,6 +780,26 @@ func makeReader(r io.Reader) Reader {
return bufio.NewReader(r)
}
func fixedHuffmanDecoderInit() {
fixedOnce.Do(func() {
// These come from the RFC section 3.2.6.
var bits [288]int
for i := 0; i < 144; i++ {
bits[i] = 8
}
for i := 144; i < 256; i++ {
bits[i] = 9
}
for i := 256; i < 280; i++ {
bits[i] = 7
}
for i := 280; i < 288; i++ {
bits[i] = 8
}
fixedHuffmanDecoder.init(bits[:])
})
}
func (f *decompressor) Reset(r io.Reader, dict []byte) error {
*f = decompressor{
r: makeReader(r),
@ -783,11 +823,13 @@ func (f *decompressor) Reset(r io.Reader, dict []byte) error {
//
// The ReadCloser returned by NewReader also implements Resetter.
func NewReader(r io.Reader) io.ReadCloser {
fixedHuffmanDecoderInit()
var f decompressor
f.bits = new([maxNumLit + maxNumDist]int)
f.codebits = new([numCodes]int)
f.r = makeReader(r)
f.hist = new([maxHist]byte)
f.bits = new([maxNumLit + maxNumDist]int)
f.codebits = new([numCodes]int)
f.step = (*decompressor).nextBlock
return &f
}
@ -800,6 +842,8 @@ func NewReader(r io.Reader) io.ReadCloser {
//
// The ReadCloser returned by NewReader also implements Resetter.
func NewReaderDict(r io.Reader, dict []byte) io.ReadCloser {
fixedHuffmanDecoderInit()
var f decompressor
f.r = makeReader(r)
f.hist = new([maxHist]byte)

2
libgo/go/compress/flate/reader_test.go
View file

@ -32,7 +32,7 @@ var testfiles = []string{
// does not repeat, but there are only 10 possible digits, so it should be
// reasonably compressible.
digits: "../testdata/e.txt",
// Twain is Project Gutenberg's edition of Mark Twain's classic English novel.
// Twain is Mark Twain's classic English novel.
twain: "../testdata/Mark.Twain-Tom.Sawyer.txt",
}

12
libgo/go/compress/flate/token.go
View file

@ -90,13 +90,11 @@ func lengthCode(len uint32) uint32 { return lengthCodes[len] }
// Returns the offset code corresponding to a specific offset
func offsetCode(off uint32) uint32 {
const n = uint32(len(offsetCodes))
switch {
case off < n:
if off < uint32(len(offsetCodes)) {
return offsetCodes[off]
case off>>7 < n:
return offsetCodes[off>>7] + 14
default:
return offsetCodes[off>>14] + 28
}
if off>>7 < uint32(len(offsetCodes)) {
return offsetCodes[off>>7] + 14
}
return offsetCodes[off>>14] + 28
}

24
libgo/go/compress/gzip/gunzip.go
View file

@ -43,6 +43,9 @@ var (
// The gzip file stores a header giving metadata about the compressed file.
// That header is exposed as the fields of the Writer and Reader structs.
//
// Strings must be UTF-8 encoded and may only contain Unicode code points
// U+0001 through U+00FF, due to limitations of the GZIP file format.
type Header struct {
Comment string // comment
Extra []byte // "extra data"
@ -66,7 +69,7 @@ type Header struct {
// returned by Read as tentative until they receive the io.EOF
// marking the end of the data.
type Reader struct {
Header
Header // valid after NewReader or Reader.Reset
r flate.Reader
decompressor io.ReadCloser
digest hash.Hash32
@ -80,7 +83,10 @@ type Reader struct {
// NewReader creates a new Reader reading the given reader.
// If r does not also implement io.ByteReader,
// the decompressor may read more data than necessary from r.
//
// It is the caller's responsibility to call Close on the Reader when done.
//
// The Reader.Header fields will be valid in the Reader returned.
func NewReader(r io.Reader) (*Reader, error) {
z := new(Reader)
z.r = makeReader(r)
@ -164,6 +170,9 @@ func (z *Reader) readString() (string, error) {
func (z *Reader) read2() (uint32, error) {
_, err := io.ReadFull(z.r, z.buf[0:2])
if err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
return 0, err
}
return uint32(z.buf[0]) | uint32(z.buf[1])<<8, nil
@ -172,6 +181,13 @@ func (z *Reader) read2() (uint32, error) {
func (z *Reader) readHeader(save bool) error {
_, err := io.ReadFull(z.r, z.buf[0:10])
if err != nil {
// RFC1952 section 2.2 says the following:
// A gzip file consists of a series of "members" (compressed data sets).
//
// Other than this, the specification does not clarify whether a
// "series" is defined as "one or more" or "zero or more". To err on the
// side of caution, Go interprets this to mean "zero or more".
// Thus, it is okay to return io.EOF here.
return err
}
if z.buf[0] != gzipID1 || z.buf[1] != gzipID2 || z.buf[2] != gzipDeflate {
@ -193,6 +209,9 @@ func (z *Reader) readHeader(save bool) error {
}
data := make([]byte, n)
if _, err = io.ReadFull(z.r, data); err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
return err
}
if save {
@ -257,6 +276,9 @@ func (z *Reader) Read(p []byte) (n int, err error) {
// Finished file; check checksum + size.
if _, err := io.ReadFull(z.r, z.buf[0:8]); err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
z.err = err
return 0, err
}

32
libgo/go/compress/gzip/gunzip_test.go
View file

@ -6,6 +6,7 @@ package gzip
import (
"bytes"
"compress/flate"
"io"
"io/ioutil"
"os"
@ -408,3 +409,34 @@ Found:
t.Fatalf("third reset: err=%v, want io.EOF", err)
}
}
func TestNilStream(t *testing.T) {
// Go liberally interprets RFC1952 section 2.2 to mean that a gzip file
// consist of zero or more members. Thus, we test that a nil stream is okay.
_, err := NewReader(bytes.NewReader(nil))
if err != io.EOF {
t.Fatalf("NewReader(nil) on empty stream: got %v, want io.EOF", err)
}
}
func TestTruncatedStreams(t *testing.T) {
const data = "\x1f\x8b\b\x04\x00\tn\x88\x00\xff\a\x00foo bar\xcbH\xcd\xc9\xc9\xd7Q(\xcf/\xcaI\x01\x04:r\xab\xff\f\x00\x00\x00"
// Intentionally iterate starting with at least one byte in the stream.
for i := 1; i < len(data)-1; i++ {
r, err := NewReader(strings.NewReader(data[:i]))
if err != nil {
if err != io.ErrUnexpectedEOF {
t.Errorf("NewReader(%d) on truncated stream: got %v, want %v", i, err, io.ErrUnexpectedEOF)
}
continue
}
_, err = io.Copy(ioutil.Discard, r)
if ferr, ok := err.(*flate.ReadError); ok {
err = ferr.Err
}
if err != io.ErrUnexpectedEOF {
t.Errorf("io.Copy(%d) on truncated stream: got %v, want %v", i, err, io.ErrUnexpectedEOF)
}
}
}

7
libgo/go/compress/gzip/gzip.go
View file

@ -25,7 +25,7 @@ const (
// A Writer is an io.WriteCloser.
// Writes to a Writer are compressed and written to w.
type Writer struct {
Header
Header // written at first call to Write, Flush, or Close
w io.Writer
level int
wroteHeader bool
@ -44,10 +44,7 @@ type Writer struct {
// Writes may be buffered and not flushed until Close.
//
// Callers that wish to set the fields in Writer.Header must do so before
// the first call to Write or Close. The Comment and Name header fields are
// UTF-8 strings in Go, but the underlying format requires NUL-terminated ISO
// 8859-1 (Latin-1). NUL or non-Latin-1 runes in those strings will lead to an
// error on Write.
// the first call to Write, Flush, or Close.
func NewWriter(w io.Writer) *Writer {
z, _ := NewWriterLevel(w, DefaultCompression)
return z

17
libgo/go/compress/lzw/reader.go
View file

@ -132,6 +132,7 @@ func (d *decoder) Read(b []byte) (int, error) {
// litWidth is the width in bits of literal codes.
func (d *decoder) decode() {
// Loop over the code stream, converting codes into decompressed bytes.
loop:
for {
code, err := d.read(d)
if err != nil {
@ -139,8 +140,7 @@ func (d *decoder) decode() {
err = io.ErrUnexpectedEOF
}
d.err = err
d.flush()
return
break
}
switch {
case code < d.clear:
@ -159,9 +159,8 @@ func (d *decoder) decode() {
d.last = decoderInvalidCode
continue
case code == d.eof:
d.flush()
d.err = io.EOF
return
break loop
case code <= d.hi:
c, i := code, len(d.output)-1
if code == d.hi {
@ -191,8 +190,7 @@ func (d *decoder) decode() {
}
default:
d.err = errors.New("lzw: invalid code")
d.flush()
return
break loop
}
d.last, d.hi = code, d.hi+1
if d.hi >= d.overflow {
@ -204,13 +202,10 @@ func (d *decoder) decode() {
}
}
if d.o >= flushBuffer {
d.flush()
return
break
}
}
}
func (d *decoder) flush() {
// Flush pending output.
d.toRead = d.output[:d.o]
d.o = 0
}

10
libgo/go/compress/lzw/writer_test.go
View file

@ -5,6 +5,7 @@
package lzw
import (
"internal/testenv"
"io"
"io/ioutil"
"os"
@ -13,6 +14,7 @@ import (
)
var filenames = []string{
"../testdata/gettysburg.txt",
"../testdata/e.txt",
"../testdata/pi.txt",
}
@ -89,10 +91,16 @@ func TestWriter(t *testing.T) {
for _, filename := range filenames {
for _, order := range [...]Order{LSB, MSB} {
// The test data "2.71828 etcetera" is ASCII text requiring at least 6 bits.
for _, litWidth := range [...]int{6, 7, 8} {
for litWidth := 6; litWidth <= 8; litWidth++ {
if filename == "../testdata/gettysburg.txt" && litWidth == 6 {
continue
}
testFile(t, filename, order, litWidth)
}
}
if testing.Short() && testenv.Builder() == "" {
break
}
}
}

393
libgo/go/compress/testdata/Mark.Twain-Tom.Sawyer.txt vendored
View file

@ -1,27 +1,3 @@
The Project Gutenberg EBook of The Adventures of Tom Sawyer, Complete
by Mark Twain (Samuel Clemens)
This eBook is for the use of anyone anywhere at no cost and with
almost no restrictions whatsoever. You may copy it, give it away or
re-use it under the terms of the Project Gutenberg License included
with this eBook or online at www.gutenberg.net
Title: The Adventures of Tom Sawyer, Complete
Author: Mark Twain (Samuel Clemens)
Release Date: August 20, 2006 [EBook #74]
[Last updated: May 3, 2011]
Language: English
*** START OF THIS PROJECT GUTENBERG EBOOK TOM SAWYER ***
Produced by David Widger. The previous edition was updated by Jose
Menendez.
@ -8487,372 +8463,3 @@ prosperous and happy. Some day it may seem worth while to take up the
story of the younger ones again and see what sort of men and women they
turned out to be; therefore it will be wisest not to reveal any of that
part of their lives at present.
End of the Project Gutenberg EBook of The Adventures of Tom Sawyer, Complete
by Mark Twain (Samuel Clemens)
*** END OF THIS PROJECT GUTENBERG EBOOK TOM SAWYER ***
***** This file should be named 74.txt or 74.zip *****
This and all associated files of various formats will be found in:
http://www.gutenberg.net/7/74/
Produced by David Widger. The previous edition was update by Jose
Menendez.
Updated editions will replace the previous one--the old editions
will be renamed.
Creating the works from public domain print editions means that no
one owns a United States copyright in these works, so the Foundation
(and you!) can copy and distribute it in the United States without
permission and without paying copyright royalties. Special rules,
set forth in the General Terms of Use part of this license, apply to
copying and distributing Project Gutenberg-tm electronic works to
protect the PROJECT GUTENBERG-tm concept and trademark. Project
Gutenberg is a registered trademark, and may not be used if you
charge for the eBooks, unless you receive specific permission. If you
do not charge anything for copies of this eBook, complying with the
rules is very easy. You may use this eBook for nearly any purpose
such as creation of derivative works, reports, performances and
research. They may be modified and printed and given away--you may do
practically ANYTHING with public domain eBooks. Redistribution is
subject to the trademark license, especially commercial
redistribution.
*** START: FULL LICENSE ***
THE FULL PROJECT GUTENBERG LICENSE
PLEASE READ THIS BEFORE YOU DISTRIBUTE OR USE THIS WORK
To protect the Project Gutenberg-tm mission of promoting the free
distribution of electronic works, by using or distributing this work
(or any other work associated in any way with the phrase "Project
Gutenberg"), you agree to comply with all the terms of the Full Project
Gutenberg-tm License (available with this file or online at
http://gutenberg.net/license).
Section 1. General Terms of Use and Redistributing Project Gutenberg-tm
electronic works
1.A. By reading or using any part of this Project Gutenberg-tm
electronic work, you indicate that you have read, understand, agree to
and accept all the terms of this license and intellectual property
(trademark/copyright) agreement. If you do not agree to abide by all
the terms of this agreement, you must cease using and return or destroy
all copies of Project Gutenberg-tm electronic works in your possession.
If you paid a fee for obtaining a copy of or access to a Project
Gutenberg-tm electronic work and you do not agree to be bound by the
terms of this agreement, you may obtain a refund from the person or
entity to whom you paid the fee as set forth in paragraph 1.E.8.
1.B. "Project Gutenberg" is a registered trademark. It may only be
used on or associated in any way with an electronic work by people who
agree to be bound by the terms of this agreement. There are a few
things that you can do with most Project Gutenberg-tm electronic works
even without complying with the full terms of this agreement. See
paragraph 1.C below. There are a lot of things you can do with Project
Gutenberg-tm electronic works if you follow the terms of this agreement
and help preserve free future access to Project Gutenberg-tm electronic
works. See paragraph 1.E below.
1.C. The Project Gutenberg Literary Archive Foundation ("the Foundation"
or PGLAF), owns a compilation copyright in the collection of Project
Gutenberg-tm electronic works. Nearly all the individual works in the
collection are in the public domain in the United States. If an
individual work is in the public domain in the United States and you are
located in the United States, we do not claim a right to prevent you from
copying, distributing, performing, displaying or creating derivative
works based on the work as long as all references to Project Gutenberg
are removed. Of course, we hope that you will support the Project
Gutenberg-tm mission of promoting free access to electronic works by
freely sharing Project Gutenberg-tm works in compliance with the terms of
this agreement for keeping the Project Gutenberg-tm name associated with
the work. You can easily comply with the terms of this agreement by
keeping this work in the same format with its attached full Project
Gutenberg-tm License when you share it without charge with others.
1.D. The copyright laws of the place where you are located also govern
what you can do with this work. Copyright laws in most countries are in
a constant state of change. If you are outside the United States, check
the laws of your country in addition to the terms of this agreement
before downloading, copying, displaying, performing, distributing or
creating derivative works based on this work or any other Project
Gutenberg-tm work. The Foundation makes no representations concerning
the copyright status of any work in any country outside the United
States.
1.E. Unless you have removed all references to Project Gutenberg:
1.E.1. The following sentence, with active links to, or other immediate
access to, the full Project Gutenberg-tm License must appear prominently
whenever any copy of a Project Gutenberg-tm work (any work on which the
phrase "Project Gutenberg" appears, or with which the phrase "Project
Gutenberg" is associated) is accessed, displayed, performed, viewed,
copied or distributed:
This eBook is for the use of anyone anywhere at no cost and with
almost no restrictions whatsoever. You may copy it, give it away or
re-use it under the terms of the Project Gutenberg License included
with this eBook or online at www.gutenberg.net
1.E.2. If an individual Project Gutenberg-tm electronic work is derived
from the public domain (does not contain a notice indicating that it is
posted with permission of the copyright holder), the work can be copied
and distributed to anyone in the United States without paying any fees
or charges. If you are redistributing or providing access to a work
with the phrase "Project Gutenberg" associated with or appearing on the
work, you must comply either with the requirements of paragraphs 1.E.1
through 1.E.7 or obtain permission for the use of the work and the
Project Gutenberg-tm trademark as set forth in paragraphs 1.E.8 or
1.E.9.
1.E.3. If an individual Project Gutenberg-tm electronic work is posted
with the permission of the copyright holder, your use and distribution
must comply with both paragraphs 1.E.1 through 1.E.7 and any additional
terms imposed by the copyright holder. Additional terms will be linked
to the Project Gutenberg-tm License for all works posted with the
permission of the copyright holder found at the beginning of this work.
1.E.4. Do not unlink or detach or remove the full Project Gutenberg-tm
License terms from this work, or any files containing a part of this
work or any other work associated with Project Gutenberg-tm.
1.E.5. Do not copy, display, perform, distribute or redistribute this
electronic work, or any part of this electronic work, without
prominently displaying the sentence set forth in paragraph 1.E.1 with
active links or immediate access to the full terms of the Project
Gutenberg-tm License.
1.E.6. You may convert to and distribute this work in any binary,
compressed, marked up, nonproprietary or proprietary form, including any
word processing or hypertext form. However, if you provide access to or
distribute copies of a Project Gutenberg-tm work in a format other than
"Plain Vanilla ASCII" or other format used in the official version
posted on the official Project Gutenberg-tm web site (www.gutenberg.net),
you must, at no additional cost, fee or expense to the user, provide a
copy, a means of exporting a copy, or a means of obtaining a copy upon
request, of the work in its original "Plain Vanilla ASCII" or other
form. Any alternate format must include the full Project Gutenberg-tm
License as specified in paragraph 1.E.1.
1.E.7. Do not charge a fee for access to, viewing, displaying,
performing, copying or distributing any Project Gutenberg-tm works
unless you comply with paragraph 1.E.8 or 1.E.9.
1.E.8. You may charge a reasonable fee for copies of or providing
access to or distributing Project Gutenberg-tm electronic works provided
that
- You pay a royalty fee of 20% of the gross profits you derive from
the use of Project Gutenberg-tm works calculated using the method
you already use to calculate your applicable taxes. The fee is
owed to the owner of the Project Gutenberg-tm trademark, but he
has agreed to donate royalties under this paragraph to the
Project Gutenberg Literary Archive Foundation. Royalty payments
must be paid within 60 days following each date on which you
prepare (or are legally required to prepare) your periodic tax
returns. Royalty payments should be clearly marked as such and
sent to the Project Gutenberg Literary Archive Foundation at the
address specified in Section 4, "Information about donations to
the Project Gutenberg Literary Archive Foundation."
- You provide a full refund of any money paid by a user who notifies
you in writing (or by e-mail) within 30 days of receipt that s/he
does not agree to the terms of the full Project Gutenberg-tm
License. You must require such a user to return or
destroy all copies of the works possessed in a physical medium
and discontinue all use of and all access to other copies of
Project Gutenberg-tm works.
- You provide, in accordance with paragraph 1.F.3, a full refund of any
money paid for a work or a replacement copy, if a defect in the
electronic work is discovered and reported to you within 90 days
of receipt of the work.
- You comply with all other terms of this agreement for free
distribution of Project Gutenberg-tm works.
1.E.9. If you wish to charge a fee or distribute a Project Gutenberg-tm
electronic work or group of works on different terms than are set
forth in this agreement, you must obtain permission in writing from
both the Project Gutenberg Literary Archive Foundation and Michael
Hart, the owner of the Project Gutenberg-tm trademark. Contact the
Foundation as set forth in Section 3 below.
1.F.
1.F.1. Project Gutenberg volunteers and employees expend considerable
effort to identify, do copyright research on, transcribe and proofread
public domain works in creating the Project Gutenberg-tm
collection. Despite these efforts, Project Gutenberg-tm electronic
works, and the medium on which they may be stored, may contain
"Defects," such as, but not limited to, incomplete, inaccurate or
corrupt data, transcription errors, a copyright or other intellectual
property infringement, a defective or damaged disk or other medium, a
computer virus, or computer codes that damage or cannot be read by
your equipment.
1.F.2. LIMITED WARRANTY, DISCLAIMER OF DAMAGES - Except for the "Right
of Replacement or Refund" described in paragraph 1.F.3, the Project
Gutenberg Literary Archive Foundation, the owner of the Project
Gutenberg-tm trademark, and any other party distributing a Project
Gutenberg-tm electronic work under this agreement, disclaim all
liability to you for damages, costs and expenses, including legal
fees. YOU AGREE THAT YOU HAVE NO REMEDIES FOR NEGLIGENCE, STRICT
LIABILITY, BREACH OF WARRANTY OR BREACH OF CONTRACT EXCEPT THOSE
PROVIDED IN PARAGRAPH F3. YOU AGREE THAT THE FOUNDATION, THE
TRADEMARK OWNER, AND ANY DISTRIBUTOR UNDER THIS AGREEMENT WILL NOT BE
LIABLE TO YOU FOR ACTUAL, DIRECT, INDIRECT, CONSEQUENTIAL, PUNITIVE OR
INCIDENTAL DAMAGES EVEN IF YOU GIVE NOTICE OF THE POSSIBILITY OF SUCH
DAMAGE.
1.F.3. LIMITED RIGHT OF REPLACEMENT OR REFUND - If you discover a
defect in this electronic work within 90 days of receiving it, you can
receive a refund of the money (if any) you paid for it by sending a
written explanation to the person you received the work from. If you
received the work on a physical medium, you must return the medium with
your written explanation. The person or entity that provided you with
the defective work may elect to provide a replacement copy in lieu of a
refund. If you received the work electronically, the person or entity
providing it to you may choose to give you a second opportunity to
receive the work electronically in lieu of a refund. If the second copy
is also defective, you may demand a refund in writing without further
opportunities to fix the problem.
1.F.4. Except for the limited right of replacement or refund set forth
in paragraph 1.F.3, this work is provided to you 'AS-IS' WITH NO OTHER
WARRANTIES OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
WARRANTIES OF MERCHANTIBILITY OR FITNESS FOR ANY PURPOSE.
1.F.5. Some states do not allow disclaimers of certain implied
warranties or the exclusion or limitation of certain types of damages.
If any disclaimer or limitation set forth in this agreement violates the
law of the state applicable to this agreement, the agreement shall be
interpreted to make the maximum disclaimer or limitation permitted by
the applicable state law. The invalidity or unenforceability of any
provision of this agreement shall not void the remaining provisions.
1.F.6. INDEMNITY - You agree to indemnify and hold the Foundation, the
trademark owner, any agent or employee of the Foundation, anyone
providing copies of Project Gutenberg-tm electronic works in accordance
with this agreement, and any volunteers associated with the production,
promotion and distribution of Project Gutenberg-tm electronic works,
harmless from all liability, costs and expenses, including legal fees,
that arise directly or indirectly from any of the following which you do
or cause to occur: (a) distribution of this or any Project Gutenberg-tm
work, (b) alteration, modification, or additions or deletions to any
Project Gutenberg-tm work, and (c) any Defect you cause.
Section 2. Information about the Mission of Project Gutenberg-tm
Project Gutenberg-tm is synonymous with the free distribution of
electronic works in formats readable by the widest variety of computers
including obsolete, old, middle-aged and new computers. It exists
because of the efforts of hundreds of volunteers and donations from
people in all walks of life.
Volunteers and financial support to provide volunteers with the
assistance they need, is critical to reaching Project Gutenberg-tm's
goals and ensuring that the Project Gutenberg-tm collection will
remain freely available for generations to come. In 2001, the Project
Gutenberg Literary Archive Foundation was created to provide a secure
and permanent future for Project Gutenberg-tm and future generations.
To learn more about the Project Gutenberg Literary Archive Foundation
and how your efforts and donations can help, see Sections 3 and 4
and the Foundation web page at http://www.pglaf.org.
Section 3. Information about the Project Gutenberg Literary Archive
Foundation
The Project Gutenberg Literary Archive Foundation is a non profit
501(c)(3) educational corporation organized under the laws of the
state of Mississippi and granted tax exempt status by the Internal
Revenue Service. The Foundation's EIN or federal tax identification
number is 64-6221541. Its 501(c)(3) letter is posted at
http://pglaf.org/fundraising. Contributions to the Project Gutenberg
Literary Archive Foundation are tax deductible to the full extent
permitted by U.S. federal laws and your state's laws.
The Foundation's principal office is located at 4557 Melan Dr. S.
Fairbanks, AK, 99712., but its volunteers and employees are scattered
throughout numerous locations. Its business office is located at
809 North 1500 West, Salt Lake City, UT 84116, (801) 596-1887, email
business@pglaf.org. Email contact links and up to date contact
information can be found at the Foundation's web site and official
page at http://pglaf.org
For additional contact information:
Dr. Gregory B. Newby
Chief Executive and Director
gbnewby@pglaf.org
Section 4. Information about Donations to the Project Gutenberg
Literary Archive Foundation
Project Gutenberg-tm depends upon and cannot survive without wide
spread public support and donations to carry out its mission of
increasing the number of public domain and licensed works that can be
freely distributed in machine readable form accessible by the widest
array of equipment including outdated equipment. Many small donations
($1 to $5,000) are particularly important to maintaining tax exempt
status with the IRS.
The Foundation is committed to complying with the laws regulating
charities and charitable donations in all 50 states of the United
States. Compliance requirements are not uniform and it takes a
considerable effort, much paperwork and many fees to meet and keep up
with these requirements. We do not solicit donations in locations
where we have not received written confirmation of compliance. To
SEND DONATIONS or determine the status of compliance for any
particular state visit http://pglaf.org
While we cannot and do not solicit contributions from states where we
have not met the solicitation requirements, we know of no prohibition
against accepting unsolicited donations from donors in such states who
approach us with offers to donate.
International donations are gratefully accepted, but we cannot make
any statements concerning tax treatment of donations received from
outside the United States. U.S. laws alone swamp our small staff.
Please check the Project Gutenberg Web pages for current donation
methods and addresses. Donations are accepted in a number of other
ways including including checks, online payments and credit card
donations. To donate, please visit: http://pglaf.org/donate
Section 5. General Information About Project Gutenberg-tm electronic
works.
Professor Michael S. Hart is the originator of the Project Gutenberg-tm
concept of a library of electronic works that could be freely shared
with anyone. For thirty years, he produced and distributed Project
Gutenberg-tm eBooks with only a loose network of volunteer support.
Project Gutenberg-tm eBooks are often created from several printed
editions, all of which are confirmed as Public Domain in the U.S.
unless a copyright notice is included. Thus, we do not necessarily
keep eBooks in compliance with any particular paper edition.
Most people start at our Web site which has the main PG search facility:
http://www.gutenberg.net
This Web site includes information about Project Gutenberg-tm,
including how to make donations to the Project Gutenberg Literary
Archive Foundation, how to help produce our new eBooks, and how to
subscribe to our email newsletter to hear about new eBooks.

29
libgo/go/compress/testdata/gettysburg.txt vendored Normal file
View file

@ -0,0 +1,29 @@
Four score and seven years ago our fathers brought forth on
this continent, a new nation, conceived in Liberty, and dedicated
to the proposition that all men are created equal.
Now we are engaged in a great Civil War, testing whether that
nation, or any nation so conceived and so dedicated, can long
endure.
We are met on a great battle-field of that war.
We have come to dedicate a portion of that field, as a final
resting place for those who here gave their lives that that
nation might live. It is altogether fitting and proper that
we should do this.
But, in a larger sense, we can not dedicate - we can not
consecrate - we can not hallow - this ground.
The brave men, living and dead, who struggled here, have
consecrated it, far above our poor power to add or detract.
The world will little note, nor long remember what we say here,
but it can never forget what they did here.
It is for us the living, rather, to be dedicated here to the
unfinished work which they who fought here have thus far so
nobly advanced. It is rather for us to be here dedicated to
the great task remaining before us - that from these honored
dead we take increased devotion to that cause for which they
gave the last full measure of devotion -
that we here highly resolve that these dead shall not have
died in vain - that this nation, under God, shall have a new
birth of freedom - and that government of the people, by the
people, for the people, shall not perish from this earth.
Abraham Lincoln, November 19, 1863, Gettysburg, Pennsylvania

9
libgo/go/compress/zlib/reader.go
View file

@ -101,6 +101,9 @@ func (z *reader) Read(p []byte) (n int, err error) {
// Finished file; check checksum.
if _, err := io.ReadFull(z.r, z.scratch[0:4]); err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
z.err = err
return 0, err
}
@ -130,6 +133,9 @@ func (z *reader) Reset(r io.Reader, dict []byte) error {
}
_, err := io.ReadFull(z.r, z.scratch[0:2])
if err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
return err
}
h := uint(z.scratch[0])<<8 | uint(z.scratch[1])
@ -140,6 +146,9 @@ func (z *reader) Reset(r io.Reader, dict []byte) error {
if haveDict {
_, err = io.ReadFull(z.r, z.scratch[0:4])
if err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
return err
}
checksum := uint32(z.scratch[0])<<24 | uint32(z.scratch[1])<<16 | uint32(z.scratch[2])<<8 | uint32(z.scratch[3])

24
libgo/go/compress/zlib/reader_test.go
View file

@ -22,6 +22,30 @@ type zlibTest struct {
// http://www.zlib.net/zpipe.c
var zlibTests = []zlibTest{
{
"truncated empty",
"",
[]byte{},
nil,
io.ErrUnexpectedEOF,
},
{
"truncated dict",
"",
[]byte{0x78, 0xbb},
[]byte{0x00},
io.ErrUnexpectedEOF,
},
{
"truncated checksum",
"",
[]byte{0x78, 0xbb, 0x00, 0x01, 0x00, 0x01, 0xca, 0x48,
0xcd, 0xc9, 0xc9, 0xd7, 0x51, 0x28, 0xcf, 0x2f,
0xca, 0x49, 0x01, 0x04, 0x00, 0x00, 0xff, 0xff,
},
[]byte{0x00},
io.ErrUnexpectedEOF,
},
{
"empty",
"",

27
libgo/go/compress/zlib/writer_test.go
View file

@ -7,6 +7,7 @@ package zlib
import (
"bytes"
"fmt"
"internal/testenv"
"io"
"io/ioutil"
"os"
@ -14,6 +15,7 @@ import (
)
var filenames = []string{
"../testdata/gettysburg.txt",
"../testdata/e.txt",
"../testdata/pi.txt",
}
@ -152,22 +154,34 @@ func TestWriter(t *testing.T) {
}
func TestWriterBig(t *testing.T) {
for _, fn := range filenames {
for i, fn := range filenames {
testFileLevelDict(t, fn, DefaultCompression, "")
testFileLevelDict(t, fn, NoCompression, "")
for level := BestSpeed; level <= BestCompression; level++ {
testFileLevelDict(t, fn, level, "")
if level >= 1 && testing.Short() && testenv.Builder() == "" {
break
}
}
if i == 0 && testing.Short() && testenv.Builder() == "" {
break
}
}
}
func TestWriterDict(t *testing.T) {
const dictionary = "0123456789."
for _, fn := range filenames {
for i, fn := range filenames {
testFileLevelDict(t, fn, DefaultCompression, dictionary)
testFileLevelDict(t, fn, NoCompression, dictionary)
for level := BestSpeed; level <= BestCompression; level++ {
testFileLevelDict(t, fn, level, dictionary)
if level >= 1 && testing.Short() && testenv.Builder() == "" {
break
}
}
if i == 0 && testing.Short() && testenv.Builder() == "" {
break
}
}
}
@ -179,10 +193,11 @@ func TestWriterReset(t *testing.T) {
testFileLevelDictReset(t, fn, DefaultCompression, nil)
testFileLevelDictReset(t, fn, NoCompression, []byte(dictionary))
testFileLevelDictReset(t, fn, DefaultCompression, []byte(dictionary))
if !testing.Short() {
for level := BestSpeed; level <= BestCompression; level++ {
testFileLevelDictReset(t, fn, level, nil)
}
if testing.Short() {
break
}
for level := BestSpeed; level <= BestCompression; level++ {
testFileLevelDictReset(t, fn, level, nil)
}
}
}

172
libgo/go/crypto/aes/aes_gcm.go Normal file
View file

@ -0,0 +1,172 @@
// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build amd64
package aes
import (
"crypto/cipher"
"crypto/subtle"
"errors"
)
// The following functions are defined in gcm_amd64.s.
func hasGCMAsm() bool
//go:noescape
func aesEncBlock(dst, src *[16]byte, ks []uint32)
//go:noescape
func gcmAesInit(productTable *[256]byte, ks []uint32)
//go:noescape
func gcmAesData(productTable *[256]byte, data []byte, T *[16]byte)
//go:noescape
func gcmAesEnc(productTable *[256]byte, dst, src []byte, ctr, T *[16]byte, ks []uint32)
//go:noescape
func gcmAesDec(productTable *[256]byte, dst, src []byte, ctr, T *[16]byte, ks []uint32)
//go:noescape
func gcmAesFinish(productTable *[256]byte, tagMask, T *[16]byte, pLen, dLen uint64)
const (
gcmBlockSize = 16
gcmTagSize = 16
gcmStandardNonceSize = 12
)
var errOpen = errors.New("cipher: message authentication failed")
// aesCipherGCM implements crypto/cipher.gcmAble so that crypto/cipher.NewGCM
// will use the optimised implementation in this file when possible. Instances
// of this type only exist when hasGCMAsm returns true.
type aesCipherGCM struct {
aesCipher
}
// NewGCM returns the AES cipher wrapped in Galois Counter Mode. This is only
// called by crypto/cipher.NewGCM via the gcmAble interface.
func (c *aesCipherGCM) NewGCM(nonceSize int) (cipher.AEAD, error) {
g := &gcmAsm{ks: c.enc, nonceSize: nonceSize}
gcmAesInit(&g.productTable, g.ks)
return g, nil
}
type gcmAsm struct {
// ks is the key schedule, the length of which depends on the size of
// the AES key.
ks []uint32
// productTable contains pre-computed multiples of the binary-field
// element used in GHASH.
productTable [256]byte
// nonceSize contains the expected size of the nonce, in bytes.
nonceSize int
}
func (g *gcmAsm) NonceSize() int {
return g.nonceSize
}
func (*gcmAsm) Overhead() int {
return gcmTagSize
}
// sliceForAppend takes a slice and a requested number of bytes. It returns a
// slice with the contents of the given slice followed by that many bytes and a
// second slice that aliases into it and contains only the extra bytes. If the
// original slice has sufficient capacity then no allocation is performed.
func sliceForAppend(in []byte, n int) (head, tail []byte) {
if total := len(in) + n; cap(in) >= total {
head = in[:total]
} else {
head = make([]byte, total)
copy(head, in)
}
tail = head[len(in):]
return
}
// Seal encrypts and authenticates plaintext. See the cipher.AEAD interface for
// details.
func (g *gcmAsm) Seal(dst, nonce, plaintext, data []byte) []byte {
if len(nonce) != g.nonceSize {
panic("cipher: incorrect nonce length given to GCM")
}
var counter, tagMask [gcmBlockSize]byte
if len(nonce) == gcmStandardNonceSize {
// Init counter to nonce||1
copy(counter[:], nonce)
counter[gcmBlockSize-1] = 1
} else {
// Otherwise counter = GHASH(nonce)
gcmAesData(&g.productTable, nonce, &counter)
gcmAesFinish(&g.productTable, &tagMask, &counter, uint64(len(nonce)), uint64(0))
}
aesEncBlock(&tagMask, &counter, g.ks)
var tagOut [gcmTagSize]byte
gcmAesData(&g.productTable, data, &tagOut)
ret, out := sliceForAppend(dst, len(plaintext)+gcmTagSize)
if len(plaintext) > 0 {
gcmAesEnc(&g.productTable, out, plaintext, &counter, &tagOut, g.ks)
}
gcmAesFinish(&g.productTable, &tagMask, &tagOut, uint64(len(plaintext)), uint64(len(data)))
copy(out[len(plaintext):], tagOut[:])
return ret
}
// Open authenticates and decrypts ciphertext. See the cipher.AEAD interface
// for details.
func (g *gcmAsm) Open(dst, nonce, ciphertext, data []byte) ([]byte, error) {
if len(nonce) != g.nonceSize {
panic("cipher: incorrect nonce length given to GCM")
}
if len(ciphertext) < gcmTagSize {
return nil, errOpen
}
tag := ciphertext[len(ciphertext)-gcmTagSize:]
ciphertext = ciphertext[:len(ciphertext)-gcmTagSize]
// See GCM spec, section 7.1.
var counter, tagMask [gcmBlockSize]byte
if len(nonce) == gcmStandardNonceSize {
// Init counter to nonce||1
copy(counter[:], nonce)
counter[gcmBlockSize-1] = 1
} else {
// Otherwise counter = GHASH(nonce)
gcmAesData(&g.productTable, nonce, &counter)
gcmAesFinish(&g.productTable, &tagMask, &counter, uint64(len(nonce)), uint64(0))
}
aesEncBlock(&tagMask, &counter, g.ks)
var expectedTag [gcmTagSize]byte
gcmAesData(&g.productTable, data, &expectedTag)
ret, out := sliceForAppend(dst, len(ciphertext))
if len(ciphertext) > 0 {
gcmAesDec(&g.productTable, out, ciphertext, &counter, &expectedTag, g.ks)
}
gcmAesFinish(&g.productTable, &tagMask, &expectedTag, uint64(len(ciphertext)), uint64(len(data)))
if subtle.ConstantTimeCompare(expectedTag[:], tag) != 1 {
for i := range out {
out[i] = 0
}
return nil, errOpen
}
return ret, nil
}

Some files were not shown because too many files have changed in this diff Show more