libgo: update to Go1.10beta1

Update the Go library to the 1.10beta1 release.
    
    Requires a few changes to the compiler for modifications to the map
    runtime code, and to handle some nowritebarrier cases in the runtime.
    
    Reviewed-on: https://go-review.googlesource.com/86455

gotools/:
	* Makefile.am (go_cmd_vet_files): New variable.
	(go_cmd_buildid_files, go_cmd_test2json_files): New variables.
	(s-zdefaultcc): Change from constants to functions.
	(noinst_PROGRAMS): Add vet, buildid, and test2json.
	(cgo$(EXEEXT)): Link against $(LIBGOTOOL).
	(vet$(EXEEXT)): New target.
	(buildid$(EXEEXT)): New target.
	(test2json$(EXEEXT)): New target.
	(install-exec-local): Install all $(noinst_PROGRAMS).
	(uninstall-local): Uninstasll all $(noinst_PROGRAMS).
	(check-go-tool): Depend on $(noinst_PROGRAMS).  Copy down
	objabi.go.
	(check-runtime): Depend on $(noinst_PROGRAMS).
	(check-cgo-test, check-carchive-test): Likewise.
	(check-vet): New target.
	(check): Depend on check-vet.  Look at cmd_vet-testlog.
	(.PHONY): Add check-vet.
	* Makefile.in: Rebuild.

From-SVN: r256365

gcc/go/gofrontend/MERGE

@@ -1,4 +1,4 @@
-1319f36ccc65cf802b8e17ddd3d2da3ca6d82f4c
+dbc0c7e4329aada2ae3554c20cfb8cfa48041213
 
 The first line of this file holds the git revision number of the last
 merge done from the gofrontend repository.

gcc/go/gofrontend/expressions.cc

@@ -7483,6 +7483,7 @@ Builtin_call_expression::lower_make(Statement_inserter* inserter)
 	  return Expression::make_error(this->location());
 	}
       len_arg = Expression::make_integer_ul(0, NULL, loc);
+      len_small = true;
     }
   else
     {
@@ -7551,9 +7552,23 @@ Builtin_call_expression::lower_make(Statement_inserter* inserter)
   else if (is_map)
     {
       Expression* type_arg = Expression::make_type_descriptor(type, type_loc);
-      call = Runtime::make_call(Runtime::MAKEMAP, loc, 4, type_arg, len_arg,
+      if (!len_small)
-				Expression::make_nil(loc),
+	call = Runtime::make_call(Runtime::MAKEMAP64, loc, 3, type_arg,
+				  len_arg,
 				  Expression::make_nil(loc));
+      else
+	{
+	  Numeric_constant nclen;
+	  unsigned long vlen;
+	  if (len_arg->numeric_constant_value(&nclen)
+	      && nclen.to_unsigned_long(&vlen) == Numeric_constant::NC_UL_VALID
+	      && vlen <= Map_type::bucket_size)
+	    call = Runtime::make_call(Runtime::MAKEMAP_SMALL, loc, 0);
+	  else
+	    call = Runtime::make_call(Runtime::MAKEMAP, loc, 3, type_arg,
+				      len_arg,
+				      Expression::make_nil(loc));
+	}
     }
   else if (is_chan)
     {
@@ -9503,14 +9518,8 @@ Call_expression::do_lower(Gogo* gogo, Named_object* function,
   // could implement them in normal code, but then we would have to
   // explicitly unwind the stack.  These functions are intended to be
   // efficient.  Note that this technique obviously only works for
-  // direct calls, but that is the only way they are used.  The actual
+  // direct calls, but that is the only way they are used.
-  // argument to these functions is always the address of a parameter;
+  if (gogo->compiling_runtime() && gogo->package_name() == "runtime")
-  // we don't need that for the GCC builtin functions, so we just
-  // ignore it.
-  if (gogo->compiling_runtime()
-      && this->args_ != NULL
-      && this->args_->size() == 1
-      && gogo->package_name() == "runtime")
     {
       Func_expression* fe = this->fn_->func_expression();
       if (fe != NULL
@@ -9518,15 +9527,21 @@ Call_expression::do_lower(Gogo* gogo, Named_object* function,
 	  && fe->named_object()->package() == NULL)
 	{
 	  std::string n = Gogo::unpack_hidden_name(fe->named_object()->name());
-	  if (n == "getcallerpc")
+	  if ((this->args_ == NULL || this->args_->size() == 0)
+	      && n == "getcallerpc")
 	    {
 	      static Named_object* builtin_return_address;
 	      return this->lower_to_builtin(&builtin_return_address,
 					    "__builtin_return_address",
 					    0);
 	    }
-	  else if (n == "getcallersp")
+	  else if (this->args_ != NULL
+		   && this->args_->size() == 1
+		   && n == "getcallersp")
 	    {
+	      // The actual argument to getcallersp is always the
+	      // address of a parameter; we don't need that for the
+	      // GCC builtin function, so we just ignore it.
 	      static Named_object* builtin_frame_address;
 	      return this->lower_to_builtin(&builtin_frame_address,
 					    "__builtin_frame_address",
@@ -10027,7 +10042,7 @@ Call_expression::do_check_types(Gogo*)
     }
 
   const Typed_identifier_list* parameters = fntype->parameters();
-  if (this->args_ == NULL)
+  if (this->args_ == NULL || this->args_->size() == 0)
     {
       if (parameters != NULL && !parameters->empty())
 	this->report_error(_("not enough arguments"));

gcc/go/gofrontend/runtime.def

@@ -91,9 +91,14 @@ DEF_GO_RUNTIME(MAKESLICE64, "runtime.makeslice64", P3(TYPE, INT64, INT64),
 	       R1(SLICE))
 
 
-// Make a map.
+// Make a map with a hint and an (optional, unused) map structure.
-DEF_GO_RUNTIME(MAKEMAP, "runtime.makemap", P4(TYPE, INT64, POINTER, POINTER),
+DEF_GO_RUNTIME(MAKEMAP, "runtime.makemap", P3(TYPE, INT, POINTER),
 		R1(MAP))
+DEF_GO_RUNTIME(MAKEMAP64, "runtime.makemap64", P3(TYPE, INT64, POINTER),
+		R1(MAP))
+
+// Make a map with no hint, or a small constant hint.
+DEF_GO_RUNTIME(MAKEMAP_SMALL, "runtime.makemap_small", P0(), R1(MAP))
 
 // Build a map from a composite literal.
 DEF_GO_RUNTIME(CONSTRUCT_MAP, "__go_construct_map",

gcc/go/gofrontend/types.cc

@@ -7830,7 +7830,7 @@ Map_type::do_get_backend(Gogo* gogo)
       bfields[7].btype = uintptr_type->get_backend(gogo);
       bfields[7].location = bloc;
 
-      bfields[8].name = "overflow";
+      bfields[8].name = "extra";
       bfields[8].btype = bpvt;
       bfields[8].location = bloc;
 
@@ -8144,21 +8144,23 @@ Map_type::hmap_type(Type* bucket_type)
 
   Type* int_type = Type::lookup_integer_type("int");
   Type* uint8_type = Type::lookup_integer_type("uint8");
+  Type* uint16_type = Type::lookup_integer_type("uint16");
   Type* uint32_type = Type::lookup_integer_type("uint32");
   Type* uintptr_type = Type::lookup_integer_type("uintptr");
   Type* void_ptr_type = Type::make_pointer_type(Type::make_void_type());
 
   Type* ptr_bucket_type = Type::make_pointer_type(bucket_type);
 
-  Struct_type* ret = make_builtin_struct_type(8,
+  Struct_type* ret = make_builtin_struct_type(9,
 					      "count", int_type,
 					      "flags", uint8_type,
 					      "B", uint8_type,
+					      "noverflow", uint16_type,
 					      "hash0", uint32_type,
 					      "buckets", ptr_bucket_type,
 					      "oldbuckets", ptr_bucket_type,
 					      "nevacuate", uintptr_type,
-					      "overflow", void_ptr_type);
+					      "extra", void_ptr_type);
   ret->set_is_struct_incomparable();
   this->hmap_type_ = ret;
   return ret;
@@ -8191,18 +8193,22 @@ Map_type::hiter_type(Gogo* gogo)
   Type* hmap_type = this->hmap_type(bucket_type);
   Type* hmap_ptr_type = Type::make_pointer_type(hmap_type);
   Type* void_ptr_type = Type::make_pointer_type(Type::make_void_type());
+  Type* bool_type = Type::lookup_bool_type();
 
-  Struct_type* ret = make_builtin_struct_type(12,
+  Struct_type* ret = make_builtin_struct_type(15,
 					      "key", key_ptr_type,
 					      "val", val_ptr_type,
 					      "t", uint8_ptr_type,
 					      "h", hmap_ptr_type,
 					      "buckets", bucket_ptr_type,
 					      "bptr", bucket_ptr_type,
-					      "overflow0", void_ptr_type,
+					      "overflow", void_ptr_type,
-					      "overflow1", void_ptr_type,
+					      "oldoverflow", void_ptr_type,
 					      "startBucket", uintptr_type,
-					      "stuff", uintptr_type,
+					      "offset", uint8_type,
+					      "wrapped", bool_type,
+					      "B", uint8_type,
+					      "i", uint8_type,
 					      "bucket", uintptr_type,
 					      "checkBucket", uintptr_type);
   ret->set_is_struct_incomparable();

gcc/go/gofrontend/types.h

@@ -2826,6 +2826,9 @@ class Map_type : public Type
   static Type*
   make_map_type_descriptor_type();
 
+  // This must be in  sync with libgo/go/runtime/hashmap.go.
+  static const int bucket_size = 8;
+
  protected:
   int
   do_traverse(Traverse*);
@@ -2867,7 +2870,6 @@ class Map_type : public Type
 
  private:
   // These must be in sync with libgo/go/runtime/hashmap.go.
-  static const int bucket_size = 8;
   static const int max_key_size = 128;
   static const int max_val_size = 128;
   static const int max_zero_size = 1024;

gcc/go/gofrontend/wb.cc

@@ -331,6 +331,25 @@ Gogo::assign_needs_write_barrier(Expression* lhs)
   if (!lhs->type()->has_pointer())
     return false;
 
+  // An assignment to a field is handled like an assignment to the
+  // struct.
+  while (true)
+    {
+      // Nothing to do for a type that can not be in the heap, or a
+      // pointer to a type that can not be in the heap.  We check this
+      // at each level of a struct.
+      if (!lhs->type()->in_heap())
+	return false;
+      if (lhs->type()->points_to() != NULL
+	  && !lhs->type()->points_to()->in_heap())
+	return false;
+
+      Field_reference_expression* fre = lhs->field_reference_expression();
+      if (fre == NULL)
+	break;
+      lhs = fre->expr();
+    }
+
   // Nothing to do for an assignment to a temporary.
   if (lhs->temporary_reference_expression() != NULL)
     return false;
@@ -359,12 +378,30 @@ Gogo::assign_needs_write_barrier(Expression* lhs)
 	}
     }
 
-  // Nothing to do for a type that can not be in the heap, or a
+  // For a struct assignment, we don't need a write barrier if all the
-  // pointer to a type that can not be in the heap.
+  // pointer types can not be in the heap.
-  if (!lhs->type()->in_heap())
+  Struct_type* st = lhs->type()->struct_type();
-    return false;
+  if (st != NULL)
-  if (lhs->type()->points_to() != NULL && !lhs->type()->points_to()->in_heap())
+    {
+      bool in_heap = false;
+      const Struct_field_list* fields = st->fields();
+      for (Struct_field_list::const_iterator p = fields->begin();
+	   p != fields->end();
+	   p++)
+	{
+	  Type* ft = p->type();
+	  if (!ft->has_pointer())
+	    continue;
+	  if (!ft->in_heap())
+	    continue;
+	  if (ft->points_to() != NULL && !ft->points_to()->in_heap())
+	    continue;
+	  in_heap = true;
+	  break;
+	}
+      if (!in_heap)
 	return false;
+    }
 
   // Write barrier needed in other cases.
   return true;

gotools/ChangeLog

@@ -1,3 +1,24 @@
+2018-01-08  Ian Lance Taylor  <iant@golang.org>
+
+	* Makefile.am (go_cmd_vet_files): New variable.
+	(go_cmd_buildid_files, go_cmd_test2json_files): New variables.
+	(s-zdefaultcc): Change from constants to functions.
+	(noinst_PROGRAMS): Add vet, buildid, and test2json.
+	(cgo$(EXEEXT)): Link against $(LIBGOTOOL).
+	(vet$(EXEEXT)): New target.
+	(buildid$(EXEEXT)): New target.
+	(test2json$(EXEEXT)): New target.
+	(install-exec-local): Install all $(noinst_PROGRAMS).
+	(uninstall-local): Uninstasll all $(noinst_PROGRAMS).
+	(check-go-tool): Depend on $(noinst_PROGRAMS).  Copy down
+	objabi.go.
+	(check-runtime): Depend on $(noinst_PROGRAMS).
+	(check-cgo-test, check-carchive-test): Likewise.
+	(check-vet): New target.
+	(check): Depend on check-vet.  Look at cmd_vet-testlog.
+	(.PHONY): Add check-vet.
+	* Makefile.in: Rebuild.
+
 2017-10-25  Ian Lance Taylor  <iant@golang.org>
 
 	* Makefile.am (check-go-tool): Output colon after ${fl}.

gotools/Makefile.am

@@ -69,6 +69,40 @@ go_cmd_cgo_files = \
 	$(cmdsrcdir)/cgo/out.go \
 	$(cmdsrcdir)/cgo/util.go
 
+go_cmd_vet_files = \
+	$(cmdsrcdir)/vet/asmdecl.go \
+	$(cmdsrcdir)/vet/assign.go \
+	$(cmdsrcdir)/vet/atomic.go \
+	$(cmdsrcdir)/vet/bool.go \
+	$(cmdsrcdir)/vet/buildtag.go \
+	$(cmdsrcdir)/vet/cgo.go \
+	$(cmdsrcdir)/vet/composite.go \
+	$(cmdsrcdir)/vet/copylock.go \
+	$(cmdsrcdir)/vet/deadcode.go \
+	$(cmdsrcdir)/vet/dead.go \
+	$(cmdsrcdir)/vet/doc.go \
+	$(cmdsrcdir)/vet/httpresponse.go \
+	$(cmdsrcdir)/vet/lostcancel.go \
+	$(cmdsrcdir)/vet/main.go \
+	$(cmdsrcdir)/vet/method.go \
+	$(cmdsrcdir)/vet/nilfunc.go \
+	$(cmdsrcdir)/vet/print.go \
+	$(cmdsrcdir)/vet/rangeloop.go \
+	$(cmdsrcdir)/vet/shadow.go \
+	$(cmdsrcdir)/vet/shift.go \
+	$(cmdsrcdir)/vet/structtag.go \
+	$(cmdsrcdir)/vet/tests.go \
+	$(cmdsrcdir)/vet/types.go \
+	$(cmdsrcdir)/vet/unsafeptr.go \
+	$(cmdsrcdir)/vet/unused.go
+
+go_cmd_buildid_files = \
+	$(cmdsrcdir)/buildid/buildid.go \
+	$(cmdsrcdir)/buildid/doc.go
+
+go_cmd_test2json_files = \
+	$(cmdsrcdir)/test2json/main.go
+
 GCCGO_INSTALL_NAME := $(shell echo gccgo|sed '$(program_transform_name)')
 GCC_INSTALL_NAME := $(shell echo gcc|sed '$(program_transform_name)')
 GXX_INSTALL_NAME := $(shell echo g++|sed '$(program_transform_name)')
@@ -76,9 +110,9 @@ GXX_INSTALL_NAME := $(shell echo g++|sed '$(program_transform_name)')
 zdefaultcc.go: s-zdefaultcc; @true
 s-zdefaultcc: Makefile
 	echo 'package main' > zdefaultcc.go.tmp
-	echo 'const defaultGCCGO = "$(bindir)/$(GCCGO_INSTALL_NAME)"' >> zdefaultcc.go.tmp
+	echo 'func defaultGCCGO(goos, goarch string) string { return "$(bindir)/$(GCCGO_INSTALL_NAME)" }' >> zdefaultcc.go.tmp
-	echo 'const defaultCC = "$(GCC_INSTALL_NAME)"' >> zdefaultcc.go.tmp
+	echo 'func defaultCC(goos, goarch string) string { return "$(GCC_INSTALL_NAME)" }' >> zdefaultcc.go.tmp
-	echo 'const defaultCXX = "$(GXX_INSTALL_NAME)"' >> zdefaultcc.go.tmp
+	echo 'func defaultCXX(goos, goarch string) string { return "$(GXX_INSTALL_NAME)" }' >> zdefaultcc.go.tmp
 	echo 'const defaultPkgConfig = "pkg-config"' >> zdefaultcc.go.tmp
 	$(SHELL) $(srcdir)/../move-if-change zdefaultcc.go.tmp zdefaultcc.go
 	$(STAMP) $@ 
@@ -97,23 +131,33 @@ if NATIVE
 # and install them as regular programs.
 
 bin_PROGRAMS = go$(EXEEXT) gofmt$(EXEEXT)
-noinst_PROGRAMS = cgo$(EXEEXT)
+noinst_PROGRAMS = cgo$(EXEEXT) vet$(EXEEXT) buildid$(EXEEXT) test2json$(EXEEXT)
 man_MANS = go.1 gofmt.1
 
 go$(EXEEXT): $(go_cmd_go_files) $(LIBGOTOOL) $(LIBGODEP)
 	$(GOLINK) $(go_cmd_go_files) $(LIBGOTOOL) $(LIBS) $(NET_LIBS)
 gofmt$(EXEEXT): $(go_cmd_gofmt_files) $(LIBGODEP)
 	$(GOLINK) $(go_cmd_gofmt_files) $(LIBS) $(NET_LIBS)
-cgo$(EXEEXT): $(go_cmd_cgo_files) zdefaultcc.go $(LIBGODEP)
+cgo$(EXEEXT): $(go_cmd_cgo_files) zdefaultcc.go $(LIBGOTOOL) $(LIBGODEP)
-	$(GOLINK) $(go_cmd_cgo_files) zdefaultcc.go $(LIBS) $(NET_LIBS)
+	$(GOLINK) $(go_cmd_cgo_files) zdefaultcc.go $(LIBGOTOOL) $(LIBS) $(NET_LIBS)
+vet$(EXEEXT): $(go_cmd_vet_files) $(LIBGOTOOL) $(LIBGODEP)
+	$(GOLINK) $(go_cmd_vet_files) $(LIBGOTOOL) $(LIBS) $(NET_LIBS)
+buildid$(EXEEXT): $(go_cmd_buildid_files) $(LIBGOTOOL) $(LIBGODEP)
+	$(GOLINK) $(go_cmd_buildid_files) $(LIBGOTOOL) $(LIBS) $(NET_LIBS)
+test2json$(EXEEXT): $(go_cmd_test2json_files) $(LIBGOTOOL) $(LIBGODEP)
+	$(GOLINK) $(go_cmd_test2json_files) $(LIBGOTOOL) $(LIBS) $(NET_LIBS)
 
-install-exec-local: cgo$(EXEEXT)
+install-exec-local: $(noinst_PROGRAMS)
 	$(MKDIR_P) $(DESTDIR)$(libexecsubdir)
-	rm -f $(DESTDIR)$(libexecsubdir)/cgo$(exeext)
+	for f in $(noinst_PROGRAMS); do \
-	$(INSTALL_PROGRAM) cgo$(exeext) $(DESTDIR)$(libexecsubdir)/cgo$(exeext)
+	  rm -f $(DESTDIR)$(libexecsubdir)/$$f; \
+	  $(INSTALL_PROGRAM) $$f $(DESTDIR)$(libexecsubdir)/$$f; \
+	done
 
 uninstall-local:
-	rm -f $(DESTDIR)$(libexecsubdir)/cgo$(exeext)
+	for f in $(noinst_PROGRAMS); do \
+	  rm -f $(DESTDIR)$(libexecsubdir)/$$f; \
+	done
 
 GOTESTFLAGS =
 
@@ -177,8 +221,8 @@ CHECK_ENV = \
 # It assumes that abs_libgodir is set.
 ECHO_ENV = PATH=`echo $(abs_builddir):$${PATH} | sed 's,::*,:,g;s,^:*,,;s,:*$$,,'` GCCGO='$(abs_builddir)/check-gccgo' CC='$(abs_builddir)/check-gcc' GCCGOTOOLDIR='$(abs_builddir)' GO_TESTING_GOTOOLS=yes LD_LIBRARY_PATH=`echo $${abs_libgodir}/.libs:$${LD_LIBRARY_PATH} | sed 's,::*,:,g;s,^:*,,;s,:*$$,,'` GOROOT=`echo $${abs_libgodir}`
 
-# check-go-tools runs `go test cmd/go` in our environment.
+# check-go-tool runs `go test cmd/go` in our environment.
-check-go-tool: go$(EXEEXT) cgo$(EXEEXT) check-head check-gccgo check-gcc
+check-go-tool: go$(EXEEXT) $(noinst_PROGRAMS) check-head check-gccgo check-gcc
 	rm -rf check-go-dir cmd_go-testlog
 	$(MKDIR_P) check-go-dir/src/cmd/go
 	cp $(cmdsrcdir)/go/*.go check-go-dir/src/cmd/go/
@@ -187,6 +231,7 @@ check-go-tool: go$(EXEEXT) cgo$(EXEEXT) check-head check-gccgo check-gcc
 	cp $(libgodir)/zdefaultcc.go check-go-dir/src/cmd/go/internal/cfg/
 	cp -r $(cmdsrcdir)/go/testdata check-go-dir/src/cmd/go/
 	cp -r $(cmdsrcdir)/internal check-go-dir/src/cmd/
+	cp $(libgodir)/objabi.go check-go-dir/src/cmd/internal/objabi/
 	@abs_libgodir=`cd $(libgodir) && $(PWD_COMMAND)`; \
 	abs_checkdir=`cd check-go-dir && $(PWD_COMMAND)`; \
 	echo "cd check-go-dir/src/cmd/go && $(ECHO_ENV) GOPATH=$${abs_checkdir} $(abs_builddir)/go$(EXEEXT) test -test.short -test.v" > cmd_go-testlog
@@ -200,7 +245,7 @@ check-go-tool: go$(EXEEXT) cgo$(EXEEXT) check-head check-gccgo check-gcc
 # The runtime package is also tested as part of libgo,
 # but the runtime tests use the go tool heavily, so testing
 # here too will catch more problems.
-check-runtime: go$(EXEEXT) cgo$(EXEEXT) check-head check-gccgo check-gcc
+check-runtime: go$(EXEEXT) $(noinst_PROGRAMS) check-head check-gccgo check-gcc
 	rm -rf check-runtime-dir runtime-testlog
 	$(MKDIR_P) check-runtime-dir
 	@abs_libgodir=`cd $(libgodir) && $(PWD_COMMAND)`; \
@@ -219,7 +264,7 @@ check-runtime: go$(EXEEXT) cgo$(EXEEXT) check-head check-gccgo check-gcc
 	grep '^--- ' runtime-testlog | sed -e 's/^--- \(.*\) ([^)]*)$$/\1/' | sort -k 2
 
 # check-cgo-test runs `go test misc/cgo/test` in our environment.
-check-cgo-test: go$(EXEEXT) cgo$(EXEEXT) check-head check-gccgo check-gcc
+check-cgo-test: go$(EXEEXT) $(noinst_PROGRAMS) check-head check-gccgo check-gcc
 	rm -rf cgo-test-dir cgo-testlog
 	$(MKDIR_P) cgo-test-dir/misc/cgo
 	cp -r $(libgomiscdir)/cgo/test cgo-test-dir/misc/cgo/
@@ -233,7 +278,7 @@ check-cgo-test: go$(EXEEXT) cgo$(EXEEXT) check-head check-gccgo check-gcc
 
 # check-carchive-test runs `go test misc/cgo/testcarchive/carchive_test.go`
 # in our environment.
-check-carchive-test: go$(EXEEXT) cgo$(EXEEXT) check-head check-gccgo check-gcc
+check-carchive-test: go$(EXEEXT) $(noinst_PROGRAMS) check-head check-gccgo check-gcc
 	rm -rf carchive-test-dir carchive-testlog
 	$(MKDIR_P) carchive-test-dir/misc/cgo
 	cp -r $(libgomiscdir)/cgo/testcarchive carchive-test-dir/misc/cgo/
@@ -245,11 +290,25 @@ check-carchive-test: go$(EXEEXT) cgo$(EXEEXT) check-head check-gccgo check-gcc
 	(cd carchive-test-dir/misc/cgo/testcarchive && $(abs_builddir)/go$(EXEEXT) test -test.v carchive_test.go) >> carchive-testlog 2>&1 || echo "--- $${fl}: go test misc/cgo/testcarchive (0.00s)" >> carchive-testlog
 	grep '^--- ' carchive-testlog | sed -e 's/^--- \(.*\) ([^)]*)$$/\1/' | sort -k 2
 
+# check-vet runs `go test cmd/vet` in our environment.
+check-vet: go$(EXEEXT) $(noinst_PROGRAMS) check-head check-gccgo check-gcc
+	rm -rf check-vet-dir cmd_vet-testlog
+	$(MKDIR_P) check-vet-dir/src/cmd
+	cp -r $(cmdsrcdir)/vet check-vet-dir/src/cmd/
+	@abs_libgodir=`cd $(libgodir) && $(PWD_COMMAND)`; \
+	abs_checkdir=`cd check-vet-dir && $(PWD_COMMAND)`; \
+	echo "cd check-vet-dir/src/cmd/vet && $(ECHO_ENV) GOPATH=$${abs_checkdir} $(abs_builddir)/go$(EXEEXT) test -test.short -test.v" > cmd_vet-testlog
+	$(CHECK_ENV) \
+	GOPATH=`cd check-vet-dir && $(PWD_COMMAND)`; \
+	export GOPATH; \
+	(cd check-vet-dir/src/cmd/vet && $(abs_builddir)/go$(EXEEXT) test -test.short -test.v) >> cmd_vet-testlog 2>&1 || echo "--- $${fl}: go test cmd/vet (0.00s)" >> cmd_vet-testlog
+	grep '^--- ' cmd_vet-testlog | sed -e 's/^--- \(.*\) ([^)]*)$$/\1/' | sort -k 2
+
 # The check targets runs the tests and assembles the output files.
-check: check-head check-go-tool check-runtime check-cgo-test check-carchive-test
+check: check-head check-go-tool check-runtime check-cgo-test check-carchive-test check-vet
 	@mv gotools.head gotools.sum
 	@cp gotools.sum gotools.log
-	@for file in cmd_go-testlog runtime-testlog cgo-testlog carchive-testlog; do \
+	@for file in cmd_go-testlog runtime-testlog cgo-testlog carchive-testlog cmd_vet-testlog; do \
 	  testname=`echo $${file} | sed -e 's/-testlog//' -e 's|_|/|'`; \
 	  echo "Running $${testname}" >> gotools.sum; \
 	  echo "Running $${testname}" >> gotools.log; \
@@ -275,7 +334,7 @@ check: check-head check-go-tool check-runtime check-cgo-test check-carchive-test
 	@echo "runtest completed at `date`" >> gotools.log
 	@if grep '^FAIL' gotools.sum >/dev/null 2>&1; then exit 1; fi
 
-.PHONY: check check-head check-go-tool check-runtime check-cgo-test check-carchive-test
+.PHONY: check check-head check-go-tool check-runtime check-cgo-test check-carchive-test check-vet
 
 else
 

gotools/Makefile.in

@@ -88,6 +88,9 @@ CONFIG_CLEAN_FILES =
 CONFIG_CLEAN_VPATH_FILES =
 am__installdirs = "$(DESTDIR)$(bindir)" "$(DESTDIR)$(man1dir)"
 PROGRAMS = $(bin_PROGRAMS) $(noinst_PROGRAMS)
+buildid_SOURCES = buildid.c
+buildid_OBJECTS = buildid.$(OBJEXT)
+buildid_LDADD = $(LDADD)
 cgo_SOURCES = cgo.c
 cgo_OBJECTS = cgo.$(OBJEXT)
 cgo_LDADD = $(LDADD)
@@ -97,6 +100,12 @@ go_LDADD = $(LDADD)
 gofmt_SOURCES = gofmt.c
 gofmt_OBJECTS = gofmt.$(OBJEXT)
 gofmt_LDADD = $(LDADD)
+test2json_SOURCES = test2json.c
+test2json_OBJECTS = test2json.$(OBJEXT)
+test2json_LDADD = $(LDADD)
+vet_SOURCES = vet.c
+vet_OBJECTS = vet.$(OBJEXT)
+vet_LDADD = $(LDADD)
 DEFAULT_INCLUDES = -I.@am__isrc@
 depcomp = $(SHELL) $(top_srcdir)/../depcomp
 am__depfiles_maybe = depfiles
@@ -105,7 +114,7 @@ COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \
 	$(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
 CCLD = $(CC)
 LINK = $(CCLD) $(AM_CFLAGS) $(CFLAGS) $(AM_LDFLAGS) $(LDFLAGS) -o $@
-SOURCES = cgo.c go.c gofmt.c
+SOURCES = buildid.c cgo.c go.c gofmt.c test2json.c vet.c
 am__can_run_installinfo = \
   case $$AM_UPDATE_INFO_DIR in \
     n|no|NO) false;; \
@@ -288,6 +297,40 @@ go_cmd_cgo_files = \
 	$(cmdsrcdir)/cgo/out.go \
 	$(cmdsrcdir)/cgo/util.go
 
+go_cmd_vet_files = \
+	$(cmdsrcdir)/vet/asmdecl.go \
+	$(cmdsrcdir)/vet/assign.go \
+	$(cmdsrcdir)/vet/atomic.go \
+	$(cmdsrcdir)/vet/bool.go \
+	$(cmdsrcdir)/vet/buildtag.go \
+	$(cmdsrcdir)/vet/cgo.go \
+	$(cmdsrcdir)/vet/composite.go \
+	$(cmdsrcdir)/vet/copylock.go \
+	$(cmdsrcdir)/vet/deadcode.go \
+	$(cmdsrcdir)/vet/dead.go \
+	$(cmdsrcdir)/vet/doc.go \
+	$(cmdsrcdir)/vet/httpresponse.go \
+	$(cmdsrcdir)/vet/lostcancel.go \
+	$(cmdsrcdir)/vet/main.go \
+	$(cmdsrcdir)/vet/method.go \
+	$(cmdsrcdir)/vet/nilfunc.go \
+	$(cmdsrcdir)/vet/print.go \
+	$(cmdsrcdir)/vet/rangeloop.go \
+	$(cmdsrcdir)/vet/shadow.go \
+	$(cmdsrcdir)/vet/shift.go \
+	$(cmdsrcdir)/vet/structtag.go \
+	$(cmdsrcdir)/vet/tests.go \
+	$(cmdsrcdir)/vet/types.go \
+	$(cmdsrcdir)/vet/unsafeptr.go \
+	$(cmdsrcdir)/vet/unused.go
+
+go_cmd_buildid_files = \
+	$(cmdsrcdir)/buildid/buildid.go \
+	$(cmdsrcdir)/buildid/doc.go
+
+go_cmd_test2json_files = \
+	$(cmdsrcdir)/test2json/main.go
+
 GCCGO_INSTALL_NAME := $(shell echo gccgo|sed '$(program_transform_name)')
 GCC_INSTALL_NAME := $(shell echo gcc|sed '$(program_transform_name)')
 GXX_INSTALL_NAME := $(shell echo g++|sed '$(program_transform_name)')
@@ -300,7 +343,7 @@ MOSTLYCLEANFILES = \
 # For a native build we build the programs using the newly built libgo
 # and install them as regular programs.
 @NATIVE_TRUE@bin_PROGRAMS = go$(EXEEXT) gofmt$(EXEEXT)
-@NATIVE_TRUE@noinst_PROGRAMS = cgo$(EXEEXT)
+@NATIVE_TRUE@noinst_PROGRAMS = cgo$(EXEEXT) vet$(EXEEXT) buildid$(EXEEXT) test2json$(EXEEXT)
 @NATIVE_TRUE@man_MANS = go.1 gofmt.1
 @NATIVE_TRUE@GOTESTFLAGS = 
 
@@ -411,6 +454,9 @@ clean-binPROGRAMS:
 
 clean-noinstPROGRAMS:
 	-test -z "$(noinst_PROGRAMS)" || rm -f $(noinst_PROGRAMS)
+@NATIVE_FALSE@buildid$(EXEEXT): $(buildid_OBJECTS) $(buildid_DEPENDENCIES) $(EXTRA_buildid_DEPENDENCIES) 
+@NATIVE_FALSE@	@rm -f buildid$(EXEEXT)
+@NATIVE_FALSE@	$(LINK) $(buildid_OBJECTS) $(buildid_LDADD) $(LIBS)
 @NATIVE_FALSE@cgo$(EXEEXT): $(cgo_OBJECTS) $(cgo_DEPENDENCIES) $(EXTRA_cgo_DEPENDENCIES) 
 @NATIVE_FALSE@	@rm -f cgo$(EXEEXT)
 @NATIVE_FALSE@	$(LINK) $(cgo_OBJECTS) $(cgo_LDADD) $(LIBS)
@@ -420,6 +466,12 @@ clean-noinstPROGRAMS:
 @NATIVE_FALSE@gofmt$(EXEEXT): $(gofmt_OBJECTS) $(gofmt_DEPENDENCIES) $(EXTRA_gofmt_DEPENDENCIES) 
 @NATIVE_FALSE@	@rm -f gofmt$(EXEEXT)
 @NATIVE_FALSE@	$(LINK) $(gofmt_OBJECTS) $(gofmt_LDADD) $(LIBS)
+@NATIVE_FALSE@test2json$(EXEEXT): $(test2json_OBJECTS) $(test2json_DEPENDENCIES) $(EXTRA_test2json_DEPENDENCIES) 
+@NATIVE_FALSE@	@rm -f test2json$(EXEEXT)
+@NATIVE_FALSE@	$(LINK) $(test2json_OBJECTS) $(test2json_LDADD) $(LIBS)
+@NATIVE_FALSE@vet$(EXEEXT): $(vet_OBJECTS) $(vet_DEPENDENCIES) $(EXTRA_vet_DEPENDENCIES) 
+@NATIVE_FALSE@	@rm -f vet$(EXEEXT)
+@NATIVE_FALSE@	$(LINK) $(vet_OBJECTS) $(vet_LDADD) $(LIBS)
 
 mostlyclean-compile:
 	-rm -f *.$(OBJEXT)
@@ -427,9 +479,12 @@ mostlyclean-compile:
 distclean-compile:
 	-rm -f *.tab.c
 
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/buildid.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/cgo.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/go.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/gofmt.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test2json.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/vet.Po@am__quote@
 
 .c.o:
 @am__fastdepCC_TRUE@	$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
@@ -676,9 +731,9 @@ uninstall-man: uninstall-man1
 zdefaultcc.go: s-zdefaultcc; @true
 s-zdefaultcc: Makefile
 	echo 'package main' > zdefaultcc.go.tmp
-	echo 'const defaultGCCGO = "$(bindir)/$(GCCGO_INSTALL_NAME)"' >> zdefaultcc.go.tmp
+	echo 'func defaultGCCGO(goos, goarch string) string { return "$(bindir)/$(GCCGO_INSTALL_NAME)" }' >> zdefaultcc.go.tmp
-	echo 'const defaultCC = "$(GCC_INSTALL_NAME)"' >> zdefaultcc.go.tmp
+	echo 'func defaultCC(goos, goarch string) string { return "$(GCC_INSTALL_NAME)" }' >> zdefaultcc.go.tmp
-	echo 'const defaultCXX = "$(GXX_INSTALL_NAME)"' >> zdefaultcc.go.tmp
+	echo 'func defaultCXX(goos, goarch string) string { return "$(GXX_INSTALL_NAME)" }' >> zdefaultcc.go.tmp
 	echo 'const defaultPkgConfig = "pkg-config"' >> zdefaultcc.go.tmp
 	$(SHELL) $(srcdir)/../move-if-change zdefaultcc.go.tmp zdefaultcc.go
 	$(STAMP) $@ 
@@ -690,16 +745,26 @@ mostlyclean-local:
 @NATIVE_TRUE@	$(GOLINK) $(go_cmd_go_files) $(LIBGOTOOL) $(LIBS) $(NET_LIBS)
 @NATIVE_TRUE@gofmt$(EXEEXT): $(go_cmd_gofmt_files) $(LIBGODEP)
 @NATIVE_TRUE@	$(GOLINK) $(go_cmd_gofmt_files) $(LIBS) $(NET_LIBS)
-@NATIVE_TRUE@cgo$(EXEEXT): $(go_cmd_cgo_files) zdefaultcc.go $(LIBGODEP)
+@NATIVE_TRUE@cgo$(EXEEXT): $(go_cmd_cgo_files) zdefaultcc.go $(LIBGOTOOL) $(LIBGODEP)
-@NATIVE_TRUE@	$(GOLINK) $(go_cmd_cgo_files) zdefaultcc.go $(LIBS) $(NET_LIBS)
+@NATIVE_TRUE@	$(GOLINK) $(go_cmd_cgo_files) zdefaultcc.go $(LIBGOTOOL) $(LIBS) $(NET_LIBS)
+@NATIVE_TRUE@vet$(EXEEXT): $(go_cmd_vet_files) $(LIBGOTOOL) $(LIBGODEP)
+@NATIVE_TRUE@	$(GOLINK) $(go_cmd_vet_files) $(LIBGOTOOL) $(LIBS) $(NET_LIBS)
+@NATIVE_TRUE@buildid$(EXEEXT): $(go_cmd_buildid_files) $(LIBGOTOOL) $(LIBGODEP)
+@NATIVE_TRUE@	$(GOLINK) $(go_cmd_buildid_files) $(LIBGOTOOL) $(LIBS) $(NET_LIBS)
+@NATIVE_TRUE@test2json$(EXEEXT): $(go_cmd_test2json_files) $(LIBGOTOOL) $(LIBGODEP)
+@NATIVE_TRUE@	$(GOLINK) $(go_cmd_test2json_files) $(LIBGOTOOL) $(LIBS) $(NET_LIBS)
 
-@NATIVE_TRUE@install-exec-local: cgo$(EXEEXT)
+@NATIVE_TRUE@install-exec-local: $(noinst_PROGRAMS)
 @NATIVE_TRUE@	$(MKDIR_P) $(DESTDIR)$(libexecsubdir)
-@NATIVE_TRUE@	rm -f $(DESTDIR)$(libexecsubdir)/cgo$(exeext)
+@NATIVE_TRUE@	for f in $(noinst_PROGRAMS); do \
-@NATIVE_TRUE@	$(INSTALL_PROGRAM) cgo$(exeext) $(DESTDIR)$(libexecsubdir)/cgo$(exeext)
+@NATIVE_TRUE@	  rm -f $(DESTDIR)$(libexecsubdir)/$$f; \
+@NATIVE_TRUE@	  $(INSTALL_PROGRAM) $$f $(DESTDIR)$(libexecsubdir)/$$f; \
+@NATIVE_TRUE@	done
 
 @NATIVE_TRUE@uninstall-local:
-@NATIVE_TRUE@	rm -f $(DESTDIR)$(libexecsubdir)/cgo$(exeext)
+@NATIVE_TRUE@	for f in $(noinst_PROGRAMS); do \
+@NATIVE_TRUE@	  rm -f $(DESTDIR)$(libexecsubdir)/$$f; \
+@NATIVE_TRUE@	done
 
 # Run tests using the go tool, and frob the output to look like that
 # generated by DejaGNU.  The main output of this is two files:
@@ -735,8 +800,8 @@ mostlyclean-local:
 @NATIVE_TRUE@	chmod +x $@.tmp
 @NATIVE_TRUE@	mv -f $@.tmp $@
 
-# check-go-tools runs `go test cmd/go` in our environment.
+# check-go-tool runs `go test cmd/go` in our environment.
-@NATIVE_TRUE@check-go-tool: go$(EXEEXT) cgo$(EXEEXT) check-head check-gccgo check-gcc
+@NATIVE_TRUE@check-go-tool: go$(EXEEXT) $(noinst_PROGRAMS) check-head check-gccgo check-gcc
 @NATIVE_TRUE@	rm -rf check-go-dir cmd_go-testlog
 @NATIVE_TRUE@	$(MKDIR_P) check-go-dir/src/cmd/go
 @NATIVE_TRUE@	cp $(cmdsrcdir)/go/*.go check-go-dir/src/cmd/go/
@@ -745,6 +810,7 @@ mostlyclean-local:
 @NATIVE_TRUE@	cp $(libgodir)/zdefaultcc.go check-go-dir/src/cmd/go/internal/cfg/
 @NATIVE_TRUE@	cp -r $(cmdsrcdir)/go/testdata check-go-dir/src/cmd/go/
 @NATIVE_TRUE@	cp -r $(cmdsrcdir)/internal check-go-dir/src/cmd/
+@NATIVE_TRUE@	cp $(libgodir)/objabi.go check-go-dir/src/cmd/internal/objabi/
 @NATIVE_TRUE@	@abs_libgodir=`cd $(libgodir) && $(PWD_COMMAND)`; \
 @NATIVE_TRUE@	abs_checkdir=`cd check-go-dir && $(PWD_COMMAND)`; \
 @NATIVE_TRUE@	echo "cd check-go-dir/src/cmd/go && $(ECHO_ENV) GOPATH=$${abs_checkdir} $(abs_builddir)/go$(EXEEXT) test -test.short -test.v" > cmd_go-testlog
@@ -758,7 +824,7 @@ mostlyclean-local:
 # The runtime package is also tested as part of libgo,
 # but the runtime tests use the go tool heavily, so testing
 # here too will catch more problems.
-@NATIVE_TRUE@check-runtime: go$(EXEEXT) cgo$(EXEEXT) check-head check-gccgo check-gcc
+@NATIVE_TRUE@check-runtime: go$(EXEEXT) $(noinst_PROGRAMS) check-head check-gccgo check-gcc
 @NATIVE_TRUE@	rm -rf check-runtime-dir runtime-testlog
 @NATIVE_TRUE@	$(MKDIR_P) check-runtime-dir
 @NATIVE_TRUE@	@abs_libgodir=`cd $(libgodir) && $(PWD_COMMAND)`; \
@@ -777,7 +843,7 @@ mostlyclean-local:
 @NATIVE_TRUE@	grep '^--- ' runtime-testlog | sed -e 's/^--- \(.*\) ([^)]*)$$/\1/' | sort -k 2
 
 # check-cgo-test runs `go test misc/cgo/test` in our environment.
-@NATIVE_TRUE@check-cgo-test: go$(EXEEXT) cgo$(EXEEXT) check-head check-gccgo check-gcc
+@NATIVE_TRUE@check-cgo-test: go$(EXEEXT) $(noinst_PROGRAMS) check-head check-gccgo check-gcc
 @NATIVE_TRUE@	rm -rf cgo-test-dir cgo-testlog
 @NATIVE_TRUE@	$(MKDIR_P) cgo-test-dir/misc/cgo
 @NATIVE_TRUE@	cp -r $(libgomiscdir)/cgo/test cgo-test-dir/misc/cgo/
@@ -791,7 +857,7 @@ mostlyclean-local:
 
 # check-carchive-test runs `go test misc/cgo/testcarchive/carchive_test.go`
 # in our environment.
-@NATIVE_TRUE@check-carchive-test: go$(EXEEXT) cgo$(EXEEXT) check-head check-gccgo check-gcc
+@NATIVE_TRUE@check-carchive-test: go$(EXEEXT) $(noinst_PROGRAMS) check-head check-gccgo check-gcc
 @NATIVE_TRUE@	rm -rf carchive-test-dir carchive-testlog
 @NATIVE_TRUE@	$(MKDIR_P) carchive-test-dir/misc/cgo
 @NATIVE_TRUE@	cp -r $(libgomiscdir)/cgo/testcarchive carchive-test-dir/misc/cgo/
@@ -803,11 +869,25 @@ mostlyclean-local:
 @NATIVE_TRUE@	(cd carchive-test-dir/misc/cgo/testcarchive && $(abs_builddir)/go$(EXEEXT) test -test.v carchive_test.go) >> carchive-testlog 2>&1 || echo "--- $${fl}: go test misc/cgo/testcarchive (0.00s)" >> carchive-testlog
 @NATIVE_TRUE@	grep '^--- ' carchive-testlog | sed -e 's/^--- \(.*\) ([^)]*)$$/\1/' | sort -k 2
 
+# check-vet runs `go test cmd/vet` in our environment.
+@NATIVE_TRUE@check-vet: go$(EXEEXT) $(noinst_PROGRAMS) check-head check-gccgo check-gcc
+@NATIVE_TRUE@	rm -rf check-vet-dir cmd_vet-testlog
+@NATIVE_TRUE@	$(MKDIR_P) check-vet-dir/src/cmd
+@NATIVE_TRUE@	cp -r $(cmdsrcdir)/vet check-vet-dir/src/cmd/
+@NATIVE_TRUE@	@abs_libgodir=`cd $(libgodir) && $(PWD_COMMAND)`; \
+@NATIVE_TRUE@	abs_checkdir=`cd check-vet-dir && $(PWD_COMMAND)`; \
+@NATIVE_TRUE@	echo "cd check-vet-dir/src/cmd/vet && $(ECHO_ENV) GOPATH=$${abs_checkdir} $(abs_builddir)/go$(EXEEXT) test -test.short -test.v" > cmd_vet-testlog
+@NATIVE_TRUE@	$(CHECK_ENV) \
+@NATIVE_TRUE@	GOPATH=`cd check-vet-dir && $(PWD_COMMAND)`; \
+@NATIVE_TRUE@	export GOPATH; \
+@NATIVE_TRUE@	(cd check-vet-dir/src/cmd/vet && $(abs_builddir)/go$(EXEEXT) test -test.short -test.v) >> cmd_vet-testlog 2>&1 || echo "--- $${fl}: go test cmd/vet (0.00s)" >> cmd_vet-testlog
+@NATIVE_TRUE@	grep '^--- ' cmd_vet-testlog | sed -e 's/^--- \(.*\) ([^)]*)$$/\1/' | sort -k 2
+
 # The check targets runs the tests and assembles the output files.
-@NATIVE_TRUE@check: check-head check-go-tool check-runtime check-cgo-test check-carchive-test
+@NATIVE_TRUE@check: check-head check-go-tool check-runtime check-cgo-test check-carchive-test check-vet
 @NATIVE_TRUE@	@mv gotools.head gotools.sum
 @NATIVE_TRUE@	@cp gotools.sum gotools.log
-@NATIVE_TRUE@	@for file in cmd_go-testlog runtime-testlog cgo-testlog carchive-testlog; do \
+@NATIVE_TRUE@	@for file in cmd_go-testlog runtime-testlog cgo-testlog carchive-testlog cmd_vet-testlog; do \
 @NATIVE_TRUE@	  testname=`echo $${file} | sed -e 's/-testlog//' -e 's|_|/|'`; \
 @NATIVE_TRUE@	  echo "Running $${testname}" >> gotools.sum; \
 @NATIVE_TRUE@	  echo "Running $${testname}" >> gotools.log; \
@@ -833,7 +913,7 @@ mostlyclean-local:
 @NATIVE_TRUE@	@echo "runtest completed at `date`" >> gotools.log
 @NATIVE_TRUE@	@if grep '^FAIL' gotools.sum >/dev/null 2>&1; then exit 1; fi
 
-@NATIVE_TRUE@.PHONY: check check-head check-go-tool check-runtime check-cgo-test check-carchive-test
+@NATIVE_TRUE@.PHONY: check check-head check-go-tool check-runtime check-cgo-test check-carchive-test check-vet
 
 # For a non-native build we have to build the programs using a
 # previously built host (or build -> host) Go compiler.  We should

libgo/MERGE

@@ -1,4 +1,4 @@
-c8aec4095e089ff6ac50d18e97c3f46561f14f48
+9ce6b5c2ed5d3d5251b9a6a0c548d5fb2c8567e8
 
 The first line of this file holds the git revision number of the
 last merge done from the master library sources.

libgo/Makefile.am

@@ -400,8 +400,11 @@ toolexeclibgounicode_DATA = \
 # internal packages nothing will explicitly depend on them.
 # Force them to be built.
 noinst_DATA = \
+	golang_org/x/net/internal/nettest.gox \
+	golang_org/x/net/nettest.gox \
 	internal/testenv.gox \
-	net/internal/socktest.gox
+	net/internal/socktest.gox \
+	os/signal/internal/pty.gox
 
 if LIBGO_IS_RTEMS
 rtems_task_variable_add_file = runtime/rtems-task-variable-add.c
@@ -533,6 +536,24 @@ s-version: Makefile
 	$(SHELL) $(srcdir)/mvifdiff.sh version.go.tmp version.go
 	$(STAMP) $@
 
+objabi.go: s-objabi; @true
+s-objabi: Makefile
+	rm -f objabi.go.tmp
+	echo "package objabi" > objabi.go.tmp
+	echo "import \"runtime\"" >> objabi.go.tmp
+	echo 'const defaultGOROOT = `$(prefix)`' >> objabi.go.tmp
+	echo 'const defaultGO386 = `sse2`' >> objabi.go.tmp
+	echo 'const defaultGOARM = `5`' >> objabi.go.tmp
+	echo 'const defaultGOMIPS = `hardfloat`' >> objabi.go.tmp
+	echo 'const defaultGOOS = runtime.GOOS' >> objabi.go.tmp
+	echo 'const defaultGOARCH = runtime.GOARCH' >> objabi.go.tmp
+	echo 'const defaultGO_EXTLINK_ENABLED = ``' >> objabi.go.tmp
+	echo 'const version = `'`cat $(srcdir)/VERSION | sed 1q`' '`$(GOC) --version | sed 1q`'`' >> objabi.go.tmp
+	echo 'const stackGuardMultiplier = 1' >> objabi.go.tmp
+	echo 'const goexperiment = ``' >> objabi.go.tmp
+	$(SHELL) $(srcdir)/mvifdiff.sh objabi.go.tmp objabi.go
+	$(STAMP) $@
+
 runtime_sysinfo.go: s-runtime_sysinfo; @true
 s-runtime_sysinfo: $(srcdir)/mkrsysinfo.sh gen-sysinfo.go
 	GOARCH=$(GOARCH) GOOS=$(GOOS) $(SHELL) $(srcdir)/mkrsysinfo.sh
@@ -553,10 +574,11 @@ zdefaultcc.go: s-zdefaultcc; @true
 s-zdefaultcc: Makefile
 	echo 'package cfg' > zdefaultcc.go.tmp
 	echo >> zdefaultcc.go.tmp
-	echo 'const DefaultGCCGO = "$(bindir)/$(GCCGO_INSTALL_NAME)"' >> zdefaultcc.go.tmp
+	echo 'func DefaultGCCGO(goos, goarch string) string { return "$(bindir)/$(GCCGO_INSTALL_NAME)" }' >> zdefaultcc.go.tmp
-	echo 'const DefaultCC = "$(GCC_INSTALL_NAME)"' >> zdefaultcc.go.tmp
+	echo 'func DefaultCC(goos, goarch string) string { return "$(GCC_INSTALL_NAME)" }' >> zdefaultcc.go.tmp
-	echo 'const DefaultCXX = "$(GXX_INSTALL_NAME)"' >> zdefaultcc.go.tmp
+	echo 'func DefaultCXX(goos, goarch string) string { return "$(GXX_INSTALL_NAME)" }' >> zdefaultcc.go.tmp
 	echo 'const DefaultPkgConfig = "pkg-config"' >> zdefaultcc.go.tmp
+	echo 'var OSArchSupportsCgo = map[string]bool{}' >> zdefaultcc.go.tmp
 	$(SHELL) $(srcdir)/../move-if-change zdefaultcc.go.tmp zdefaultcc.go
 	$(STAMP) $@ 
 
@@ -758,11 +780,15 @@ PACKAGES = \
 	go/types \
 	golang_org/x/crypto/chacha20poly1305 \
 	golang_org/x/crypto/chacha20poly1305/internal/chacha20 \
+	golang_org/x/crypto/cryptobyte \
+	golang_org/x/crypto/cryptobyte/asn1 \
 	golang_org/x/crypto/curve25519 \
 	golang_org/x/crypto/poly1305 \
 	golang_org/x/net/http2/hpack \
 	golang_org/x/net/idna \
+	golang_org/x/net/internal/nettest \
 	golang_org/x/net/lex/httplex \
+	golang_org/x/net/nettest \
 	golang_org/x/net/proxy \
 	golang_org/x/text/secure/bidirule \
 	golang_org/x/text/transform \
@@ -824,6 +850,7 @@ PACKAGES = \
 	os \
 	os/exec \
 	os/signal \
+	os/signal/internal/pty \
 	os/user \
 	path \
 	path/filepath \
@@ -905,7 +932,7 @@ libgolibbegin_a_CFLAGS = $(AM_CFLAGS) -fPIC
 GOTOOL_PACKAGES = \
 	cmd/go/internal/base \
 	cmd/go/internal/bug \
-	cmd/go/internal/buildid \
+	cmd/go/internal/cache \
 	cmd/go/internal/cfg \
 	cmd/go/internal/clean \
 	cmd/go/internal/cmdflag \
@@ -927,7 +954,12 @@ GOTOOL_PACKAGES = \
 	cmd/go/internal/web \
 	cmd/go/internal/work \
 	cmd/internal/browser \
-	cmd/internal/objabi
+	cmd/internal/buildid \
+	cmd/internal/edit \
+	cmd/internal/objabi \
+	cmd/internal/test2json \
+	cmd/vet/internal/cfg \
+	cmd/vet/internal/whitelist
 
 libgotool_a_SOURCES =
 libgotool_a_DEPENDENCIES = $(addsuffix .lo,$(GOTOOL_PACKAGES))
@@ -1136,17 +1168,23 @@ runtime_pprof_check_GOCFLAGS = -static-libgo -fno-inline
 extra_go_files_runtime_internal_sys = version.go
 runtime/internal/sys.lo.dep: $(extra_go_files_runtime_internal_sys)
 
+extra_go_files_cmd_internal_objabi = objabi.go
+cmd/internal/objabi.lo.dep: $(extra_go_files_cmd_internal_objabi)
+
 extra_go_files_cmd_go_internal_cfg = zdefaultcc.go
 cmd/go/internal/cfg.lo.dep: $(extra_go_files_cmd_go_internal_cfg)
 
 extra_go_files_cmd_go_internal_load = zstdpkglist.go
 cmd/go/internal/load.lo.dep: $(extra_go_files_cmd_go_internal_load)
 
+extra_check_libs_cmd_go_internal_cache = $(abs_builddir)/libgotool.a
 extra_check_libs_cmd_go_internal_generate = $(abs_builddir)/libgotool.a
 extra_check_libs_cmd_go_internal_get = $(abs_builddir)/libgotool.a
 extra_check_libs_cmd_go_internal_load = $(abs_builddir)/libgotool.a
 extra_check_libs_cmd_go_internal_work = $(abs_builddir)/libgotool.a
 
+extra_check_libs_cmd_vet_internal_cfg = $(abs_builddir)/libgotool.a
+
 # FIXME: The following C files may as well move to the runtime
 # directory and be treated like other C files.
 
@@ -1233,10 +1271,12 @@ TEST_PACKAGES = \
 	bufio/check \
 	bytes/check \
 	context/check \
+	crypto/check \
 	errors/check \
 	expvar/check \
 	flag/check \
 	fmt/check \
+	hash/check \
 	html/check \
 	image/check \
 	io/check \
@@ -1258,11 +1298,16 @@ TEST_PACKAGES = \
 	unicode/check \
 	archive/tar/check \
 	archive/zip/check \
+	cmd/go/internal/cache/check \
 	cmd/go/internal/generate/check \
 	cmd/go/internal/get/check \
 	cmd/go/internal/load/check \
 	cmd/go/internal/work/check \
+	cmd/internal/buildid/check \
+	cmd/internal/edit/check \
 	cmd/internal/objabi/check \
+	cmd/internal/test2json/check \
+	cmd/vet/internal/cfg/check \
 	compress/bzip2/check \
 	compress/flate/check \
 	compress/gzip/check \
@@ -1315,6 +1360,7 @@ TEST_PACKAGES = \
 	go/constant/check \
 	go/doc/check \
 	go/format/check \
+	go/importer/check \
 	go/internal/gcimporter/check \
 	go/internal/gccgoimporter/check \
 	go/internal/srcimporter/check \
@@ -1325,6 +1371,7 @@ TEST_PACKAGES = \
 	go/types/check \
 	golang_org/x/crypto/chacha20poly1305/check \
 	golang_org/x/crypto/chacha20poly1305/internal/chacha20/check \
+	golang_org/x/crypto/cryptobyte/check \
 	golang_org/x/crypto/curve25519/check \
 	golang_org/x/crypto/poly1305/check \
 	golang_org/x/net/http2/hpack/check \

libgo/Makefile.in

@@ -770,7 +770,9 @@ toolexeclibgounicode_DATA = \
 # Some packages are only needed for tests, so unlike the other
 # internal packages nothing will explicitly depend on them.
 # Force them to be built.
-noinst_DATA = internal/testenv.gox net/internal/socktest.gox \
+noinst_DATA = golang_org/x/net/internal/nettest.gox \
+	golang_org/x/net/nettest.gox internal/testenv.gox \
+	net/internal/socktest.gox os/signal/internal/pty.gox \
 	zstdpkglist.go zdefaultcc.go
 @LIBGO_IS_RTEMS_FALSE@rtems_task_variable_add_file = 
 @LIBGO_IS_RTEMS_TRUE@rtems_task_variable_add_file = runtime/rtems-task-variable-add.c
@@ -909,11 +911,15 @@ PACKAGES = \
 	go/types \
 	golang_org/x/crypto/chacha20poly1305 \
 	golang_org/x/crypto/chacha20poly1305/internal/chacha20 \
+	golang_org/x/crypto/cryptobyte \
+	golang_org/x/crypto/cryptobyte/asn1 \
 	golang_org/x/crypto/curve25519 \
 	golang_org/x/crypto/poly1305 \
 	golang_org/x/net/http2/hpack \
 	golang_org/x/net/idna \
+	golang_org/x/net/internal/nettest \
 	golang_org/x/net/lex/httplex \
+	golang_org/x/net/nettest \
 	golang_org/x/net/proxy \
 	golang_org/x/text/secure/bidirule \
 	golang_org/x/text/transform \
@@ -975,6 +981,7 @@ PACKAGES = \
 	os \
 	os/exec \
 	os/signal \
+	os/signal/internal/pty \
 	os/user \
 	path \
 	path/filepath \
@@ -1053,7 +1060,7 @@ libgolibbegin_a_CFLAGS = $(AM_CFLAGS) -fPIC
 GOTOOL_PACKAGES = \
 	cmd/go/internal/base \
 	cmd/go/internal/bug \
-	cmd/go/internal/buildid \
+	cmd/go/internal/cache \
 	cmd/go/internal/cfg \
 	cmd/go/internal/clean \
 	cmd/go/internal/cmdflag \
@@ -1075,7 +1082,12 @@ GOTOOL_PACKAGES = \
 	cmd/go/internal/web \
 	cmd/go/internal/work \
 	cmd/internal/browser \
-	cmd/internal/objabi
+	cmd/internal/buildid \
+	cmd/internal/edit \
+	cmd/internal/objabi \
+	cmd/internal/test2json \
+	cmd/vet/internal/cfg \
+	cmd/vet/internal/whitelist
 
 libgotool_a_SOURCES = 
 libgotool_a_DEPENDENCIES = $(addsuffix .lo,$(GOTOOL_PACKAGES))
@@ -1210,12 +1222,15 @@ runtime_internal_sys_lo_check_GOCFLAGS = -fgo-compiling-runtime
 # Also use -fno-inline to get better results from the memory profiler.
 runtime_pprof_check_GOCFLAGS = -static-libgo -fno-inline
 extra_go_files_runtime_internal_sys = version.go
+extra_go_files_cmd_internal_objabi = objabi.go
 extra_go_files_cmd_go_internal_cfg = zdefaultcc.go
 extra_go_files_cmd_go_internal_load = zstdpkglist.go
+extra_check_libs_cmd_go_internal_cache = $(abs_builddir)/libgotool.a
 extra_check_libs_cmd_go_internal_generate = $(abs_builddir)/libgotool.a
 extra_check_libs_cmd_go_internal_get = $(abs_builddir)/libgotool.a
 extra_check_libs_cmd_go_internal_load = $(abs_builddir)/libgotool.a
 extra_check_libs_cmd_go_internal_work = $(abs_builddir)/libgotool.a
+extra_check_libs_cmd_vet_internal_cfg = $(abs_builddir)/libgotool.a
 @HAVE_STAT_TIMESPEC_FALSE@@LIBGO_IS_SOLARIS_TRUE@matchargs_os = 
 
 # Solaris 11.4 changed the type of fields in struct stat.
@@ -1238,10 +1253,12 @@ TEST_PACKAGES = \
 	bufio/check \
 	bytes/check \
 	context/check \
+	crypto/check \
 	errors/check \
 	expvar/check \
 	flag/check \
 	fmt/check \
+	hash/check \
 	html/check \
 	image/check \
 	io/check \
@@ -1263,11 +1280,16 @@ TEST_PACKAGES = \
 	unicode/check \
 	archive/tar/check \
 	archive/zip/check \
+	cmd/go/internal/cache/check \
 	cmd/go/internal/generate/check \
 	cmd/go/internal/get/check \
 	cmd/go/internal/load/check \
 	cmd/go/internal/work/check \
+	cmd/internal/buildid/check \
+	cmd/internal/edit/check \
 	cmd/internal/objabi/check \
+	cmd/internal/test2json/check \
+	cmd/vet/internal/cfg/check \
 	compress/bzip2/check \
 	compress/flate/check \
 	compress/gzip/check \
@@ -1320,6 +1342,7 @@ TEST_PACKAGES = \
 	go/constant/check \
 	go/doc/check \
 	go/format/check \
+	go/importer/check \
 	go/internal/gcimporter/check \
 	go/internal/gccgoimporter/check \
 	go/internal/srcimporter/check \
@@ -1330,6 +1353,7 @@ TEST_PACKAGES = \
 	go/types/check \
 	golang_org/x/crypto/chacha20poly1305/check \
 	golang_org/x/crypto/chacha20poly1305/internal/chacha20/check \
+	golang_org/x/crypto/cryptobyte/check \
 	golang_org/x/crypto/curve25519/check \
 	golang_org/x/crypto/poly1305/check \
 	golang_org/x/net/http2/hpack/check \
@@ -3130,6 +3154,24 @@ s-version: Makefile
 	$(SHELL) $(srcdir)/mvifdiff.sh version.go.tmp version.go
 	$(STAMP) $@
 
+objabi.go: s-objabi; @true
+s-objabi: Makefile
+	rm -f objabi.go.tmp
+	echo "package objabi" > objabi.go.tmp
+	echo "import \"runtime\"" >> objabi.go.tmp
+	echo 'const defaultGOROOT = `$(prefix)`' >> objabi.go.tmp
+	echo 'const defaultGO386 = `sse2`' >> objabi.go.tmp
+	echo 'const defaultGOARM = `5`' >> objabi.go.tmp
+	echo 'const defaultGOMIPS = `hardfloat`' >> objabi.go.tmp
+	echo 'const defaultGOOS = runtime.GOOS' >> objabi.go.tmp
+	echo 'const defaultGOARCH = runtime.GOARCH' >> objabi.go.tmp
+	echo 'const defaultGO_EXTLINK_ENABLED = ``' >> objabi.go.tmp
+	echo 'const version = `'`cat $(srcdir)/VERSION | sed 1q`' '`$(GOC) --version | sed 1q`'`' >> objabi.go.tmp
+	echo 'const stackGuardMultiplier = 1' >> objabi.go.tmp
+	echo 'const goexperiment = ``' >> objabi.go.tmp
+	$(SHELL) $(srcdir)/mvifdiff.sh objabi.go.tmp objabi.go
+	$(STAMP) $@
+
 runtime_sysinfo.go: s-runtime_sysinfo; @true
 s-runtime_sysinfo: $(srcdir)/mkrsysinfo.sh gen-sysinfo.go
 	GOARCH=$(GOARCH) GOOS=$(GOOS) $(SHELL) $(srcdir)/mkrsysinfo.sh
@@ -3146,10 +3188,11 @@ zdefaultcc.go: s-zdefaultcc; @true
 s-zdefaultcc: Makefile
 	echo 'package cfg' > zdefaultcc.go.tmp
 	echo >> zdefaultcc.go.tmp
-	echo 'const DefaultGCCGO = "$(bindir)/$(GCCGO_INSTALL_NAME)"' >> zdefaultcc.go.tmp
+	echo 'func DefaultGCCGO(goos, goarch string) string { return "$(bindir)/$(GCCGO_INSTALL_NAME)" }' >> zdefaultcc.go.tmp
-	echo 'const DefaultCC = "$(GCC_INSTALL_NAME)"' >> zdefaultcc.go.tmp
+	echo 'func DefaultCC(goos, goarch string) string { return "$(GCC_INSTALL_NAME)" }' >> zdefaultcc.go.tmp
-	echo 'const DefaultCXX = "$(GXX_INSTALL_NAME)"' >> zdefaultcc.go.tmp
+	echo 'func DefaultCXX(goos, goarch string) string { return "$(GXX_INSTALL_NAME)" }' >> zdefaultcc.go.tmp
 	echo 'const DefaultPkgConfig = "pkg-config"' >> zdefaultcc.go.tmp
+	echo 'var OSArchSupportsCgo = map[string]bool{}' >> zdefaultcc.go.tmp
 	$(SHELL) $(srcdir)/../move-if-change zdefaultcc.go.tmp zdefaultcc.go
 	$(STAMP) $@ 
 
@@ -3305,6 +3348,7 @@ $(foreach package,$(GOTOOL_PACKAGES),$(eval $(call PACKAGE_template,$(package)))
 runtime.lo.dep: $(extra_go_files_runtime)
 syscall.lo.dep: $(extra_go_files_syscall)
 runtime/internal/sys.lo.dep: $(extra_go_files_runtime_internal_sys)
+cmd/internal/objabi.lo.dep: $(extra_go_files_cmd_internal_objabi)
 cmd/go/internal/cfg.lo.dep: $(extra_go_files_cmd_go_internal_cfg)
 cmd/go/internal/load.lo.dep: $(extra_go_files_cmd_go_internal_load)
 

libgo/VERSION

@@ -1 +1 @@
-go1.9
+go1.10beta1

libgo/configure

@@ -2494,7 +2494,7 @@ ac_compiler_gnu=$ac_cv_c_compiler_gnu
 ac_config_headers="$ac_config_headers config.h"
 
 
-libtool_VERSION=12:0:0
+libtool_VERSION=13:0:0
 
 
 # Default to --enable-multilib
@@ -13652,7 +13652,7 @@ ALLGOARCHFAMILY="I386 ALPHA AMD64 ARM ARM64 IA64 M68K MIPS MIPS64 PPC PPC64 S390
 
 GOARCH=unknown
 GOARCH_FAMILY=unknown
-GOARCH_BIGENDIAN=0
+GOARCH_BIGENDIAN=false
 GOARCH_CACHELINESIZE=64
 GOARCH_PHYSPAGESIZE=4096
 GOARCH_PCQUANTUM=1
@@ -13680,6 +13680,12 @@ case ${host} in
     GOARCH_CACHELINESIZE=32
     GOARCH_PCQUANTUM=4
     GOARCH_MINFRAMESIZE=4
+    case ${host} in
+      arm*b*-*-*)
+	GOARCH=armbe
+        GOARCH_BIGENDIAN=true
+	;;
+    esac
     ;;
   i[34567]86-*-* | x86_64-*-*)
     cat confdefs.h - <<_ACEOF >conftest.$ac_ext
@@ -13712,7 +13718,7 @@ rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
   m68k*-*-*)
     GOARCH=m68k
     GOARCH_FAMILY=M68K
-    GOARCH_BIGENDIAN=1
+    GOARCH_BIGENDIAN=true
     GOARCH_CACHELINESIZE=16
     GOARCH_PCQUANTUM=4
     GOARCH_INT64ALIGN=2
@@ -13776,7 +13782,7 @@ rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
         GOARCH="${GOARCH}le"
         ;;
     *)
-	GOARCH_BIGENDIAN=1
+	GOARCH_BIGENDIAN=true
         ;;
     esac
     GOARCH_CACHELINESIZE=32
@@ -13794,7 +13800,7 @@ _ACEOF
 if ac_fn_c_try_compile "$LINENO"; then :
   GOARCH=ppc
 GOARCH_FAMILY=PPC
-GOARCH_BIGENDIAN=1
+GOARCH_BIGENDIAN=true
 
 else
 
@@ -13811,7 +13817,7 @@ if ac_fn_c_try_compile "$LINENO"; then :
 
 else
   GOARCH=ppc64
-GOARCH_BIGENDIAN=1
+GOARCH_BIGENDIAN=true
 
 fi
 rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
@@ -13841,7 +13847,7 @@ GOARCH_MINFRAMESIZE=8
 
 fi
 rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
-    GOARCH_BIGENDIAN=1
+    GOARCH_BIGENDIAN=true
     GOARCH_CACHELINESIZE=256
     GOARCH_PCQUANTUM=2
     ;;
@@ -13863,7 +13869,7 @@ GOARCH_FAMILY=SPARC64
 
 fi
 rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
-    GOARCH_BIGENDIAN=1
+    GOARCH_BIGENDIAN=true
     GOARCH_PHYSPAGESIZE=8192
     GOARCH_PCQUANTUM=4
     ;;
@@ -15142,7 +15148,7 @@ fi
 $as_echo "$libgo_cv_c_fancymath" >&6; }
 MATH_FLAG=
 if test "$libgo_cv_c_fancymath" = yes; then
-  MATH_FLAG="-mfancy-math-387 -funsafe-math-optimizations"
+  MATH_FLAG="-mfancy-math-387 -funsafe-math-optimizations -fno-math-errno"
 else
   MATH_FLAG="-ffp-contract=off"
 fi

libgo/configure.ac

@@ -11,7 +11,7 @@ AC_INIT(package-unused, version-unused,, libgo)
 AC_CONFIG_SRCDIR(Makefile.am)
 AC_CONFIG_HEADER(config.h)
 
-libtool_VERSION=12:0:0
+libtool_VERSION=13:0:0
 AC_SUBST(libtool_VERSION)
 
 AM_ENABLE_MULTILIB(, ..)
@@ -215,7 +215,7 @@ ALLGOARCHFAMILY="I386 ALPHA AMD64 ARM ARM64 IA64 M68K MIPS MIPS64 PPC PPC64 S390
 
 GOARCH=unknown
 GOARCH_FAMILY=unknown
-GOARCH_BIGENDIAN=0
+GOARCH_BIGENDIAN=false
 GOARCH_CACHELINESIZE=64
 GOARCH_PHYSPAGESIZE=4096
 GOARCH_PCQUANTUM=1
@@ -243,6 +243,12 @@ case ${host} in
     GOARCH_CACHELINESIZE=32
     GOARCH_PCQUANTUM=4
     GOARCH_MINFRAMESIZE=4
+    case ${host} in
+      arm*b*-*-*)
+	GOARCH=armbe
+        GOARCH_BIGENDIAN=true
+	;;
+    esac
     ;;
 changequote(,)dnl
   i[34567]86-*-* | x86_64-*-*)
@@ -270,7 +276,7 @@ GOARCH_HUGEPAGESIZE="1 << 21"
   m68k*-*-*)
     GOARCH=m68k
     GOARCH_FAMILY=M68K
-    GOARCH_BIGENDIAN=1
+    GOARCH_BIGENDIAN=true
     GOARCH_CACHELINESIZE=16
     GOARCH_PCQUANTUM=4
     GOARCH_INT64ALIGN=2
@@ -313,7 +319,7 @@ GOARCH_HUGEPAGESIZE="1 << 21"
         GOARCH="${GOARCH}le"
         ;;
     *)
-	GOARCH_BIGENDIAN=1
+	GOARCH_BIGENDIAN=true
         ;;
     esac
     GOARCH_CACHELINESIZE=32
@@ -327,7 +333,7 @@ GOARCH_HUGEPAGESIZE="1 << 21"
 #endif],
 [GOARCH=ppc
 GOARCH_FAMILY=PPC
-GOARCH_BIGENDIAN=1
+GOARCH_BIGENDIAN=true
 ],
     [
 GOARCH_FAMILY=PPC64
@@ -338,7 +344,7 @@ AC_COMPILE_IFELSE([
 [GOARCH=ppc64le
 ],
 [GOARCH=ppc64
-GOARCH_BIGENDIAN=1
+GOARCH_BIGENDIAN=true
 ])])
     GOARCH_PHYSPAGESIZE=65536
     GOARCH_PCQUANTUM=4
@@ -356,7 +362,7 @@ GOARCH_MINFRAMESIZE=4
 GOARCH_FAMILY=S390X
 GOARCH_MINFRAMESIZE=8
 ])
-    GOARCH_BIGENDIAN=1
+    GOARCH_BIGENDIAN=true
     GOARCH_CACHELINESIZE=256
     GOARCH_PCQUANTUM=2
     ;;
@@ -371,7 +377,7 @@ GOARCH_FAMILY=SPARC
 [GOARCH=sparc64
 GOARCH_FAMILY=SPARC64
 ])
-    GOARCH_BIGENDIAN=1
+    GOARCH_BIGENDIAN=true
     GOARCH_PHYSPAGESIZE=8192
     GOARCH_PCQUANTUM=4
     ;;
@@ -718,7 +724,7 @@ AC_COMPILE_IFELSE([int i;],
 CFLAGS=$CFLAGS_hold])
 MATH_FLAG=
 if test "$libgo_cv_c_fancymath" = yes; then
-  MATH_FLAG="-mfancy-math-387 -funsafe-math-optimizations"
+  MATH_FLAG="-mfancy-math-387 -funsafe-math-optimizations -fno-math-errno"
 else
   MATH_FLAG="-ffp-contract=off"
 fi

libgo/go/archive/tar/common.go

@@ -3,20 +3,22 @@
 // license that can be found in the LICENSE file.
 
 // Package tar implements access to tar archives.
-// It aims to cover most of the variations, including those produced
-// by GNU and BSD tars.
 //
-// References:
+// Tape archives (tar) are a file format for storing a sequence of files that
-//   http://www.freebsd.org/cgi/man.cgi?query=tar&sektion=5
+// can be read and written in a streaming manner.
-//   http://www.gnu.org/software/tar/manual/html_node/Standard.html
+// This package aims to cover most variations of the format,
-//   http://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html
+// including those produced by GNU and BSD tar tools.
 package tar
 
 import (
 	"errors"
 	"fmt"
+	"math"
 	"os"
 	"path"
+	"reflect"
+	"strconv"
+	"strings"
 	"time"
 )
 
@@ -24,42 +26,500 @@ import (
 // architectures. If a large value is encountered when decoding, the result
 // stored in Header will be the truncated version.
 
-// Header type flags.
+var (
-const (
+	ErrHeader          = errors.New("archive/tar: invalid tar header")
-	TypeReg           = '0'    // regular file
+	ErrWriteTooLong    = errors.New("archive/tar: write too long")
-	TypeRegA          = '\x00' // regular file
+	ErrFieldTooLong    = errors.New("archive/tar: header field too long")
-	TypeLink          = '1'    // hard link
+	ErrWriteAfterClose = errors.New("archive/tar: write after close")
-	TypeSymlink       = '2'    // symbolic link
+	errMissData        = errors.New("archive/tar: sparse file references non-existent data")
-	TypeChar          = '3'    // character device node
+	errUnrefData       = errors.New("archive/tar: sparse file contains unreferenced data")
-	TypeBlock         = '4'    // block device node
+	errWriteHole       = errors.New("archive/tar: write non-NUL byte in sparse hole")
-	TypeDir           = '5'    // directory
-	TypeFifo          = '6'    // fifo node
-	TypeCont          = '7'    // reserved
-	TypeXHeader       = 'x'    // extended header
-	TypeXGlobalHeader = 'g'    // global extended header
-	TypeGNULongName   = 'L'    // Next file has a long name
-	TypeGNULongLink   = 'K'    // Next file symlinks to a file w/ a long name
-	TypeGNUSparse     = 'S'    // sparse file
 )
 
+type headerError []string
+
+func (he headerError) Error() string {
+	const prefix = "archive/tar: cannot encode header"
+	var ss []string
+	for _, s := range he {
+		if s != "" {
+			ss = append(ss, s)
+		}
+	}
+	if len(ss) == 0 {
+		return prefix
+	}
+	return fmt.Sprintf("%s: %v", prefix, strings.Join(ss, "; and "))
+}
+
+// Type flags for Header.Typeflag.
+const (
+	// Type '0' indicates a regular file.
+	TypeReg  = '0'
+	TypeRegA = '\x00' // For legacy support; use TypeReg instead
+
+	// Type '1' to '6' are header-only flags and may not have a data body.
+	TypeLink    = '1' // Hard link
+	TypeSymlink = '2' // Symbolic link
+	TypeChar    = '3' // Character device node
+	TypeBlock   = '4' // Block device node
+	TypeDir     = '5' // Directory
+	TypeFifo    = '6' // FIFO node
+
+	// Type '7' is reserved.
+	TypeCont = '7'
+
+	// Type 'x' is used by the PAX format to store key-value records that
+	// are only relevant to the next file.
+	// This package transparently handles these types.
+	TypeXHeader = 'x'
+
+	// Type 'g' is used by the PAX format to store key-value records that
+	// are relevant to all subsequent files.
+	// This package only supports parsing and composing such headers,
+	// but does not currently support persisting the global state across files.
+	TypeXGlobalHeader = 'g'
+
+	// Type 'S' indicates a sparse file in the GNU format.
+	TypeGNUSparse = 'S'
+
+	// Types 'L' and 'K' are used by the GNU format for a meta file
+	// used to store the path or link name for the next file.
+	// This package transparently handles these types.
+	TypeGNULongName = 'L'
+	TypeGNULongLink = 'K'
+)
+
+// Keywords for PAX extended header records.
+const (
+	paxNone     = "" // Indicates that no PAX key is suitable
+	paxPath     = "path"
+	paxLinkpath = "linkpath"
+	paxSize     = "size"
+	paxUid      = "uid"
+	paxGid      = "gid"
+	paxUname    = "uname"
+	paxGname    = "gname"
+	paxMtime    = "mtime"
+	paxAtime    = "atime"
+	paxCtime    = "ctime"   // Removed from later revision of PAX spec, but was valid
+	paxCharset  = "charset" // Currently unused
+	paxComment  = "comment" // Currently unused
+
+	paxSchilyXattr = "SCHILY.xattr."
+
+	// Keywords for GNU sparse files in a PAX extended header.
+	paxGNUSparse          = "GNU.sparse."
+	paxGNUSparseNumBlocks = "GNU.sparse.numblocks"
+	paxGNUSparseOffset    = "GNU.sparse.offset"
+	paxGNUSparseNumBytes  = "GNU.sparse.numbytes"
+	paxGNUSparseMap       = "GNU.sparse.map"
+	paxGNUSparseName      = "GNU.sparse.name"
+	paxGNUSparseMajor     = "GNU.sparse.major"
+	paxGNUSparseMinor     = "GNU.sparse.minor"
+	paxGNUSparseSize      = "GNU.sparse.size"
+	paxGNUSparseRealSize  = "GNU.sparse.realsize"
+)
+
+// basicKeys is a set of the PAX keys for which we have built-in support.
+// This does not contain "charset" or "comment", which are both PAX-specific,
+// so adding them as first-class features of Header is unlikely.
+// Users can use the PAXRecords field to set it themselves.
+var basicKeys = map[string]bool{
+	paxPath: true, paxLinkpath: true, paxSize: true, paxUid: true, paxGid: true,
+	paxUname: true, paxGname: true, paxMtime: true, paxAtime: true, paxCtime: true,
+}
+
 // A Header represents a single header in a tar archive.
 // Some fields may not be populated.
+//
+// For forward compatibility, users that retrieve a Header from Reader.Next,
+// mutate it in some ways, and then pass it back to Writer.WriteHeader
+// should do so by creating a new Header and copying the fields
+// that they are interested in preserving.
 type Header struct {
-	Name       string    // name of header file entry
+	Typeflag byte // Type of header entry (should be TypeReg for most files)
-	Mode       int64     // permission and mode bits
+
-	Uid        int       // user id of owner
+	Name     string // Name of file entry
-	Gid        int       // group id of owner
+	Linkname string // Target name of link (valid for TypeLink or TypeSymlink)
-	Size       int64     // length in bytes
+
-	ModTime    time.Time // modified time
+	Size  int64  // Logical file size in bytes
-	Typeflag   byte      // type of header entry
+	Mode  int64  // Permission and mode bits
-	Linkname   string    // target name of link
+	Uid   int    // User ID of owner
-	Uname      string    // user name of owner
+	Gid   int    // Group ID of owner
-	Gname      string    // group name of owner
+	Uname string // User name of owner
-	Devmajor   int64     // major number of character or block device
+	Gname string // Group name of owner
-	Devminor   int64     // minor number of character or block device
+
-	AccessTime time.Time // access time
+	// If the Format is unspecified, then Writer.WriteHeader rounds ModTime
-	ChangeTime time.Time // status change time
+	// to the nearest second and ignores the AccessTime and ChangeTime fields.
+	//
+	// To use AccessTime or ChangeTime, specify the Format as PAX or GNU.
+	// To use sub-second resolution, specify the Format as PAX.
+	ModTime    time.Time // Modification time
+	AccessTime time.Time // Access time (requires either PAX or GNU support)
+	ChangeTime time.Time // Change time (requires either PAX or GNU support)
+
+	Devmajor int64 // Major device number (valid for TypeChar or TypeBlock)
+	Devminor int64 // Minor device number (valid for TypeChar or TypeBlock)
+
+	// Xattrs stores extended attributes as PAX records under the
+	// "SCHILY.xattr." namespace.
+	//
+	// The following are semantically equivalent:
+	//  h.Xattrs[key] = value
+	//  h.PAXRecords["SCHILY.xattr."+key] = value
+	//
+	// When Writer.WriteHeader is called, the contents of Xattrs will take
+	// precedence over those in PAXRecords.
+	//
+	// Deprecated: Use PAXRecords instead.
 	Xattrs map[string]string
+
+	// PAXRecords is a map of PAX extended header records.
+	//
+	// User-defined records should have keys of the following form:
+	//	VENDOR.keyword
+	// Where VENDOR is some namespace in all uppercase, and keyword may
+	// not contain the '=' character (e.g., "GOLANG.pkg.version").
+	// The key and value should be non-empty UTF-8 strings.
+	//
+	// When Writer.WriteHeader is called, PAX records derived from the
+	// the other fields in Header take precedence over PAXRecords.
+	PAXRecords map[string]string
+
+	// Format specifies the format of the tar header.
+	//
+	// This is set by Reader.Next as a best-effort guess at the format.
+	// Since the Reader liberally reads some non-compliant files,
+	// it is possible for this to be FormatUnknown.
+	//
+	// If the format is unspecified when Writer.WriteHeader is called,
+	// then it uses the first format (in the order of USTAR, PAX, GNU)
+	// capable of encoding this Header (see Format).
+	Format Format
+}
+
+// sparseEntry represents a Length-sized fragment at Offset in the file.
+type sparseEntry struct{ Offset, Length int64 }
+
+func (s sparseEntry) endOffset() int64 { return s.Offset + s.Length }
+
+// A sparse file can be represented as either a sparseDatas or a sparseHoles.
+// As long as the total size is known, they are equivalent and one can be
+// converted to the other form and back. The various tar formats with sparse
+// file support represent sparse files in the sparseDatas form. That is, they
+// specify the fragments in the file that has data, and treat everything else as
+// having zero bytes. As such, the encoding and decoding logic in this package
+// deals with sparseDatas.
+//
+// However, the external API uses sparseHoles instead of sparseDatas because the
+// zero value of sparseHoles logically represents a normal file (i.e., there are
+// no holes in it). On the other hand, the zero value of sparseDatas implies
+// that the file has no data in it, which is rather odd.
+//
+// As an example, if the underlying raw file contains the 10-byte data:
+//	var compactFile = "abcdefgh"
+//
+// And the sparse map has the following entries:
+//	var spd sparseDatas = []sparseEntry{
+//		{Offset: 2,  Length: 5},  // Data fragment for 2..6
+//		{Offset: 18, Length: 3},  // Data fragment for 18..20
+//	}
+//	var sph sparseHoles = []sparseEntry{
+//		{Offset: 0,  Length: 2},  // Hole fragment for 0..1
+//		{Offset: 7,  Length: 11}, // Hole fragment for 7..17
+//		{Offset: 21, Length: 4},  // Hole fragment for 21..24
+//	}
+//
+// Then the content of the resulting sparse file with a Header.Size of 25 is:
+//	var sparseFile = "\x00"*2 + "abcde" + "\x00"*11 + "fgh" + "\x00"*4
+type (
+	sparseDatas []sparseEntry
+	sparseHoles []sparseEntry
+)
+
+// validateSparseEntries reports whether sp is a valid sparse map.
+// It does not matter whether sp represents data fragments or hole fragments.
+func validateSparseEntries(sp []sparseEntry, size int64) bool {
+	// Validate all sparse entries. These are the same checks as performed by
+	// the BSD tar utility.
+	if size < 0 {
+		return false
+	}
+	var pre sparseEntry
+	for _, cur := range sp {
+		switch {
+		case cur.Offset < 0 || cur.Length < 0:
+			return false // Negative values are never okay
+		case cur.Offset > math.MaxInt64-cur.Length:
+			return false // Integer overflow with large length
+		case cur.endOffset() > size:
+			return false // Region extends beyond the actual size
+		case pre.endOffset() > cur.Offset:
+			return false // Regions cannot overlap and must be in order
+		}
+		pre = cur
+	}
+	return true
+}
+
+// alignSparseEntries mutates src and returns dst where each fragment's
+// starting offset is aligned up to the nearest block edge, and each
+// ending offset is aligned down to the nearest block edge.
+//
+// Even though the Go tar Reader and the BSD tar utility can handle entries
+// with arbitrary offsets and lengths, the GNU tar utility can only handle
+// offsets and lengths that are multiples of blockSize.
+func alignSparseEntries(src []sparseEntry, size int64) []sparseEntry {
+	dst := src[:0]
+	for _, s := range src {
+		pos, end := s.Offset, s.endOffset()
+		pos += blockPadding(+pos) // Round-up to nearest blockSize
+		if end != size {
+			end -= blockPadding(-end) // Round-down to nearest blockSize
+		}
+		if pos < end {
+			dst = append(dst, sparseEntry{Offset: pos, Length: end - pos})
+		}
+	}
+	return dst
+}
+
+// invertSparseEntries converts a sparse map from one form to the other.
+// If the input is sparseHoles, then it will output sparseDatas and vice-versa.
+// The input must have been already validated.
+//
+// This function mutates src and returns a normalized map where:
+//	* adjacent fragments are coalesced together
+//	* only the last fragment may be empty
+//	* the endOffset of the last fragment is the total size
+func invertSparseEntries(src []sparseEntry, size int64) []sparseEntry {
+	dst := src[:0]
+	var pre sparseEntry
+	for _, cur := range src {
+		if cur.Length == 0 {
+			continue // Skip empty fragments
+		}
+		pre.Length = cur.Offset - pre.Offset
+		if pre.Length > 0 {
+			dst = append(dst, pre) // Only add non-empty fragments
+		}
+		pre.Offset = cur.endOffset()
+	}
+	pre.Length = size - pre.Offset // Possibly the only empty fragment
+	return append(dst, pre)
+}
+
+// fileState tracks the number of logical (includes sparse holes) and physical
+// (actual in tar archive) bytes remaining for the current file.
+//
+// Invariant: LogicalRemaining >= PhysicalRemaining
+type fileState interface {
+	LogicalRemaining() int64
+	PhysicalRemaining() int64
+}
+
+// allowedFormats determines which formats can be used.
+// The value returned is the logical OR of multiple possible formats.
+// If the value is FormatUnknown, then the input Header cannot be encoded
+// and an error is returned explaining why.
+//
+// As a by-product of checking the fields, this function returns paxHdrs, which
+// contain all fields that could not be directly encoded.
+// A value receiver ensures that this method does not mutate the source Header.
+func (h Header) allowedFormats() (format Format, paxHdrs map[string]string, err error) {
+	format = FormatUSTAR | FormatPAX | FormatGNU
+	paxHdrs = make(map[string]string)
+
+	var whyNoUSTAR, whyNoPAX, whyNoGNU string
+	var preferPAX bool // Prefer PAX over USTAR
+	verifyString := func(s string, size int, name, paxKey string) {
+		// NUL-terminator is optional for path and linkpath.
+		// Technically, it is required for uname and gname,
+		// but neither GNU nor BSD tar checks for it.
+		tooLong := len(s) > size
+		allowLongGNU := paxKey == paxPath || paxKey == paxLinkpath
+		if hasNUL(s) || (tooLong && !allowLongGNU) {
+			whyNoGNU = fmt.Sprintf("GNU cannot encode %s=%q", name, s)
+			format.mustNotBe(FormatGNU)
+		}
+		if !isASCII(s) || tooLong {
+			canSplitUSTAR := paxKey == paxPath
+			if _, _, ok := splitUSTARPath(s); !canSplitUSTAR || !ok {
+				whyNoUSTAR = fmt.Sprintf("USTAR cannot encode %s=%q", name, s)
+				format.mustNotBe(FormatUSTAR)
+			}
+			if paxKey == paxNone {
+				whyNoPAX = fmt.Sprintf("PAX cannot encode %s=%q", name, s)
+				format.mustNotBe(FormatPAX)
+			} else {
+				paxHdrs[paxKey] = s
+			}
+		}
+		if v, ok := h.PAXRecords[paxKey]; ok && v == s {
+			paxHdrs[paxKey] = v
+		}
+	}
+	verifyNumeric := func(n int64, size int, name, paxKey string) {
+		if !fitsInBase256(size, n) {
+			whyNoGNU = fmt.Sprintf("GNU cannot encode %s=%d", name, n)
+			format.mustNotBe(FormatGNU)
+		}
+		if !fitsInOctal(size, n) {
+			whyNoUSTAR = fmt.Sprintf("USTAR cannot encode %s=%d", name, n)
+			format.mustNotBe(FormatUSTAR)
+			if paxKey == paxNone {
+				whyNoPAX = fmt.Sprintf("PAX cannot encode %s=%d", name, n)
+				format.mustNotBe(FormatPAX)
+			} else {
+				paxHdrs[paxKey] = strconv.FormatInt(n, 10)
+			}
+		}
+		if v, ok := h.PAXRecords[paxKey]; ok && v == strconv.FormatInt(n, 10) {
+			paxHdrs[paxKey] = v
+		}
+	}
+	verifyTime := func(ts time.Time, size int, name, paxKey string) {
+		if ts.IsZero() {
+			return // Always okay
+		}
+		if !fitsInBase256(size, ts.Unix()) {
+			whyNoGNU = fmt.Sprintf("GNU cannot encode %s=%v", name, ts)
+			format.mustNotBe(FormatGNU)
+		}
+		isMtime := paxKey == paxMtime
+		fitsOctal := fitsInOctal(size, ts.Unix())
+		if (isMtime && !fitsOctal) || !isMtime {
+			whyNoUSTAR = fmt.Sprintf("USTAR cannot encode %s=%v", name, ts)
+			format.mustNotBe(FormatUSTAR)
+		}
+		needsNano := ts.Nanosecond() != 0
+		if !isMtime || !fitsOctal || needsNano {
+			preferPAX = true // USTAR may truncate sub-second measurements
+			if paxKey == paxNone {
+				whyNoPAX = fmt.Sprintf("PAX cannot encode %s=%v", name, ts)
+				format.mustNotBe(FormatPAX)
+			} else {
+				paxHdrs[paxKey] = formatPAXTime(ts)
+			}
+		}
+		if v, ok := h.PAXRecords[paxKey]; ok && v == formatPAXTime(ts) {
+			paxHdrs[paxKey] = v
+		}
+	}
+
+	// Check basic fields.
+	var blk block
+	v7 := blk.V7()
+	ustar := blk.USTAR()
+	gnu := blk.GNU()
+	verifyString(h.Name, len(v7.Name()), "Name", paxPath)
+	verifyString(h.Linkname, len(v7.LinkName()), "Linkname", paxLinkpath)
+	verifyString(h.Uname, len(ustar.UserName()), "Uname", paxUname)
+	verifyString(h.Gname, len(ustar.GroupName()), "Gname", paxGname)
+	verifyNumeric(h.Mode, len(v7.Mode()), "Mode", paxNone)
+	verifyNumeric(int64(h.Uid), len(v7.UID()), "Uid", paxUid)
+	verifyNumeric(int64(h.Gid), len(v7.GID()), "Gid", paxGid)
+	verifyNumeric(h.Size, len(v7.Size()), "Size", paxSize)
+	verifyNumeric(h.Devmajor, len(ustar.DevMajor()), "Devmajor", paxNone)
+	verifyNumeric(h.Devminor, len(ustar.DevMinor()), "Devminor", paxNone)
+	verifyTime(h.ModTime, len(v7.ModTime()), "ModTime", paxMtime)
+	verifyTime(h.AccessTime, len(gnu.AccessTime()), "AccessTime", paxAtime)
+	verifyTime(h.ChangeTime, len(gnu.ChangeTime()), "ChangeTime", paxCtime)
+
+	// Check for header-only types.
+	var whyOnlyPAX, whyOnlyGNU string
+	switch h.Typeflag {
+	case TypeReg, TypeChar, TypeBlock, TypeFifo, TypeGNUSparse:
+		// Exclude TypeLink and TypeSymlink, since they may reference directories.
+		if strings.HasSuffix(h.Name, "/") {
+			return FormatUnknown, nil, headerError{"filename may not have trailing slash"}
+		}
+	case TypeXHeader, TypeGNULongName, TypeGNULongLink:
+		return FormatUnknown, nil, headerError{"cannot manually encode TypeXHeader, TypeGNULongName, or TypeGNULongLink headers"}
+	case TypeXGlobalHeader:
+		h2 := Header{Name: h.Name, Typeflag: h.Typeflag, Xattrs: h.Xattrs, PAXRecords: h.PAXRecords, Format: h.Format}
+		if !reflect.DeepEqual(h, h2) {
+			return FormatUnknown, nil, headerError{"only PAXRecords should be set for TypeXGlobalHeader"}
+		}
+		whyOnlyPAX = "only PAX supports TypeXGlobalHeader"
+		format.mayOnlyBe(FormatPAX)
+	}
+	if !isHeaderOnlyType(h.Typeflag) && h.Size < 0 {
+		return FormatUnknown, nil, headerError{"negative size on header-only type"}
+	}
+
+	// Check PAX records.
+	if len(h.Xattrs) > 0 {
+		for k, v := range h.Xattrs {
+			paxHdrs[paxSchilyXattr+k] = v
+		}
+		whyOnlyPAX = "only PAX supports Xattrs"
+		format.mayOnlyBe(FormatPAX)
+	}
+	if len(h.PAXRecords) > 0 {
+		for k, v := range h.PAXRecords {
+			switch _, exists := paxHdrs[k]; {
+			case exists:
+				continue // Do not overwrite existing records
+			case h.Typeflag == TypeXGlobalHeader:
+				paxHdrs[k] = v // Copy all records
+			case !basicKeys[k] && !strings.HasPrefix(k, paxGNUSparse):
+				paxHdrs[k] = v // Ignore local records that may conflict
+			}
+		}
+		whyOnlyPAX = "only PAX supports PAXRecords"
+		format.mayOnlyBe(FormatPAX)
+	}
+	for k, v := range paxHdrs {
+		if !validPAXRecord(k, v) {
+			return FormatUnknown, nil, headerError{fmt.Sprintf("invalid PAX record: %q", k+" = "+v)}
+		}
+	}
+
+	// TODO(dsnet): Re-enable this when adding sparse support.
+	// See https://golang.org/issue/22735
+	/*
+		// Check sparse files.
+		if len(h.SparseHoles) > 0 || h.Typeflag == TypeGNUSparse {
+			if isHeaderOnlyType(h.Typeflag) {
+				return FormatUnknown, nil, headerError{"header-only type cannot be sparse"}
+			}
+			if !validateSparseEntries(h.SparseHoles, h.Size) {
+				return FormatUnknown, nil, headerError{"invalid sparse holes"}
+			}
+			if h.Typeflag == TypeGNUSparse {
+				whyOnlyGNU = "only GNU supports TypeGNUSparse"
+				format.mayOnlyBe(FormatGNU)
+			} else {
+				whyNoGNU = "GNU supports sparse files only with TypeGNUSparse"
+				format.mustNotBe(FormatGNU)
+			}
+			whyNoUSTAR = "USTAR does not support sparse files"
+			format.mustNotBe(FormatUSTAR)
+		}
+	*/
+
+	// Check desired format.
+	if wantFormat := h.Format; wantFormat != FormatUnknown {
+		if wantFormat.has(FormatPAX) && !preferPAX {
+			wantFormat.mayBe(FormatUSTAR) // PAX implies USTAR allowed too
+		}
+		format.mayOnlyBe(wantFormat) // Set union of formats allowed and format wanted
+	}
+	if format == FormatUnknown {
+		switch h.Format {
+		case FormatUSTAR:
+			err = headerError{"Format specifies USTAR", whyNoUSTAR, whyOnlyPAX, whyOnlyGNU}
+		case FormatPAX:
+			err = headerError{"Format specifies PAX", whyNoPAX, whyOnlyGNU}
+		case FormatGNU:
+			err = headerError{"Format specifies GNU", whyNoGNU, whyOnlyPAX}
+		default:
+			err = headerError{whyNoUSTAR, whyNoPAX, whyNoGNU, whyOnlyPAX, whyOnlyGNU}
+		}
+	}
+	return format, paxHdrs, err
 }
 
 // FileInfo returns an os.FileInfo for the Header.
@@ -92,63 +552,43 @@ func (fi headerFileInfo) Mode() (mode os.FileMode) {
 
 	// Set setuid, setgid and sticky bits.
 	if fi.h.Mode&c_ISUID != 0 {
-		// setuid
 		mode |= os.ModeSetuid
 	}
 	if fi.h.Mode&c_ISGID != 0 {
-		// setgid
 		mode |= os.ModeSetgid
 	}
 	if fi.h.Mode&c_ISVTX != 0 {
-		// sticky
 		mode |= os.ModeSticky
 	}
 
-	// Set file mode bits.
+	// Set file mode bits; clear perm, setuid, setgid, and sticky bits.
-	// clear perm, setuid, setgid and sticky bits.
+	switch m := os.FileMode(fi.h.Mode) &^ 07777; m {
-	m := os.FileMode(fi.h.Mode) &^ 07777
+	case c_ISDIR:
-	if m == c_ISDIR {
-		// directory
 		mode |= os.ModeDir
-	}
+	case c_ISFIFO:
-	if m == c_ISFIFO {
-		// named pipe (FIFO)
 		mode |= os.ModeNamedPipe
-	}
+	case c_ISLNK:
-	if m == c_ISLNK {
-		// symbolic link
 		mode |= os.ModeSymlink
-	}
+	case c_ISBLK:
-	if m == c_ISBLK {
-		// device file
 		mode |= os.ModeDevice
-	}
+	case c_ISCHR:
-	if m == c_ISCHR {
-		// Unix character device
 		mode |= os.ModeDevice
 		mode |= os.ModeCharDevice
-	}
+	case c_ISSOCK:
-	if m == c_ISSOCK {
-		// Unix domain socket
 		mode |= os.ModeSocket
 	}
 
 	switch fi.h.Typeflag {
 	case TypeSymlink:
-		// symbolic link
 		mode |= os.ModeSymlink
 	case TypeChar:
-		// character device node
 		mode |= os.ModeDevice
 		mode |= os.ModeCharDevice
 	case TypeBlock:
-		// block device node
 		mode |= os.ModeDevice
 	case TypeDir:
-		// directory
 		mode |= os.ModeDir
 	case TypeFifo:
-		// fifo node
 		mode |= os.ModeNamedPipe
 	}
 
@@ -176,33 +616,16 @@ const (
 	c_ISSOCK = 0140000 // Socket
 )
 
-// Keywords for the PAX Extended Header
-const (
-	paxAtime    = "atime"
-	paxCharset  = "charset"
-	paxComment  = "comment"
-	paxCtime    = "ctime" // please note that ctime is not a valid pax header.
-	paxGid      = "gid"
-	paxGname    = "gname"
-	paxLinkpath = "linkpath"
-	paxMtime    = "mtime"
-	paxPath     = "path"
-	paxSize     = "size"
-	paxUid      = "uid"
-	paxUname    = "uname"
-	paxXattr    = "SCHILY.xattr."
-	paxNone     = ""
-)
-
 // FileInfoHeader creates a partially-populated Header from fi.
 // If fi describes a symlink, FileInfoHeader records link as the link target.
 // If fi describes a directory, a slash is appended to the name.
-// Because os.FileInfo's Name method returns only the base name of
+//
-// the file it describes, it may be necessary to modify the Name field
+// Since os.FileInfo's Name method only returns the base name of
-// of the returned header to provide the full path name of the file.
+// the file it describes, it may be necessary to modify Header.Name
+// to provide the full path name of the file.
 func FileInfoHeader(fi os.FileInfo, link string) (*Header, error) {
 	if fi == nil {
-		return nil, errors.New("tar: FileInfo is nil")
+		return nil, errors.New("archive/tar: FileInfo is nil")
 	}
 	fm := fi.Mode()
 	h := &Header{
@@ -265,6 +688,12 @@ func FileInfoHeader(fi os.FileInfo, link string) (*Header, error) {
 			h.Size = 0
 			h.Linkname = sys.Linkname
 		}
+		if sys.PAXRecords != nil {
+			h.PAXRecords = make(map[string]string)
+			for k, v := range sys.PAXRecords {
+				h.PAXRecords[k] = v
+			}
+		}
 	}
 	if sysStat != nil {
 		return h, sysStat(fi, h)
@@ -282,3 +711,10 @@ func isHeaderOnlyType(flag byte) bool {
 		return false
 	}
 }
+
+func min(a, b int64) int64 {
+	if a < b {
+		return a
+	}
+	return b
+}

libgo/go/archive/tar/format.go

@@ -4,38 +4,133 @@
 
 package tar
 
+import "strings"
+
+// Format represents the tar archive format.
+//
+// The original tar format was introduced in Unix V7.
+// Since then, there have been multiple competing formats attempting to
+// standardize or extend the V7 format to overcome its limitations.
+// The most common formats are the USTAR, PAX, and GNU formats,
+// each with their own advantages and limitations.
+//
+// The following table captures the capabilities of each format:
+//
+//	                  |  USTAR |       PAX |       GNU
+//	------------------+--------+-----------+----------
+//	Name              |   256B | unlimited | unlimited
+//	Linkname          |   100B | unlimited | unlimited
+//	Size              | uint33 | unlimited |    uint89
+//	Mode              | uint21 |    uint21 |    uint57
+//	Uid/Gid           | uint21 | unlimited |    uint57
+//	Uname/Gname       |    32B | unlimited |       32B
+//	ModTime           | uint33 | unlimited |     int89
+//	AccessTime        |    n/a | unlimited |     int89
+//	ChangeTime        |    n/a | unlimited |     int89
+//	Devmajor/Devminor | uint21 |    uint21 |    uint57
+//	------------------+--------+-----------+----------
+//	string encoding   |  ASCII |     UTF-8 |    binary
+//	sub-second times  |     no |       yes |        no
+//	sparse files      |     no |       yes |       yes
+//
+// The table's upper portion shows the Header fields, where each format reports
+// the maximum number of bytes allowed for each string field and
+// the integer type used to store each numeric field
+// (where timestamps are stored as the number of seconds since the Unix epoch).
+//
+// The table's lower portion shows specialized features of each format,
+// such as supported string encodings, support for sub-second timestamps,
+// or support for sparse files.
+//
+// The Writer currently provides no support for sparse files.
+type Format int
+
 // Constants to identify various tar formats.
 const (
-	// The format is unknown.
+	// Deliberately hide the meaning of constants from public API.
-	formatUnknown = (1 << iota) / 2 // Sequence of 0, 1, 2, 4, 8, etc...
+	_ Format = (1 << iota) / 4 // Sequence of 0, 0, 1, 2, 4, 8, etc...
+
+	// FormatUnknown indicates that the format is unknown.
+	FormatUnknown
 
 	// The format of the original Unix V7 tar tool prior to standardization.
 	formatV7
 
-	// The old and new GNU formats, which are incompatible with USTAR.
+	// FormatUSTAR represents the USTAR header format defined in POSIX.1-1988.
-	// This does cover the old GNU sparse extension.
+	//
-	// This does not cover the GNU sparse extensions using PAX headers,
+	// While this format is compatible with most tar readers,
-	// versions 0.0, 0.1, and 1.0; these fall under the PAX format.
+	// the format has several limitations making it unsuitable for some usages.
-	formatGNU
+	// Most notably, it cannot support sparse files, files larger than 8GiB,
+	// filenames larger than 256 characters, and non-ASCII filenames.
+	//
+	// Reference:
+	//	http://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html#tag_20_92_13_06
+	FormatUSTAR
+
+	// FormatPAX represents the PAX header format defined in POSIX.1-2001.
+	//
+	// PAX extends USTAR by writing a special file with Typeflag TypeXHeader
+	// preceding the original header. This file contains a set of key-value
+	// records, which are used to overcome USTAR's shortcomings, in addition to
+	// providing the ability to have sub-second resolution for timestamps.
+	//
+	// Some newer formats add their own extensions to PAX by defining their
+	// own keys and assigning certain semantic meaning to the associated values.
+	// For example, sparse file support in PAX is implemented using keys
+	// defined by the GNU manual (e.g., "GNU.sparse.map").
+	//
+	// Reference:
+	//	http://pubs.opengroup.org/onlinepubs/009695399/utilities/pax.html
+	FormatPAX
+
+	// FormatGNU represents the GNU header format.
+	//
+	// The GNU header format is older than the USTAR and PAX standards and
+	// is not compatible with them. The GNU format supports
+	// arbitrary file sizes, filenames of arbitrary encoding and length,
+	// sparse files, and other features.
+	//
+	// It is recommended that PAX be chosen over GNU unless the target
+	// application can only parse GNU formatted archives.
+	//
+	// Reference:
+	//	http://www.gnu.org/software/tar/manual/html_node/Standard.html
+	FormatGNU
 
 	// Schily's tar format, which is incompatible with USTAR.
 	// This does not cover STAR extensions to the PAX format; these fall under
 	// the PAX format.
 	formatSTAR
 
-	// USTAR is the former standardization of tar defined in POSIX.1-1988.
+	formatMax
-	// This is incompatible with the GNU and STAR formats.
-	formatUSTAR
-
-	// PAX is the latest standardization of tar defined in POSIX.1-2001.
-	// This is an extension of USTAR and is "backwards compatible" with it.
-	//
-	// Some newer formats add their own extensions to PAX, such as GNU sparse
-	// files and SCHILY extended attributes. Since they are backwards compatible
-	// with PAX, they will be labelled as "PAX".
-	formatPAX
 )
 
+func (f Format) has(f2 Format) bool   { return f&f2 != 0 }
+func (f *Format) mayBe(f2 Format)     { *f |= f2 }
+func (f *Format) mayOnlyBe(f2 Format) { *f &= f2 }
+func (f *Format) mustNotBe(f2 Format) { *f &^= f2 }
+
+var formatNames = map[Format]string{
+	formatV7: "V7", FormatUSTAR: "USTAR", FormatPAX: "PAX", FormatGNU: "GNU", formatSTAR: "STAR",
+}
+
+func (f Format) String() string {
+	var ss []string
+	for f2 := Format(1); f2 < formatMax; f2 <<= 1 {
+		if f.has(f2) {
+			ss = append(ss, formatNames[f2])
+		}
+	}
+	switch len(ss) {
+	case 0:
+		return "<unknown>"
+	case 1:
+		return ss[0]
+	default:
+		return "(" + strings.Join(ss, " | ") + ")"
+	}
+}
+
 // Magics used to identify various formats.
 const (
 	magicGNU, versionGNU     = "ustar ", " \x00"
@@ -50,6 +145,12 @@ const (
 	prefixSize = 155 // Max length of the prefix field in USTAR format
 )
 
+// blockPadding computes the number of bytes needed to pad offset up to the
+// nearest block edge where 0 <= n < blockSize.
+func blockPadding(offset int64) (n int64) {
+	return -offset & (blockSize - 1)
+}
+
 var zeroBlock block
 
 type block [blockSize]byte
@@ -63,14 +164,14 @@ func (b *block) Sparse() sparseArray { return (sparseArray)(b[:]) }
 
 // GetFormat checks that the block is a valid tar header based on the checksum.
 // It then attempts to guess the specific format based on magic values.
-// If the checksum fails, then formatUnknown is returned.
+// If the checksum fails, then FormatUnknown is returned.
-func (b *block) GetFormat() (format int) {
+func (b *block) GetFormat() Format {
 	// Verify checksum.
 	var p parser
 	value := p.parseOctal(b.V7().Chksum())
 	chksum1, chksum2 := b.ComputeChecksum()
 	if p.err != nil || (value != chksum1 && value != chksum2) {
-		return formatUnknown
+		return FormatUnknown
 	}
 
 	// Guess the magic values.
@@ -81,9 +182,9 @@ func (b *block) GetFormat() (format int) {
 	case magic == magicUSTAR && trailer == trailerSTAR:
 		return formatSTAR
 	case magic == magicUSTAR:
-		return formatUSTAR
+		return FormatUSTAR | FormatPAX
 	case magic == magicGNU && version == versionGNU:
-		return formatGNU
+		return FormatGNU
 	default:
 		return formatV7
 	}
@@ -91,19 +192,19 @@ func (b *block) GetFormat() (format int) {
 
 // SetFormat writes the magic values necessary for specified format
 // and then updates the checksum accordingly.
-func (b *block) SetFormat(format int) {
+func (b *block) SetFormat(format Format) {
 	// Set the magic values.
-	switch format {
+	switch {
-	case formatV7:
+	case format.has(formatV7):
 		// Do nothing.
-	case formatGNU:
+	case format.has(FormatGNU):
 		copy(b.GNU().Magic(), magicGNU)
 		copy(b.GNU().Version(), versionGNU)
-	case formatSTAR:
+	case format.has(formatSTAR):
 		copy(b.STAR().Magic(), magicUSTAR)
 		copy(b.STAR().Version(), versionUSTAR)
 		copy(b.STAR().Trailer(), trailerSTAR)
-	case formatUSTAR, formatPAX:
+	case format.has(FormatUSTAR | FormatPAX):
 		copy(b.USTAR().Magic(), magicUSTAR)
 		copy(b.USTAR().Version(), versionUSTAR)
 	default:
@@ -128,12 +229,17 @@ func (b *block) ComputeChecksum() (unsigned, signed int64) {
 		if 148 <= i && i < 156 {
 			c = ' ' // Treat the checksum field itself as all spaces.
 		}
-		unsigned += int64(uint8(c))
+		unsigned += int64(c)
 		signed += int64(int8(c))
 	}
 	return unsigned, signed
 }
 
+// Reset clears the block with all zeros.
+func (b *block) Reset() {
+	*b = block{}
+}
+
 type headerV7 [blockSize]byte
 
 func (h *headerV7) Name() []byte     { return h[000:][:100] }
@@ -187,11 +293,11 @@ func (h *headerUSTAR) Prefix() []byte    { return h[345:][:155] }
 
 type sparseArray []byte
 
-func (s sparseArray) Entry(i int) sparseNode { return (sparseNode)(s[i*24:]) }
+func (s sparseArray) Entry(i int) sparseElem { return (sparseElem)(s[i*24:]) }
 func (s sparseArray) IsExtended() []byte     { return s[24*s.MaxEntries():][:1] }
 func (s sparseArray) MaxEntries() int        { return len(s) / 24 }
 
-type sparseNode []byte
+type sparseElem []byte
 
-func (s sparseNode) Offset() []byte   { return s[00:][:12] }
+func (s sparseElem) Offset() []byte { return s[00:][:12] }
-func (s sparseNode) NumBytes() []byte { return s[12:][:12] }
+func (s sparseElem) Length() []byte { return s[12:][:12] }

libgo/go/archive/tar/stat_unix.go

@@ -8,6 +8,10 @@ package tar
 
 import (
 	"os"
+	"os/user"
+	"runtime"
+	"strconv"
+	"sync"
 	"syscall"
 )
 
@@ -15,6 +19,10 @@ func init() {
 	sysStat = statUnix
 }
 
+// userMap and groupMap caches UID and GID lookups for performance reasons.
+// The downside is that renaming uname or gname by the OS never takes effect.
+var userMap, groupMap sync.Map // map[int]string
+
 func statUnix(fi os.FileInfo, h *Header) error {
 	sys, ok := fi.Sys().(*syscall.Stat_t)
 	if !ok {
@@ -22,11 +30,67 @@ func statUnix(fi os.FileInfo, h *Header) error {
 	}
 	h.Uid = int(sys.Uid)
 	h.Gid = int(sys.Gid)
-	// TODO(bradfitz): populate username & group.  os/user
+
-	// doesn't cache LookupId lookups, and lacks group
+	// Best effort at populating Uname and Gname.
-	// lookup functions.
+	// The os/user functions may fail for any number of reasons
+	// (not implemented on that platform, cgo not enabled, etc).
+	if u, ok := userMap.Load(h.Uid); ok {
+		h.Uname = u.(string)
+	} else if u, err := user.LookupId(strconv.Itoa(h.Uid)); err == nil {
+		h.Uname = u.Username
+		userMap.Store(h.Uid, h.Uname)
+	}
+	if g, ok := groupMap.Load(h.Gid); ok {
+		h.Gname = g.(string)
+	} else if g, err := user.LookupGroupId(strconv.Itoa(h.Gid)); err == nil {
+		h.Gname = g.Name
+		groupMap.Store(h.Gid, h.Gname)
+	}
+
 	h.AccessTime = statAtime(sys)
 	h.ChangeTime = statCtime(sys)
-	// TODO(bradfitz): major/minor device numbers?
+
+	// Best effort at populating Devmajor and Devminor.
+	if h.Typeflag == TypeChar || h.Typeflag == TypeBlock {
+		dev := uint64(sys.Rdev) // May be int32 or uint32
+		switch runtime.GOOS {
+		case "linux":
+			// Copied from golang.org/x/sys/unix/dev_linux.go.
+			major := uint32((dev & 0x00000000000fff00) >> 8)
+			major |= uint32((dev & 0xfffff00000000000) >> 32)
+			minor := uint32((dev & 0x00000000000000ff) >> 0)
+			minor |= uint32((dev & 0x00000ffffff00000) >> 12)
+			h.Devmajor, h.Devminor = int64(major), int64(minor)
+		case "darwin":
+			// Copied from golang.org/x/sys/unix/dev_darwin.go.
+			major := uint32((dev >> 24) & 0xff)
+			minor := uint32(dev & 0xffffff)
+			h.Devmajor, h.Devminor = int64(major), int64(minor)
+		case "dragonfly":
+			// Copied from golang.org/x/sys/unix/dev_dragonfly.go.
+			major := uint32((dev >> 8) & 0xff)
+			minor := uint32(dev & 0xffff00ff)
+			h.Devmajor, h.Devminor = int64(major), int64(minor)
+		case "freebsd":
+			// Copied from golang.org/x/sys/unix/dev_freebsd.go.
+			major := uint32((dev >> 8) & 0xff)
+			minor := uint32(dev & 0xffff00ff)
+			h.Devmajor, h.Devminor = int64(major), int64(minor)
+		case "netbsd":
+			// Copied from golang.org/x/sys/unix/dev_netbsd.go.
+			major := uint32((dev & 0x000fff00) >> 8)
+			minor := uint32((dev & 0x000000ff) >> 0)
+			minor |= uint32((dev & 0xfff00000) >> 12)
+			h.Devmajor, h.Devminor = int64(major), int64(minor)
+		case "openbsd":
+			// Copied from golang.org/x/sys/unix/dev_openbsd.go.
+			major := uint32((dev & 0x0000ff00) >> 8)
+			minor := uint32((dev & 0x000000ff) >> 0)
+			minor |= uint32((dev & 0xffff0000) >> 8)
+			h.Devmajor, h.Devminor = int64(major), int64(minor)
+		default:
+			// TODO: Implement solaris (see https://golang.org/issue/8106)
+		}
+	}
 	return nil
 }

libgo/go/archive/tar/strconv.go

@@ -12,26 +12,34 @@ import (
 	"time"
 )
 
+// hasNUL reports whether the NUL character exists within s.
+func hasNUL(s string) bool {
+	return strings.IndexByte(s, 0) >= 0
+}
+
+// isASCII reports whether the input is an ASCII C-style string.
 func isASCII(s string) bool {
 	for _, c := range s {
-		if c >= 0x80 {
+		if c >= 0x80 || c == 0x00 {
 			return false
 		}
 	}
 	return true
 }
 
+// toASCII converts the input to an ASCII C-style string.
+// This a best effort conversion, so invalid characters are dropped.
 func toASCII(s string) string {
 	if isASCII(s) {
 		return s
 	}
-	var buf bytes.Buffer
+	b := make([]byte, 0, len(s))
 	for _, c := range s {
-		if c < 0x80 {
+		if c < 0x80 && c != 0x00 {
-			buf.WriteByte(byte(c))
+			b = append(b, byte(c))
 		}
 	}
-	return buf.String()
+	return string(b)
 }
 
 type parser struct {
@@ -45,23 +53,28 @@ type formatter struct {
 // parseString parses bytes as a NUL-terminated C-style string.
 // If a NUL byte is not found then the whole slice is returned as a string.
 func (*parser) parseString(b []byte) string {
-	n := 0
+	if i := bytes.IndexByte(b, 0); i >= 0 {
-	for n < len(b) && b[n] != 0 {
+		return string(b[:i])
-		n++
 	}
-	return string(b[0:n])
+	return string(b)
 }
 
-// Write s into b, terminating it with a NUL if there is room.
+// formatString copies s into b, NUL-terminating if possible.
 func (f *formatter) formatString(b []byte, s string) {
 	if len(s) > len(b) {
 		f.err = ErrFieldTooLong
-		return
 	}
-	ascii := toASCII(s)
+	copy(b, s)
-	copy(b, ascii)
+	if len(s) < len(b) {
-	if len(ascii) < len(b) {
+		b[len(s)] = 0
-		b[len(ascii)] = 0
+	}
+
+	// Some buggy readers treat regular files with a trailing slash
+	// in the V7 path field as a directory even though the full path
+	// recorded elsewhere (e.g., via PAX record) contains no trailing slash.
+	if len(s) > len(b) && b[len(b)-1] == '/' {
+		n := len(strings.TrimRight(s[:len(b)], "/"))
+		b[n] = 0 // Replace trailing slash with NUL terminator
 	}
 }
 
@@ -73,7 +86,7 @@ func (f *formatter) formatString(b []byte, s string) {
 // that the first byte can only be either 0x80 or 0xff. Thus, the first byte is
 // equivalent to the sign bit in two's complement form.
 func fitsInBase256(n int, x int64) bool {
-	var binBits = uint(n-1) * 8
+	binBits := uint(n-1) * 8
 	return n >= 9 || (x >= -1<<binBits && x < 1<<binBits)
 }
 
@@ -121,8 +134,14 @@ func (p *parser) parseNumeric(b []byte) int64 {
 	return p.parseOctal(b)
 }
 
-// Write x into b, as binary (GNUtar/star extension).
+// formatNumeric encodes x into b using base-8 (octal) encoding if possible.
+// Otherwise it will attempt to use base-256 (binary) encoding.
 func (f *formatter) formatNumeric(b []byte, x int64) {
+	if fitsInOctal(len(b), x) {
+		f.formatOctal(b, x)
+		return
+	}
+
 	if fitsInBase256(len(b), x) {
 		for i := len(b) - 1; i >= 0; i-- {
 			b[i] = byte(x)
@@ -155,6 +174,11 @@ func (p *parser) parseOctal(b []byte) int64 {
 }
 
 func (f *formatter) formatOctal(b []byte, x int64) {
+	if !fitsInOctal(len(b), x) {
+		x = 0 // Last resort, just write zero
+		f.err = ErrFieldTooLong
+	}
+
 	s := strconv.FormatInt(x, 8)
 	// Add leading zeros, but leave room for a NUL.
 	if n := len(b) - len(s) - 1; n > 0 {
@@ -163,6 +187,13 @@ func (f *formatter) formatOctal(b []byte, x int64) {
 	f.formatString(b, s)
 }
 
+// fitsInOctal reports whether the integer x fits in a field n-bytes long
+// using octal encoding with the appropriate NUL terminator.
+func fitsInOctal(n int, x int64) bool {
+	octBits := uint(n-1) * 3
+	return x >= 0 && (n >= 22 || x < 1<<octBits)
+}
+
 // parsePAXTime takes a string of the form %d.%d as described in the PAX
 // specification. Note that this implementation allows for negative timestamps,
 // which is allowed for by the PAX specification, but not always portable.
@@ -195,19 +226,32 @@ func parsePAXTime(s string) (time.Time, error) {
 	}
 	nsecs, _ := strconv.ParseInt(sn, 10, 64) // Must succeed
 	if len(ss) > 0 && ss[0] == '-' {
-		return time.Unix(secs, -1*int64(nsecs)), nil // Negative correction
+		return time.Unix(secs, -1*nsecs), nil // Negative correction
 	}
-	return time.Unix(secs, int64(nsecs)), nil
+	return time.Unix(secs, nsecs), nil
 }
 
-// TODO(dsnet): Implement formatPAXTime.
+// formatPAXTime converts ts into a time of the form %d.%d as described in the
+// PAX specification. This function is capable of negative timestamps.
+func formatPAXTime(ts time.Time) (s string) {
+	secs, nsecs := ts.Unix(), ts.Nanosecond()
+	if nsecs == 0 {
+		return strconv.FormatInt(secs, 10)
+	}
+
+	// If seconds is negative, then perform correction.
+	sign := ""
+	if secs < 0 {
+		sign = "-"             // Remember sign
+		secs = -(secs + 1)     // Add a second to secs
+		nsecs = -(nsecs - 1E9) // Take that second away from nsecs
+	}
+	return strings.TrimRight(fmt.Sprintf("%s%d.%09d", sign, secs, nsecs), "0")
+}
 
 // parsePAXRecord parses the input PAX record string into a key-value pair.
 // If parsing is successful, it will slice off the currently read record and
 // return the remainder as r.
-//
-// A PAX record is of the following form:
-//	"%d %s=%s\n" % (size, key, value)
 func parsePAXRecord(s string) (k, v, r string, err error) {
 	// The size field ends at the first space.
 	sp := strings.IndexByte(s, ' ')
@@ -232,21 +276,51 @@ func parsePAXRecord(s string) (k, v, r string, err error) {
 	if eq == -1 {
 		return "", "", s, ErrHeader
 	}
-	return rec[:eq], rec[eq+1:], rem, nil
+	k, v = rec[:eq], rec[eq+1:]
+
+	if !validPAXRecord(k, v) {
+		return "", "", s, ErrHeader
+	}
+	return k, v, rem, nil
 }
 
 // formatPAXRecord formats a single PAX record, prefixing it with the
 // appropriate length.
-func formatPAXRecord(k, v string) string {
+func formatPAXRecord(k, v string) (string, error) {
+	if !validPAXRecord(k, v) {
+		return "", ErrHeader
+	}
+
 	const padding = 3 // Extra padding for ' ', '=', and '\n'
 	size := len(k) + len(v) + padding
 	size += len(strconv.Itoa(size))
-	record := fmt.Sprintf("%d %s=%s\n", size, k, v)
+	record := strconv.Itoa(size) + " " + k + "=" + v + "\n"
 
 	// Final adjustment if adding size field increased the record size.
 	if len(record) != size {
 		size = len(record)
-		record = fmt.Sprintf("%d %s=%s\n", size, k, v)
+		record = strconv.Itoa(size) + " " + k + "=" + v + "\n"
+	}
+	return record, nil
+}
+
+// validPAXRecord reports whether the key-value pair is valid where each
+// record is formatted as:
+//	"%d %s=%s\n" % (size, key, value)
+//
+// Keys and values should be UTF-8, but the number of bad writers out there
+// forces us to be a more liberal.
+// Thus, we only reject all keys with NUL, and only reject NULs in values
+// for the PAX version of the USTAR string fields.
+// The key must not contain an '=' character.
+func validPAXRecord(k, v string) bool {
+	if k == "" || strings.IndexByte(k, '=') >= 0 {
+		return false
+	}
+	switch k {
+	case paxPath, paxLinkpath, paxUname, paxGname:
+		return !hasNUL(v)
+	default:
+		return !hasNUL(k)
 	}
-	return record
 }

libgo/go/archive/tar/strconv_test.go

@@ -110,6 +110,25 @@ func TestFormatNumeric(t *testing.T) {
 		want string
 		ok   bool
 	}{
+		// Test base-8 (octal) encoded values.
+		{0, "0\x00", true},
+		{7, "7\x00", true},
+		{8, "\x80\x08", true},
+		{077, "77\x00", true},
+		{0100, "\x80\x00\x40", true},
+		{0, "0000000\x00", true},
+		{0123, "0000123\x00", true},
+		{07654321, "7654321\x00", true},
+		{07777777, "7777777\x00", true},
+		{010000000, "\x80\x00\x00\x00\x00\x20\x00\x00", true},
+		{0, "00000000000\x00", true},
+		{000001234567, "00001234567\x00", true},
+		{076543210321, "76543210321\x00", true},
+		{012345670123, "12345670123\x00", true},
+		{077777777777, "77777777777\x00", true},
+		{0100000000000, "\x80\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00", true},
+		{math.MaxInt64, "777777777777777777777\x00", true},
+
 		// Test base-256 (binary) encoded values.
 		{-1, "\xff", true},
 		{-1, "\xff\xff", true},
@@ -155,6 +174,45 @@ func TestFormatNumeric(t *testing.T) {
 	}
 }
 
+func TestFitsInOctal(t *testing.T) {
+	vectors := []struct {
+		input int64
+		width int
+		ok    bool
+	}{
+		{-1, 1, false},
+		{-1, 2, false},
+		{-1, 3, false},
+		{0, 1, true},
+		{0 + 1, 1, false},
+		{0, 2, true},
+		{07, 2, true},
+		{07 + 1, 2, false},
+		{0, 4, true},
+		{0777, 4, true},
+		{0777 + 1, 4, false},
+		{0, 8, true},
+		{07777777, 8, true},
+		{07777777 + 1, 8, false},
+		{0, 12, true},
+		{077777777777, 12, true},
+		{077777777777 + 1, 12, false},
+		{math.MaxInt64, 22, true},
+		{012345670123, 12, true},
+		{01564164, 12, true},
+		{-012345670123, 12, false},
+		{-01564164, 12, false},
+		{-1564164, 30, false},
+	}
+
+	for _, v := range vectors {
+		ok := fitsInOctal(v.width, v.input)
+		if ok != v.ok {
+			t.Errorf("checkOctal(%d, %d): got %v, want %v", v.input, v.width, ok, v.ok)
+		}
+	}
+}
+
 func TestParsePAXTime(t *testing.T) {
 	vectors := []struct {
 		in   string
@@ -236,6 +294,51 @@ func TestParsePAXTime(t *testing.T) {
 	}
 }
 
+func TestFormatPAXTime(t *testing.T) {
+	vectors := []struct {
+		sec, nsec int64
+		want      string
+	}{
+		{1350244992, 0, "1350244992"},
+		{1350244992, 300000000, "1350244992.3"},
+		{1350244992, 23960100, "1350244992.0239601"},
+		{1350244992, 23960108, "1350244992.023960108"},
+		{+1, +1E9 - 1E0, "1.999999999"},
+		{+1, +1E9 - 1E3, "1.999999"},
+		{+1, +1E9 - 1E6, "1.999"},
+		{+1, +0E0 - 0E0, "1"},
+		{+1, +1E6 - 0E0, "1.001"},
+		{+1, +1E3 - 0E0, "1.000001"},
+		{+1, +1E0 - 0E0, "1.000000001"},
+		{0, 1E9 - 1E0, "0.999999999"},
+		{0, 1E9 - 1E3, "0.999999"},
+		{0, 1E9 - 1E6, "0.999"},
+		{0, 0E0, "0"},
+		{0, 1E6 + 0E0, "0.001"},
+		{0, 1E3 + 0E0, "0.000001"},
+		{0, 1E0 + 0E0, "0.000000001"},
+		{-1, -1E9 + 1E0, "-1.999999999"},
+		{-1, -1E9 + 1E3, "-1.999999"},
+		{-1, -1E9 + 1E6, "-1.999"},
+		{-1, -0E0 + 0E0, "-1"},
+		{-1, -1E6 + 0E0, "-1.001"},
+		{-1, -1E3 + 0E0, "-1.000001"},
+		{-1, -1E0 + 0E0, "-1.000000001"},
+		{-1350244992, 0, "-1350244992"},
+		{-1350244992, -300000000, "-1350244992.3"},
+		{-1350244992, -23960100, "-1350244992.0239601"},
+		{-1350244992, -23960108, "-1350244992.023960108"},
+	}
+
+	for _, v := range vectors {
+		got := formatPAXTime(time.Unix(v.sec, v.nsec))
+		if got != v.want {
+			t.Errorf("formatPAXTime(%ds, %dns): got %q, want %q",
+				v.sec, v.nsec, got, v.want)
+		}
+	}
+}
+
 func TestParsePAXRecord(t *testing.T) {
 	medName := strings.Repeat("CD", 50)
 	longName := strings.Repeat("AB", 100)
@@ -256,7 +359,7 @@ func TestParsePAXRecord(t *testing.T) {
 		{"18 foo=b=\nar=\n==\x00\n", "", "foo", "b=\nar=\n==\x00", true},
 		{"27 foo=hello9 foo=ba\nworld\n", "", "foo", "hello9 foo=ba\nworld", true},
 		{"27 ☺☻☹=日a本b語ç\nmeow mix", "meow mix", "☺☻☹", "日a本b語ç", true},
-		{"17 \x00hello=\x00world\n", "", "\x00hello", "\x00world", true},
+		{"17 \x00hello=\x00world\n", "17 \x00hello=\x00world\n", "", "", false},
 		{"1 k=1\n", "1 k=1\n", "", "", false},
 		{"6 k~1\n", "6 k~1\n", "", "", false},
 		{"6_k=1\n", "6_k=1\n", "", "", false},
@@ -296,21 +399,33 @@ func TestFormatPAXRecord(t *testing.T) {
 		inKey string
 		inVal string
 		want  string
+		ok    bool
 	}{
-		{"k", "v", "6 k=v\n"},
+		{"k", "v", "6 k=v\n", true},
-		{"path", "/etc/hosts", "19 path=/etc/hosts\n"},
+		{"path", "/etc/hosts", "19 path=/etc/hosts\n", true},
-		{"path", longName, "210 path=" + longName + "\n"},
+		{"path", longName, "210 path=" + longName + "\n", true},
-		{"path", medName, "110 path=" + medName + "\n"},
+		{"path", medName, "110 path=" + medName + "\n", true},
-		{"foo", "ba", "9 foo=ba\n"},
+		{"foo", "ba", "9 foo=ba\n", true},
-		{"foo", "bar", "11 foo=bar\n"},
+		{"foo", "bar", "11 foo=bar\n", true},
-		{"foo", "b=\nar=\n==\x00", "18 foo=b=\nar=\n==\x00\n"},
+		{"foo", "b=\nar=\n==\x00", "18 foo=b=\nar=\n==\x00\n", true},
-		{"foo", "hello9 foo=ba\nworld", "27 foo=hello9 foo=ba\nworld\n"},
+		{"foo", "hello9 foo=ba\nworld", "27 foo=hello9 foo=ba\nworld\n", true},
-		{"☺☻☹", "日a本b語ç", "27 ☺☻☹=日a本b語ç\n"},
+		{"☺☻☹", "日a本b語ç", "27 ☺☻☹=日a本b語ç\n", true},
-		{"\x00hello", "\x00world", "17 \x00hello=\x00world\n"},
+		{"xhello", "\x00world", "17 xhello=\x00world\n", true},
+		{"path", "null\x00", "", false},
+		{"null\x00", "value", "", false},
+		{paxSchilyXattr + "key", "null\x00", "26 SCHILY.xattr.key=null\x00\n", true},
 	}
 
 	for _, v := range vectors {
-		got := formatPAXRecord(v.inKey, v.inVal)
+		got, err := formatPAXRecord(v.inKey, v.inVal)
+		ok := (err == nil)
+		if ok != v.ok {
+			if v.ok {
+				t.Errorf("formatPAXRecord(%q, %q): got format failure, want success", v.inKey, v.inVal)
+			} else {
+				t.Errorf("formatPAXRecord(%q, %q): got format success, want failure", v.inKey, v.inVal)
+			}
+		}
 		if got != v.want {
 			t.Errorf("formatPAXRecord(%q, %q): got %q, want %q",
 				v.inKey, v.inVal, got, v.want)

libgo/go/archive/tar/tar_test.go

@@ -6,8 +6,12 @@ package tar
 
 import (
 	"bytes"
+	"errors"
+	"fmt"
 	"internal/testenv"
+	"io"
 	"io/ioutil"
+	"math"
 	"os"
 	"path"
 	"path/filepath"
@@ -17,6 +21,193 @@ import (
 	"time"
 )
 
+type testError struct{ error }
+
+type fileOps []interface{} // []T where T is (string | int64)
+
+// testFile is an io.ReadWriteSeeker where the IO operations performed
+// on it must match the list of operations in ops.
+type testFile struct {
+	ops fileOps
+	pos int64
+}
+
+func (f *testFile) Read(b []byte) (int, error) {
+	if len(b) == 0 {
+		return 0, nil
+	}
+	if len(f.ops) == 0 {
+		return 0, io.EOF
+	}
+	s, ok := f.ops[0].(string)
+	if !ok {
+		return 0, errors.New("unexpected Read operation")
+	}
+
+	n := copy(b, s)
+	if len(s) > n {
+		f.ops[0] = s[n:]
+	} else {
+		f.ops = f.ops[1:]
+	}
+	f.pos += int64(len(b))
+	return n, nil
+}
+
+func (f *testFile) Write(b []byte) (int, error) {
+	if len(b) == 0 {
+		return 0, nil
+	}
+	if len(f.ops) == 0 {
+		return 0, errors.New("unexpected Write operation")
+	}
+	s, ok := f.ops[0].(string)
+	if !ok {
+		return 0, errors.New("unexpected Write operation")
+	}
+
+	if !strings.HasPrefix(s, string(b)) {
+		return 0, testError{fmt.Errorf("got Write(%q), want Write(%q)", b, s)}
+	}
+	if len(s) > len(b) {
+		f.ops[0] = s[len(b):]
+	} else {
+		f.ops = f.ops[1:]
+	}
+	f.pos += int64(len(b))
+	return len(b), nil
+}
+
+func (f *testFile) Seek(pos int64, whence int) (int64, error) {
+	if pos == 0 && whence == io.SeekCurrent {
+		return f.pos, nil
+	}
+	if len(f.ops) == 0 {
+		return 0, errors.New("unexpected Seek operation")
+	}
+	s, ok := f.ops[0].(int64)
+	if !ok {
+		return 0, errors.New("unexpected Seek operation")
+	}
+
+	if s != pos || whence != io.SeekCurrent {
+		return 0, testError{fmt.Errorf("got Seek(%d, %d), want Seek(%d, %d)", pos, whence, s, io.SeekCurrent)}
+	}
+	f.pos += s
+	f.ops = f.ops[1:]
+	return f.pos, nil
+}
+
+func equalSparseEntries(x, y []sparseEntry) bool {
+	return (len(x) == 0 && len(y) == 0) || reflect.DeepEqual(x, y)
+}
+
+func TestSparseEntries(t *testing.T) {
+	vectors := []struct {
+		in   []sparseEntry
+		size int64
+
+		wantValid    bool          // Result of validateSparseEntries
+		wantAligned  []sparseEntry // Result of alignSparseEntries
+		wantInverted []sparseEntry // Result of invertSparseEntries
+	}{{
+		in: []sparseEntry{}, size: 0,
+		wantValid:    true,
+		wantInverted: []sparseEntry{{0, 0}},
+	}, {
+		in: []sparseEntry{}, size: 5000,
+		wantValid:    true,
+		wantInverted: []sparseEntry{{0, 5000}},
+	}, {
+		in: []sparseEntry{{0, 5000}}, size: 5000,
+		wantValid:    true,
+		wantAligned:  []sparseEntry{{0, 5000}},
+		wantInverted: []sparseEntry{{5000, 0}},
+	}, {
+		in: []sparseEntry{{1000, 4000}}, size: 5000,
+		wantValid:    true,
+		wantAligned:  []sparseEntry{{1024, 3976}},
+		wantInverted: []sparseEntry{{0, 1000}, {5000, 0}},
+	}, {
+		in: []sparseEntry{{0, 3000}}, size: 5000,
+		wantValid:    true,
+		wantAligned:  []sparseEntry{{0, 2560}},
+		wantInverted: []sparseEntry{{3000, 2000}},
+	}, {
+		in: []sparseEntry{{3000, 2000}}, size: 5000,
+		wantValid:    true,
+		wantAligned:  []sparseEntry{{3072, 1928}},
+		wantInverted: []sparseEntry{{0, 3000}, {5000, 0}},
+	}, {
+		in: []sparseEntry{{2000, 2000}}, size: 5000,
+		wantValid:    true,
+		wantAligned:  []sparseEntry{{2048, 1536}},
+		wantInverted: []sparseEntry{{0, 2000}, {4000, 1000}},
+	}, {
+		in: []sparseEntry{{0, 2000}, {8000, 2000}}, size: 10000,
+		wantValid:    true,
+		wantAligned:  []sparseEntry{{0, 1536}, {8192, 1808}},
+		wantInverted: []sparseEntry{{2000, 6000}, {10000, 0}},
+	}, {
+		in: []sparseEntry{{0, 2000}, {2000, 2000}, {4000, 0}, {4000, 3000}, {7000, 1000}, {8000, 0}, {8000, 2000}}, size: 10000,
+		wantValid:    true,
+		wantAligned:  []sparseEntry{{0, 1536}, {2048, 1536}, {4096, 2560}, {7168, 512}, {8192, 1808}},
+		wantInverted: []sparseEntry{{10000, 0}},
+	}, {
+		in: []sparseEntry{{0, 0}, {1000, 0}, {2000, 0}, {3000, 0}, {4000, 0}, {5000, 0}}, size: 5000,
+		wantValid:    true,
+		wantInverted: []sparseEntry{{0, 5000}},
+	}, {
+		in: []sparseEntry{{1, 0}}, size: 0,
+		wantValid: false,
+	}, {
+		in: []sparseEntry{{-1, 0}}, size: 100,
+		wantValid: false,
+	}, {
+		in: []sparseEntry{{0, -1}}, size: 100,
+		wantValid: false,
+	}, {
+		in: []sparseEntry{{0, 0}}, size: -100,
+		wantValid: false,
+	}, {
+		in: []sparseEntry{{math.MaxInt64, 3}, {6, -5}}, size: 35,
+		wantValid: false,
+	}, {
+		in: []sparseEntry{{1, 3}, {6, -5}}, size: 35,
+		wantValid: false,
+	}, {
+		in: []sparseEntry{{math.MaxInt64, math.MaxInt64}}, size: math.MaxInt64,
+		wantValid: false,
+	}, {
+		in: []sparseEntry{{3, 3}}, size: 5,
+		wantValid: false,
+	}, {
+		in: []sparseEntry{{2, 0}, {1, 0}, {0, 0}}, size: 3,
+		wantValid: false,
+	}, {
+		in: []sparseEntry{{1, 3}, {2, 2}}, size: 10,
+		wantValid: false,
+	}}
+
+	for i, v := range vectors {
+		gotValid := validateSparseEntries(v.in, v.size)
+		if gotValid != v.wantValid {
+			t.Errorf("test %d, validateSparseEntries() = %v, want %v", i, gotValid, v.wantValid)
+		}
+		if !v.wantValid {
+			continue
+		}
+		gotAligned := alignSparseEntries(append([]sparseEntry{}, v.in...), v.size)
+		if !equalSparseEntries(gotAligned, v.wantAligned) {
+			t.Errorf("test %d, alignSparseEntries():\ngot  %v\nwant %v", i, gotAligned, v.wantAligned)
+		}
+		gotInverted := invertSparseEntries(append([]sparseEntry{}, v.in...), v.size)
+		if !equalSparseEntries(gotInverted, v.wantInverted) {
+			t.Errorf("test %d, inverseSparseEntries():\ngot  %v\nwant %v", i, gotInverted, v.wantInverted)
+		}
+	}
+}
+
 func TestFileInfoHeader(t *testing.T) {
 	fi, err := os.Stat("testdata/small.txt")
 	if err != nil {
@@ -110,14 +301,11 @@ func TestRoundTrip(t *testing.T) {
 	tw := NewWriter(&b)
 	hdr := &Header{
 		Name:       "file.txt",
-		Uid:  1 << 21, // too big for 8 octal digits
+		Uid:        1 << 21, // Too big for 8 octal digits
 		Size:       int64(len(data)),
-		// AddDate to strip monotonic clock reading,
+		ModTime:    time.Now().Round(time.Second),
-		// and Round to discard sub-second precision,
+		PAXRecords: map[string]string{"uid": "2097152"},
-		// both of which are not included in the tar header
+		Format:     FormatPAX,
-		// and would otherwise break the round-trip check
-		// below.
-		ModTime: time.Now().AddDate(0, 0, 0).Round(1 * time.Second),
 	}
 	if err := tw.WriteHeader(hdr); err != nil {
 		t.Fatalf("tw.WriteHeader: %v", err)
@@ -329,3 +517,338 @@ func TestHeaderRoundTrip(t *testing.T) {
 		}
 	}
 }
+
+func TestHeaderAllowedFormats(t *testing.T) {
+	vectors := []struct {
+		header  *Header           // Input header
+		paxHdrs map[string]string // Expected PAX headers that may be needed
+		formats Format            // Expected formats that can encode the header
+	}{{
+		header:  &Header{},
+		formats: FormatUSTAR | FormatPAX | FormatGNU,
+	}, {
+		header:  &Header{Size: 077777777777},
+		formats: FormatUSTAR | FormatPAX | FormatGNU,
+	}, {
+		header:  &Header{Size: 077777777777, Format: FormatUSTAR},
+		formats: FormatUSTAR,
+	}, {
+		header:  &Header{Size: 077777777777, Format: FormatPAX},
+		formats: FormatUSTAR | FormatPAX,
+	}, {
+		header:  &Header{Size: 077777777777, Format: FormatGNU},
+		formats: FormatGNU,
+	}, {
+		header:  &Header{Size: 077777777777 + 1},
+		paxHdrs: map[string]string{paxSize: "8589934592"},
+		formats: FormatPAX | FormatGNU,
+	}, {
+		header:  &Header{Size: 077777777777 + 1, Format: FormatPAX},
+		paxHdrs: map[string]string{paxSize: "8589934592"},
+		formats: FormatPAX,
+	}, {
+		header:  &Header{Size: 077777777777 + 1, Format: FormatGNU},
+		paxHdrs: map[string]string{paxSize: "8589934592"},
+		formats: FormatGNU,
+	}, {
+		header:  &Header{Mode: 07777777},
+		formats: FormatUSTAR | FormatPAX | FormatGNU,
+	}, {
+		header:  &Header{Mode: 07777777 + 1},
+		formats: FormatGNU,
+	}, {
+		header:  &Header{Devmajor: -123},
+		formats: FormatGNU,
+	}, {
+		header:  &Header{Devmajor: 1<<56 - 1},
+		formats: FormatGNU,
+	}, {
+		header:  &Header{Devmajor: 1 << 56},
+		formats: FormatUnknown,
+	}, {
+		header:  &Header{Devmajor: -1 << 56},
+		formats: FormatGNU,
+	}, {
+		header:  &Header{Devmajor: -1<<56 - 1},
+		formats: FormatUnknown,
+	}, {
+		header:  &Header{Name: "用戶名", Devmajor: -1 << 56},
+		formats: FormatGNU,
+	}, {
+		header:  &Header{Size: math.MaxInt64},
+		paxHdrs: map[string]string{paxSize: "9223372036854775807"},
+		formats: FormatPAX | FormatGNU,
+	}, {
+		header:  &Header{Size: math.MinInt64},
+		paxHdrs: map[string]string{paxSize: "-9223372036854775808"},
+		formats: FormatUnknown,
+	}, {
+		header:  &Header{Uname: "0123456789abcdef0123456789abcdef"},
+		formats: FormatUSTAR | FormatPAX | FormatGNU,
+	}, {
+		header:  &Header{Uname: "0123456789abcdef0123456789abcdefx"},
+		paxHdrs: map[string]string{paxUname: "0123456789abcdef0123456789abcdefx"},
+		formats: FormatPAX,
+	}, {
+		header:  &Header{Name: "foobar"},
+		formats: FormatUSTAR | FormatPAX | FormatGNU,
+	}, {
+		header:  &Header{Name: strings.Repeat("a", nameSize)},
+		formats: FormatUSTAR | FormatPAX | FormatGNU,
+	}, {
+		header:  &Header{Name: strings.Repeat("a", nameSize+1)},
+		paxHdrs: map[string]string{paxPath: strings.Repeat("a", nameSize+1)},
+		formats: FormatPAX | FormatGNU,
+	}, {
+		header:  &Header{Linkname: "用戶名"},
+		paxHdrs: map[string]string{paxLinkpath: "用戶名"},
+		formats: FormatPAX | FormatGNU,
+	}, {
+		header:  &Header{Linkname: strings.Repeat("用戶名\x00", nameSize)},
+		paxHdrs: map[string]string{paxLinkpath: strings.Repeat("用戶名\x00", nameSize)},
+		formats: FormatUnknown,
+	}, {
+		header:  &Header{Linkname: "\x00hello"},
+		paxHdrs: map[string]string{paxLinkpath: "\x00hello"},
+		formats: FormatUnknown,
+	}, {
+		header:  &Header{Uid: 07777777},
+		formats: FormatUSTAR | FormatPAX | FormatGNU,
+	}, {
+		header:  &Header{Uid: 07777777 + 1},
+		paxHdrs: map[string]string{paxUid: "2097152"},
+		formats: FormatPAX | FormatGNU,
+	}, {
+		header:  &Header{Xattrs: nil},
+		formats: FormatUSTAR | FormatPAX | FormatGNU,
+	}, {
+		header:  &Header{Xattrs: map[string]string{"foo": "bar"}},
+		paxHdrs: map[string]string{paxSchilyXattr + "foo": "bar"},
+		formats: FormatPAX,
+	}, {
+		header:  &Header{Xattrs: map[string]string{"foo": "bar"}, Format: FormatGNU},
+		paxHdrs: map[string]string{paxSchilyXattr + "foo": "bar"},
+		formats: FormatUnknown,
+	}, {
+		header:  &Header{Xattrs: map[string]string{"用戶名": "\x00hello"}},
+		paxHdrs: map[string]string{paxSchilyXattr + "用戶名": "\x00hello"},
+		formats: FormatPAX,
+	}, {
+		header:  &Header{Xattrs: map[string]string{"foo=bar": "baz"}},
+		formats: FormatUnknown,
+	}, {
+		header:  &Header{Xattrs: map[string]string{"foo": ""}},
+		paxHdrs: map[string]string{paxSchilyXattr + "foo": ""},
+		formats: FormatPAX,
+	}, {
+		header:  &Header{ModTime: time.Unix(0, 0)},
+		formats: FormatUSTAR | FormatPAX | FormatGNU,
+	}, {
+		header:  &Header{ModTime: time.Unix(077777777777, 0)},
+		formats: FormatUSTAR | FormatPAX | FormatGNU,
+	}, {
+		header:  &Header{ModTime: time.Unix(077777777777+1, 0)},
+		paxHdrs: map[string]string{paxMtime: "8589934592"},
+		formats: FormatPAX | FormatGNU,
+	}, {
+		header:  &Header{ModTime: time.Unix(math.MaxInt64, 0)},
+		paxHdrs: map[string]string{paxMtime: "9223372036854775807"},
+		formats: FormatPAX | FormatGNU,
+	}, {
+		header:  &Header{ModTime: time.Unix(math.MaxInt64, 0), Format: FormatUSTAR},
+		paxHdrs: map[string]string{paxMtime: "9223372036854775807"},
+		formats: FormatUnknown,
+	}, {
+		header:  &Header{ModTime: time.Unix(-1, 0)},
+		paxHdrs: map[string]string{paxMtime: "-1"},
+		formats: FormatPAX | FormatGNU,
+	}, {
+		header:  &Header{ModTime: time.Unix(1, 500)},
+		paxHdrs: map[string]string{paxMtime: "1.0000005"},
+		formats: FormatUSTAR | FormatPAX | FormatGNU,
+	}, {
+		header:  &Header{ModTime: time.Unix(1, 0)},
+		formats: FormatUSTAR | FormatPAX | FormatGNU,
+	}, {
+		header:  &Header{ModTime: time.Unix(1, 0), Format: FormatPAX},
+		formats: FormatUSTAR | FormatPAX,
+	}, {
+		header:  &Header{ModTime: time.Unix(1, 500), Format: FormatUSTAR},
+		paxHdrs: map[string]string{paxMtime: "1.0000005"},
+		formats: FormatUSTAR,
+	}, {
+		header:  &Header{ModTime: time.Unix(1, 500), Format: FormatPAX},
+		paxHdrs: map[string]string{paxMtime: "1.0000005"},
+		formats: FormatPAX,
+	}, {
+		header:  &Header{ModTime: time.Unix(1, 500), Format: FormatGNU},
+		paxHdrs: map[string]string{paxMtime: "1.0000005"},
+		formats: FormatGNU,
+	}, {
+		header:  &Header{ModTime: time.Unix(-1, 500)},
+		paxHdrs: map[string]string{paxMtime: "-0.9999995"},
+		formats: FormatPAX | FormatGNU,
+	}, {
+		header:  &Header{ModTime: time.Unix(-1, 500), Format: FormatGNU},
+		paxHdrs: map[string]string{paxMtime: "-0.9999995"},
+		formats: FormatGNU,
+	}, {
+		header:  &Header{AccessTime: time.Unix(0, 0)},
+		paxHdrs: map[string]string{paxAtime: "0"},
+		formats: FormatPAX | FormatGNU,
+	}, {
+		header:  &Header{AccessTime: time.Unix(0, 0), Format: FormatUSTAR},
+		paxHdrs: map[string]string{paxAtime: "0"},
+		formats: FormatUnknown,
+	}, {
+		header:  &Header{AccessTime: time.Unix(0, 0), Format: FormatPAX},
+		paxHdrs: map[string]string{paxAtime: "0"},
+		formats: FormatPAX,
+	}, {
+		header:  &Header{AccessTime: time.Unix(0, 0), Format: FormatGNU},
+		paxHdrs: map[string]string{paxAtime: "0"},
+		formats: FormatGNU,
+	}, {
+		header:  &Header{AccessTime: time.Unix(-123, 0)},
+		paxHdrs: map[string]string{paxAtime: "-123"},
+		formats: FormatPAX | FormatGNU,
+	}, {
+		header:  &Header{AccessTime: time.Unix(-123, 0), Format: FormatPAX},
+		paxHdrs: map[string]string{paxAtime: "-123"},
+		formats: FormatPAX,
+	}, {
+		header:  &Header{ChangeTime: time.Unix(123, 456)},
+		paxHdrs: map[string]string{paxCtime: "123.000000456"},
+		formats: FormatPAX | FormatGNU,
+	}, {
+		header:  &Header{ChangeTime: time.Unix(123, 456), Format: FormatUSTAR},
+		paxHdrs: map[string]string{paxCtime: "123.000000456"},
+		formats: FormatUnknown,
+	}, {
+		header:  &Header{ChangeTime: time.Unix(123, 456), Format: FormatGNU},
+		paxHdrs: map[string]string{paxCtime: "123.000000456"},
+		formats: FormatGNU,
+	}, {
+		header:  &Header{ChangeTime: time.Unix(123, 456), Format: FormatPAX},
+		paxHdrs: map[string]string{paxCtime: "123.000000456"},
+		formats: FormatPAX,
+	}, {
+		header:  &Header{Name: "foo/", Typeflag: TypeDir},
+		formats: FormatUSTAR | FormatPAX | FormatGNU,
+	}, {
+		header:  &Header{Name: "foo/", Typeflag: TypeReg},
+		formats: FormatUnknown,
+	}, {
+		header:  &Header{Name: "foo/", Typeflag: TypeSymlink},
+		formats: FormatUSTAR | FormatPAX | FormatGNU,
+	}}
+
+	for i, v := range vectors {
+		formats, paxHdrs, err := v.header.allowedFormats()
+		if formats != v.formats {
+			t.Errorf("test %d, allowedFormats(): got %v, want %v", i, formats, v.formats)
+		}
+		if formats&FormatPAX > 0 && !reflect.DeepEqual(paxHdrs, v.paxHdrs) && !(len(paxHdrs) == 0 && len(v.paxHdrs) == 0) {
+			t.Errorf("test %d, allowedFormats():\ngot  %v\nwant %s", i, paxHdrs, v.paxHdrs)
+		}
+		if (formats != FormatUnknown) && (err != nil) {
+			t.Errorf("test %d, unexpected error: %v", i, err)
+		}
+		if (formats == FormatUnknown) && (err == nil) {
+			t.Errorf("test %d, got nil-error, want non-nil error", i)
+		}
+	}
+}
+
+func Benchmark(b *testing.B) {
+	type file struct {
+		hdr  *Header
+		body []byte
+	}
+
+	vectors := []struct {
+		label string
+		files []file
+	}{{
+		"USTAR",
+		[]file{{
+			&Header{Name: "bar", Mode: 0640, Size: int64(3)},
+			[]byte("foo"),
+		}, {
+			&Header{Name: "world", Mode: 0640, Size: int64(5)},
+			[]byte("hello"),
+		}},
+	}, {
+		"GNU",
+		[]file{{
+			&Header{Name: "bar", Mode: 0640, Size: int64(3), Devmajor: -1},
+			[]byte("foo"),
+		}, {
+			&Header{Name: "world", Mode: 0640, Size: int64(5), Devmajor: -1},
+			[]byte("hello"),
+		}},
+	}, {
+		"PAX",
+		[]file{{
+			&Header{Name: "bar", Mode: 0640, Size: int64(3), Xattrs: map[string]string{"foo": "bar"}},
+			[]byte("foo"),
+		}, {
+			&Header{Name: "world", Mode: 0640, Size: int64(5), Xattrs: map[string]string{"foo": "bar"}},
+			[]byte("hello"),
+		}},
+	}}
+
+	b.Run("Writer", func(b *testing.B) {
+		for _, v := range vectors {
+			b.Run(v.label, func(b *testing.B) {
+				b.ReportAllocs()
+				for i := 0; i < b.N; i++ {
+					// Writing to ioutil.Discard because we want to
+					// test purely the writer code and not bring in disk performance into this.
+					tw := NewWriter(ioutil.Discard)
+					for _, file := range v.files {
+						if err := tw.WriteHeader(file.hdr); err != nil {
+							b.Errorf("unexpected WriteHeader error: %v", err)
+						}
+						if _, err := tw.Write(file.body); err != nil {
+							b.Errorf("unexpected Write error: %v", err)
+						}
+					}
+					if err := tw.Close(); err != nil {
+						b.Errorf("unexpected Close error: %v", err)
+					}
+				}
+			})
+		}
+	})
+
+	b.Run("Reader", func(b *testing.B) {
+		for _, v := range vectors {
+			var buf bytes.Buffer
+			var r bytes.Reader
+
+			// Write the archive to a byte buffer.
+			tw := NewWriter(&buf)
+			for _, file := range v.files {
+				tw.WriteHeader(file.hdr)
+				tw.Write(file.body)
+			}
+			tw.Close()
+			b.Run(v.label, func(b *testing.B) {
+				b.ReportAllocs()
+				// Read from the byte buffer.
+				for i := 0; i < b.N; i++ {
+					r.Reset(buf.Bytes())
+					tr := NewReader(&r)
+					if _, err := tr.Next(); err != nil {
+						b.Errorf("unexpected Next error: %v", err)
+					}
+					if _, err := io.Copy(ioutil.Discard, tr); err != nil {
+						b.Errorf("unexpected Copy error : %v", err)
+					}
+				}
+			})
+		}
+	})
+
+}

libgo/go/archive/tar/writer.go

@@ -4,255 +4,391 @@
 
 package tar
 
-// TODO(dsymonds):
-// - catch more errors (no first header, etc.)
-
 import (
 	"bytes"
-	"errors"
 	"fmt"
 	"io"
 	"path"
 	"sort"
-	"strconv"
 	"strings"
 	"time"
 )
 
-var (
+// Writer provides sequential writing of a tar archive.
-	ErrWriteTooLong    = errors.New("archive/tar: write too long")
+// Write.WriteHeader begins a new file with the provided Header,
-	ErrFieldTooLong    = errors.New("archive/tar: header field too long")
+// and then Writer can be treated as an io.Writer to supply that file's data.
-	ErrWriteAfterClose = errors.New("archive/tar: write after close")
-	errInvalidHeader   = errors.New("archive/tar: header field too long or contains invalid values")
-)
-
-// A Writer provides sequential writing of a tar archive in POSIX.1 format.
-// A tar archive consists of a sequence of files.
-// Call WriteHeader to begin a new file, and then call Write to supply that file's data,
-// writing at most hdr.Size bytes in total.
 type Writer struct {
 	w    io.Writer
+	pad  int64      // Amount of padding to write after current file entry
+	curr fileWriter // Writer for current file entry
+	hdr  Header     // Shallow copy of Header that is safe for mutations
+	blk  block      // Buffer to use as temporary local storage
+
+	// err is a persistent error.
+	// It is only the responsibility of every exported method of Writer to
+	// ensure that this error is sticky.
 	err error
-	nb         int64 // number of unwritten bytes for current file entry
-	pad        int64 // amount of padding to write after current file entry
-	closed     bool
-	usedBinary bool  // whether the binary numeric field extension was used
-	preferPax  bool  // use PAX header instead of binary numeric header
-	hdrBuff    block // buffer to use in writeHeader when writing a regular header
-	paxHdrBuff block // buffer to use in writeHeader when writing a PAX header
 }
 
 // NewWriter creates a new Writer writing to w.
-func NewWriter(w io.Writer) *Writer { return &Writer{w: w} }
+func NewWriter(w io.Writer) *Writer {
+	return &Writer{w: w, curr: &regFileWriter{w, 0}}
+}
 
-// Flush finishes writing the current file (optional).
+type fileWriter interface {
+	io.Writer
+	fileState
+
+	ReadFrom(io.Reader) (int64, error)
+}
+
+// Flush finishes writing the current file's block padding.
+// The current file must be fully written before Flush can be called.
+//
+// This is unnecessary as the next call to WriteHeader or Close
+// will implicitly flush out the file's padding.
 func (tw *Writer) Flush() error {
-	if tw.nb > 0 {
-		tw.err = fmt.Errorf("archive/tar: missed writing %d bytes", tw.nb)
-		return tw.err
-	}
-
-	n := tw.nb + tw.pad
-	for n > 0 && tw.err == nil {
-		nr := n
-		if nr > blockSize {
-			nr = blockSize
-		}
-		var nw int
-		nw, tw.err = tw.w.Write(zeroBlock[0:nr])
-		n -= int64(nw)
-	}
-	tw.nb = 0
-	tw.pad = 0
-	return tw.err
-}
-
-var (
-	minTime = time.Unix(0, 0)
-	// There is room for 11 octal digits (33 bits) of mtime.
-	maxTime = minTime.Add((1<<33 - 1) * time.Second)
-)
-
-// WriteHeader writes hdr and prepares to accept the file's contents.
-// WriteHeader calls Flush if it is not the first header.
-// Calling after a Close will return ErrWriteAfterClose.
-func (tw *Writer) WriteHeader(hdr *Header) error {
-	return tw.writeHeader(hdr, true)
-}
-
-// WriteHeader writes hdr and prepares to accept the file's contents.
-// WriteHeader calls Flush if it is not the first header.
-// Calling after a Close will return ErrWriteAfterClose.
-// As this method is called internally by writePax header to allow it to
-// suppress writing the pax header.
-func (tw *Writer) writeHeader(hdr *Header, allowPax bool) error {
-	if tw.closed {
-		return ErrWriteAfterClose
-	}
-	if tw.err == nil {
-		tw.Flush()
-	}
 	if tw.err != nil {
 		return tw.err
 	}
-
+	if nb := tw.curr.LogicalRemaining(); nb > 0 {
-	// a map to hold pax header records, if any are needed
+		return fmt.Errorf("archive/tar: missed writing %d bytes", nb)
-	paxHeaders := make(map[string]string)
-
-	// TODO(dsnet): we might want to use PAX headers for
-	// subsecond time resolution, but for now let's just capture
-	// too long fields or non ascii characters
-
-	// We need to select which scratch buffer to use carefully,
-	// since this method is called recursively to write PAX headers.
-	// If allowPax is true, this is the non-recursive call, and we will use hdrBuff.
-	// If allowPax is false, we are being called by writePAXHeader, and hdrBuff is
-	// already being used by the non-recursive call, so we must use paxHdrBuff.
-	header := &tw.hdrBuff
-	if !allowPax {
-		header = &tw.paxHdrBuff
 	}
-	copy(header[:], zeroBlock[:])
+	if _, tw.err = tw.w.Write(zeroBlock[:tw.pad]); tw.err != nil {
-
-	// Wrappers around formatter that automatically sets paxHeaders if the
-	// argument extends beyond the capacity of the input byte slice.
-	var f formatter
-	var formatString = func(b []byte, s string, paxKeyword string) {
-		needsPaxHeader := paxKeyword != paxNone && len(s) > len(b) || !isASCII(s)
-		if needsPaxHeader {
-			paxHeaders[paxKeyword] = s
-		}
-
-		// Write string in a best-effort manner to satisfy readers that expect
-		// the field to be non-empty.
-		s = toASCII(s)
-		if len(s) > len(b) {
-			s = s[:len(b)]
-		}
-		f.formatString(b, s) // Should never error
-	}
-	var formatNumeric = func(b []byte, x int64, paxKeyword string) {
-		// Try octal first.
-		s := strconv.FormatInt(x, 8)
-		if len(s) < len(b) {
-			f.formatOctal(b, x)
-			return
-		}
-
-		// If it is too long for octal, and PAX is preferred, use a PAX header.
-		if paxKeyword != paxNone && tw.preferPax {
-			f.formatOctal(b, 0)
-			s := strconv.FormatInt(x, 10)
-			paxHeaders[paxKeyword] = s
-			return
-		}
-
-		tw.usedBinary = true
-		f.formatNumeric(b, x)
-	}
-
-	// Handle out of range ModTime carefully.
-	var modTime int64
-	if !hdr.ModTime.Before(minTime) && !hdr.ModTime.After(maxTime) {
-		modTime = hdr.ModTime.Unix()
-	}
-
-	v7 := header.V7()
-	formatString(v7.Name(), hdr.Name, paxPath)
-	// TODO(dsnet): The GNU format permits the mode field to be encoded in
-	// base-256 format. Thus, we can use formatNumeric instead of formatOctal.
-	f.formatOctal(v7.Mode(), hdr.Mode)
-	formatNumeric(v7.UID(), int64(hdr.Uid), paxUid)
-	formatNumeric(v7.GID(), int64(hdr.Gid), paxGid)
-	formatNumeric(v7.Size(), hdr.Size, paxSize)
-	// TODO(dsnet): Consider using PAX for finer time granularity.
-	formatNumeric(v7.ModTime(), modTime, paxNone)
-	v7.TypeFlag()[0] = hdr.Typeflag
-	formatString(v7.LinkName(), hdr.Linkname, paxLinkpath)
-
-	ustar := header.USTAR()
-	formatString(ustar.UserName(), hdr.Uname, paxUname)
-	formatString(ustar.GroupName(), hdr.Gname, paxGname)
-	formatNumeric(ustar.DevMajor(), hdr.Devmajor, paxNone)
-	formatNumeric(ustar.DevMinor(), hdr.Devminor, paxNone)
-
-	// TODO(dsnet): The logic surrounding the prefix field is broken when trying
-	// to encode the header as GNU format. The challenge with the current logic
-	// is that we are unsure what format we are using at any given moment until
-	// we have processed *all* of the fields. The problem is that by the time
-	// all fields have been processed, some work has already been done to handle
-	// each field under the assumption that it is for one given format or
-	// another. In some situations, this causes the Writer to be confused and
-	// encode a prefix field when the format being used is GNU. Thus, producing
-	// an invalid tar file.
-	//
-	// As a short-term fix, we disable the logic to use the prefix field, which
-	// will force the badly generated GNU files to become encoded as being
-	// the PAX format.
-	//
-	// As an alternative fix, we could hard-code preferPax to be true. However,
-	// this is problematic for the following reasons:
-	//	* The preferPax functionality is not tested at all.
-	//	* This can result in headers that try to use both the GNU and PAX
-	//	features at the same time, which is also wrong.
-	//
-	// The proper fix for this is to use a two-pass method:
-	//	* The first pass simply determines what set of formats can possibly
-	//	encode the given header.
-	//	* The second pass actually encodes the header as that given format
-	//	without worrying about violating the format.
-	//
-	// See the following:
-	//	https://golang.org/issue/12594
-	//	https://golang.org/issue/17630
-	//	https://golang.org/issue/9683
-	const usePrefix = false
-
-	// try to use a ustar header when only the name is too long
-	_, paxPathUsed := paxHeaders[paxPath]
-	if usePrefix && !tw.preferPax && len(paxHeaders) == 1 && paxPathUsed {
-		prefix, suffix, ok := splitUSTARPath(hdr.Name)
-		if ok {
-			// Since we can encode in USTAR format, disable PAX header.
-			delete(paxHeaders, paxPath)
-
-			// Update the path fields
-			formatString(v7.Name(), suffix, paxNone)
-			formatString(ustar.Prefix(), prefix, paxNone)
-		}
-	}
-
-	if tw.usedBinary {
-		header.SetFormat(formatGNU)
-	} else {
-		header.SetFormat(formatUSTAR)
-	}
-
-	// Check if there were any formatting errors.
-	if f.err != nil {
-		tw.err = f.err
 		return tw.err
 	}
+	tw.pad = 0
+	return nil
+}
 
-	if allowPax {
+// WriteHeader writes hdr and prepares to accept the file's contents.
-		for k, v := range hdr.Xattrs {
+// The Header.Size determines how many bytes can be written for the next file.
-			paxHeaders[paxXattr+k] = v
+// If the current file is not fully written, then this returns an error.
+// This implicitly flushes any padding necessary before writing the header.
+func (tw *Writer) WriteHeader(hdr *Header) error {
+	if err := tw.Flush(); err != nil {
+		return err
+	}
+	tw.hdr = *hdr // Shallow copy of Header
+
+	// Round ModTime and ignore AccessTime and ChangeTime unless
+	// the format is explicitly chosen.
+	// This ensures nominal usage of WriteHeader (without specifying the format)
+	// does not always result in the PAX format being chosen, which
+	// causes a 1KiB increase to every header.
+	if tw.hdr.Format == FormatUnknown {
+		tw.hdr.ModTime = tw.hdr.ModTime.Round(time.Second)
+		tw.hdr.AccessTime = time.Time{}
+		tw.hdr.ChangeTime = time.Time{}
+	}
+
+	allowedFormats, paxHdrs, err := tw.hdr.allowedFormats()
+	switch {
+	case allowedFormats.has(FormatUSTAR):
+		tw.err = tw.writeUSTARHeader(&tw.hdr)
+		return tw.err
+	case allowedFormats.has(FormatPAX):
+		tw.err = tw.writePAXHeader(&tw.hdr, paxHdrs)
+		return tw.err
+	case allowedFormats.has(FormatGNU):
+		tw.err = tw.writeGNUHeader(&tw.hdr)
+		return tw.err
+	default:
+		return err // Non-fatal error
+	}
+}
+
+func (tw *Writer) writeUSTARHeader(hdr *Header) error {
+	// Check if we can use USTAR prefix/suffix splitting.
+	var namePrefix string
+	if prefix, suffix, ok := splitUSTARPath(hdr.Name); ok {
+		namePrefix, hdr.Name = prefix, suffix
+	}
+
+	// Pack the main header.
+	var f formatter
+	blk := tw.templateV7Plus(hdr, f.formatString, f.formatOctal)
+	f.formatString(blk.USTAR().Prefix(), namePrefix)
+	blk.SetFormat(FormatUSTAR)
+	if f.err != nil {
+		return f.err // Should never happen since header is validated
+	}
+	return tw.writeRawHeader(blk, hdr.Size, hdr.Typeflag)
+}
+
+func (tw *Writer) writePAXHeader(hdr *Header, paxHdrs map[string]string) error {
+	realName, realSize := hdr.Name, hdr.Size
+
+	// TODO(dsnet): Re-enable this when adding sparse support.
+	// See https://golang.org/issue/22735
+	/*
+		// Handle sparse files.
+		var spd sparseDatas
+		var spb []byte
+		if len(hdr.SparseHoles) > 0 {
+			sph := append([]sparseEntry{}, hdr.SparseHoles...) // Copy sparse map
+			sph = alignSparseEntries(sph, hdr.Size)
+			spd = invertSparseEntries(sph, hdr.Size)
+
+			// Format the sparse map.
+			hdr.Size = 0 // Replace with encoded size
+			spb = append(strconv.AppendInt(spb, int64(len(spd)), 10), '\n')
+			for _, s := range spd {
+				hdr.Size += s.Length
+				spb = append(strconv.AppendInt(spb, s.Offset, 10), '\n')
+				spb = append(strconv.AppendInt(spb, s.Length, 10), '\n')
+			}
+			pad := blockPadding(int64(len(spb)))
+			spb = append(spb, zeroBlock[:pad]...)
+			hdr.Size += int64(len(spb)) // Accounts for encoded sparse map
+
+			// Add and modify appropriate PAX records.
+			dir, file := path.Split(realName)
+			hdr.Name = path.Join(dir, "GNUSparseFile.0", file)
+			paxHdrs[paxGNUSparseMajor] = "1"
+			paxHdrs[paxGNUSparseMinor] = "0"
+			paxHdrs[paxGNUSparseName] = realName
+			paxHdrs[paxGNUSparseRealSize] = strconv.FormatInt(realSize, 10)
+			paxHdrs[paxSize] = strconv.FormatInt(hdr.Size, 10)
+			delete(paxHdrs, paxPath) // Recorded by paxGNUSparseName
+		}
+	*/
+	_ = realSize
+
+	// Write PAX records to the output.
+	isGlobal := hdr.Typeflag == TypeXGlobalHeader
+	if len(paxHdrs) > 0 || isGlobal {
+		// Sort keys for deterministic ordering.
+		var keys []string
+		for k := range paxHdrs {
+			keys = append(keys, k)
+		}
+		sort.Strings(keys)
+
+		// Write each record to a buffer.
+		var buf bytes.Buffer
+		for _, k := range keys {
+			rec, err := formatPAXRecord(k, paxHdrs[k])
+			if err != nil {
+				return err
+			}
+			buf.WriteString(rec)
+		}
+
+		// Write the extended header file.
+		var name string
+		var flag byte
+		if isGlobal {
+			name = realName
+			if name == "" {
+				name = "GlobalHead.0.0"
+			}
+			flag = TypeXGlobalHeader
+		} else {
+			dir, file := path.Split(realName)
+			name = path.Join(dir, "PaxHeaders.0", file)
+			flag = TypeXHeader
+		}
+		data := buf.String()
+		if err := tw.writeRawFile(name, data, flag, FormatPAX); err != nil || isGlobal {
+			return err // Global headers return here
 		}
 	}
 
-	if len(paxHeaders) > 0 {
+	// Pack the main header.
-		if !allowPax {
+	var f formatter // Ignore errors since they are expected
-			return errInvalidHeader
+	fmtStr := func(b []byte, s string) { f.formatString(b, toASCII(s)) }
+	blk := tw.templateV7Plus(hdr, fmtStr, f.formatOctal)
+	blk.SetFormat(FormatPAX)
+	if err := tw.writeRawHeader(blk, hdr.Size, hdr.Typeflag); err != nil {
+		return err
 	}
-		if err := tw.writePAXHeader(hdr, paxHeaders); err != nil {
+
+	// TODO(dsnet): Re-enable this when adding sparse support.
+	// See https://golang.org/issue/22735
+	/*
+		// Write the sparse map and setup the sparse writer if necessary.
+		if len(spd) > 0 {
+			// Use tw.curr since the sparse map is accounted for in hdr.Size.
+			if _, err := tw.curr.Write(spb); err != nil {
+				return err
+			}
+			tw.curr = &sparseFileWriter{tw.curr, spd, 0}
+		}
+	*/
+	return nil
+}
+
+func (tw *Writer) writeGNUHeader(hdr *Header) error {
+	// Use long-link files if Name or Linkname exceeds the field size.
+	const longName = "././@LongLink"
+	if len(hdr.Name) > nameSize {
+		data := hdr.Name + "\x00"
+		if err := tw.writeRawFile(longName, data, TypeGNULongName, FormatGNU); err != nil {
+			return err
+		}
+	}
+	if len(hdr.Linkname) > nameSize {
+		data := hdr.Linkname + "\x00"
+		if err := tw.writeRawFile(longName, data, TypeGNULongLink, FormatGNU); err != nil {
 			return err
 		}
 	}
-	tw.nb = hdr.Size
-	tw.pad = (blockSize - (tw.nb % blockSize)) % blockSize
 
-	_, tw.err = tw.w.Write(header[:])
+	// Pack the main header.
-	return tw.err
+	var f formatter // Ignore errors since they are expected
+	var spd sparseDatas
+	var spb []byte
+	blk := tw.templateV7Plus(hdr, f.formatString, f.formatNumeric)
+	if !hdr.AccessTime.IsZero() {
+		f.formatNumeric(blk.GNU().AccessTime(), hdr.AccessTime.Unix())
+	}
+	if !hdr.ChangeTime.IsZero() {
+		f.formatNumeric(blk.GNU().ChangeTime(), hdr.ChangeTime.Unix())
+	}
+	// TODO(dsnet): Re-enable this when adding sparse support.
+	// See https://golang.org/issue/22735
+	/*
+		if hdr.Typeflag == TypeGNUSparse {
+			sph := append([]sparseEntry{}, hdr.SparseHoles...) // Copy sparse map
+			sph = alignSparseEntries(sph, hdr.Size)
+			spd = invertSparseEntries(sph, hdr.Size)
+
+			// Format the sparse map.
+			formatSPD := func(sp sparseDatas, sa sparseArray) sparseDatas {
+				for i := 0; len(sp) > 0 && i < sa.MaxEntries(); i++ {
+					f.formatNumeric(sa.Entry(i).Offset(), sp[0].Offset)
+					f.formatNumeric(sa.Entry(i).Length(), sp[0].Length)
+					sp = sp[1:]
+				}
+				if len(sp) > 0 {
+					sa.IsExtended()[0] = 1
+				}
+				return sp
+			}
+			sp2 := formatSPD(spd, blk.GNU().Sparse())
+			for len(sp2) > 0 {
+				var spHdr block
+				sp2 = formatSPD(sp2, spHdr.Sparse())
+				spb = append(spb, spHdr[:]...)
+			}
+
+			// Update size fields in the header block.
+			realSize := hdr.Size
+			hdr.Size = 0 // Encoded size; does not account for encoded sparse map
+			for _, s := range spd {
+				hdr.Size += s.Length
+			}
+			copy(blk.V7().Size(), zeroBlock[:]) // Reset field
+			f.formatNumeric(blk.V7().Size(), hdr.Size)
+			f.formatNumeric(blk.GNU().RealSize(), realSize)
+		}
+	*/
+	blk.SetFormat(FormatGNU)
+	if err := tw.writeRawHeader(blk, hdr.Size, hdr.Typeflag); err != nil {
+		return err
+	}
+
+	// Write the extended sparse map and setup the sparse writer if necessary.
+	if len(spd) > 0 {
+		// Use tw.w since the sparse map is not accounted for in hdr.Size.
+		if _, err := tw.w.Write(spb); err != nil {
+			return err
+		}
+		tw.curr = &sparseFileWriter{tw.curr, spd, 0}
+	}
+	return nil
+}
+
+type (
+	stringFormatter func([]byte, string)
+	numberFormatter func([]byte, int64)
+)
+
+// templateV7Plus fills out the V7 fields of a block using values from hdr.
+// It also fills out fields (uname, gname, devmajor, devminor) that are
+// shared in the USTAR, PAX, and GNU formats using the provided formatters.
+//
+// The block returned is only valid until the next call to
+// templateV7Plus or writeRawFile.
+func (tw *Writer) templateV7Plus(hdr *Header, fmtStr stringFormatter, fmtNum numberFormatter) *block {
+	tw.blk.Reset()
+
+	modTime := hdr.ModTime
+	if modTime.IsZero() {
+		modTime = time.Unix(0, 0)
+	}
+
+	v7 := tw.blk.V7()
+	v7.TypeFlag()[0] = hdr.Typeflag
+	fmtStr(v7.Name(), hdr.Name)
+	fmtStr(v7.LinkName(), hdr.Linkname)
+	fmtNum(v7.Mode(), hdr.Mode)
+	fmtNum(v7.UID(), int64(hdr.Uid))
+	fmtNum(v7.GID(), int64(hdr.Gid))
+	fmtNum(v7.Size(), hdr.Size)
+	fmtNum(v7.ModTime(), modTime.Unix())
+
+	ustar := tw.blk.USTAR()
+	fmtStr(ustar.UserName(), hdr.Uname)
+	fmtStr(ustar.GroupName(), hdr.Gname)
+	fmtNum(ustar.DevMajor(), hdr.Devmajor)
+	fmtNum(ustar.DevMinor(), hdr.Devminor)
+
+	return &tw.blk
+}
+
+// writeRawFile writes a minimal file with the given name and flag type.
+// It uses format to encode the header format and will write data as the body.
+// It uses default values for all of the other fields (as BSD and GNU tar does).
+func (tw *Writer) writeRawFile(name, data string, flag byte, format Format) error {
+	tw.blk.Reset()
+
+	// Best effort for the filename.
+	name = toASCII(name)
+	if len(name) > nameSize {
+		name = name[:nameSize]
+	}
+	name = strings.TrimRight(name, "/")
+
+	var f formatter
+	v7 := tw.blk.V7()
+	v7.TypeFlag()[0] = flag
+	f.formatString(v7.Name(), name)
+	f.formatOctal(v7.Mode(), 0)
+	f.formatOctal(v7.UID(), 0)
+	f.formatOctal(v7.GID(), 0)
+	f.formatOctal(v7.Size(), int64(len(data))) // Must be < 8GiB
+	f.formatOctal(v7.ModTime(), 0)
+	tw.blk.SetFormat(format)
+	if f.err != nil {
+		return f.err // Only occurs if size condition is violated
+	}
+
+	// Write the header and data.
+	if err := tw.writeRawHeader(&tw.blk, int64(len(data)), flag); err != nil {
+		return err
+	}
+	_, err := io.WriteString(tw, data)
+	return err
+}
+
+// writeRawHeader writes the value of blk, regardless of its value.
+// It sets up the Writer such that it can accept a file of the given size.
+// If the flag is a special header-only flag, then the size is treated as zero.
+func (tw *Writer) writeRawHeader(blk *block, size int64, flag byte) error {
+	if err := tw.Flush(); err != nil {
+		return err
+	}
+	if _, err := tw.w.Write(blk[:]); err != nil {
+		return err
+	}
+	if isHeaderOnlyType(flag) {
+		size = 0
+	}
+	tw.curr = &regFileWriter{tw.w, size}
+	tw.pad = blockPadding(size)
+	return nil
 }
 
 // splitUSTARPath splits a path according to USTAR prefix and suffix rules.
@@ -276,95 +412,233 @@ func splitUSTARPath(name string) (prefix, suffix string, ok bool) {
 	return name[:i], name[i+1:], true
 }
 
-// writePaxHeader writes an extended pax header to the
+// Write writes to the current file in the tar archive.
-// archive.
-func (tw *Writer) writePAXHeader(hdr *Header, paxHeaders map[string]string) error {
-	// Prepare extended header
-	ext := new(Header)
-	ext.Typeflag = TypeXHeader
-	// Setting ModTime is required for reader parsing to
-	// succeed, and seems harmless enough.
-	ext.ModTime = hdr.ModTime
-	// The spec asks that we namespace our pseudo files
-	// with the current pid. However, this results in differing outputs
-	// for identical inputs. As such, the constant 0 is now used instead.
-	// golang.org/issue/12358
-	dir, file := path.Split(hdr.Name)
-	fullName := path.Join(dir, "PaxHeaders.0", file)
-
-	ascii := toASCII(fullName)
-	if len(ascii) > nameSize {
-		ascii = ascii[:nameSize]
-	}
-	ext.Name = ascii
-	// Construct the body
-	var buf bytes.Buffer
-
-	// Keys are sorted before writing to body to allow deterministic output.
-	keys := make([]string, 0, len(paxHeaders))
-	for k := range paxHeaders {
-		keys = append(keys, k)
-	}
-	sort.Strings(keys)
-
-	for _, k := range keys {
-		fmt.Fprint(&buf, formatPAXRecord(k, paxHeaders[k]))
-	}
-
-	ext.Size = int64(len(buf.Bytes()))
-	if err := tw.writeHeader(ext, false); err != nil {
-		return err
-	}
-	if _, err := tw.Write(buf.Bytes()); err != nil {
-		return err
-	}
-	if err := tw.Flush(); err != nil {
-		return err
-	}
-	return nil
-}
-
-// Write writes to the current entry in the tar archive.
 // Write returns the error ErrWriteTooLong if more than
-// hdr.Size bytes are written after WriteHeader.
+// Header.Size bytes are written after WriteHeader.
-func (tw *Writer) Write(b []byte) (n int, err error) {
+//
-	if tw.closed {
+// Calling Write on special types like TypeLink, TypeSymlink, TypeChar,
-		err = ErrWriteAfterClose
+// TypeBlock, TypeDir, and TypeFifo returns (0, ErrWriteTooLong) regardless
-		return
+// of what the Header.Size claims.
-	}
+func (tw *Writer) Write(b []byte) (int, error) {
-	overwrite := false
+	if tw.err != nil {
-	if int64(len(b)) > tw.nb {
+		return 0, tw.err
-		b = b[0:tw.nb]
-		overwrite = true
-	}
-	n, err = tw.w.Write(b)
-	tw.nb -= int64(n)
-	if err == nil && overwrite {
-		err = ErrWriteTooLong
-		return
 	}
+	n, err := tw.curr.Write(b)
+	if err != nil && err != ErrWriteTooLong {
 		tw.err = err
-	return
+	}
+	return n, err
 }
 
-// Close closes the tar archive, flushing any unwritten
+// readFrom populates the content of the current file by reading from r.
-// data to the underlying writer.
+// The bytes read must match the number of remaining bytes in the current file.
+//
+// If the current file is sparse and r is an io.ReadSeeker,
+// then readFrom uses Seek to skip past holes defined in Header.SparseHoles,
+// assuming that skipped regions are all NULs.
+// This always reads the last byte to ensure r is the right size.
+//
+// TODO(dsnet): Re-export this when adding sparse file support.
+// See https://golang.org/issue/22735
+func (tw *Writer) readFrom(r io.Reader) (int64, error) {
+	if tw.err != nil {
+		return 0, tw.err
+	}
+	n, err := tw.curr.ReadFrom(r)
+	if err != nil && err != ErrWriteTooLong {
+		tw.err = err
+	}
+	return n, err
+}
+
+// Close closes the tar archive by flushing the padding, and writing the footer.
+// If the current file (from a prior call to WriteHeader) is not fully written,
+// then this returns an error.
 func (tw *Writer) Close() error {
-	if tw.err != nil || tw.closed {
+	if tw.err == ErrWriteAfterClose {
-		return tw.err
+		return nil
 	}
-	tw.Flush()
-	tw.closed = true
 	if tw.err != nil {
 		return tw.err
 	}
 
-	// trailer: two zero blocks
+	// Trailer: two zero blocks.
-	for i := 0; i < 2; i++ {
+	err := tw.Flush()
-		_, tw.err = tw.w.Write(zeroBlock[:])
+	for i := 0; i < 2 && err == nil; i++ {
-		if tw.err != nil {
+		_, err = tw.w.Write(zeroBlock[:])
-			break
+	}
+
+	// Ensure all future actions are invalid.
+	tw.err = ErrWriteAfterClose
+	return err // Report IO errors
+}
+
+// regFileWriter is a fileWriter for writing data to a regular file entry.
+type regFileWriter struct {
+	w  io.Writer // Underlying Writer
+	nb int64     // Number of remaining bytes to write
+}
+
+func (fw *regFileWriter) Write(b []byte) (n int, err error) {
+	overwrite := int64(len(b)) > fw.nb
+	if overwrite {
+		b = b[:fw.nb]
+	}
+	if len(b) > 0 {
+		n, err = fw.w.Write(b)
+		fw.nb -= int64(n)
+	}
+	switch {
+	case err != nil:
+		return n, err
+	case overwrite:
+		return n, ErrWriteTooLong
+	default:
+		return n, nil
+	}
+}
+
+func (fw *regFileWriter) ReadFrom(r io.Reader) (int64, error) {
+	return io.Copy(struct{ io.Writer }{fw}, r)
+}
+
+func (fw regFileWriter) LogicalRemaining() int64 {
+	return fw.nb
+}
+func (fw regFileWriter) PhysicalRemaining() int64 {
+	return fw.nb
+}
+
+// sparseFileWriter is a fileWriter for writing data to a sparse file entry.
+type sparseFileWriter struct {
+	fw  fileWriter  // Underlying fileWriter
+	sp  sparseDatas // Normalized list of data fragments
+	pos int64       // Current position in sparse file
+}
+
+func (sw *sparseFileWriter) Write(b []byte) (n int, err error) {
+	overwrite := int64(len(b)) > sw.LogicalRemaining()
+	if overwrite {
+		b = b[:sw.LogicalRemaining()]
+	}
+
+	b0 := b
+	endPos := sw.pos + int64(len(b))
+	for endPos > sw.pos && err == nil {
+		var nf int // Bytes written in fragment
+		dataStart, dataEnd := sw.sp[0].Offset, sw.sp[0].endOffset()
+		if sw.pos < dataStart { // In a hole fragment
+			bf := b[:min(int64(len(b)), dataStart-sw.pos)]
+			nf, err = zeroWriter{}.Write(bf)
+		} else { // In a data fragment
+			bf := b[:min(int64(len(b)), dataEnd-sw.pos)]
+			nf, err = sw.fw.Write(bf)
+		}
+		b = b[nf:]
+		sw.pos += int64(nf)
+		if sw.pos >= dataEnd && len(sw.sp) > 1 {
+			sw.sp = sw.sp[1:] // Ensure last fragment always remains
+		}
+	}
+
+	n = len(b0) - len(b)
+	switch {
+	case err == ErrWriteTooLong:
+		return n, errMissData // Not possible; implies bug in validation logic
+	case err != nil:
+		return n, err
+	case sw.LogicalRemaining() == 0 && sw.PhysicalRemaining() > 0:
+		return n, errUnrefData // Not possible; implies bug in validation logic
+	case overwrite:
+		return n, ErrWriteTooLong
+	default:
+		return n, nil
+	}
+}
+
+func (sw *sparseFileWriter) ReadFrom(r io.Reader) (n int64, err error) {
+	rs, ok := r.(io.ReadSeeker)
+	if ok {
+		if _, err := rs.Seek(0, io.SeekCurrent); err != nil {
+			ok = false // Not all io.Seeker can really seek
+		}
+	}
+	if !ok {
+		return io.Copy(struct{ io.Writer }{sw}, r)
+	}
+
+	var readLastByte bool
+	pos0 := sw.pos
+	for sw.LogicalRemaining() > 0 && !readLastByte && err == nil {
+		var nf int64 // Size of fragment
+		dataStart, dataEnd := sw.sp[0].Offset, sw.sp[0].endOffset()
+		if sw.pos < dataStart { // In a hole fragment
+			nf = dataStart - sw.pos
+			if sw.PhysicalRemaining() == 0 {
+				readLastByte = true
+				nf--
+			}
+			_, err = rs.Seek(nf, io.SeekCurrent)
+		} else { // In a data fragment
+			nf = dataEnd - sw.pos
+			nf, err = io.CopyN(sw.fw, rs, nf)
+		}
+		sw.pos += nf
+		if sw.pos >= dataEnd && len(sw.sp) > 1 {
+			sw.sp = sw.sp[1:] // Ensure last fragment always remains
+		}
+	}
+
+	// If the last fragment is a hole, then seek to 1-byte before EOF, and
+	// read a single byte to ensure the file is the right size.
+	if readLastByte && err == nil {
+		_, err = mustReadFull(rs, []byte{0})
+		sw.pos++
+	}
+
+	n = sw.pos - pos0
+	switch {
+	case err == io.EOF:
+		return n, io.ErrUnexpectedEOF
+	case err == ErrWriteTooLong:
+		return n, errMissData // Not possible; implies bug in validation logic
+	case err != nil:
+		return n, err
+	case sw.LogicalRemaining() == 0 && sw.PhysicalRemaining() > 0:
+		return n, errUnrefData // Not possible; implies bug in validation logic
+	default:
+		return n, ensureEOF(rs)
+	}
+}
+
+func (sw sparseFileWriter) LogicalRemaining() int64 {
+	return sw.sp[len(sw.sp)-1].endOffset() - sw.pos
+}
+func (sw sparseFileWriter) PhysicalRemaining() int64 {
+	return sw.fw.PhysicalRemaining()
+}
+
+// zeroWriter may only be written with NULs, otherwise it returns errWriteHole.
+type zeroWriter struct{}
+
+func (zeroWriter) Write(b []byte) (int, error) {
+	for i, c := range b {
+		if c != 0 {
+			return i, errWriteHole
+		}
+	}
+	return len(b), nil
+}
+
+// ensureEOF checks whether r is at EOF, reporting ErrWriteTooLong if not so.
+func ensureEOF(r io.Reader) error {
+	n, err := tryReadFull(r, []byte{0})
+	switch {
+	case n > 0:
+		return ErrWriteTooLong
+	case err == io.EOF:
+		return nil
+	default:
+		return err
 	}
-	}
-	return tw.err
 }

libgo/go/archive/zip/reader.go

@@ -13,6 +13,7 @@ import (
 	"hash/crc32"
 	"io"
 	"os"
+	"time"
 )
 
 var (
@@ -94,7 +95,7 @@ func (z *Reader) init(r io.ReaderAt, size int64) error {
 
 	// The count of files inside a zip is truncated to fit in a uint16.
 	// Gloss over this by reading headers until we encounter
-	// a bad one, and then only report a ErrFormat or UnexpectedEOF if
+	// a bad one, and then only report an ErrFormat or UnexpectedEOF if
 	// the file count modulo 65536 is incorrect.
 	for {
 		f := &File{zip: z, zipr: r, zipsize: size}
@@ -280,52 +281,128 @@ func readDirectoryHeader(f *File, r io.Reader) error {
 	f.Extra = d[filenameLen : filenameLen+extraLen]
 	f.Comment = string(d[filenameLen+extraLen:])
 
+	// Determine the character encoding.
+	utf8Valid1, utf8Require1 := detectUTF8(f.Name)
+	utf8Valid2, utf8Require2 := detectUTF8(f.Comment)
+	switch {
+	case !utf8Valid1 || !utf8Valid2:
+		// Name and Comment definitely not UTF-8.
+		f.NonUTF8 = true
+	case !utf8Require1 && !utf8Require2:
+		// Name and Comment use only single-byte runes that overlap with UTF-8.
+		f.NonUTF8 = false
+	default:
+		// Might be UTF-8, might be some other encoding; preserve existing flag.
+		// Some ZIP writers use UTF-8 encoding without setting the UTF-8 flag.
+		// Since it is impossible to always distinguish valid UTF-8 from some
+		// other encoding (e.g., GBK or Shift-JIS), we trust the flag.
+		f.NonUTF8 = f.Flags&0x800 == 0
+	}
+
 	needUSize := f.UncompressedSize == ^uint32(0)
 	needCSize := f.CompressedSize == ^uint32(0)
 	needHeaderOffset := f.headerOffset == int64(^uint32(0))
 
-	if len(f.Extra) > 0 {
 	// Best effort to find what we need.
 	// Other zip authors might not even follow the basic format,
 	// and we'll just ignore the Extra content in that case.
-		b := readBuf(f.Extra)
+	var modified time.Time
-		for len(b) >= 4 { // need at least tag and size
+parseExtras:
-			tag := b.uint16()
+	for extra := readBuf(f.Extra); len(extra) >= 4; { // need at least tag and size
-			size := b.uint16()
+		fieldTag := extra.uint16()
-			if int(size) > len(b) {
+		fieldSize := int(extra.uint16())
+		if len(extra) < fieldSize {
 			break
 		}
-			if tag == zip64ExtraId {
+		fieldBuf := extra.sub(fieldSize)
+
+		switch fieldTag {
+		case zip64ExtraID:
 			// update directory values from the zip64 extra block.
 			// They should only be consulted if the sizes read earlier
 			// are maxed out.
 			// See golang.org/issue/13367.
-				eb := readBuf(b[:size])
-
 			if needUSize {
 				needUSize = false
-					if len(eb) < 8 {
+				if len(fieldBuf) < 8 {
 					return ErrFormat
 				}
-					f.UncompressedSize64 = eb.uint64()
+				f.UncompressedSize64 = fieldBuf.uint64()
 			}
 			if needCSize {
 				needCSize = false
-					if len(eb) < 8 {
+				if len(fieldBuf) < 8 {
 					return ErrFormat
 				}
-					f.CompressedSize64 = eb.uint64()
+				f.CompressedSize64 = fieldBuf.uint64()
 			}
 			if needHeaderOffset {
 				needHeaderOffset = false
-					if len(eb) < 8 {
+				if len(fieldBuf) < 8 {
 					return ErrFormat
 				}
-					f.headerOffset = int64(eb.uint64())
+				f.headerOffset = int64(fieldBuf.uint64())
 			}
-				break
+		case ntfsExtraID:
+			if len(fieldBuf) < 4 {
+				continue parseExtras
 			}
-			b = b[size:]
+			fieldBuf.uint32()        // reserved (ignored)
+			for len(fieldBuf) >= 4 { // need at least tag and size
+				attrTag := fieldBuf.uint16()
+				attrSize := int(fieldBuf.uint16())
+				if len(fieldBuf) < attrSize {
+					continue parseExtras
+				}
+				attrBuf := fieldBuf.sub(attrSize)
+				if attrTag != 1 || attrSize != 24 {
+					continue // Ignore irrelevant attributes
+				}
+
+				const ticksPerSecond = 1e7    // Windows timestamp resolution
+				ts := int64(attrBuf.uint64()) // ModTime since Windows epoch
+				secs := int64(ts / ticksPerSecond)
+				nsecs := (1e9 / ticksPerSecond) * int64(ts%ticksPerSecond)
+				epoch := time.Date(1601, time.January, 1, 0, 0, 0, 0, time.UTC)
+				modified = time.Unix(epoch.Unix()+secs, nsecs)
+			}
+		case unixExtraID:
+			if len(fieldBuf) < 8 {
+				continue parseExtras
+			}
+			fieldBuf.uint32()              // AcTime (ignored)
+			ts := int64(fieldBuf.uint32()) // ModTime since Unix epoch
+			modified = time.Unix(ts, 0)
+		case extTimeExtraID:
+			if len(fieldBuf) < 5 || fieldBuf.uint8()&1 == 0 {
+				continue parseExtras
+			}
+			ts := int64(fieldBuf.uint32()) // ModTime since Unix epoch
+			modified = time.Unix(ts, 0)
+		case infoZipUnixExtraID:
+			if len(fieldBuf) < 4 {
+				continue parseExtras
+			}
+			ts := int64(fieldBuf.uint32()) // ModTime since Unix epoch
+			modified = time.Unix(ts, 0)
+		}
+	}
+
+	msdosModified := msDosTimeToTime(f.ModifiedDate, f.ModifiedTime)
+	f.Modified = msdosModified
+	if !modified.IsZero() {
+		f.Modified = modified.UTC()
+
+		// If legacy MS-DOS timestamps are set, we can use the delta between
+		// the legacy and extended versions to estimate timezone offset.
+		//
+		// A non-UTC timezone is always used (even if offset is zero).
+		// Thus, FileHeader.Modified.Location() == time.UTC is useful for
+		// determining whether extended timestamps are present.
+		// This is necessary for users that need to do additional time
+		// calculations when dealing with legacy ZIP formats.
+		if f.ModifiedTime != 0 || f.ModifiedDate != 0 {
+			f.Modified = modified.In(timeZone(msdosModified.Sub(modified)))
 		}
 	}
 
@@ -508,6 +585,12 @@ func findSignatureInBlock(b []byte) int {
 
 type readBuf []byte
 
+func (b *readBuf) uint8() uint8 {
+	v := (*b)[0]
+	*b = (*b)[1:]
+	return v
+}
+
 func (b *readBuf) uint16() uint16 {
 	v := binary.LittleEndian.Uint16(*b)
 	*b = (*b)[2:]
@@ -525,3 +608,9 @@ func (b *readBuf) uint64() uint64 {
 	*b = (*b)[8:]
 	return v
 }
+
+func (b *readBuf) sub(n int) readBuf {
+	b2 := (*b)[:n]
+	*b = (*b)[n:]
+	return b2
+}

libgo/go/archive/zip/reader_test.go

@@ -29,7 +29,9 @@ type ZipTest struct {
 type ZipTestFile struct {
 	Name     string
 	Mode     os.FileMode
-	Mtime string // optional, modified time in format "mm-dd-yy hh:mm:ss"
+	NonUTF8  bool
+	ModTime  time.Time
+	Modified time.Time
 
 	// Information describing expected zip file content.
 	// First, reading the entire content should produce the error ContentErr.
@@ -47,16 +49,6 @@ type ZipTestFile struct {
 	Size       uint64
 }
 
-// Caution: The Mtime values found for the test files should correspond to
-//          the values listed with unzip -l <zipfile>. However, the values
-//          listed by unzip appear to be off by some hours. When creating
-//          fresh test files and testing them, this issue is not present.
-//          The test files were created in Sydney, so there might be a time
-//          zone issue. The time zone information does have to be encoded
-//          somewhere, because otherwise unzip -l could not provide a different
-//          time from what the archive/zip package provides, but there appears
-//          to be no documentation about this.
-
 var tests = []ZipTest{
 	{
 		Name:    "test.zip",
@@ -65,13 +57,13 @@ var tests = []ZipTest{
 			{
 				Name:     "test.txt",
 				Content:  []byte("This is a test text file.\n"),
-				Mtime:   "09-05-10 12:12:02",
+				Modified: time.Date(2010, 9, 5, 12, 12, 1, 0, timeZone(+10*time.Hour)),
 				Mode:     0644,
 			},
 			{
 				Name:     "gophercolor16x16.png",
 				File:     "gophercolor16x16.png",
-				Mtime: "09-05-10 15:52:58",
+				Modified: time.Date(2010, 9, 5, 15, 52, 58, 0, timeZone(+10*time.Hour)),
 				Mode:     0644,
 			},
 		},
@@ -83,13 +75,13 @@ var tests = []ZipTest{
 			{
 				Name:     "test.txt",
 				Content:  []byte("This is a test text file.\n"),
-				Mtime:   "09-05-10 12:12:02",
+				Modified: time.Date(2010, 9, 5, 12, 12, 1, 0, timeZone(+10*time.Hour)),
 				Mode:     0644,
 			},
 			{
 				Name:     "gophercolor16x16.png",
 				File:     "gophercolor16x16.png",
-				Mtime: "09-05-10 15:52:58",
+				Modified: time.Date(2010, 9, 5, 15, 52, 58, 0, timeZone(+10*time.Hour)),
 				Mode:     0644,
 			},
 		},
@@ -101,7 +93,7 @@ var tests = []ZipTest{
 			{
 				Name:     "r/r.zip",
 				Content:  rZipBytes(),
-				Mtime:   "03-04-10 00:24:16",
+				Modified: time.Date(2010, 3, 4, 0, 24, 16, 0, time.UTC),
 				Mode:     0666,
 			},
 		},
@@ -112,6 +104,7 @@ var tests = []ZipTest{
 			{
 				Name:     "symlink",
 				Content:  []byte("../target"),
+				Modified: time.Date(2012, 2, 3, 19, 56, 48, 0, timeZone(-2*time.Hour)),
 				Mode:     0777 | os.ModeSymlink,
 			},
 		},
@@ -129,7 +122,7 @@ var tests = []ZipTest{
 			{
 				Name:     "filename",
 				Content:  []byte("This is a test textfile.\n"),
-				Mtime:   "02-02-11 13:06:20",
+				Modified: time.Date(2011, 2, 2, 13, 6, 20, 0, time.UTC),
 				Mode:     0666,
 			},
 		},
@@ -137,12 +130,62 @@ var tests = []ZipTest{
 	{
 		// created in windows XP file manager.
 		Name: "winxp.zip",
-		File: crossPlatform,
+		File: []ZipTestFile{
+			{
+				Name:     "hello",
+				Content:  []byte("world \r\n"),
+				Modified: time.Date(2011, 12, 8, 10, 4, 24, 0, time.UTC),
+				Mode:     0666,
+			},
+			{
+				Name:     "dir/bar",
+				Content:  []byte("foo \r\n"),
+				Modified: time.Date(2011, 12, 8, 10, 4, 50, 0, time.UTC),
+				Mode:     0666,
+			},
+			{
+				Name:     "dir/empty/",
+				Content:  []byte{},
+				Modified: time.Date(2011, 12, 8, 10, 8, 6, 0, time.UTC),
+				Mode:     os.ModeDir | 0777,
+			},
+			{
+				Name:     "readonly",
+				Content:  []byte("important \r\n"),
+				Modified: time.Date(2011, 12, 8, 10, 6, 8, 0, time.UTC),
+				Mode:     0444,
+			},
+		},
 	},
 	{
 		// created by Zip 3.0 under Linux
 		Name: "unix.zip",
-		File: crossPlatform,
+		File: []ZipTestFile{
+			{
+				Name:     "hello",
+				Content:  []byte("world \r\n"),
+				Modified: time.Date(2011, 12, 8, 10, 4, 24, 0, timeZone(0)),
+				Mode:     0666,
+			},
+			{
+				Name:     "dir/bar",
+				Content:  []byte("foo \r\n"),
+				Modified: time.Date(2011, 12, 8, 10, 4, 50, 0, timeZone(0)),
+				Mode:     0666,
+			},
+			{
+				Name:     "dir/empty/",
+				Content:  []byte{},
+				Modified: time.Date(2011, 12, 8, 10, 8, 6, 0, timeZone(0)),
+				Mode:     os.ModeDir | 0777,
+			},
+			{
+				Name:     "readonly",
+				Content:  []byte("important \r\n"),
+				Modified: time.Date(2011, 12, 8, 10, 6, 8, 0, timeZone(0)),
+				Mode:     0444,
+			},
+		},
 	},
 	{
 		// created by Go, before we wrote the "optional" data
@@ -152,13 +195,13 @@ var tests = []ZipTest{
 			{
 				Name:     "foo.txt",
 				Content:  []byte("foo\n"),
-				Mtime:   "03-08-12 16:59:10",
+				Modified: time.Date(2012, 3, 8, 16, 59, 10, 0, timeZone(-8*time.Hour)),
 				Mode:     0644,
 			},
 			{
 				Name:     "bar.txt",
 				Content:  []byte("bar\n"),
-				Mtime:   "03-08-12 16:59:12",
+				Modified: time.Date(2012, 3, 8, 16, 59, 12, 0, timeZone(-8*time.Hour)),
 				Mode:     0644,
 			},
 		},
@@ -171,11 +214,13 @@ var tests = []ZipTest{
 			{
 				Name:     "foo.txt",
 				Content:  []byte("foo\n"),
+				Modified: time.Date(1979, 11, 30, 0, 0, 0, 0, time.UTC),
 				Mode:     0666,
 			},
 			{
 				Name:     "bar.txt",
 				Content:  []byte("bar\n"),
+				Modified: time.Date(1979, 11, 30, 0, 0, 0, 0, time.UTC),
 				Mode:     0666,
 			},
 		},
@@ -187,12 +232,14 @@ var tests = []ZipTest{
 			{
 				Name:       "foo.txt",
 				Content:    []byte("foo\n"),
+				Modified:   time.Date(1979, 11, 30, 0, 0, 0, 0, time.UTC),
 				Mode:       0666,
 				ContentErr: ErrChecksum,
 			},
 			{
 				Name:     "bar.txt",
 				Content:  []byte("bar\n"),
+				Modified: time.Date(1979, 11, 30, 0, 0, 0, 0, time.UTC),
 				Mode:     0666,
 			},
 		},
@@ -205,13 +252,13 @@ var tests = []ZipTest{
 			{
 				Name:     "foo.txt",
 				Content:  []byte("foo\n"),
-				Mtime:   "03-08-12 16:59:10",
+				Modified: time.Date(2012, 3, 8, 16, 59, 10, 0, timeZone(-8*time.Hour)),
 				Mode:     0644,
 			},
 			{
 				Name:     "bar.txt",
 				Content:  []byte("bar\n"),
-				Mtime:   "03-08-12 16:59:12",
+				Modified: time.Date(2012, 3, 8, 16, 59, 12, 0, timeZone(-8*time.Hour)),
 				Mode:     0644,
 			},
 		},
@@ -225,14 +272,14 @@ var tests = []ZipTest{
 			{
 				Name:       "foo.txt",
 				Content:    []byte("foo\n"),
-				Mtime:      "03-08-12 16:59:10",
+				Modified:   time.Date(2012, 3, 8, 16, 59, 10, 0, timeZone(-8*time.Hour)),
 				Mode:       0644,
 				ContentErr: ErrChecksum,
 			},
 			{
 				Name:     "bar.txt",
 				Content:  []byte("bar\n"),
-				Mtime:   "03-08-12 16:59:12",
+				Modified: time.Date(2012, 3, 8, 16, 59, 12, 0, timeZone(-8*time.Hour)),
 				Mode:     0644,
 			},
 		},
@@ -243,7 +290,7 @@ var tests = []ZipTest{
 			{
 				Name:     "README",
 				Content:  []byte("This small file is in ZIP64 format.\n"),
-				Mtime:   "08-10-12 14:33:32",
+				Modified: time.Date(2012, 8, 10, 14, 33, 32, 0, time.UTC),
 				Mode:     0644,
 			},
 		},
@@ -255,7 +302,7 @@ var tests = []ZipTest{
 			{
 				Name:     "README",
 				Content:  []byte("This small file is in ZIP64 format.\n"),
-				Mtime:   "08-10-12 14:33:32",
+				Modified: time.Date(2012, 8, 10, 14, 33, 32, 0, timeZone(-4*time.Hour)),
 				Mode:     0644,
 			},
 		},
@@ -269,38 +316,176 @@ var tests = []ZipTest{
 				Name:     "big.file",
 				Content:  nil,
 				Size:     1<<32 - 1,
+				Modified: time.Date(1979, 11, 30, 0, 0, 0, 0, time.UTC),
 				Mode:     0666,
 			},
 		},
 	},
-}
-
-var crossPlatform = []ZipTestFile{
 	{
-		Name:    "hello",
+		Name: "utf8-7zip.zip",
-		Content: []byte("world \r\n"),
+		File: []ZipTestFile{
-		Mode:    0666,
-	},
 			{
-		Name:    "dir/bar",
+				Name:     "世界",
-		Content: []byte("foo \r\n"),
-		Mode:    0666,
-	},
-	{
-		Name:    "dir/empty/",
 				Content:  []byte{},
-		Mode:    os.ModeDir | 0777,
+				Mode:     0666,
+				Modified: time.Date(2017, 11, 6, 13, 9, 27, 867862500, timeZone(-8*time.Hour)),
+			},
+		},
 	},
 	{
-		Name:    "readonly",
+		Name: "utf8-infozip.zip",
-		Content: []byte("important \r\n"),
+		File: []ZipTestFile{
-		Mode:    0444,
+			{
+				Name:    "世界",
+				Content: []byte{},
+				Mode:    0644,
+				// Name is valid UTF-8, but format does not have UTF-8 flag set.
+				// We don't do UTF-8 detection for multi-byte runes due to
+				// false-positives with other encodings (e.g., Shift-JIS).
+				// Format says encoding is not UTF-8, so we trust it.
+				NonUTF8:  true,
+				Modified: time.Date(2017, 11, 6, 13, 9, 27, 0, timeZone(-8*time.Hour)),
+			},
+		},
+	},
+	{
+		Name: "utf8-osx.zip",
+		File: []ZipTestFile{
+			{
+				Name:    "世界",
+				Content: []byte{},
+				Mode:    0644,
+				// Name is valid UTF-8, but format does not have UTF-8 set.
+				NonUTF8:  true,
+				Modified: time.Date(2017, 11, 6, 13, 9, 27, 0, timeZone(-8*time.Hour)),
+			},
+		},
+	},
+	{
+		Name: "utf8-winrar.zip",
+		File: []ZipTestFile{
+			{
+				Name:     "世界",
+				Content:  []byte{},
+				Mode:     0666,
+				Modified: time.Date(2017, 11, 6, 13, 9, 27, 867862500, timeZone(-8*time.Hour)),
+			},
+		},
+	},
+	{
+		Name: "utf8-winzip.zip",
+		File: []ZipTestFile{
+			{
+				Name:     "世界",
+				Content:  []byte{},
+				Mode:     0666,
+				Modified: time.Date(2017, 11, 6, 13, 9, 27, 867000000, timeZone(-8*time.Hour)),
+			},
+		},
+	},
+	{
+		Name: "time-7zip.zip",
+		File: []ZipTestFile{
+			{
+				Name:     "test.txt",
+				Content:  []byte{},
+				Size:     1<<32 - 1,
+				Modified: time.Date(2017, 10, 31, 21, 11, 57, 244817900, timeZone(-7*time.Hour)),
+				Mode:     0666,
+			},
+		},
+	},
+	{
+		Name: "time-infozip.zip",
+		File: []ZipTestFile{
+			{
+				Name:     "test.txt",
+				Content:  []byte{},
+				Size:     1<<32 - 1,
+				Modified: time.Date(2017, 10, 31, 21, 11, 57, 0, timeZone(-7*time.Hour)),
+				Mode:     0644,
+			},
+		},
+	},
+	{
+		Name: "time-osx.zip",
+		File: []ZipTestFile{
+			{
+				Name:     "test.txt",
+				Content:  []byte{},
+				Size:     1<<32 - 1,
+				Modified: time.Date(2017, 10, 31, 21, 17, 27, 0, timeZone(-7*time.Hour)),
+				Mode:     0644,
+			},
+		},
+	},
+	{
+		Name: "time-win7.zip",
+		File: []ZipTestFile{
+			{
+				Name:     "test.txt",
+				Content:  []byte{},
+				Size:     1<<32 - 1,
+				Modified: time.Date(2017, 10, 31, 21, 11, 58, 0, time.UTC),
+				Mode:     0666,
+			},
+		},
+	},
+	{
+		Name: "time-winrar.zip",
+		File: []ZipTestFile{
+			{
+				Name:     "test.txt",
+				Content:  []byte{},
+				Size:     1<<32 - 1,
+				Modified: time.Date(2017, 10, 31, 21, 11, 57, 244817900, timeZone(-7*time.Hour)),
+				Mode:     0666,
+			},
+		},
+	},
+	{
+		Name: "time-winzip.zip",
+		File: []ZipTestFile{
+			{
+				Name:     "test.txt",
+				Content:  []byte{},
+				Size:     1<<32 - 1,
+				Modified: time.Date(2017, 10, 31, 21, 11, 57, 244000000, timeZone(-7*time.Hour)),
+				Mode:     0666,
+			},
+		},
+	},
+	{
+		Name: "time-go.zip",
+		File: []ZipTestFile{
+			{
+				Name:     "test.txt",
+				Content:  []byte{},
+				Size:     1<<32 - 1,
+				Modified: time.Date(2017, 10, 31, 21, 11, 57, 0, timeZone(-7*time.Hour)),
+				Mode:     0666,
+			},
+		},
+	},
+	{
+		Name: "time-22738.zip",
+		File: []ZipTestFile{
+			{
+				Name:     "file",
+				Content:  []byte{},
+				Mode:     0666,
+				Modified: time.Date(1999, 12, 31, 19, 0, 0, 0, timeZone(-5*time.Hour)),
+				ModTime:  time.Date(1999, 12, 31, 19, 0, 0, 0, time.UTC),
+			},
+		},
 	},
 }
 
 func TestReader(t *testing.T) {
 	for _, zt := range tests {
+		t.Run(zt.Name, func(t *testing.T) {
 			readTestZip(t, zt)
+		})
 	}
 }
 
@@ -319,7 +504,7 @@ func readTestZip(t *testing.T, zt ZipTest) {
 		}
 	}
 	if err != zt.Error {
-		t.Errorf("%s: error=%v, want %v", zt.Name, err, zt.Error)
+		t.Errorf("error=%v, want %v", err, zt.Error)
 		return
 	}
 
@@ -335,16 +520,19 @@ func readTestZip(t *testing.T, zt ZipTest) {
 	}
 
 	if z.Comment != zt.Comment {
-		t.Errorf("%s: comment=%q, want %q", zt.Name, z.Comment, zt.Comment)
+		t.Errorf("comment=%q, want %q", z.Comment, zt.Comment)
 	}
 	if len(z.File) != len(zt.File) {
-		t.Fatalf("%s: file count=%d, want %d", zt.Name, len(z.File), len(zt.File))
+		t.Fatalf("file count=%d, want %d", len(z.File), len(zt.File))
 	}
 
 	// test read of each file
 	for i, ft := range zt.File {
 		readTestFile(t, zt, ft, z.File[i])
 	}
+	if t.Failed() {
+		return
+	}
 
 	// test simultaneous reads
 	n := 0
@@ -363,23 +551,24 @@ func readTestZip(t *testing.T, zt ZipTest) {
 	}
 }
 
+func equalTimeAndZone(t1, t2 time.Time) bool {
+	name1, offset1 := t1.Zone()
+	name2, offset2 := t2.Zone()
+	return t1.Equal(t2) && name1 == name2 && offset1 == offset2
+}
+
 func readTestFile(t *testing.T, zt ZipTest, ft ZipTestFile, f *File) {
 	if f.Name != ft.Name {
-		t.Errorf("%s: name=%q, want %q", zt.Name, f.Name, ft.Name)
+		t.Errorf("name=%q, want %q", f.Name, ft.Name)
+	}
+	if !ft.Modified.IsZero() && !equalTimeAndZone(f.Modified, ft.Modified) {
+		t.Errorf("%s: Modified=%s, want %s", f.Name, f.Modified, ft.Modified)
+	}
+	if !ft.ModTime.IsZero() && !equalTimeAndZone(f.ModTime(), ft.ModTime) {
+		t.Errorf("%s: ModTime=%s, want %s", f.Name, f.ModTime(), ft.ModTime)
 	}
 
-	if ft.Mtime != "" {
+	testFileMode(t, f, ft.Mode)
-		mtime, err := time.Parse("01-02-06 15:04:05", ft.Mtime)
-		if err != nil {
-			t.Error(err)
-			return
-		}
-		if ft := f.ModTime(); !ft.Equal(mtime) {
-			t.Errorf("%s: %s: mtime=%s, want %s", zt.Name, f.Name, ft, mtime)
-		}
-	}
-
-	testFileMode(t, zt.Name, f, ft.Mode)
 
 	size := uint64(f.UncompressedSize)
 	if size == uint32max {
@@ -390,7 +579,7 @@ func readTestFile(t *testing.T, zt ZipTest, ft ZipTestFile, f *File) {
 
 	r, err := f.Open()
 	if err != nil {
-		t.Errorf("%s: %v", zt.Name, err)
+		t.Errorf("%v", err)
 		return
 	}
 
@@ -408,7 +597,7 @@ func readTestFile(t *testing.T, zt ZipTest, ft ZipTestFile, f *File) {
 	var b bytes.Buffer
 	_, err = io.Copy(&b, r)
 	if err != ft.ContentErr {
-		t.Errorf("%s: copying contents: %v (want %v)", zt.Name, err, ft.ContentErr)
+		t.Errorf("copying contents: %v (want %v)", err, ft.ContentErr)
 	}
 	if err != nil {
 		return
@@ -440,12 +629,12 @@ func readTestFile(t *testing.T, zt ZipTest, ft ZipTestFile, f *File) {
 	}
 }
 
-func testFileMode(t *testing.T, zipName string, f *File, want os.FileMode) {
+func testFileMode(t *testing.T, f *File, want os.FileMode) {
 	mode := f.Mode()
 	if want == 0 {
-		t.Errorf("%s: %s mode: got %v, want none", zipName, f.Name, mode)
+		t.Errorf("%s mode: got %v, want none", f.Name, mode)
 	} else if mode != want {
-		t.Errorf("%s: %s mode: want %v, got %v", zipName, f.Name, want, mode)
+		t.Errorf("%s mode: want %v, got %v", f.Name, want, mode)
 	}
 }
 

libgo/go/archive/zip/struct.go

@@ -27,8 +27,8 @@ import (
 
 // Compression methods.
 const (
-	Store   uint16 = 0
+	Store   uint16 = 0 // no compression
-	Deflate uint16 = 8
+	Deflate uint16 = 8 // DEFLATE compressed
 )
 
 const (
@@ -46,40 +46,79 @@ const (
 	directory64LocLen        = 20         //
 	directory64EndLen        = 56         // + extra
 
-	// Constants for the first byte in CreatorVersion
+	// Constants for the first byte in CreatorVersion.
 	creatorFAT    = 0
 	creatorUnix   = 3
 	creatorNTFS   = 11
 	creatorVFAT   = 14
 	creatorMacOSX = 19
 
-	// version numbers
+	// Version numbers.
 	zipVersion20 = 20 // 2.0
 	zipVersion45 = 45 // 4.5 (reads and writes zip64 archives)
 
-	// limits for non zip64 files
+	// Limits for non zip64 files.
 	uint16max = (1 << 16) - 1
 	uint32max = (1 << 32) - 1
 
-	// extra header id's
+	// Extra header IDs.
-	zip64ExtraId = 0x0001 // zip64 Extended Information Extra Field
+	//
+	// IDs 0..31 are reserved for official use by PKWARE.
+	// IDs above that range are defined by third-party vendors.
+	// Since ZIP lacked high precision timestamps (nor a official specification
+	// of the timezone used for the date fields), many competing extra fields
+	// have been invented. Pervasive use effectively makes them "official".
+	//
+	// See http://mdfs.net/Docs/Comp/Archiving/Zip/ExtraField
+	zip64ExtraID       = 0x0001 // Zip64 extended information
+	ntfsExtraID        = 0x000a // NTFS
+	unixExtraID        = 0x000d // UNIX
+	extTimeExtraID     = 0x5455 // Extended timestamp
+	infoZipUnixExtraID = 0x5855 // Info-ZIP Unix extension
 )
 
 // FileHeader describes a file within a zip file.
 // See the zip spec for details.
 type FileHeader struct {
 	// Name is the name of the file.
-	// It must be a relative path: it must not start with a drive
+	// It must be a relative path, not start with a drive letter (e.g. C:),
-	// letter (e.g. C:) or leading slash, and only forward slashes
+	// and must use forward slashes instead of back slashes.
-	// are allowed.
 	Name string
 
+	// Comment is any arbitrary user-defined string shorter than 64KiB.
+	Comment string
+
+	// NonUTF8 indicates that Name and Comment are not encoded in UTF-8.
+	//
+	// By specification, the only other encoding permitted should be CP-437,
+	// but historically many ZIP readers interpret Name and Comment as whatever
+	// the system's local character encoding happens to be.
+	//
+	// This flag should only be set if the user intends to encode a non-portable
+	// ZIP file for a specific localized region. Otherwise, the Writer
+	// automatically sets the ZIP format's UTF-8 flag for valid UTF-8 strings.
+	NonUTF8 bool
+
 	CreatorVersion uint16
 	ReaderVersion  uint16
 	Flags          uint16
+
+	// Method is the compression method. If zero, Store is used.
 	Method uint16
-	ModifiedTime       uint16 // MS-DOS time
+
-	ModifiedDate       uint16 // MS-DOS date
+	// Modified is the modified time of the file.
+	//
+	// When reading, an extended timestamp is preferred over the legacy MS-DOS
+	// date field, and the offset between the times is used as the timezone.
+	// If only the MS-DOS date is present, the timezone is assumed to be UTC.
+	//
+	// When writing, an extended timestamp (which is timezone-agnostic) is
+	// always emitted. The legacy MS-DOS date field is encoded according to the
+	// location of the Modified time.
+	Modified     time.Time
+	ModifiedTime uint16 // Deprecated: Legacy MS-DOS date; use Modified instead.
+	ModifiedDate uint16 // Deprecated: Legacy MS-DOS time; use Modified instead.
+
 	CRC32              uint32
 	CompressedSize     uint32 // Deprecated: Use CompressedSize64 instead.
 	UncompressedSize   uint32 // Deprecated: Use UncompressedSize64 instead.
@@ -87,7 +126,6 @@ type FileHeader struct {
 	UncompressedSize64 uint64
 	Extra              []byte
 	ExternalAttrs      uint32 // Meaning depends on CreatorVersion
-	Comment            string
 }
 
 // FileInfo returns an os.FileInfo for the FileHeader.
@@ -117,6 +155,8 @@ func (fi headerFileInfo) Sys() interface{}   { return fi.fh }
 // Because os.FileInfo's Name method returns only the base name of
 // the file it describes, it may be necessary to modify the Name field
 // of the returned header to provide the full path name of the file.
+// If compression is desired, callers should set the FileHeader.Method
+// field; it is unset by default.
 func FileInfoHeader(fi os.FileInfo) (*FileHeader, error) {
 	size := fi.Size()
 	fh := &FileHeader{
@@ -144,6 +184,21 @@ type directoryEnd struct {
 	comment            string
 }
 
+// timeZone returns a *time.Location based on the provided offset.
+// If the offset is non-sensible, then this uses an offset of zero.
+func timeZone(offset time.Duration) *time.Location {
+	const (
+		minOffset   = -12 * time.Hour  // E.g., Baker island at -12:00
+		maxOffset   = +14 * time.Hour  // E.g., Line island at +14:00
+		offsetAlias = 15 * time.Minute // E.g., Nepal at +5:45
+	)
+	offset = offset.Round(offsetAlias)
+	if offset < minOffset || maxOffset < offset {
+		offset = 0
+	}
+	return time.FixedZone("", int(offset/time.Second))
+}
+
 // msDosTimeToTime converts an MS-DOS date and time into a time.Time.
 // The resolution is 2s.
 // See: http://msdn.microsoft.com/en-us/library/ms724247(v=VS.85).aspx
@@ -168,21 +223,26 @@ func msDosTimeToTime(dosDate, dosTime uint16) time.Time {
 // The resolution is 2s.
 // See: http://msdn.microsoft.com/en-us/library/ms724274(v=VS.85).aspx
 func timeToMsDosTime(t time.Time) (fDate uint16, fTime uint16) {
-	t = t.In(time.UTC)
 	fDate = uint16(t.Day() + int(t.Month())<<5 + (t.Year()-1980)<<9)
 	fTime = uint16(t.Second()/2 + t.Minute()<<5 + t.Hour()<<11)
 	return
 }
 
-// ModTime returns the modification time in UTC.
+// ModTime returns the modification time in UTC using the legacy
-// The resolution is 2s.
+// ModifiedDate and ModifiedTime fields.
+//
+// Deprecated: Use Modified instead.
 func (h *FileHeader) ModTime() time.Time {
 	return msDosTimeToTime(h.ModifiedDate, h.ModifiedTime)
 }
 
-// SetModTime sets the ModifiedTime and ModifiedDate fields to the given time in UTC.
+// SetModTime sets the Modified, ModifiedTime, and ModifiedDate fields
-// The resolution is 2s.
+// to the given time in UTC.
+//
+// Deprecated: Use Modified instead.
 func (h *FileHeader) SetModTime(t time.Time) {
+	t = t.UTC() // Convert to UTC for compatibility
+	h.Modified = t
 	h.ModifiedDate, h.ModifiedTime = timeToMsDosTime(t)
 }
 

libgo/go/archive/zip/writer.go

@@ -14,6 +14,11 @@ import (
 	"unicode/utf8"
 )
 
+var (
+	errLongName  = errors.New("zip: FileHeader.Name too long")
+	errLongExtra = errors.New("zip: FileHeader.Extra too long")
+)
+
 // Writer implements a zip file writer.
 type Writer struct {
 	cw          *countWriter
@@ -21,6 +26,7 @@ type Writer struct {
 	last        *fileWriter
 	closed      bool
 	compressors map[uint16]Compressor
+	comment     string
 
 	// testHookCloseSizeOffset if non-nil is called with the size
 	// of offset of the central directory at Close.
@@ -54,6 +60,16 @@ func (w *Writer) Flush() error {
 	return w.cw.w.(*bufio.Writer).Flush()
 }
 
+// SetComment sets the end-of-central-directory comment field.
+// It can only be called before Close.
+func (w *Writer) SetComment(comment string) error {
+	if len(comment) > uint16max {
+		return errors.New("zip: Writer.Comment too long")
+	}
+	w.comment = comment
+	return nil
+}
+
 // Close finishes writing the zip file by writing the central directory.
 // It does not (and cannot) close the underlying writer.
 func (w *Writer) Close() error {
@@ -91,7 +107,7 @@ func (w *Writer) Close() error {
 			// append a zip64 extra block to Extra
 			var buf [28]byte // 2x uint16 + 3x uint64
 			eb := writeBuf(buf[:])
-			eb.uint16(zip64ExtraId)
+			eb.uint16(zip64ExtraID)
 			eb.uint16(24) // size = 3x uint64
 			eb.uint64(h.UncompressedSize64)
 			eb.uint64(h.CompressedSize64)
@@ -177,16 +193,20 @@ func (w *Writer) Close() error {
 	b.uint16(uint16(records))        // number of entries total
 	b.uint32(uint32(size))           // size of directory
 	b.uint32(uint32(offset))         // start of directory
-	// skipped size of comment (always zero)
+	b.uint16(uint16(len(w.comment))) // byte size of EOCD comment
 	if _, err := w.cw.Write(buf[:]); err != nil {
 		return err
 	}
+	if _, err := io.WriteString(w.cw, w.comment); err != nil {
+		return err
+	}
 
 	return w.cw.w.(*bufio.Writer).Flush()
 }
 
 // Create adds a file to the zip file using the provided name.
 // It returns a Writer to which the file contents should be written.
+// The file contents will be compressed using the Deflate method.
 // The name must be a relative path: it must not start with a drive
 // letter (e.g. C:) or leading slash, and only forward slashes are
 // allowed.
@@ -200,27 +220,36 @@ func (w *Writer) Create(name string) (io.Writer, error) {
 	return w.CreateHeader(header)
 }
 
-func hasValidUTF8(s string) bool {
+// detectUTF8 reports whether s is a valid UTF-8 string, and whether the string
-	n := 0
+// must be considered UTF-8 encoding (i.e., not compatible with CP-437, ASCII,
-	for _, r := range s {
+// or any other common encoding).
-		// By default, ZIP uses CP437, which is only identical to ASCII for the printable characters.
+func detectUTF8(s string) (valid, require bool) {
-		if r < 0x20 || r >= 0x7f {
+	for i := 0; i < len(s); {
-			if !utf8.ValidRune(r) {
+		r, size := utf8.DecodeRuneInString(s[i:])
-				return false
+		i += size
+		// Officially, ZIP uses CP-437, but many readers use the system's
+		// local character encoding. Most encoding are compatible with a large
+		// subset of CP-437, which itself is ASCII-like.
+		//
+		// Forbid 0x7e and 0x5c since EUC-KR and Shift-JIS replace those
+		// characters with localized currency and overline characters.
+		if r < 0x20 || r > 0x7d || r == 0x5c {
+			if !utf8.ValidRune(r) || (r == utf8.RuneError && size == 1) {
+				return false, false
 			}
-			n++
+			require = true
 		}
 	}
-	return n > 0
+	return true, require
 }
 
-// CreateHeader adds a file to the zip file using the provided FileHeader
+// CreateHeader adds a file to the zip archive using the provided FileHeader
-// for the file metadata.
+// for the file metadata. Writer takes ownership of fh and may mutate
-// It returns a Writer to which the file contents should be written.
+// its fields. The caller must not modify fh after calling CreateHeader.
 //
+// This returns a Writer to which the file contents should be written.
 // The file's contents must be written to the io.Writer before the next
-// call to Create, CreateHeader, or Close. The provided FileHeader fh
+// call to Create, CreateHeader, or Close.
-// must not be modified after a call to CreateHeader.
 func (w *Writer) CreateHeader(fh *FileHeader) (io.Writer, error) {
 	if w.last != nil && !w.last.closed {
 		if err := w.last.close(); err != nil {
@@ -234,13 +263,62 @@ func (w *Writer) CreateHeader(fh *FileHeader) (io.Writer, error) {
 
 	fh.Flags |= 0x8 // we will write a data descriptor
 
-	if hasValidUTF8(fh.Name) || hasValidUTF8(fh.Comment) {
+	// The ZIP format has a sad state of affairs regarding character encoding.
-		fh.Flags |= 0x800 // filename or comment have valid utf-8 string
+	// Officially, the name and comment fields are supposed to be encoded
+	// in CP-437 (which is mostly compatible with ASCII), unless the UTF-8
+	// flag bit is set. However, there are several problems:
+	//
+	//	* Many ZIP readers still do not support UTF-8.
+	//	* If the UTF-8 flag is cleared, several readers simply interpret the
+	//	name and comment fields as whatever the local system encoding is.
+	//
+	// In order to avoid breaking readers without UTF-8 support,
+	// we avoid setting the UTF-8 flag if the strings are CP-437 compatible.
+	// However, if the strings require multibyte UTF-8 encoding and is a
+	// valid UTF-8 string, then we set the UTF-8 bit.
+	//
+	// For the case, where the user explicitly wants to specify the encoding
+	// as UTF-8, they will need to set the flag bit themselves.
+	utf8Valid1, utf8Require1 := detectUTF8(fh.Name)
+	utf8Valid2, utf8Require2 := detectUTF8(fh.Comment)
+	switch {
+	case fh.NonUTF8:
+		fh.Flags &^= 0x800
+	case (utf8Require1 || utf8Require2) && (utf8Valid1 && utf8Valid2):
+		fh.Flags |= 0x800
 	}
 
 	fh.CreatorVersion = fh.CreatorVersion&0xff00 | zipVersion20 // preserve compatibility byte
 	fh.ReaderVersion = zipVersion20
 
+	// If Modified is set, this takes precedence over MS-DOS timestamp fields.
+	if !fh.Modified.IsZero() {
+		// Contrary to the FileHeader.SetModTime method, we intentionally
+		// do not convert to UTC, because we assume the user intends to encode
+		// the date using the specified timezone. A user may want this control
+		// because many legacy ZIP readers interpret the timestamp according
+		// to the local timezone.
+		//
+		// The timezone is only non-UTC if a user directly sets the Modified
+		// field directly themselves. All other approaches sets UTC.
+		fh.ModifiedDate, fh.ModifiedTime = timeToMsDosTime(fh.Modified)
+
+		// Use "extended timestamp" format since this is what Info-ZIP uses.
+		// Nearly every major ZIP implementation uses a different format,
+		// but at least most seem to be able to understand the other formats.
+		//
+		// This format happens to be identical for both local and central header
+		// if modification time is the only timestamp being encoded.
+		var mbuf [9]byte // 2*SizeOf(uint16) + SizeOf(uint8) + SizeOf(uint32)
+		mt := uint32(fh.Modified.Unix())
+		eb := writeBuf(mbuf[:])
+		eb.uint16(extTimeExtraID)
+		eb.uint16(5)  // Size: SizeOf(uint8) + SizeOf(uint32)
+		eb.uint8(1)   // Flags: ModTime
+		eb.uint32(mt) // ModTime
+		fh.Extra = append(fh.Extra, mbuf[:]...)
+	}
+
 	fw := &fileWriter{
 		zipw:      w.cw,
 		compCount: &countWriter{w: w.cw},
@@ -273,6 +351,14 @@ func (w *Writer) CreateHeader(fh *FileHeader) (io.Writer, error) {
 }
 
 func writeHeader(w io.Writer, h *FileHeader) error {
+	const maxUint16 = 1<<16 - 1
+	if len(h.Name) > maxUint16 {
+		return errLongName
+	}
+	if len(h.Extra) > maxUint16 {
+		return errLongExtra
+	}
+
 	var buf [fileHeaderLen]byte
 	b := writeBuf(buf[:])
 	b.uint32(uint32(fileHeaderSignature))
@@ -402,6 +488,11 @@ func (w nopCloser) Close() error {
 
 type writeBuf []byte
 
+func (b *writeBuf) uint8(v uint8) {
+	(*b)[0] = v
+	*b = (*b)[1:]
+}
+
 func (b *writeBuf) uint16(v uint16) {
 	binary.LittleEndian.PutUint16(*b, v)
 	*b = (*b)[2:]

libgo/go/archive/zip/writer_test.go

@@ -6,11 +6,14 @@ package zip
 
 import (
 	"bytes"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"math/rand"
 	"os"
+	"strings"
 	"testing"
+	"time"
 )
 
 // TODO(adg): a more sophisticated test suite
@@ -57,8 +60,8 @@ var writeTests = []WriteTest{
 
 func TestWriter(t *testing.T) {
 	largeData := make([]byte, 1<<17)
-	for i := range largeData {
+	if _, err := rand.Read(largeData); err != nil {
-		largeData[i] = byte(rand.Int())
+		t.Fatal("rand.Read failed:", err)
 	}
 	writeTests[1].Data = largeData
 	defer func() {
@@ -87,31 +90,100 @@ func TestWriter(t *testing.T) {
 	}
 }
 
+// TestWriterComment is test for EOCD comment read/write.
+func TestWriterComment(t *testing.T) {
+	var tests = []struct {
+		comment string
+		ok      bool
+	}{
+		{"hi, hello", true},
+		{"hi, こんにちわ", true},
+		{strings.Repeat("a", uint16max), true},
+		{strings.Repeat("a", uint16max+1), false},
+	}
+
+	for _, test := range tests {
+		// write a zip file
+		buf := new(bytes.Buffer)
+		w := NewWriter(buf)
+		if err := w.SetComment(test.comment); err != nil {
+			if test.ok {
+				t.Fatalf("SetComment: unexpected error %v", err)
+			}
+			continue
+		} else {
+			if !test.ok {
+				t.Fatalf("SetComment: unexpected success, want error")
+			}
+		}
+
+		if err := w.Close(); test.ok == (err != nil) {
+			t.Fatal(err)
+		}
+
+		if w.closed != test.ok {
+			t.Fatalf("Writer.closed: got %v, want %v", w.closed, test.ok)
+		}
+
+		// skip read test in failure cases
+		if !test.ok {
+			continue
+		}
+
+		// read it back
+		r, err := NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len()))
+		if err != nil {
+			t.Fatal(err)
+		}
+		if r.Comment != test.comment {
+			t.Fatalf("Reader.Comment: got %v, want %v", r.Comment, test.comment)
+		}
+	}
+}
+
 func TestWriterUTF8(t *testing.T) {
 	var utf8Tests = []struct {
 		name    string
 		comment string
-		expect  uint16
+		nonUTF8 bool
+		flags   uint16
 	}{
 		{
 			name:    "hi, hello",
 			comment: "in the world",
-			expect:  0x8,
+			flags:   0x8,
 		},
 		{
 			name:    "hi, こんにちわ",
 			comment: "in the world",
-			expect:  0x808,
+			flags:   0x808,
+		},
+		{
+			name:    "hi, こんにちわ",
+			comment: "in the world",
+			nonUTF8: true,
+			flags:   0x8,
 		},
 		{
 			name:    "hi, hello",
 			comment: "in the 世界",
-			expect:  0x808,
+			flags:   0x808,
 		},
 		{
 			name:    "hi, こんにちわ",
 			comment: "in the 世界",
-			expect:  0x808,
+			flags:   0x808,
+		},
+		{
+			name:    "the replacement rune is <20>",
+			comment: "the replacement rune is <20>",
+			flags:   0x808,
+		},
+		{
+			// Name is Japanese encoded in Shift JIS.
+			name:    "\x93\xfa\x96{\x8c\xea.txt",
+			comment: "in the 世界",
+			flags:   0x008, // UTF-8 must not be set
 		},
 	}
 
@@ -123,6 +195,7 @@ func TestWriterUTF8(t *testing.T) {
 		h := &FileHeader{
 			Name:    test.name,
 			Comment: test.comment,
+			NonUTF8: test.nonUTF8,
 			Method:  Deflate,
 		}
 		w, err := w.CreateHeader(h)
@@ -142,18 +215,41 @@ func TestWriterUTF8(t *testing.T) {
 		t.Fatal(err)
 	}
 	for i, test := range utf8Tests {
-		got := r.File[i].Flags
+		flags := r.File[i].Flags
-		t.Logf("name %v, comment %v", test.name, test.comment)
+		if flags != test.flags {
-		if got != test.expect {
+			t.Errorf("CreateHeader(name=%q comment=%q nonUTF8=%v): flags=%#x, want %#x", test.name, test.comment, test.nonUTF8, flags, test.flags)
-			t.Fatalf("Flags: got %v, want %v", got, test.expect)
 		}
 	}
 }
 
+func TestWriterTime(t *testing.T) {
+	var buf bytes.Buffer
+	h := &FileHeader{
+		Name:     "test.txt",
+		Modified: time.Date(2017, 10, 31, 21, 11, 57, 0, timeZone(-7*time.Hour)),
+	}
+	w := NewWriter(&buf)
+	if _, err := w.CreateHeader(h); err != nil {
+		t.Fatalf("unexpected CreateHeader error: %v", err)
+	}
+	if err := w.Close(); err != nil {
+		t.Fatalf("unexpected Close error: %v", err)
+	}
+
+	want, err := ioutil.ReadFile("testdata/time-go.zip")
+	if err != nil {
+		t.Fatalf("unexpected ReadFile error: %v", err)
+	}
+	if got := buf.Bytes(); !bytes.Equal(got, want) {
+		fmt.Printf("%x\n%x\n", got, want)
+		t.Error("contents of time-go.zip differ")
+	}
+}
+
 func TestWriterOffset(t *testing.T) {
 	largeData := make([]byte, 1<<17)
-	for i := range largeData {
+	if _, err := rand.Read(largeData); err != nil {
-		largeData[i] = byte(rand.Int())
+		t.Fatal("rand.Read failed:", err)
 	}
 	writeTests[1].Data = largeData
 	defer func() {
@@ -225,7 +321,7 @@ func testReadFile(t *testing.T, f *File, wt *WriteTest) {
 	if f.Name != wt.Name {
 		t.Fatalf("File name: got %q, want %q", f.Name, wt.Name)
 	}
-	testFileMode(t, wt.Name, f, wt.Mode)
+	testFileMode(t, f, wt.Mode)
 	rc, err := f.Open()
 	if err != nil {
 		t.Fatal("opening:", err)

libgo/go/archive/zip/zip_test.go

@@ -645,16 +645,54 @@ func TestHeaderTooShort(t *testing.T) {
 	h := FileHeader{
 		Name:   "foo.txt",
 		Method: Deflate,
-		Extra:  []byte{zip64ExtraId}, // missing size and second half of tag, but Extra is best-effort parsing
+		Extra:  []byte{zip64ExtraID}, // missing size and second half of tag, but Extra is best-effort parsing
 	}
 	testValidHeader(&h, t)
 }
 
+func TestHeaderTooLongErr(t *testing.T) {
+	var headerTests = []struct {
+		name    string
+		extra   []byte
+		wanterr error
+	}{
+		{
+			name:    strings.Repeat("x", 1<<16),
+			extra:   []byte{},
+			wanterr: errLongName,
+		},
+		{
+			name:    "long_extra",
+			extra:   bytes.Repeat([]byte{0xff}, 1<<16),
+			wanterr: errLongExtra,
+		},
+	}
+
+	// write a zip file
+	buf := new(bytes.Buffer)
+	w := NewWriter(buf)
+
+	for _, test := range headerTests {
+		h := &FileHeader{
+			Name:  test.name,
+			Extra: test.extra,
+		}
+		_, err := w.CreateHeader(h)
+		if err != test.wanterr {
+			t.Errorf("error=%v, want %v", err, test.wanterr)
+		}
+	}
+
+	if err := w.Close(); err != nil {
+		t.Fatal(err)
+	}
+}
+
 func TestHeaderIgnoredSize(t *testing.T) {
 	h := FileHeader{
 		Name:   "foo.txt",
 		Method: Deflate,
-		Extra:  []byte{zip64ExtraId & 0xFF, zip64ExtraId >> 8, 24, 0, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8}, // bad size but shouldn't be consulted
+		Extra:  []byte{zip64ExtraID & 0xFF, zip64ExtraID >> 8, 24, 0, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8}, // bad size but shouldn't be consulted
 	}
 	testValidHeader(&h, t)
 }

libgo/go/bufio/bufio.go

@@ -62,6 +62,9 @@ func NewReader(rd io.Reader) *Reader {
 	return NewReaderSize(rd, defaultBufSize)
 }
 
+// Size returns the size of the underlying buffer in bytes.
+func (r *Reader) Size() int { return len(r.buf) }
+
 // Reset discards any buffered data, resets all state, and switches
 // the buffered reader to read from r.
 func (b *Reader) Reset(r io.Reader) {
@@ -548,6 +551,9 @@ func NewWriter(w io.Writer) *Writer {
 	return NewWriterSize(w, defaultBufSize)
 }
 
+// Size returns the size of the underlying buffer in bytes.
+func (b *Writer) Size() int { return len(b.buf) }
+
 // Reset discards any unflushed buffered data, clears any error, and
 // resets b to write its output to w.
 func (b *Writer) Reset(w io.Writer) {

libgo/go/bufio/bufio_test.go

@@ -1418,6 +1418,24 @@ func TestReaderDiscard(t *testing.T) {
 
 }
 
+func TestReaderSize(t *testing.T) {
+	if got, want := NewReader(nil).Size(), DefaultBufSize; got != want {
+		t.Errorf("NewReader's Reader.Size = %d; want %d", got, want)
+	}
+	if got, want := NewReaderSize(nil, 1234).Size(), 1234; got != want {
+		t.Errorf("NewReaderSize's Reader.Size = %d; want %d", got, want)
+	}
+}
+
+func TestWriterSize(t *testing.T) {
+	if got, want := NewWriter(nil).Size(), DefaultBufSize; got != want {
+		t.Errorf("NewWriter's Writer.Size = %d; want %d", got, want)
+	}
+	if got, want := NewWriterSize(nil, 1234).Size(), 1234; got != want {
+		t.Errorf("NewWriterSize's Writer.Size = %d; want %d", got, want)
+	}
+}
+
 // An onlyReader only implements io.Reader, no matter what other methods the underlying implementation may have.
 type onlyReader struct {
 	io.Reader

libgo/go/bufio/export_test.go

@@ -11,6 +11,8 @@ import (
 
 var IsSpace = isSpace
 
+const DefaultBufSize = defaultBufSize
+
 func (s *Scanner) MaxTokenSize(n int) {
 	if n < utf8.UTFMax || n > 1e9 {
 		panic("bad max token size")

libgo/go/bufio/scan.go

@@ -123,8 +123,9 @@ var ErrFinalToken = errors.New("final token")
 // After Scan returns false, the Err method will return any error that
 // occurred during scanning, except that if it was io.EOF, Err
 // will return nil.
-// Scan panics if the split function returns 100 empty tokens without
+// Scan panics if the split function returns too many empty
-// advancing the input. This is a common error mode for scanners.
+// tokens without advancing the input. This is a common error mode for
+// scanners.
 func (s *Scanner) Scan() bool {
 	if s.done {
 		return false
@@ -156,8 +157,8 @@ func (s *Scanner) Scan() bool {
 				} else {
 					// Returning tokens not advancing input at EOF.
 					s.empties++
-					if s.empties > 100 {
+					if s.empties > maxConsecutiveEmptyReads {
-						panic("bufio.Scan: 100 empty tokens without progressing")
+						panic("bufio.Scan: too many empty tokens without progressing")
 					}
 				}
 				return true

libgo/go/builtin/builtin.go

@@ -171,8 +171,9 @@ func cap(v Type) int
 //	Slice: The size specifies the length. The capacity of the slice is
 //	equal to its length. A second integer argument may be provided to
 //	specify a different capacity; it must be no smaller than the
-//	length, so make([]int, 0, 10) allocates a slice of length 0 and
+//	length. For example, make([]int, 0, 10) allocates an underlying array
-//	capacity 10.
+//	of size 10 and returns a slice of length 0 and capacity 10 that is
+//	backed by this underlying array.
 //	Map: An empty map is allocated with enough space to hold the
 //	specified number of elements. The size may be omitted, in which case
 //	a small starting size is allocated.

libgo/go/bytes/boundary_test.go

@@ -0,0 +1,84 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+//
+// +build linux
+
+package bytes_test
+
+import (
+	. "bytes"
+	"syscall"
+	"testing"
+)
+
+// This file tests the situation where byte operations are checking
+// data very near to a page boundary. We want to make sure those
+// operations do not read across the boundary and cause a page
+// fault where they shouldn't.
+
+// These tests run only on linux. The code being tested is
+// not OS-specific, so it does not need to be tested on all
+// operating systems.
+
+// dangerousSlice returns a slice which is immediately
+// preceded and followed by a faulting page.
+func dangerousSlice(t *testing.T) []byte {
+	pagesize := syscall.Getpagesize()
+	b, err := syscall.Mmap(0, 0, 3*pagesize, syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_ANONYMOUS|syscall.MAP_PRIVATE)
+	if err != nil {
+		t.Fatalf("mmap failed %s", err)
+	}
+	err = syscall.Mprotect(b[:pagesize], syscall.PROT_NONE)
+	if err != nil {
+		t.Fatalf("mprotect low failed %s\n", err)
+	}
+	err = syscall.Mprotect(b[2*pagesize:], syscall.PROT_NONE)
+	if err != nil {
+		t.Fatalf("mprotect high failed %s\n", err)
+	}
+	return b[pagesize : 2*pagesize]
+}
+
+func TestEqualNearPageBoundary(t *testing.T) {
+	t.Parallel()
+	b := dangerousSlice(t)
+	for i := range b {
+		b[i] = 'A'
+	}
+	for i := 0; i <= len(b); i++ {
+		Equal(b[:i], b[len(b)-i:])
+		Equal(b[len(b)-i:], b[:i])
+	}
+}
+
+func TestIndexByteNearPageBoundary(t *testing.T) {
+	t.Parallel()
+	b := dangerousSlice(t)
+	for i := range b {
+		idx := IndexByte(b[i:], 1)
+		if idx != -1 {
+			t.Fatalf("IndexByte(b[%d:])=%d, want -1\n", i, idx)
+		}
+	}
+}
+
+func TestIndexNearPageBoundary(t *testing.T) {
+	t.Parallel()
+	var q [64]byte
+	b := dangerousSlice(t)
+	if len(b) > 256 {
+		// Only worry about when we're near the end of a page.
+		b = b[len(b)-256:]
+	}
+	for j := 1; j < len(q); j++ {
+		q[j-1] = 1 // difference is only found on the last byte
+		for i := range b {
+			idx := Index(b[i:], q[:j])
+			if idx != -1 {
+				t.Fatalf("Index(b[%d:], q[:%d])=%d, want -1\n", i, j, idx)
+			}
+		}
+		q[j-1] = 0
+	}
+}

libgo/go/bytes/buffer.go

@@ -17,32 +17,35 @@ import (
 type Buffer struct {
 	buf       []byte   // contents are the bytes buf[off : len(buf)]
 	off       int      // read at &buf[off], write at &buf[len(buf)]
+	bootstrap [64]byte // memory to hold first slice; helps small buffers avoid allocation.
 	lastRead  readOp   // last read operation, so that Unread* can work correctly.
-	// FIXME: lastRead can fit in a single byte
 
-	// memory to hold first slice; helps small buffers avoid allocation.
 	// FIXME: it would be advisable to align Buffer to cachelines to avoid false
 	// sharing.
-	bootstrap [64]byte
 }
 
 // The readOp constants describe the last action performed on
 // the buffer, so that UnreadRune and UnreadByte can check for
 // invalid usage. opReadRuneX constants are chosen such that
 // converted to int they correspond to the rune size that was read.
-type readOp int
+type readOp int8
 
+// Don't use iota for these, as the values need to correspond with the
+// names and comments, which is easier to see when being explicit.
 const (
 	opRead      readOp = -1 // Any other read operation.
-	opInvalid          = 0  // Non-read operation.
+	opInvalid   readOp = 0  // Non-read operation.
-	opReadRune1        = 1  // Read rune of size 1.
+	opReadRune1 readOp = 1  // Read rune of size 1.
-	opReadRune2        = 2  // Read rune of size 2.
+	opReadRune2 readOp = 2  // Read rune of size 2.
-	opReadRune3        = 3  // Read rune of size 3.
+	opReadRune3 readOp = 3  // Read rune of size 3.
-	opReadRune4        = 4  // Read rune of size 4.
+	opReadRune4 readOp = 4  // Read rune of size 4.
 )
 
 // ErrTooLarge is passed to panic if memory cannot be allocated to store data in a buffer.
 var ErrTooLarge = errors.New("bytes.Buffer: too large")
+var errNegativeRead = errors.New("bytes.Buffer: reader returned negative count from Read")
+
+const maxInt = int(^uint(0) >> 1)
 
 // Bytes returns a slice of length b.Len() holding the unread portion of the buffer.
 // The slice is valid for use only until the next buffer modification (that is,
@@ -53,6 +56,8 @@ func (b *Buffer) Bytes() []byte { return b.buf[b.off:] }
 
 // String returns the contents of the unread portion of the buffer
 // as a string. If the Buffer is a nil pointer, it returns "<nil>".
+//
+// To build strings more efficiently, see the strings.Builder type.
 func (b *Buffer) String() string {
 	if b == nil {
 		// Special case, useful in debugging.
@@ -61,6 +66,9 @@ func (b *Buffer) String() string {
 	return string(b.buf[b.off:])
 }
 
+// empty returns whether the unread portion of the buffer is empty.
+func (b *Buffer) empty() bool { return len(b.buf) <= b.off }
+
 // Len returns the number of bytes of the unread portion of the buffer;
 // b.Len() == len(b.Bytes()).
 func (b *Buffer) Len() int { return len(b.buf) - b.off }
@@ -81,7 +89,7 @@ func (b *Buffer) Truncate(n int) {
 	if n < 0 || n > b.Len() {
 		panic("bytes.Buffer: truncation out of range")
 	}
-	b.buf = b.buf[0 : b.off+n]
+	b.buf = b.buf[:b.off+n]
 }
 
 // Reset resets the buffer to be empty,
@@ -97,7 +105,7 @@ func (b *Buffer) Reset() {
 // internal buffer only needs to be resliced.
 // It returns the index where bytes should be written and whether it succeeded.
 func (b *Buffer) tryGrowByReslice(n int) (int, bool) {
-	if l := len(b.buf); l+n <= cap(b.buf) {
+	if l := len(b.buf); n <= cap(b.buf)-l {
 		b.buf = b.buf[:l+n]
 		return l, true
 	}
@@ -122,15 +130,18 @@ func (b *Buffer) grow(n int) int {
 		b.buf = b.bootstrap[:n]
 		return 0
 	}
-	if m+n <= cap(b.buf)/2 {
+	c := cap(b.buf)
+	if n <= c/2-m {
 		// We can slide things down instead of allocating a new
-		// slice. We only need m+n <= cap(b.buf) to slide, but
+		// slice. We only need m+n <= c to slide, but
 		// we instead let capacity get twice as large so we
 		// don't spend all our time copying.
-		copy(b.buf[:], b.buf[b.off:])
+		copy(b.buf, b.buf[b.off:])
+	} else if c > maxInt-c-n {
+		panic(ErrTooLarge)
 	} else {
 		// Not enough space anywhere, we need to allocate.
-		buf := makeSlice(2*cap(b.buf) + n)
+		buf := makeSlice(2*c + n)
 		copy(buf, b.buf[b.off:])
 		b.buf = buf
 	}
@@ -150,7 +161,7 @@ func (b *Buffer) Grow(n int) {
 		panic("bytes.Buffer.Grow: negative count")
 	}
 	m := b.grow(n)
-	b.buf = b.buf[0:m]
+	b.buf = b.buf[:m]
 }
 
 // Write appends the contents of p to the buffer, growing the buffer as
@@ -189,34 +200,22 @@ const MinRead = 512
 // buffer becomes too large, ReadFrom will panic with ErrTooLarge.
 func (b *Buffer) ReadFrom(r io.Reader) (n int64, err error) {
 	b.lastRead = opInvalid
-	// If buffer is empty, reset to recover space.
-	if b.off >= len(b.buf) {
-		b.Reset()
-	}
 	for {
-		if free := cap(b.buf) - len(b.buf); free < MinRead {
+		i := b.grow(MinRead)
-			// not enough space at end
+		m, e := r.Read(b.buf[i:cap(b.buf)])
-			newBuf := b.buf
+		if m < 0 {
-			if b.off+free < MinRead {
+			panic(errNegativeRead)
-				// not enough space using beginning of buffer;
-				// double buffer capacity
-				newBuf = makeSlice(2*cap(b.buf) + MinRead)
 		}
-			copy(newBuf, b.buf[b.off:])
+
-			b.buf = newBuf[:len(b.buf)-b.off]
+		b.buf = b.buf[:i+m]
-			b.off = 0
-		}
-		m, e := r.Read(b.buf[len(b.buf):cap(b.buf)])
-		b.buf = b.buf[0 : len(b.buf)+m]
 		n += int64(m)
 		if e == io.EOF {
-			break
+			return n, nil // e is EOF, so return nil explicitly
 		}
 		if e != nil {
 			return n, e
 		}
 	}
-	return n, nil // err is EOF, so return nil explicitly
 }
 
 // makeSlice allocates a slice of size n. If the allocation fails, it panics
@@ -237,8 +236,7 @@ func makeSlice(n int) []byte {
 // encountered during the write is also returned.
 func (b *Buffer) WriteTo(w io.Writer) (n int64, err error) {
 	b.lastRead = opInvalid
-	if b.off < len(b.buf) {
+	if nBytes := b.Len(); nBytes > 0 {
-		nBytes := b.Len()
 		m, e := w.Write(b.buf[b.off:])
 		if m > nBytes {
 			panic("bytes.Buffer.WriteTo: invalid Write count")
@@ -256,7 +254,7 @@ func (b *Buffer) WriteTo(w io.Writer) (n int64, err error) {
 	}
 	// Buffer is now empty; reset.
 	b.Reset()
-	return
+	return n, nil
 }
 
 // WriteByte appends the byte c to the buffer, growing the buffer as needed.
@@ -298,11 +296,11 @@ func (b *Buffer) WriteRune(r rune) (n int, err error) {
 // otherwise it is nil.
 func (b *Buffer) Read(p []byte) (n int, err error) {
 	b.lastRead = opInvalid
-	if b.off >= len(b.buf) {
+	if b.empty() {
 		// Buffer is empty, reset to recover space.
 		b.Reset()
 		if len(p) == 0 {
-			return
+			return 0, nil
 		}
 		return 0, io.EOF
 	}
@@ -311,7 +309,7 @@ func (b *Buffer) Read(p []byte) (n int, err error) {
 	if n > 0 {
 		b.lastRead = opRead
 	}
-	return
+	return n, nil
 }
 
 // Next returns a slice containing the next n bytes from the buffer,
@@ -335,8 +333,7 @@ func (b *Buffer) Next(n int) []byte {
 // ReadByte reads and returns the next byte from the buffer.
 // If no byte is available, it returns error io.EOF.
 func (b *Buffer) ReadByte() (byte, error) {
-	b.lastRead = opInvalid
+	if b.empty() {
-	if b.off >= len(b.buf) {
 		// Buffer is empty, reset to recover space.
 		b.Reset()
 		return 0, io.EOF
@@ -353,8 +350,7 @@ func (b *Buffer) ReadByte() (byte, error) {
 // If the bytes are an erroneous UTF-8 encoding, it
 // consumes one byte and returns U+FFFD, 1.
 func (b *Buffer) ReadRune() (r rune, size int, err error) {
-	b.lastRead = opInvalid
+	if b.empty() {
-	if b.off >= len(b.buf) {
 		// Buffer is empty, reset to recover space.
 		b.Reset()
 		return 0, 0, io.EOF
@@ -413,7 +409,7 @@ func (b *Buffer) ReadBytes(delim byte) (line []byte, err error) {
 	// return a copy of slice. The buffer's backing array may
 	// be overwritten by later calls.
 	line = append(line, slice...)
-	return
+	return line, err
 }
 
 // readSlice is like ReadBytes but returns a reference to internal buffer data.

libgo/go/bytes/buffer_test.go

@@ -6,25 +6,27 @@ package bytes_test
 
 import (
 	. "bytes"
-	"internal/testenv"
 	"io"
 	"math/rand"
-	"os/exec"
 	"runtime"
 	"testing"
 	"unicode/utf8"
 )
 
 const N = 10000       // make this bigger for a larger (and slower) test
-var data string      // test data for write tests
+var testString string // test data for write tests
-var testBytes []byte // test data; same as data but as a slice.
+var testBytes []byte  // test data; same as testString but as a slice.
+
+type negativeReader struct{}
+
+func (r *negativeReader) Read([]byte) (int, error) { return -1, nil }
 
 func init() {
 	testBytes = make([]byte, N)
 	for i := 0; i < N; i++ {
 		testBytes[i] = 'a' + byte(i%26)
 	}
-	data = string(testBytes)
+	testString = string(testBytes)
 }
 
 // Verify that contents of buf match the string s.
@@ -88,12 +90,12 @@ func fillBytes(t *testing.T, testname string, buf *Buffer, s string, n int, fub
 
 func TestNewBuffer(t *testing.T) {
 	buf := NewBuffer(testBytes)
-	check(t, "NewBuffer", buf, data)
+	check(t, "NewBuffer", buf, testString)
 }
 
 func TestNewBufferString(t *testing.T) {
-	buf := NewBufferString(data)
+	buf := NewBufferString(testString)
-	check(t, "NewBufferString", buf, data)
+	check(t, "NewBufferString", buf, testString)
 }
 
 // Empty buf through repeated reads into fub.
@@ -128,7 +130,7 @@ func TestBasicOperations(t *testing.T) {
 		buf.Truncate(0)
 		check(t, "TestBasicOperations (3)", &buf, "")
 
-		n, err := buf.Write([]byte(data[0:1]))
+		n, err := buf.Write(testBytes[0:1])
 		if n != 1 {
 			t.Errorf("wrote 1 byte, but n == %d", n)
 		}
@@ -137,30 +139,30 @@ func TestBasicOperations(t *testing.T) {
 		}
 		check(t, "TestBasicOperations (4)", &buf, "a")
 
-		buf.WriteByte(data[1])
+		buf.WriteByte(testString[1])
 		check(t, "TestBasicOperations (5)", &buf, "ab")
 
-		n, err = buf.Write([]byte(data[2:26]))
+		n, err = buf.Write(testBytes[2:26])
 		if n != 24 {
-			t.Errorf("wrote 25 bytes, but n == %d", n)
+			t.Errorf("wrote 24 bytes, but n == %d", n)
 		}
-		check(t, "TestBasicOperations (6)", &buf, string(data[0:26]))
+		check(t, "TestBasicOperations (6)", &buf, testString[0:26])
 
 		buf.Truncate(26)
-		check(t, "TestBasicOperations (7)", &buf, string(data[0:26]))
+		check(t, "TestBasicOperations (7)", &buf, testString[0:26])
 
 		buf.Truncate(20)
-		check(t, "TestBasicOperations (8)", &buf, string(data[0:20]))
+		check(t, "TestBasicOperations (8)", &buf, testString[0:20])
 
-		empty(t, "TestBasicOperations (9)", &buf, string(data[0:20]), make([]byte, 5))
+		empty(t, "TestBasicOperations (9)", &buf, testString[0:20], make([]byte, 5))
 		empty(t, "TestBasicOperations (10)", &buf, "", make([]byte, 100))
 
-		buf.WriteByte(data[1])
+		buf.WriteByte(testString[1])
 		c, err := buf.ReadByte()
 		if err != nil {
 			t.Error("ReadByte unexpected eof")
 		}
-		if c != data[1] {
+		if c != testString[1] {
 			t.Errorf("ReadByte wrong value c=%v", c)
 		}
 		c, err = buf.ReadByte()
@@ -177,8 +179,8 @@ func TestLargeStringWrites(t *testing.T) {
 		limit = 9
 	}
 	for i := 3; i < limit; i += 3 {
-		s := fillString(t, "TestLargeWrites (1)", &buf, "", 5, data)
+		s := fillString(t, "TestLargeWrites (1)", &buf, "", 5, testString)
-		empty(t, "TestLargeStringWrites (2)", &buf, s, make([]byte, len(data)/i))
+		empty(t, "TestLargeStringWrites (2)", &buf, s, make([]byte, len(testString)/i))
 	}
 	check(t, "TestLargeStringWrites (3)", &buf, "")
 }
@@ -191,7 +193,7 @@ func TestLargeByteWrites(t *testing.T) {
 	}
 	for i := 3; i < limit; i += 3 {
 		s := fillBytes(t, "TestLargeWrites (1)", &buf, "", 5, testBytes)
-		empty(t, "TestLargeByteWrites (2)", &buf, s, make([]byte, len(data)/i))
+		empty(t, "TestLargeByteWrites (2)", &buf, s, make([]byte, len(testString)/i))
 	}
 	check(t, "TestLargeByteWrites (3)", &buf, "")
 }
@@ -199,8 +201,8 @@ func TestLargeByteWrites(t *testing.T) {
 func TestLargeStringReads(t *testing.T) {
 	var buf Buffer
 	for i := 3; i < 30; i += 3 {
-		s := fillString(t, "TestLargeReads (1)", &buf, "", 5, data[0:len(data)/i])
+		s := fillString(t, "TestLargeReads (1)", &buf, "", 5, testString[0:len(testString)/i])
-		empty(t, "TestLargeReads (2)", &buf, s, make([]byte, len(data)))
+		empty(t, "TestLargeReads (2)", &buf, s, make([]byte, len(testString)))
 	}
 	check(t, "TestLargeStringReads (3)", &buf, "")
 }
@@ -209,7 +211,7 @@ func TestLargeByteReads(t *testing.T) {
 	var buf Buffer
 	for i := 3; i < 30; i += 3 {
 		s := fillBytes(t, "TestLargeReads (1)", &buf, "", 5, testBytes[0:len(testBytes)/i])
-		empty(t, "TestLargeReads (2)", &buf, s, make([]byte, len(data)))
+		empty(t, "TestLargeReads (2)", &buf, s, make([]byte, len(testString)))
 	}
 	check(t, "TestLargeByteReads (3)", &buf, "")
 }
@@ -218,14 +220,14 @@ func TestMixedReadsAndWrites(t *testing.T) {
 	var buf Buffer
 	s := ""
 	for i := 0; i < 50; i++ {
-		wlen := rand.Intn(len(data))
+		wlen := rand.Intn(len(testString))
 		if i%2 == 0 {
-			s = fillString(t, "TestMixedReadsAndWrites (1)", &buf, s, 1, data[0:wlen])
+			s = fillString(t, "TestMixedReadsAndWrites (1)", &buf, s, 1, testString[0:wlen])
 		} else {
 			s = fillBytes(t, "TestMixedReadsAndWrites (1)", &buf, s, 1, testBytes[0:wlen])
 		}
 
-		rlen := rand.Intn(len(data))
+		rlen := rand.Intn(len(testString))
 		fub := make([]byte, rlen)
 		n, _ := buf.Read(fub)
 		s = s[n:]
@@ -263,17 +265,37 @@ func TestReadFrom(t *testing.T) {
 		s := fillBytes(t, "TestReadFrom (1)", &buf, "", 5, testBytes[0:len(testBytes)/i])
 		var b Buffer
 		b.ReadFrom(&buf)
-		empty(t, "TestReadFrom (2)", &b, s, make([]byte, len(data)))
+		empty(t, "TestReadFrom (2)", &b, s, make([]byte, len(testString)))
 	}
 }
 
+func TestReadFromNegativeReader(t *testing.T) {
+	var b Buffer
+	defer func() {
+		switch err := recover().(type) {
+		case nil:
+			t.Fatal("bytes.Buffer.ReadFrom didn't panic")
+		case error:
+			// this is the error string of errNegativeRead
+			wantError := "bytes.Buffer: reader returned negative count from Read"
+			if err.Error() != wantError {
+				t.Fatalf("recovered panic: got %v, want %v", err.Error(), wantError)
+			}
+		default:
+			t.Fatalf("unexpected panic value: %#v", err)
+		}
+	}()
+
+	b.ReadFrom(new(negativeReader))
+}
+
 func TestWriteTo(t *testing.T) {
 	var buf Buffer
 	for i := 3; i < 30; i += 3 {
 		s := fillBytes(t, "TestWriteTo (1)", &buf, "", 5, testBytes[0:len(testBytes)/i])
 		var b Buffer
 		buf.WriteTo(&b)
-		empty(t, "TestWriteTo (2)", &b, s, make([]byte, len(data)))
+		empty(t, "TestWriteTo (2)", &b, s, make([]byte, len(testString)))
 	}
 }
 
@@ -473,6 +495,18 @@ func TestGrow(t *testing.T) {
 	}
 }
 
+func TestGrowOverflow(t *testing.T) {
+	defer func() {
+		if err := recover(); err != ErrTooLarge {
+			t.Errorf("after too-large Grow, recover() = %v; want %v", err, ErrTooLarge)
+		}
+	}()
+
+	buf := NewBuffer(make([]byte, 1))
+	const maxInt = int(^uint(0) >> 1)
+	buf.Grow(maxInt)
+}
+
 // Was a bug: used to give EOF reading empty slice at EOF.
 func TestReadEmptyAtEOF(t *testing.T) {
 	b := new(Buffer)
@@ -548,26 +582,6 @@ func TestBufferGrowth(t *testing.T) {
 	}
 }
 
-// Test that tryGrowByReslice is inlined.
-// Only execute on "linux-amd64" builder in order to avoid breakage.
-func TestTryGrowByResliceInlined(t *testing.T) {
-	targetBuilder := "linux-amd64"
-	if testenv.Builder() != targetBuilder {
-		t.Skipf("%q gets executed on %q builder only", t.Name(), targetBuilder)
-	}
-	t.Parallel()
-	goBin := testenv.GoToolPath(t)
-	out, err := exec.Command(goBin, "tool", "nm", goBin).CombinedOutput()
-	if err != nil {
-		t.Fatalf("go tool nm: %v: %s", err, out)
-	}
-	// Verify this doesn't exist:
-	sym := "bytes.(*Buffer).tryGrowByReslice"
-	if Contains(out, []byte(sym)) {
-		t.Errorf("found symbol %q in cmd/go, but should be inlined", sym)
-	}
-}
-
 func BenchmarkWriteByte(b *testing.B) {
 	const n = 4 << 10
 	b.SetBytes(n)

libgo/go/bytes/bytes.go

@@ -39,7 +39,7 @@ func explode(s []byte, n int) [][]byte {
 			break
 		}
 		_, size = utf8.DecodeRune(s)
-		a[na] = s[0:size]
+		a[na] = s[0:size:size]
 		s = s[size:]
 		na++
 	}
@@ -68,12 +68,12 @@ func Contains(b, subslice []byte) bool {
 	return Index(b, subslice) != -1
 }
 
-// ContainsAny reports whether any of the UTF-8-encoded Unicode code points in chars are within b.
+// ContainsAny reports whether any of the UTF-8-encoded code points in chars are within b.
 func ContainsAny(b []byte, chars string) bool {
 	return IndexAny(b, chars) >= 0
 }
 
-// ContainsRune reports whether the Unicode code point r is within b.
+// ContainsRune reports whether the rune is contained in the UTF-8-encoded byte slice b.
 func ContainsRune(b []byte, r rune) bool {
 	return IndexRune(b, r) >= 0
 }
@@ -112,7 +112,7 @@ func LastIndexByte(s []byte, c byte) int {
 	return -1
 }
 
-// IndexRune interprets s as a sequence of UTF-8-encoded Unicode code points.
+// IndexRune interprets s as a sequence of UTF-8-encoded code points.
 // It returns the byte index of the first occurrence in s of the given rune.
 // It returns -1 if rune is not present in s.
 // If r is utf8.RuneError, it returns the first instance of any
@@ -144,7 +144,10 @@ func IndexRune(s []byte, r rune) int {
 // code points in chars. It returns -1 if chars is empty or if there is no code
 // point in common.
 func IndexAny(s []byte, chars string) int {
-	if len(chars) > 0 {
+	if chars == "" {
+		// Avoid scanning all of s.
+		return -1
+	}
 	if len(s) > 8 {
 		if as, isASCII := makeASCIISet(chars); isASCII {
 			for i, c := range s {
@@ -169,7 +172,6 @@ func IndexAny(s []byte, chars string) int {
 			}
 		}
 	}
-	}
 	return -1
 }
 
@@ -178,7 +180,10 @@ func IndexAny(s []byte, chars string) int {
 // the Unicode code points in chars. It returns -1 if chars is empty or if
 // there is no code point in common.
 func LastIndexAny(s []byte, chars string) int {
-	if len(chars) > 0 {
+	if chars == "" {
+		// Avoid scanning all of s.
+		return -1
+	}
 	if len(s) > 8 {
 		if as, isASCII := makeASCIISet(chars); isASCII {
 			for i := len(s) - 1; i >= 0; i-- {
@@ -198,7 +203,6 @@ func LastIndexAny(s []byte, chars string) int {
 			}
 		}
 	}
-	}
 	return -1
 }
 
@@ -223,7 +227,7 @@ func genSplit(s, sep []byte, sepSave, n int) [][]byte {
 		if m < 0 {
 			break
 		}
-		a[i] = s[:m+sepSave]
+		a[i] = s[: m+sepSave : m+sepSave]
 		s = s[m+len(sep):]
 		i++
 	}
@@ -265,52 +269,112 @@ func SplitAfter(s, sep []byte) [][]byte {
 	return genSplit(s, sep, len(sep), -1)
 }
 
-// Fields splits the slice s around each instance of one or more consecutive white space
+var asciiSpace = [256]uint8{'\t': 1, '\n': 1, '\v': 1, '\f': 1, '\r': 1, ' ': 1}
-// characters, returning a slice of subslices of s or an empty list if s contains only white space.
+
+// Fields interprets s as a sequence of UTF-8-encoded code points.
+// It splits the slice s around each instance of one or more consecutive white space
+// characters, as defined by unicode.IsSpace, returning a slice of subslices of s or an
+// empty slice if s contains only white space.
 func Fields(s []byte) [][]byte {
+	// First count the fields.
+	// This is an exact count if s is ASCII, otherwise it is an approximation.
+	n := 0
+	wasSpace := 1
+	// setBits is used to track which bits are set in the bytes of s.
+	setBits := uint8(0)
+	for i := 0; i < len(s); i++ {
+		r := s[i]
+		setBits |= r
+		isSpace := int(asciiSpace[r])
+		n += wasSpace & ^isSpace
+		wasSpace = isSpace
+	}
+
+	if setBits >= utf8.RuneSelf {
+		// Some runes in the input slice are not ASCII.
 		return FieldsFunc(s, unicode.IsSpace)
+	}
+
+	// ASCII fast path
+	a := make([][]byte, n)
+	na := 0
+	fieldStart := 0
+	i := 0
+	// Skip spaces in the front of the input.
+	for i < len(s) && asciiSpace[s[i]] != 0 {
+		i++
+	}
+	fieldStart = i
+	for i < len(s) {
+		if asciiSpace[s[i]] == 0 {
+			i++
+			continue
+		}
+		a[na] = s[fieldStart:i:i]
+		na++
+		i++
+		// Skip spaces in between fields.
+		for i < len(s) && asciiSpace[s[i]] != 0 {
+			i++
+		}
+		fieldStart = i
+	}
+	if fieldStart < len(s) { // Last field might end at EOF.
+		a[na] = s[fieldStart:len(s):len(s)]
+	}
+	return a
 }
 
-// FieldsFunc interprets s as a sequence of UTF-8-encoded Unicode code points.
+// FieldsFunc interprets s as a sequence of UTF-8-encoded code points.
 // It splits the slice s at each run of code points c satisfying f(c) and
 // returns a slice of subslices of s. If all code points in s satisfy f(c), or
 // len(s) == 0, an empty slice is returned.
 // FieldsFunc makes no guarantees about the order in which it calls f(c).
 // If f does not return consistent results for a given c, FieldsFunc may crash.
 func FieldsFunc(s []byte, f func(rune) bool) [][]byte {
-	n := 0
+	// A span is used to record a slice of s of the form s[start:end].
-	inField := false
+	// The start index is inclusive and the end index is exclusive.
+	type span struct {
+		start int
+		end   int
+	}
+	spans := make([]span, 0, 32)
+
+	// Find the field start and end indices.
+	wasField := false
+	fromIndex := 0
 	for i := 0; i < len(s); {
-		r, size := utf8.DecodeRune(s[i:])
+		size := 1
-		wasInField := inField
+		r := rune(s[i])
-		inField = !f(r)
+		if r >= utf8.RuneSelf {
-		if inField && !wasInField {
+			r, size = utf8.DecodeRune(s[i:])
-			n++
+		}
+		if f(r) {
+			if wasField {
+				spans = append(spans, span{start: fromIndex, end: i})
+				wasField = false
+			}
+		} else {
+			if !wasField {
+				fromIndex = i
+				wasField = true
+			}
 		}
 		i += size
 	}
 
-	a := make([][]byte, n)
+	// Last field might end at EOF.
-	na := 0
+	if wasField {
-	fieldStart := -1
+		spans = append(spans, span{fromIndex, len(s)})
-	for i := 0; i <= len(s) && na < n; {
-		r, size := utf8.DecodeRune(s[i:])
-		if fieldStart < 0 && size > 0 && !f(r) {
-			fieldStart = i
-			i += size
-			continue
 	}
-		if fieldStart >= 0 && (size == 0 || f(r)) {
+
-			a[na] = s[fieldStart:i]
+	// Create subslices from recorded field indices.
-			na++
+	a := make([][]byte, len(spans))
-			fieldStart = -1
+	for i, span := range spans {
+		a[i] = s[span.start:span.end:span.end]
 	}
-		if size == 0 {
+
-			break
+	return a
-		}
-		i += size
-	}
-	return a[0:na]
 }
 
 // Join concatenates the elements of s to create a new byte slice. The separator
@@ -349,8 +413,8 @@ func HasSuffix(s, suffix []byte) bool {
 
 // Map returns a copy of the byte slice s with all its characters modified
 // according to the mapping function. If mapping returns a negative value, the character is
-// dropped from the string with no replacement. The characters in s and the
+// dropped from the byte slice with no replacement. The characters in s and the
-// output are interpreted as UTF-8-encoded Unicode code points.
+// output are interpreted as UTF-8-encoded code points.
 func Map(mapping func(r rune) rune, s []byte) []byte {
 	// In the worst case, the slice can grow when mapped, making
 	// things unpleasant. But it's so rare we barge in assuming it's
@@ -408,28 +472,28 @@ func Repeat(b []byte, count int) []byte {
 	return nb
 }
 
-// ToUpper returns a copy of the byte slice s with all Unicode letters mapped to their upper case.
+// ToUpper treats s as UTF-8-encoded bytes and returns a copy with all the Unicode letters within it mapped to their upper case.
 func ToUpper(s []byte) []byte { return Map(unicode.ToUpper, s) }
 
-// ToLower returns a copy of the byte slice s with all Unicode letters mapped to their lower case.
+// ToLower treats s as UTF-8-encoded bytes and returns a copy with all the Unicode letters mapped to their lower case.
 func ToLower(s []byte) []byte { return Map(unicode.ToLower, s) }
 
-// ToTitle returns a copy of the byte slice s with all Unicode letters mapped to their title case.
+// ToTitle treats s as UTF-8-encoded bytes and returns a copy with all the Unicode letters mapped to their title case.
 func ToTitle(s []byte) []byte { return Map(unicode.ToTitle, s) }
 
-// ToUpperSpecial returns a copy of the byte slice s with all Unicode letters mapped to their
+// ToUpperSpecial treats s as UTF-8-encoded bytes and returns a copy with all the Unicode letters mapped to their
 // upper case, giving priority to the special casing rules.
 func ToUpperSpecial(c unicode.SpecialCase, s []byte) []byte {
 	return Map(func(r rune) rune { return c.ToUpper(r) }, s)
 }
 
-// ToLowerSpecial returns a copy of the byte slice s with all Unicode letters mapped to their
+// ToLowerSpecial treats s as UTF-8-encoded bytes and returns a copy with all the Unicode letters mapped to their
 // lower case, giving priority to the special casing rules.
 func ToLowerSpecial(c unicode.SpecialCase, s []byte) []byte {
 	return Map(func(r rune) rune { return c.ToLower(r) }, s)
 }
 
-// ToTitleSpecial returns a copy of the byte slice s with all Unicode letters mapped to their
+// ToTitleSpecial treats s as UTF-8-encoded bytes and returns a copy with all the Unicode letters mapped to their
 // title case, giving priority to the special casing rules.
 func ToTitleSpecial(c unicode.SpecialCase, s []byte) []byte {
 	return Map(func(r rune) rune { return c.ToTitle(r) }, s)
@@ -460,8 +524,8 @@ func isSeparator(r rune) bool {
 	return unicode.IsSpace(r)
 }
 
-// Title returns a copy of s with all Unicode letters that begin words
+// Title treats s as UTF-8-encoded bytes and returns a copy with all Unicode letters that begin
-// mapped to their title case.
+// words mapped to their title case.
 //
 // BUG(rsc): The rule Title uses for word boundaries does not handle Unicode punctuation properly.
 func Title(s []byte) []byte {
@@ -481,8 +545,8 @@ func Title(s []byte) []byte {
 		s)
 }
 
-// TrimLeftFunc returns a subslice of s by slicing off all leading UTF-8-encoded
+// TrimLeftFunc treats s as UTF-8-encoded bytes and returns a subslice of s by slicing off
-// Unicode code points c that satisfy f(c).
+// all leading UTF-8-encoded code points c that satisfy f(c).
 func TrimLeftFunc(s []byte, f func(r rune) bool) []byte {
 	i := indexFunc(s, f, false)
 	if i == -1 {
@@ -491,8 +555,8 @@ func TrimLeftFunc(s []byte, f func(r rune) bool) []byte {
 	return s[i:]
 }
 
-// TrimRightFunc returns a subslice of s by slicing off all trailing UTF-8
+// TrimRightFunc returns a subslice of s by slicing off all trailing
-// encoded Unicode code points c that satisfy f(c).
+// UTF-8-encoded code points c that satisfy f(c).
 func TrimRightFunc(s []byte, f func(r rune) bool) []byte {
 	i := lastIndexFunc(s, f, false)
 	if i >= 0 && s[i] >= utf8.RuneSelf {
@@ -505,7 +569,7 @@ func TrimRightFunc(s []byte, f func(r rune) bool) []byte {
 }
 
 // TrimFunc returns a subslice of s by slicing off all leading and trailing
-// UTF-8-encoded Unicode code points c that satisfy f(c).
+// UTF-8-encoded code points c that satisfy f(c).
 func TrimFunc(s []byte, f func(r rune) bool) []byte {
 	return TrimRightFunc(TrimLeftFunc(s, f), f)
 }
@@ -528,14 +592,14 @@ func TrimSuffix(s, suffix []byte) []byte {
 	return s
 }
 
-// IndexFunc interprets s as a sequence of UTF-8-encoded Unicode code points.
+// IndexFunc interprets s as a sequence of UTF-8-encoded code points.
 // It returns the byte index in s of the first Unicode
 // code point satisfying f(c), or -1 if none do.
 func IndexFunc(s []byte, f func(r rune) bool) int {
 	return indexFunc(s, f, true)
 }
 
-// LastIndexFunc interprets s as a sequence of UTF-8-encoded Unicode code points.
+// LastIndexFunc interprets s as a sequence of UTF-8-encoded code points.
 // It returns the byte index in s of the last Unicode
 // code point satisfying f(c), or -1 if none do.
 func LastIndexFunc(s []byte, f func(r rune) bool) int {
@@ -626,19 +690,19 @@ func makeCutsetFunc(cutset string) func(r rune) bool {
 }
 
 // Trim returns a subslice of s by slicing off all leading and
-// trailing UTF-8-encoded Unicode code points contained in cutset.
+// trailing UTF-8-encoded code points contained in cutset.
 func Trim(s []byte, cutset string) []byte {
 	return TrimFunc(s, makeCutsetFunc(cutset))
 }
 
 // TrimLeft returns a subslice of s by slicing off all leading
-// UTF-8-encoded Unicode code points contained in cutset.
+// UTF-8-encoded code points contained in cutset.
 func TrimLeft(s []byte, cutset string) []byte {
 	return TrimLeftFunc(s, makeCutsetFunc(cutset))
 }
 
 // TrimRight returns a subslice of s by slicing off all trailing
-// UTF-8-encoded Unicode code points that are contained in cutset.
+// UTF-8-encoded code points that are contained in cutset.
 func TrimRight(s []byte, cutset string) []byte {
 	return TrimRightFunc(s, makeCutsetFunc(cutset))
 }
@@ -649,7 +713,8 @@ func TrimSpace(s []byte) []byte {
 	return TrimFunc(s, unicode.IsSpace)
 }
 
-// Runes returns a slice of runes (Unicode code points) equivalent to s.
+// Runes interprets s as a sequence of UTF-8-encoded code points.
+// It returns a slice of runes (Unicode code points) equivalent to s.
 func Runes(s []byte) []rune {
 	t := make([]rune, utf8.RuneCount(s))
 	i := 0
@@ -758,3 +823,46 @@ func EqualFold(s, t []byte) bool {
 	// One string is empty. Are both?
 	return len(s) == len(t)
 }
+
+func indexRabinKarp(s, sep []byte) int {
+	// Rabin-Karp search
+	hashsep, pow := hashStr(sep)
+	n := len(sep)
+	var h uint32
+	for i := 0; i < n; i++ {
+		h = h*primeRK + uint32(s[i])
+	}
+	if h == hashsep && Equal(s[:n], sep) {
+		return 0
+	}
+	for i := n; i < len(s); {
+		h *= primeRK
+		h += uint32(s[i])
+		h -= pow * uint32(s[i-n])
+		i++
+		if h == hashsep && Equal(s[i-n:i], sep) {
+			return i - n
+		}
+	}
+	return -1
+}
+
+// primeRK is the prime base used in Rabin-Karp algorithm.
+const primeRK = 16777619
+
+// hashStr returns the hash and the appropriate multiplicative
+// factor for use in Rabin-Karp algorithm.
+func hashStr(sep []byte) (uint32, uint32) {
+	hash := uint32(0)
+	for i := 0; i < len(sep); i++ {
+		hash = hash*primeRK + uint32(sep[i])
+	}
+	var pow, sq uint32 = 1, primeRK
+	for i := len(sep); i > 0; i >>= 1 {
+		if i&1 != 0 {
+			pow *= sq
+		}
+		sq *= sq
+	}
+	return hash, pow
+}

libgo/go/bytes/bytes_amd64.go

@@ -77,52 +77,14 @@ func Index(s, sep []byte) int {
 		}
 		return -1
 	}
-	// Rabin-Karp search
+	return indexRabinKarp(s, sep)
-	hashsep, pow := hashStr(sep)
-	var h uint32
-	for i := 0; i < n; i++ {
-		h = h*primeRK + uint32(s[i])
-	}
-	if h == hashsep && Equal(s[:n], sep) {
-		return 0
-	}
-	for i := n; i < len(s); {
-		h *= primeRK
-		h += uint32(s[i])
-		h -= pow * uint32(s[i-n])
-		i++
-		if h == hashsep && Equal(s[i-n:i], sep) {
-			return i - n
-		}
-	}
-	return -1
 }
 
 // Count counts the number of non-overlapping instances of sep in s.
-// If sep is an empty slice, Count returns 1 + the number of Unicode code points in s.
+// If sep is an empty slice, Count returns 1 + the number of UTF-8-encoded code points in s.
 func Count(s, sep []byte) int {
 	if len(sep) == 1 && cpu.X86.HasPOPCNT {
 		return countByte(s, sep[0])
 	}
 	return countGeneric(s, sep)
 }
-
-// primeRK is the prime base used in Rabin-Karp algorithm.
-const primeRK = 16777619
-
-// hashStr returns the hash and the appropriate multiplicative
-// factor for use in Rabin-Karp algorithm.
-func hashStr(sep []byte) (uint32, uint32) {
-	hash := uint32(0)
-	for i := 0; i < len(sep); i++ {
-		hash = hash*primeRK + uint32(sep[i])
-	}
-	var pow, sq uint32 = 1, primeRK
-	for i := len(sep); i > 0; i >>= 1 {
-		if i&1 != 0 {
-			pow *= sq
-		}
-		sq *= sq
-	}
-	return hash, pow
-}

libgo/go/bytes/bytes_arm64.go

@@ -0,0 +1,70 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+package bytes
+
+func countByte(s []byte, c byte) int // bytes_arm64.s
+
+// Index returns the index of the first instance of sep in s, or -1 if sep is not present in s.
+func Index(s, sep []byte) int {
+	n := len(sep)
+	switch {
+	case n == 0:
+		return 0
+	case n == 1:
+		return IndexByte(s, sep[0])
+	case n == len(s):
+		if Equal(sep, s) {
+			return 0
+		}
+		return -1
+	case n > len(s):
+		return -1
+	}
+	c := sep[0]
+	i := 0
+	fails := 0
+	t := s[:len(s)-n+1]
+	for i < len(t) {
+		if t[i] != c {
+			o := IndexByte(t[i:], c)
+			if o < 0 {
+				break
+			}
+			i += o
+		}
+		if Equal(s[i:i+n], sep) {
+			return i
+		}
+		i++
+		fails++
+		if fails >= 4+i>>4 && i < len(t) {
+			// Give up on IndexByte, it isn't skipping ahead
+			// far enough to be better than Rabin-Karp.
+			// Experiments (using IndexPeriodic) suggest
+			// the cutover is about 16 byte skips.
+			// TODO: if large prefixes of sep are matching
+			// we should cutover at even larger average skips,
+			// because Equal becomes that much more expensive.
+			// This code does not take that effect into account.
+			j := indexRabinKarp(s[i:], sep)
+			if j < 0 {
+				return -1
+			}
+			return i + j
+		}
+	}
+	return -1
+}
+
+// Count counts the number of non-overlapping instances of sep in s.
+// If sep is an empty slice, Count returns 1 + the number of UTF-8-encoded code points in s.
+func Count(s, sep []byte) int {
+	if len(sep) == 1 {
+		return countByte(s, sep[0])
+	}
+	return countGeneric(s, sep)
+}

libgo/go/bytes/bytes_generic.go

@@ -2,27 +2,29 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// -build !amd64,!s390x
+// -build !amd64,!s390x,!arm64
 
 package bytes
 
-// TODO: implements short string optimization on non amd64 platforms
-// and get rid of bytes_amd64.go
-
 // Index returns the index of the first instance of sep in s, or -1 if sep is not present in s.
 func Index(s, sep []byte) int {
 	n := len(sep)
-	if n == 0 {
+	switch {
+	case n == 0:
+		return 0
+	case n == 1:
+		return IndexByte(s, sep[0])
+	case n == len(s):
+		if Equal(sep, s) {
 			return 0
 		}
-	if n > len(s) {
+		return -1
+	case n > len(s):
 		return -1
 	}
 	c := sep[0]
-	if n == 1 {
-		return IndexByte(s, c)
-	}
 	i := 0
+	fails := 0
 	t := s[:len(s)-n+1]
 	for i < len(t) {
 		if t[i] != c {
@@ -36,12 +38,28 @@ func Index(s, sep []byte) int {
 			return i
 		}
 		i++
+		fails++
+		if fails >= 4+i>>4 && i < len(t) {
+			// Give up on IndexByte, it isn't skipping ahead
+			// far enough to be better than Rabin-Karp.
+			// Experiments (using IndexPeriodic) suggest
+			// the cutover is about 16 byte skips.
+			// TODO: if large prefixes of sep are matching
+			// we should cutover at even larger average skips,
+			// because Equal becomes that much more expensive.
+			// This code does not take that effect into account.
+			j := indexRabinKarp(s[i:], sep)
+			if j < 0 {
+				return -1
+			}
+			return i + j
+		}
 	}
 	return -1
 }
 
 // Count counts the number of non-overlapping instances of sep in s.
-// If sep is an empty slice, Count returns 1 + the number of Unicode code points in s.
+// If sep is an empty slice, Count returns 1 + the number of UTF-8-encoded code points in s.
 func Count(s, sep []byte) int {
 	return countGeneric(s, sep)
 }

libgo/go/bytes/bytes_s390x.go

@@ -78,49 +78,11 @@ func Index(s, sep []byte) int {
 		}
 		return -1
 	}
-	// Rabin-Karp search
+	return indexRabinKarp(s, sep)
-	hashsep, pow := hashStr(sep)
-	var h uint32
-	for i := 0; i < n; i++ {
-		h = h*primeRK + uint32(s[i])
-	}
-	if h == hashsep && Equal(s[:n], sep) {
-		return 0
-	}
-	for i := n; i < len(s); {
-		h *= primeRK
-		h += uint32(s[i])
-		h -= pow * uint32(s[i-n])
-		i++
-		if h == hashsep && Equal(s[i-n:i], sep) {
-			return i - n
-		}
-	}
-	return -1
 }
 
 // Count counts the number of non-overlapping instances of sep in s.
-// If sep is an empty slice, Count returns 1 + the number of Unicode code points in s.
+// If sep is an empty slice, Count returns 1 + the number of UTF-8-encoded code points in s.
 func Count(s, sep []byte) int {
 	return countGeneric(s, sep)
 }
-
-// primeRK is the prime base used in Rabin-Karp algorithm.
-const primeRK = 16777619
-
-// hashStr returns the hash and the appropriate multiplicative
-// factor for use in Rabin-Karp algorithm.
-func hashStr(sep []byte) (uint32, uint32) {
-	hash := uint32(0)
-	for i := 0; i < len(sep); i++ {
-		hash = hash*primeRK + uint32(sep[i])
-	}
-	var pow, sq uint32 = 1, primeRK
-	for i := len(sep); i > 0; i >>= 1 {
-		if i&1 != 0 {
-			pow *= sq
-		}
-		sq *= sq
-	}
-	return hash, pow
-}

libgo/go/bytes/bytes_test.go

@@ -140,6 +140,9 @@ var indexTests = []BinOpTest{
 	{"barfoobarfooyyyzzzyyyzzzyyyzzzyyyxxxzzzyyy", "x", 33},
 	{"foofyfoobarfoobar", "y", 4},
 	{"oooooooooooooooooooooo", "r", -1},
+	// test fallback to Rabin-Karp.
+	{"oxoxoxoxoxoxoxoxoxoxoxoy", "oy", 22},
+	{"oxoxoxoxoxoxoxoxoxoxoxox", "oy", -1},
 }
 
 var lastIndexTests = []BinOpTest{
@@ -741,6 +744,13 @@ var splittests = []SplitTest{
 func TestSplit(t *testing.T) {
 	for _, tt := range splittests {
 		a := SplitN([]byte(tt.s), []byte(tt.sep), tt.n)
+
+		// Appending to the results should not change future results.
+		var x []byte
+		for _, v := range a {
+			x = append(v, 'z')
+		}
+
 		result := sliceOfString(a)
 		if !eq(result, tt.a) {
 			t.Errorf(`Split(%q, %q, %d) = %v; want %v`, tt.s, tt.sep, tt.n, result, tt.a)
@@ -749,6 +759,11 @@ func TestSplit(t *testing.T) {
 		if tt.n == 0 {
 			continue
 		}
+
+		if want := tt.a[len(tt.a)-1] + "z"; string(x) != want {
+			t.Errorf("last appended result was %s; want %s", x, want)
+		}
+
 		s := Join(a, []byte(tt.sep))
 		if string(s) != tt.s {
 			t.Errorf(`Join(Split(%q, %q, %d), %q) = %q`, tt.s, tt.sep, tt.n, tt.sep, s)
@@ -787,11 +802,23 @@ var splitaftertests = []SplitTest{
 func TestSplitAfter(t *testing.T) {
 	for _, tt := range splitaftertests {
 		a := SplitAfterN([]byte(tt.s), []byte(tt.sep), tt.n)
+
+		// Appending to the results should not change future results.
+		var x []byte
+		for _, v := range a {
+			x = append(v, 'z')
+		}
+
 		result := sliceOfString(a)
 		if !eq(result, tt.a) {
 			t.Errorf(`Split(%q, %q, %d) = %v; want %v`, tt.s, tt.sep, tt.n, result, tt.a)
 			continue
 		}
+
+		if want := tt.a[len(tt.a)-1] + "z"; string(x) != want {
+			t.Errorf("last appended result was %s; want %s", x, want)
+		}
+
 		s := Join(a, nil)
 		if string(s) != tt.s {
 			t.Errorf(`Join(Split(%q, %q, %d), %q) = %q`, tt.s, tt.sep, tt.n, tt.sep, s)
@@ -826,12 +853,29 @@ var fieldstests = []FieldsTest{
 
 func TestFields(t *testing.T) {
 	for _, tt := range fieldstests {
-		a := Fields([]byte(tt.s))
+		b := []byte(tt.s)
+		a := Fields(b)
+
+		// Appending to the results should not change future results.
+		var x []byte
+		for _, v := range a {
+			x = append(v, 'z')
+		}
+
 		result := sliceOfString(a)
 		if !eq(result, tt.a) {
 			t.Errorf("Fields(%q) = %v; want %v", tt.s, a, tt.a)
 			continue
 		}
+
+		if string(b) != tt.s {
+			t.Errorf("slice changed to %s; want %s", string(b), tt.s)
+		}
+		if len(tt.a) > 0 {
+			if want := tt.a[len(tt.a)-1] + "z"; string(x) != want {
+				t.Errorf("last appended result was %s; want %s", x, want)
+			}
+		}
 	}
 }
 
@@ -852,11 +896,28 @@ func TestFieldsFunc(t *testing.T) {
 		{"aXXbXXXcX", []string{"a", "b", "c"}},
 	}
 	for _, tt := range fieldsFuncTests {
-		a := FieldsFunc([]byte(tt.s), pred)
+		b := []byte(tt.s)
+		a := FieldsFunc(b, pred)
+
+		// Appending to the results should not change future results.
+		var x []byte
+		for _, v := range a {
+			x = append(v, 'z')
+		}
+
 		result := sliceOfString(a)
 		if !eq(result, tt.a) {
 			t.Errorf("FieldsFunc(%q) = %v, want %v", tt.s, a, tt.a)
 		}
+
+		if string(b) != tt.s {
+			t.Errorf("slice changed to %s; want %s", b, tt.s)
+		}
+		if len(tt.a) > 0 {
+			if want := tt.a[len(tt.a)-1] + "z"; string(x) != want {
+				t.Errorf("last appended result was %s; want %s", x, want)
+			}
+		}
 	}
 }
 
@@ -1507,19 +1568,58 @@ var makeFieldsInput = func() []byte {
 	return x
 }
 
-var fieldsInput = makeFieldsInput()
+var makeFieldsInputASCII = func() []byte {
+	x := make([]byte, 1<<20)
+	// Input is ~10% space, rest ASCII non-space.
+	for i := range x {
+		if rand.Intn(10) == 0 {
+			x[i] = ' '
+		} else {
+			x[i] = 'x'
+		}
+	}
+	return x
+}
+
+var bytesdata = []struct {
+	name string
+	data []byte
+}{
+	{"ASCII", makeFieldsInputASCII()},
+	{"Mixed", makeFieldsInput()},
+}
 
 func BenchmarkFields(b *testing.B) {
-	b.SetBytes(int64(len(fieldsInput)))
+	for _, sd := range bytesdata {
+		b.Run(sd.name, func(b *testing.B) {
+			for j := 1 << 4; j <= 1<<20; j <<= 4 {
+				b.Run(fmt.Sprintf("%d", j), func(b *testing.B) {
+					b.ReportAllocs()
+					b.SetBytes(int64(j))
+					data := sd.data[:j]
 					for i := 0; i < b.N; i++ {
-		Fields(fieldsInput)
+						Fields(data)
+					}
+				})
+			}
+		})
 	}
 }
 
 func BenchmarkFieldsFunc(b *testing.B) {
-	b.SetBytes(int64(len(fieldsInput)))
+	for _, sd := range bytesdata {
+		b.Run(sd.name, func(b *testing.B) {
+			for j := 1 << 4; j <= 1<<20; j <<= 4 {
+				b.Run(fmt.Sprintf("%d", j), func(b *testing.B) {
+					b.ReportAllocs()
+					b.SetBytes(int64(j))
+					data := sd.data[:j]
 					for i := 0; i < b.N; i++ {
-		FieldsFunc(fieldsInput, unicode.IsSpace)
+						FieldsFunc(data, unicode.IsSpace)
+					}
+				})
+			}
+		})
 	}
 }
 
@@ -1638,3 +1738,18 @@ func BenchmarkTrimASCII(b *testing.B) {
 		}
 	}
 }
+
+func BenchmarkIndexPeriodic(b *testing.B) {
+	key := []byte{1, 1}
+	for _, skip := range [...]int{2, 4, 8, 16, 32, 64} {
+		b.Run(fmt.Sprintf("IndexPeriodic%d", skip), func(b *testing.B) {
+			buf := make([]byte, 1<<16)
+			for i := 0; i < len(buf); i += skip {
+				buf[i] = 1
+			}
+			for i := 0; i < b.N; i++ {
+				Index(buf, key)
+			}
+		})
+	}
+}

libgo/go/bytes/equal_test.go

@@ -1,47 +0,0 @@
-// Copyright 2013 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-//
-// +build linux
-
-package bytes_test
-
-import (
-	. "bytes"
-	"syscall"
-	"testing"
-	"unsafe"
-)
-
-// This file tests the situation where memeq is checking
-// data very near to a page boundary. We want to make sure
-// equal does not read across the boundary and cause a page
-// fault where it shouldn't.
-
-// This test runs only on linux. The code being tested is
-// not OS-specific, so it does not need to be tested on all
-// operating systems.
-
-func TestEqualNearPageBoundary(t *testing.T) {
-	pagesize := syscall.Getpagesize()
-	b := make([]byte, 4*pagesize)
-	i := pagesize
-	for ; uintptr(unsafe.Pointer(&b[i]))%uintptr(pagesize) != 0; i++ {
-	}
-	syscall.Mprotect(b[i-pagesize:i], 0)
-	syscall.Mprotect(b[i+pagesize:i+2*pagesize], 0)
-	defer syscall.Mprotect(b[i-pagesize:i], syscall.PROT_READ|syscall.PROT_WRITE)
-	defer syscall.Mprotect(b[i+pagesize:i+2*pagesize], syscall.PROT_READ|syscall.PROT_WRITE)
-
-	// both of these should fault
-	//pagesize += int(b[i-1])
-	//pagesize += int(b[i+pagesize])
-
-	for j := 0; j < pagesize; j++ {
-		b[i+j] = 'A'
-	}
-	for j := 0; j <= pagesize; j++ {
-		Equal(b[i:i+j], b[i+pagesize-j:i+pagesize])
-		Equal(b[i+pagesize-j:i+pagesize], b[i:i+j])
-	}
-}

libgo/go/bytes/example_test.go

@@ -119,6 +119,32 @@ func ExampleContains() {
 	// true
 }
 
+func ExampleContainsAny() {
+	fmt.Println(bytes.ContainsAny([]byte("I like seafood."), "fÄo!"))
+	fmt.Println(bytes.ContainsAny([]byte("I like seafood."), "去是伟大的."))
+	fmt.Println(bytes.ContainsAny([]byte("I like seafood."), ""))
+	fmt.Println(bytes.ContainsAny([]byte(""), ""))
+	// Output:
+	// true
+	// true
+	// false
+	// false
+}
+
+func ExampleContainsRune() {
+	fmt.Println(bytes.ContainsRune([]byte("I like seafood."), 'f'))
+	fmt.Println(bytes.ContainsRune([]byte("I like seafood."), 'ö'))
+	fmt.Println(bytes.ContainsRune([]byte("去是伟大的!"), '大'))
+	fmt.Println(bytes.ContainsRune([]byte("去是伟大的!"), '!'))
+	fmt.Println(bytes.ContainsRune([]byte(""), '@'))
+	// Output:
+	// true
+	// false
+	// true
+	// true
+	// false
+}
+
 func ExampleCount() {
 	fmt.Println(bytes.Count([]byte("cheese"), []byte("e")))
 	fmt.Println(bytes.Count([]byte("five"), []byte(""))) // before & after each rune
@@ -127,6 +153,14 @@ func ExampleCount() {
 	// 5
 }
 
+func ExampleEqual() {
+	fmt.Println(bytes.Equal([]byte("Go"), []byte("Go")))
+	fmt.Println(bytes.Equal([]byte("Go"), []byte("C++")))
+	// Output:
+	// true
+	// false
+}
+
 func ExampleEqualFold() {
 	fmt.Println(bytes.EqualFold([]byte("Go"), []byte("go")))
 	// Output: true
@@ -162,6 +196,14 @@ func ExampleIndex() {
 	// -1
 }
 
+func ExampleIndexByte() {
+	fmt.Println(bytes.IndexByte([]byte("chicken"), byte('k')))
+	fmt.Println(bytes.IndexByte([]byte("chicken"), byte('g')))
+	// Output:
+	// 4
+	// -1
+}
+
 func ExampleIndexFunc() {
 	f := func(c rune) bool {
 		return unicode.Is(unicode.Han, c)
@@ -199,6 +241,36 @@ func ExampleLastIndex() {
 	// -1
 }
 
+func ExampleLastIndexAny() {
+	fmt.Println(bytes.LastIndexAny([]byte("go gopher"), "MüQp"))
+	fmt.Println(bytes.LastIndexAny([]byte("go 地鼠"), "地大"))
+	fmt.Println(bytes.LastIndexAny([]byte("go gopher"), "z,!."))
+	// Output:
+	// 5
+	// 3
+	// -1
+}
+
+func ExampleLastIndexByte() {
+	fmt.Println(bytes.LastIndexByte([]byte("go gopher"), byte('g')))
+	fmt.Println(bytes.LastIndexByte([]byte("go gopher"), byte('r')))
+	fmt.Println(bytes.LastIndexByte([]byte("go gopher"), byte('z')))
+	// Output:
+	// 3
+	// 8
+	// -1
+}
+
+func ExampleLastIndexFunc() {
+	fmt.Println(bytes.LastIndexFunc([]byte("go gopher!"), unicode.IsLetter))
+	fmt.Println(bytes.LastIndexFunc([]byte("go gopher!"), unicode.IsPunct))
+	fmt.Println(bytes.LastIndexFunc([]byte("go gopher!"), unicode.IsNumber))
+	// Output:
+	// 8
+	// 9
+	// -1
+}
+
 func ExampleJoin() {
 	s := [][]byte{[]byte("foo"), []byte("bar"), []byte("baz")}
 	fmt.Printf("%s", bytes.Join(s, []byte(", ")))
@@ -218,6 +290,23 @@ func ExampleReplace() {
 	// moo moo moo
 }
 
+func ExampleRunes() {
+	rs := bytes.Runes([]byte("go gopher"))
+	for _, r := range rs {
+		fmt.Printf("%#U\n", r)
+	}
+	// Output:
+	// U+0067 'g'
+	// U+006F 'o'
+	// U+0020 ' '
+	// U+0067 'g'
+	// U+006F 'o'
+	// U+0070 'p'
+	// U+0068 'h'
+	// U+0065 'e'
+	// U+0072 'r'
+}
+
 func ExampleSplit() {
 	fmt.Printf("%q\n", bytes.Split([]byte("a,b,c"), []byte(",")))
 	fmt.Printf("%q\n", bytes.Split([]byte("a man a plan a canal panama"), []byte("a ")))
@@ -267,6 +356,18 @@ func ExampleTrim() {
 	// Output: ["Achtung! Achtung"]
 }
 
+func ExampleTrimFunc() {
+	fmt.Println(string(bytes.TrimFunc([]byte("go-gopher!"), unicode.IsLetter)))
+	fmt.Println(string(bytes.TrimFunc([]byte("\"go-gopher!\""), unicode.IsLetter)))
+	fmt.Println(string(bytes.TrimFunc([]byte("go-gopher!"), unicode.IsPunct)))
+	fmt.Println(string(bytes.TrimFunc([]byte("1234go-gopher!567"), unicode.IsNumber)))
+	// Output:
+	// -gopher!
+	// "go-gopher!"
+	// go-gopher
+	// go-gopher!
+}
+
 func ExampleMap() {
 	rot13 := func(r rune) rune {
 		switch {
@@ -281,11 +382,43 @@ func ExampleMap() {
 	// Output: 'Gjnf oevyyvt naq gur fyvgul tbcure...
 }
 
+func ExampleTrimLeft() {
+	fmt.Print(string(bytes.TrimLeft([]byte("453gopher8257"), "0123456789")))
+	// Output:
+	// gopher8257
+}
+
+func ExampleTrimLeftFunc() {
+	fmt.Println(string(bytes.TrimLeftFunc([]byte("go-gopher"), unicode.IsLetter)))
+	fmt.Println(string(bytes.TrimLeftFunc([]byte("go-gopher!"), unicode.IsPunct)))
+	fmt.Println(string(bytes.TrimLeftFunc([]byte("1234go-gopher!567"), unicode.IsNumber)))
+	// Output:
+	// -gopher
+	// go-gopher!
+	// go-gopher!567
+}
+
 func ExampleTrimSpace() {
 	fmt.Printf("%s", bytes.TrimSpace([]byte(" \t\n a lone gopher \n\t\r\n")))
 	// Output: a lone gopher
 }
 
+func ExampleTrimRight() {
+	fmt.Print(string(bytes.TrimRight([]byte("453gopher8257"), "0123456789")))
+	// Output:
+	// 453gopher
+}
+
+func ExampleTrimRightFunc() {
+	fmt.Println(string(bytes.TrimRightFunc([]byte("go-gopher"), unicode.IsLetter)))
+	fmt.Println(string(bytes.TrimRightFunc([]byte("go-gopher!"), unicode.IsPunct)))
+	fmt.Println(string(bytes.TrimRightFunc([]byte("1234go-gopher!567"), unicode.IsNumber)))
+	// Output:
+	// go-
+	// go-gopher
+	// 1234go-gopher!
+}
+
 func ExampleToUpper() {
 	fmt.Printf("%s", bytes.ToUpper([]byte("Gopher")))
 	// Output: GOPHER
@@ -295,3 +428,11 @@ func ExampleToLower() {
 	fmt.Printf("%s", bytes.ToLower([]byte("Gopher")))
 	// Output: gopher
 }
+
+func ExampleReader_Len() {
+	fmt.Println(bytes.NewReader([]byte("Hi!")).Len())
+	fmt.Println(bytes.NewReader([]byte("こんにちは!")).Len())
+	// Output:
+	// 3
+	// 16
+}

libgo/go/bytes/reader.go

@@ -35,6 +35,7 @@ func (r *Reader) Len() int {
 // to any other method.
 func (r *Reader) Size() int64 { return int64(len(r.s)) }
 
+// Read implements the io.Reader interface.
 func (r *Reader) Read(b []byte) (n int, err error) {
 	if r.i >= int64(len(r.s)) {
 		return 0, io.EOF
@@ -45,6 +46,7 @@ func (r *Reader) Read(b []byte) (n int, err error) {
 	return
 }
 
+// ReadAt implements the io.ReaderAt interface.
 func (r *Reader) ReadAt(b []byte, off int64) (n int, err error) {
 	// cannot modify state - see io.ReaderAt
 	if off < 0 {
@@ -60,6 +62,7 @@ func (r *Reader) ReadAt(b []byte, off int64) (n int, err error) {
 	return
 }
 
+// ReadByte implements the io.ByteReader interface.
 func (r *Reader) ReadByte() (byte, error) {
 	r.prevRune = -1
 	if r.i >= int64(len(r.s)) {
@@ -70,6 +73,7 @@ func (r *Reader) ReadByte() (byte, error) {
 	return b, nil
 }
 
+// UnreadByte complements ReadByte in implementing the io.ByteScanner interface.
 func (r *Reader) UnreadByte() error {
 	r.prevRune = -1
 	if r.i <= 0 {
@@ -79,6 +83,7 @@ func (r *Reader) UnreadByte() error {
 	return nil
 }
 
+// ReadRune implements the io.RuneReader interface.
 func (r *Reader) ReadRune() (ch rune, size int, err error) {
 	if r.i >= int64(len(r.s)) {
 		r.prevRune = -1
@@ -94,6 +99,7 @@ func (r *Reader) ReadRune() (ch rune, size int, err error) {
 	return
 }
 
+// UnreadRune complements ReadRune in implementing the io.RuneScanner interface.
 func (r *Reader) UnreadRune() error {
 	if r.prevRune < 0 {
 		return errors.New("bytes.Reader.UnreadRune: previous operation was not ReadRune")

libgo/go/bytes/reader_test.go

@@ -140,9 +140,9 @@ func TestReaderWriteTo(t *testing.T) {
 	for i := 0; i < 30; i += 3 {
 		var l int
 		if i > 0 {
-			l = len(data) / i
+			l = len(testString) / i
 		}
-		s := data[:l]
+		s := testString[:l]
 		r := NewReader(testBytes[:l])
 		var b Buffer
 		n, err := r.WriteTo(&b)

libgo/go/cmd/buildid/buildid.go

@@ -0,0 +1,73 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"strings"
+
+	"cmd/internal/buildid"
+)
+
+func usage() {
+	fmt.Fprintf(os.Stderr, "usage: go tool buildid [-w] file\n")
+	flag.PrintDefaults()
+	os.Exit(2)
+}
+
+var wflag = flag.Bool("w", false, "write build ID")
+
+func main() {
+	log.SetPrefix("buildid: ")
+	log.SetFlags(0)
+	flag.Usage = usage
+	flag.Parse()
+	if flag.NArg() != 1 {
+		usage()
+	}
+
+	file := flag.Arg(0)
+	id, err := buildid.ReadFile(file)
+	if err != nil {
+		log.Fatal(err)
+	}
+	if !*wflag {
+		fmt.Printf("%s\n", id)
+		return
+	}
+
+	f, err := os.Open(file)
+	if err != nil {
+		log.Fatal(err)
+	}
+	matches, hash, err := buildid.FindAndHash(f, id, 0)
+	if err != nil {
+		log.Fatal(err)
+	}
+	f.Close()
+
+	tail := id
+	if i := strings.LastIndex(id, "."); i >= 0 {
+		tail = tail[i+1:]
+	}
+	if len(tail) != len(hash)*2 {
+		log.Fatalf("%s: cannot find %d-byte hash in id %s", file, len(hash), id)
+	}
+	newID := id[:len(id)-len(tail)] + fmt.Sprintf("%x", hash)
+
+	f, err = os.OpenFile(file, os.O_WRONLY, 0)
+	if err != nil {
+		log.Fatal(err)
+	}
+	if err := buildid.Rewrite(f, matches, newID); err != nil {
+		log.Fatal(err)
+	}
+	if err := f.Close(); err != nil {
+		log.Fatal(err)
+	}
+}

libgo/go/cmd/buildid/doc.go

@@ -0,0 +1,18 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+/*
+Buildid displays or updates the build ID stored in a Go package or binary.
+
+Usage:
+	go tool buildid [-w] file
+
+By default, buildid prints the build ID found in the named file.
+If the -w option is given, buildid rewrites the build ID found in
+the file to accurately record a content hash of the file.
+
+This tool is only intended for use by the go command or
+other build systems.
+*/
+package main

libgo/go/cmd/cgo/ast.go

@@ -58,11 +58,14 @@ func (f *File) ParseGo(name string, src []byte) {
 	// so we use ast1 to look for the doc comments on import "C"
 	// and on exported functions, and we use ast2 for translating
 	// and reprinting.
+	// In cgo mode, we ignore ast2 and just apply edits directly
+	// the text behind ast1. In godefs mode we modify and print ast2.
 	ast1 := parse(name, src, parser.ParseComments)
 	ast2 := parse(name, src, 0)
 
 	f.Package = ast1.Name.Name
 	f.Name = make(map[string]*Name)
+	f.NamePos = make(map[*Name]token.Pos)
 
 	// In ast1, find the import "C" line and get any extra C preamble.
 	sawC := false
@@ -96,6 +99,7 @@ func (f *File) ParseGo(name string, src []byte) {
 	}
 
 	// In ast2, strip the import "C" line.
+	if *godefs {
 		w := 0
 		for _, decl := range ast2.Decls {
 			d, ok := decl.(*ast.GenDecl)
@@ -120,12 +124,28 @@ func (f *File) ParseGo(name string, src []byte) {
 			w++
 		}
 		ast2.Decls = ast2.Decls[0:w]
+	} else {
+		for _, decl := range ast2.Decls {
+			d, ok := decl.(*ast.GenDecl)
+			if !ok {
+				continue
+			}
+			for _, spec := range d.Specs {
+				if s, ok := spec.(*ast.ImportSpec); ok && s.Path.Value == `"C"` {
+					// Replace "C" with _ "unsafe", to keep program valid.
+					// (Deleting import statement or clause is not safe if it is followed
+					// in the source by an explicit semicolon.)
+					f.Edit.Replace(f.offset(s.Path.Pos()), f.offset(s.Path.End()), `_ "unsafe"`)
+				}
+			}
+		}
+	}
 
 	// Accumulate pointers to uses of C.x.
 	if f.Ref == nil {
 		f.Ref = make([]*Ref, 0, 8)
 	}
-	f.walk(ast2, "prog", (*File).saveExprs)
+	f.walk(ast2, ctxProg, (*File).saveExprs)
 
 	// Accumulate exported functions.
 	// The comments are only on ast1 but we need to
@@ -133,8 +153,8 @@ func (f *File) ParseGo(name string, src []byte) {
 	// The first walk fills in ExpFunc, and the
 	// second walk changes the entries to
 	// refer to ast2 instead.
-	f.walk(ast1, "prog", (*File).saveExport)
+	f.walk(ast1, ctxProg, (*File).saveExport)
-	f.walk(ast2, "prog", (*File).saveExport2)
+	f.walk(ast2, ctxProg, (*File).saveExport2)
 
 	f.Comments = ast1.Comments
 	f.AST = ast2
@@ -143,9 +163,6 @@ func (f *File) ParseGo(name string, src []byte) {
 // Like ast.CommentGroup's Text method but preserves
 // leading blank lines, so that line numbers line up.
 func commentText(g *ast.CommentGroup) string {
-	if g == nil {
-		return ""
-	}
 	var pieces []string
 	for _, com := range g.List {
 		c := com.Text
@@ -165,7 +182,7 @@ func commentText(g *ast.CommentGroup) string {
 }
 
 // Save various references we are going to need later.
-func (f *File) saveExprs(x interface{}, context string) {
+func (f *File) saveExprs(x interface{}, context astContext) {
 	switch x := x.(type) {
 	case *ast.Expr:
 		switch (*x).(type) {
@@ -178,7 +195,7 @@ func (f *File) saveExprs(x interface{}, context string) {
 }
 
 // Save references to C.xxx for later processing.
-func (f *File) saveRef(n *ast.Expr, context string) {
+func (f *File) saveRef(n *ast.Expr, context astContext) {
 	sel := (*n).(*ast.SelectorExpr)
 	// For now, assume that the only instance of capital C is when
 	// used as the imported package identifier.
@@ -188,10 +205,10 @@ func (f *File) saveRef(n *ast.Expr, context string) {
 	if l, ok := sel.X.(*ast.Ident); !ok || l.Name != "C" {
 		return
 	}
-	if context == "as2" {
+	if context == ctxAssign2 {
-		context = "expr"
+		context = ctxExpr
 	}
-	if context == "embed-type" {
+	if context == ctxEmbedType {
 		error_(sel.Pos(), "cannot embed C type")
 	}
 	goname := sel.Sel.Name
@@ -212,6 +229,7 @@ func (f *File) saveRef(n *ast.Expr, context string) {
 			Go: goname,
 		}
 		f.Name[goname] = name
+		f.NamePos[name] = sel.Pos()
 	}
 	f.Ref = append(f.Ref, &Ref{
 		Name:    name,
@@ -221,7 +239,7 @@ func (f *File) saveRef(n *ast.Expr, context string) {
 }
 
 // Save calls to C.xxx for later processing.
-func (f *File) saveCall(call *ast.CallExpr, context string) {
+func (f *File) saveCall(call *ast.CallExpr, context astContext) {
 	sel, ok := call.Fun.(*ast.SelectorExpr)
 	if !ok {
 		return
@@ -229,12 +247,12 @@ func (f *File) saveCall(call *ast.CallExpr, context string) {
 	if l, ok := sel.X.(*ast.Ident); !ok || l.Name != "C" {
 		return
 	}
-	c := &Call{Call: call, Deferred: context == "defer"}
+	c := &Call{Call: call, Deferred: context == ctxDefer}
 	f.Calls = append(f.Calls, c)
 }
 
 // If a function should be exported add it to ExpFunc.
-func (f *File) saveExport(x interface{}, context string) {
+func (f *File) saveExport(x interface{}, context astContext) {
 	n, ok := x.(*ast.FuncDecl)
 	if !ok {
 		return
@@ -274,7 +292,7 @@ func (f *File) saveExport(x interface{}, context string) {
 }
 
 // Make f.ExpFunc[i] point at the Func from this AST instead of the other one.
-func (f *File) saveExport2(x interface{}, context string) {
+func (f *File) saveExport2(x interface{}, context astContext) {
 	n, ok := x.(*ast.FuncDecl)
 	if !ok {
 		return
@@ -288,8 +306,30 @@ func (f *File) saveExport2(x interface{}, context string) {
 	}
 }
 
+type astContext int
+
+const (
+	ctxProg astContext = iota
+	ctxEmbedType
+	ctxType
+	ctxStmt
+	ctxExpr
+	ctxField
+	ctxParam
+	ctxAssign2 // assignment of a single expression to two variables
+	ctxSwitch
+	ctxTypeSwitch
+	ctxFile
+	ctxDecl
+	ctxSpec
+	ctxDefer
+	ctxCall  // any function call other than ctxCall2
+	ctxCall2 // function call whose result is assigned to two variables
+	ctxSelector
+)
+
 // walk walks the AST x, calling visit(f, x, context) for each node.
-func (f *File) walk(x interface{}, context string, visit func(*File, interface{}, string)) {
+func (f *File) walk(x interface{}, context astContext, visit func(*File, interface{}, astContext)) {
 	visit(f, x, context)
 	switch n := x.(type) {
 	case *ast.Expr:
@@ -304,10 +344,10 @@ func (f *File) walk(x interface{}, context string, visit func(*File, interface{}
 
 	// These are ordered and grouped to match ../../go/ast/ast.go
 	case *ast.Field:
-		if len(n.Names) == 0 && context == "field" {
+		if len(n.Names) == 0 && context == ctxField {
-			f.walk(&n.Type, "embed-type", visit)
+			f.walk(&n.Type, ctxEmbedType, visit)
 		} else {
-			f.walk(&n.Type, "type", visit)
+			f.walk(&n.Type, ctxType, visit)
 		}
 	case *ast.FieldList:
 		for _, field := range n.List {
@@ -318,163 +358,163 @@ func (f *File) walk(x interface{}, context string, visit func(*File, interface{}
 	case *ast.Ellipsis:
 	case *ast.BasicLit:
 	case *ast.FuncLit:
-		f.walk(n.Type, "type", visit)
+		f.walk(n.Type, ctxType, visit)
-		f.walk(n.Body, "stmt", visit)
+		f.walk(n.Body, ctxStmt, visit)
 	case *ast.CompositeLit:
-		f.walk(&n.Type, "type", visit)
+		f.walk(&n.Type, ctxType, visit)
-		f.walk(n.Elts, "expr", visit)
+		f.walk(n.Elts, ctxExpr, visit)
 	case *ast.ParenExpr:
 		f.walk(&n.X, context, visit)
 	case *ast.SelectorExpr:
-		f.walk(&n.X, "selector", visit)
+		f.walk(&n.X, ctxSelector, visit)
 	case *ast.IndexExpr:
-		f.walk(&n.X, "expr", visit)
+		f.walk(&n.X, ctxExpr, visit)
-		f.walk(&n.Index, "expr", visit)
+		f.walk(&n.Index, ctxExpr, visit)
 	case *ast.SliceExpr:
-		f.walk(&n.X, "expr", visit)
+		f.walk(&n.X, ctxExpr, visit)
 		if n.Low != nil {
-			f.walk(&n.Low, "expr", visit)
+			f.walk(&n.Low, ctxExpr, visit)
 		}
 		if n.High != nil {
-			f.walk(&n.High, "expr", visit)
+			f.walk(&n.High, ctxExpr, visit)
 		}
 		if n.Max != nil {
-			f.walk(&n.Max, "expr", visit)
+			f.walk(&n.Max, ctxExpr, visit)
 		}
 	case *ast.TypeAssertExpr:
-		f.walk(&n.X, "expr", visit)
+		f.walk(&n.X, ctxExpr, visit)
-		f.walk(&n.Type, "type", visit)
+		f.walk(&n.Type, ctxType, visit)
 	case *ast.CallExpr:
-		if context == "as2" {
+		if context == ctxAssign2 {
-			f.walk(&n.Fun, "call2", visit)
+			f.walk(&n.Fun, ctxCall2, visit)
 		} else {
-			f.walk(&n.Fun, "call", visit)
+			f.walk(&n.Fun, ctxCall, visit)
 		}
-		f.walk(n.Args, "expr", visit)
+		f.walk(n.Args, ctxExpr, visit)
 	case *ast.StarExpr:
 		f.walk(&n.X, context, visit)
 	case *ast.UnaryExpr:
-		f.walk(&n.X, "expr", visit)
+		f.walk(&n.X, ctxExpr, visit)
 	case *ast.BinaryExpr:
-		f.walk(&n.X, "expr", visit)
+		f.walk(&n.X, ctxExpr, visit)
-		f.walk(&n.Y, "expr", visit)
+		f.walk(&n.Y, ctxExpr, visit)
 	case *ast.KeyValueExpr:
-		f.walk(&n.Key, "expr", visit)
+		f.walk(&n.Key, ctxExpr, visit)
-		f.walk(&n.Value, "expr", visit)
+		f.walk(&n.Value, ctxExpr, visit)
 
 	case *ast.ArrayType:
-		f.walk(&n.Len, "expr", visit)
+		f.walk(&n.Len, ctxExpr, visit)
-		f.walk(&n.Elt, "type", visit)
+		f.walk(&n.Elt, ctxType, visit)
 	case *ast.StructType:
-		f.walk(n.Fields, "field", visit)
+		f.walk(n.Fields, ctxField, visit)
 	case *ast.FuncType:
-		f.walk(n.Params, "param", visit)
+		f.walk(n.Params, ctxParam, visit)
 		if n.Results != nil {
-			f.walk(n.Results, "param", visit)
+			f.walk(n.Results, ctxParam, visit)
 		}
 	case *ast.InterfaceType:
-		f.walk(n.Methods, "field", visit)
+		f.walk(n.Methods, ctxField, visit)
 	case *ast.MapType:
-		f.walk(&n.Key, "type", visit)
+		f.walk(&n.Key, ctxType, visit)
-		f.walk(&n.Value, "type", visit)
+		f.walk(&n.Value, ctxType, visit)
 	case *ast.ChanType:
-		f.walk(&n.Value, "type", visit)
+		f.walk(&n.Value, ctxType, visit)
 
 	case *ast.BadStmt:
 	case *ast.DeclStmt:
-		f.walk(n.Decl, "decl", visit)
+		f.walk(n.Decl, ctxDecl, visit)
 	case *ast.EmptyStmt:
 	case *ast.LabeledStmt:
-		f.walk(n.Stmt, "stmt", visit)
+		f.walk(n.Stmt, ctxStmt, visit)
 	case *ast.ExprStmt:
-		f.walk(&n.X, "expr", visit)
+		f.walk(&n.X, ctxExpr, visit)
 	case *ast.SendStmt:
-		f.walk(&n.Chan, "expr", visit)
+		f.walk(&n.Chan, ctxExpr, visit)
-		f.walk(&n.Value, "expr", visit)
+		f.walk(&n.Value, ctxExpr, visit)
 	case *ast.IncDecStmt:
-		f.walk(&n.X, "expr", visit)
+		f.walk(&n.X, ctxExpr, visit)
 	case *ast.AssignStmt:
-		f.walk(n.Lhs, "expr", visit)
+		f.walk(n.Lhs, ctxExpr, visit)
 		if len(n.Lhs) == 2 && len(n.Rhs) == 1 {
-			f.walk(n.Rhs, "as2", visit)
+			f.walk(n.Rhs, ctxAssign2, visit)
 		} else {
-			f.walk(n.Rhs, "expr", visit)
+			f.walk(n.Rhs, ctxExpr, visit)
 		}
 	case *ast.GoStmt:
-		f.walk(n.Call, "expr", visit)
+		f.walk(n.Call, ctxExpr, visit)
 	case *ast.DeferStmt:
-		f.walk(n.Call, "defer", visit)
+		f.walk(n.Call, ctxDefer, visit)
 	case *ast.ReturnStmt:
-		f.walk(n.Results, "expr", visit)
+		f.walk(n.Results, ctxExpr, visit)
 	case *ast.BranchStmt:
 	case *ast.BlockStmt:
 		f.walk(n.List, context, visit)
 	case *ast.IfStmt:
-		f.walk(n.Init, "stmt", visit)
+		f.walk(n.Init, ctxStmt, visit)
-		f.walk(&n.Cond, "expr", visit)
+		f.walk(&n.Cond, ctxExpr, visit)
-		f.walk(n.Body, "stmt", visit)
+		f.walk(n.Body, ctxStmt, visit)
-		f.walk(n.Else, "stmt", visit)
+		f.walk(n.Else, ctxStmt, visit)
 	case *ast.CaseClause:
-		if context == "typeswitch" {
+		if context == ctxTypeSwitch {
-			context = "type"
+			context = ctxType
 		} else {
-			context = "expr"
+			context = ctxExpr
 		}
 		f.walk(n.List, context, visit)
-		f.walk(n.Body, "stmt", visit)
+		f.walk(n.Body, ctxStmt, visit)
 	case *ast.SwitchStmt:
-		f.walk(n.Init, "stmt", visit)
+		f.walk(n.Init, ctxStmt, visit)
-		f.walk(&n.Tag, "expr", visit)
+		f.walk(&n.Tag, ctxExpr, visit)
-		f.walk(n.Body, "switch", visit)
+		f.walk(n.Body, ctxSwitch, visit)
 	case *ast.TypeSwitchStmt:
-		f.walk(n.Init, "stmt", visit)
+		f.walk(n.Init, ctxStmt, visit)
-		f.walk(n.Assign, "stmt", visit)
+		f.walk(n.Assign, ctxStmt, visit)
-		f.walk(n.Body, "typeswitch", visit)
+		f.walk(n.Body, ctxTypeSwitch, visit)
 	case *ast.CommClause:
-		f.walk(n.Comm, "stmt", visit)
+		f.walk(n.Comm, ctxStmt, visit)
-		f.walk(n.Body, "stmt", visit)
+		f.walk(n.Body, ctxStmt, visit)
 	case *ast.SelectStmt:
-		f.walk(n.Body, "stmt", visit)
+		f.walk(n.Body, ctxStmt, visit)
 	case *ast.ForStmt:
-		f.walk(n.Init, "stmt", visit)
+		f.walk(n.Init, ctxStmt, visit)
-		f.walk(&n.Cond, "expr", visit)
+		f.walk(&n.Cond, ctxExpr, visit)
-		f.walk(n.Post, "stmt", visit)
+		f.walk(n.Post, ctxStmt, visit)
-		f.walk(n.Body, "stmt", visit)
+		f.walk(n.Body, ctxStmt, visit)
 	case *ast.RangeStmt:
-		f.walk(&n.Key, "expr", visit)
+		f.walk(&n.Key, ctxExpr, visit)
-		f.walk(&n.Value, "expr", visit)
+		f.walk(&n.Value, ctxExpr, visit)
-		f.walk(&n.X, "expr", visit)
+		f.walk(&n.X, ctxExpr, visit)
-		f.walk(n.Body, "stmt", visit)
+		f.walk(n.Body, ctxStmt, visit)
 
 	case *ast.ImportSpec:
 	case *ast.ValueSpec:
-		f.walk(&n.Type, "type", visit)
+		f.walk(&n.Type, ctxType, visit)
 		if len(n.Names) == 2 && len(n.Values) == 1 {
-			f.walk(&n.Values[0], "as2", visit)
+			f.walk(&n.Values[0], ctxAssign2, visit)
 		} else {
-			f.walk(n.Values, "expr", visit)
+			f.walk(n.Values, ctxExpr, visit)
 		}
 	case *ast.TypeSpec:
-		f.walk(&n.Type, "type", visit)
+		f.walk(&n.Type, ctxType, visit)
 
 	case *ast.BadDecl:
 	case *ast.GenDecl:
-		f.walk(n.Specs, "spec", visit)
+		f.walk(n.Specs, ctxSpec, visit)
 	case *ast.FuncDecl:
 		if n.Recv != nil {
-			f.walk(n.Recv, "param", visit)
+			f.walk(n.Recv, ctxParam, visit)
 		}
-		f.walk(n.Type, "type", visit)
+		f.walk(n.Type, ctxType, visit)
 		if n.Body != nil {
-			f.walk(n.Body, "stmt", visit)
+			f.walk(n.Body, ctxStmt, visit)
 		}
 
 	case *ast.File:
-		f.walk(n.Decls, "decl", visit)
+		f.walk(n.Decls, ctxDecl, visit)
 
 	case *ast.Package:
 		for _, file := range n.Files {
-			f.walk(file, "file", visit)
+			f.walk(file, ctxFile, visit)
 		}
 
 	case []ast.Decl:

libgo/go/cmd/cgo/doc.go

@@ -102,11 +102,13 @@ the use of cgo, and to 0 to disable it. The go tool will set the
 build constraint "cgo" if cgo is enabled.
 
 When cross-compiling, you must specify a C cross-compiler for cgo to
-use. You can do this by setting the CC_FOR_TARGET environment
+use. You can do this by setting the generic CC_FOR_TARGET or the
-variable when building the toolchain using make.bash, or by setting
+more specific CC_FOR_${GOOS}_${GOARCH} (for example, CC_FOR_linux_arm)
-the CC environment variable any time you run the go tool. The
+environment variable when building the toolchain using make.bash,
-CXX_FOR_TARGET and CXX environment variables work in a similar way for
+or you can set the CC environment variable any time you run the go tool.
-C++ code.
+
+The CXX_FOR_TARGET, CXX_FOR_${GOOS}_${GOARCH}, and CXX
+environment variables work in a similar way for C++ code.
 
 Go references to C
 
@@ -126,12 +128,29 @@ C.complexfloat (complex float), and C.complexdouble (complex double).
 The C type void* is represented by Go's unsafe.Pointer.
 The C types __int128_t and __uint128_t are represented by [16]byte.
 
+A few special C types which would normally be represented by a pointer
+type in Go are instead represented by a uintptr.  See the Special
+cases section below.
+
 To access a struct, union, or enum type directly, prefix it with
 struct_, union_, or enum_, as in C.struct_stat.
 
 The size of any C type T is available as C.sizeof_T, as in
 C.sizeof_struct_stat.
 
+A C function may be declared in the Go file with a parameter type of
+the special name _GoString_. This function may be called with an
+ordinary Go string value. The string length, and a pointer to the
+string contents, may be accessed by calling the C functions
+
+	size_t _GoStringLen(_GoString_ s);
+	const char *_GoStringPtr(_GoString_ s);
+
+These functions are only available in the preamble, not in other C
+files. The C code must not modify the contents of the pointer returned
+by _GoStringPtr. Note that the string contents may not have a trailing
+NUL byte.
+
 As Go doesn't have support for C's union type in the general case,
 C's union types are represented as a Go byte array with the same length.
 
@@ -241,7 +260,16 @@ They will be available in the C code as:
 found in the _cgo_export.h generated header, after any preambles
 copied from the cgo input files. Functions with multiple
 return values are mapped to functions returning a struct.
+
 Not all Go types can be mapped to C types in a useful way.
+Go struct types are not supported; use a C struct type.
+Go array types are not supported; use a C pointer.
+
+Go functions that take arguments of type string may be called with the
+C type _GoString_, described above. The _GoString_ type will be
+automatically defined in the preamble. Note that there is no way for C
+code to create a value of this type; this is only useful for passing
+string values from Go to C and back to Go.
 
 Using //export in a file places a restriction on the preamble:
 since it is copied into two different C output files, it must not
@@ -264,6 +292,14 @@ pointer is a Go pointer or a C pointer is a dynamic property
 determined by how the memory was allocated; it has nothing to do with
 the type of the pointer.
 
+Note that values of some Go types, other than the type's zero value,
+always include Go pointers. This is true of string, slice, interface,
+channel, map, and function types. A pointer type may hold a Go pointer
+or a C pointer. Array and struct types may or may not include Go
+pointers, depending on the element types. All the discussion below
+about Go pointers applies not just to pointer types, but also to other
+types that include Go pointers.
+
 Go code may pass a Go pointer to C provided the Go memory to which it
 points does not contain any Go pointers. The C code must preserve
 this property: it must not store any Go pointers in Go memory, even
@@ -274,14 +310,17 @@ the Go memory in question is the entire array or the entire backing
 array of the slice.
 
 C code may not keep a copy of a Go pointer after the call returns.
+This includes the _GoString_ type, which, as noted above, includes a
+Go pointer; _GoString_ values may not be retained by C code.
 
-A Go function called by C code may not return a Go pointer. A Go
+A Go function called by C code may not return a Go pointer (which
-function called by C code may take C pointers as arguments, and it may
+implies that it may not return a string, slice, channel, and so
-store non-pointer or C pointer data through those pointers, but it may
+forth). A Go function called by C code may take C pointers as
-not store a Go pointer in memory pointed to by a C pointer. A Go
+arguments, and it may store non-pointer or C pointer data through
-function called by C code may take a Go pointer as an argument, but it
+those pointers, but it may not store a Go pointer in memory pointed to
-must preserve the property that the Go memory to which it points does
+by a C pointer. A Go function called by C code may take a Go pointer
-not contain any Go pointers.
+as an argument, but it must preserve the property that the Go memory
+to which it points does not contain any Go pointers.
 
 Go code may not store a Go pointer in C memory. C code may store Go
 pointers in C memory, subject to the rule above: it must stop storing
@@ -299,6 +338,84 @@ and of course there is nothing stopping the C code from doing anything
 it likes. However, programs that break these rules are likely to fail
 in unexpected and unpredictable ways.
 
+Special cases
+
+A few special C types which would normally be represented by a pointer
+type in Go are instead represented by a uintptr. Those types are
+the CF*Ref types from the CoreFoundation library on Darwin, including:
+
+	CFAllocatorRef
+	CFArrayRef
+	CFAttributedStringRef
+	CFBagRef
+	CFBinaryHeapRef
+	CFBitVectorRef
+	CFBooleanRef
+	CFBundleRef
+	CFCalendarRef
+	CFCharacterSetRef
+	CFDataRef
+	CFDateFormatterRef
+	CFDateRef
+	CFDictionaryRef
+	CFErrorRef
+	CFFileDescriptorRef
+	CFFileSecurityRef
+	CFLocaleRef
+	CFMachPortRef
+	CFMessagePortRef
+	CFMutableArrayRef
+	CFMutableAttributedStringRef
+	CFMutableBagRef
+	CFMutableBitVectorRef
+	CFMutableCharacterSetRef
+	CFMutableDataRef
+	CFMutableDictionaryRef
+	CFMutableSetRef
+	CFMutableStringRef
+	CFNotificationCenterRef
+	CFNullRef
+	CFNumberFormatterRef
+	CFNumberRef
+	CFPlugInInstanceRef
+	CFPlugInRef
+	CFPropertyListRef
+	CFReadStreamRef
+	CFRunLoopObserverRef
+	CFRunLoopRef
+	CFRunLoopSourceRef
+	CFRunLoopTimerRef
+	CFSetRef
+	CFSocketRef
+	CFStringRef
+	CFStringTokenizerRef
+	CFTimeZoneRef
+	CFTreeRef
+	CFTypeRef
+	CFURLCreateFromFSRef
+	CFURLEnumeratorRef
+	CFURLGetFSRef
+	CFURLRef
+	CFUUIDRef
+	CFUserNotificationRef
+	CFWriteStreamRef
+	CFXMLNodeRef
+	CFXMLParserRef
+	CFXMLTreeRef
+
+These types are uintptr on the Go side because they would otherwise
+confuse the Go garbage collector; they are sometimes not really
+pointers but data structures encoded in a pointer type. All operations
+on these types must happen in C. The proper constant to initialize an
+empty such reference is 0, not nil.
+
+This special case was introduced in Go 1.10. For auto-updating code
+from Go 1.9 and earlier, use the cftype rewrite in the Go fix tool:
+
+	go tool fix -r cftype <pkg>
+
+It will replace nil with 0 in the appropriate places.
+
 Using cgo directly
 
 Usage:
@@ -312,32 +429,35 @@ invoking the C compiler to compile the C parts of the package.
 
 The following options are available when running cgo directly:
 
+	-V
+		Print cgo version and exit.
+	-debug-define
+		Debugging option. Print #defines.
+	-debug-gcc
+		Debugging option. Trace C compiler execution and output.
 	-dynimport file
 		Write list of symbols imported by file. Write to
 		-dynout argument or to standard output. Used by go
 		build when building a cgo package.
+	-dynlinker
+		Write dynamic linker as part of -dynimport output.
 	-dynout file
 		Write -dynimport output to file.
 	-dynpackage package
 		Set Go package for -dynimport output.
-	-dynlinker
-		Write dynamic linker as part of -dynimport output.
-	-godefs
-		Write out input file in Go syntax replacing C package
-		names with real values. Used to generate files in the
-		syscall package when bootstrapping a new target.
-	-srcdir directory
-		Find the Go input files, listed on the command line,
-		in directory.
-	-objdir directory
-		Put all generated files in directory.
-	-importpath string
-		The import path for the Go package. Optional; used for
-		nicer comments in the generated files.
 	-exportheader file
 		If there are any exported functions, write the
 		generated export declarations to file.
 		C code can #include this to see the declarations.
+	-importpath string
+		The import path for the Go package. Optional; used for
+		nicer comments in the generated files.
+	-import_runtime_cgo
+		If set (which it is by default) import runtime/cgo in
+		generated output.
+	-import_syscall
+		If set (which it is by default) import syscall in
+		generated output.
 	-gccgo
 		Generate output for the gccgo compiler rather than the
 		gc compiler.
@@ -345,16 +465,13 @@ The following options are available when running cgo directly:
 		The -fgo-prefix option to be used with gccgo.
 	-gccgopkgpath path
 		The -fgo-pkgpath option to be used with gccgo.
-	-import_runtime_cgo
+	-godefs
-		If set (which it is by default) import runtime/cgo in
+		Write out input file in Go syntax replacing C package
-		generated output.
+		names with real values. Used to generate files in the
-	-import_syscall
+		syscall package when bootstrapping a new target.
-		If set (which it is by default) import syscall in
+	-objdir directory
-		generated output.
+		Put all generated files in directory.
-	-debug-define
+	-srcdir directory
-		Debugging option. Print #defines.
-	-debug-gcc
-		Debugging option. Trace C compiler execution and output.
 */
 package main
 
@@ -403,21 +520,19 @@ about simple #defines for constants and the like. These are recorded
 for later use.
 
 Next, cgo needs to identify the kinds for each identifier. For the
-identifiers C.foo and C.bar, cgo generates this C program:
+identifiers C.foo, cgo generates this C program:
 
 	<preamble>
 	#line 1 "not-declared"
-	void __cgo_f_xxx_1(void) { __typeof__(foo) *__cgo_undefined__; }
+	void __cgo_f_1_1(void) { __typeof__(foo) *__cgo_undefined__1; }
 	#line 1 "not-type"
-	void __cgo_f_xxx_2(void) { foo *__cgo_undefined__; }
+	void __cgo_f_1_2(void) { foo *__cgo_undefined__2; }
-	#line 1 "not-const"
+	#line 1 "not-int-const"
-	void __cgo_f_xxx_3(void) { enum { __cgo_undefined__ = (foo)*1 }; }
+	void __cgo_f_1_3(void) { enum { __cgo_undefined__3 = (foo)*1 }; }
-	#line 2 "not-declared"
+	#line 1 "not-num-const"
-	void __cgo_f_xxx_1(void) { __typeof__(bar) *__cgo_undefined__; }
+	void __cgo_f_1_4(void) { static const double __cgo_undefined__4 = (foo); }
-	#line 2 "not-type"
+	#line 1 "not-str-lit"
-	void __cgo_f_xxx_2(void) { bar *__cgo_undefined__; }
+	void __cgo_f_1_5(void) { static const char __cgo_undefined__5[] = (foo); }
-	#line 2 "not-const"
-	void __cgo_f_xxx_3(void) { enum { __cgo_undefined__ = (bar)*1 }; }
 
 This program will not compile, but cgo can use the presence or absence
 of an error message on a given line to deduce the information it
@@ -427,45 +542,72 @@ errors that might stop parsing early.
 
 An error on not-declared:1 indicates that foo is undeclared.
 An error on not-type:1 indicates that foo is not a type (if declared at all, it is an identifier).
-An error on not-const:1 indicates that foo is not an integer constant.
+An error on not-int-const:1 indicates that foo is not an integer constant.
+An error on not-num-const:1 indicates that foo is not a number constant.
+An error on not-str-lit:1 indicates that foo is not a string literal.
+An error on not-signed-int-const:1 indicates that foo is not a signed integer constant.
 
-The line number specifies the name involved. In the example, 1 is foo and 2 is bar.
+The line number specifies the name involved. In the example, 1 is foo.
 
 Next, cgo must learn the details of each type, variable, function, or
 constant. It can do this by reading object files. If cgo has decided
-that t1 is a type, v2 and v3 are variables or functions, and c4, c5,
+that t1 is a type, v2 and v3 are variables or functions, and i4, i5
-and c6 are constants, it generates:
+are integer constants, u6 is an unsigned integer constant, and f7 and f8
+are float constants, and s9 and s10 are string constants, it generates:
 
 	<preamble>
 	__typeof__(t1) *__cgo__1;
 	__typeof__(v2) *__cgo__2;
 	__typeof__(v3) *__cgo__3;
-	__typeof__(c4) *__cgo__4;
+	__typeof__(i4) *__cgo__4;
-	enum { __cgo_enum__4 = c4 };
+	enum { __cgo_enum__4 = i4 };
-	__typeof__(c5) *__cgo__5;
+	__typeof__(i5) *__cgo__5;
-	enum { __cgo_enum__5 = c5 };
+	enum { __cgo_enum__5 = i5 };
-	__typeof__(c6) *__cgo__6;
+	__typeof__(u6) *__cgo__6;
-	enum { __cgo_enum__6 = c6 };
+	enum { __cgo_enum__6 = u6 };
+	__typeof__(f7) *__cgo__7;
+	__typeof__(f8) *__cgo__8;
+	__typeof__(s9) *__cgo__9;
+	__typeof__(s10) *__cgo__10;
 
-	long long __cgo_debug_data[] = {
+	long long __cgodebug_ints[] = {
 		0, // t1
 		0, // v2
 		0, // v3
-		c4,
+		i4,
-		c5,
+		i5,
-		c6,
+		u6,
+		0, // f7
+		0, // f8
+		0, // s9
+		0, // s10
 		1
 	};
 
+	double __cgodebug_floats[] = {
+		0, // t1
+		0, // v2
+		0, // v3
+		0, // i4
+		0, // i5
+		0, // u6
+		f7,
+		f8,
+		0, // s9
+		0, // s10
+		1
+	};
+
+	const char __cgodebug_str__9[] = s9;
+	const unsigned long long __cgodebug_strlen__9 = sizeof(s9)-1;
+	const char __cgodebug_str__10[] = s10;
+	const unsigned long long __cgodebug_strlen__10 = sizeof(s10)-1;
+
 and again invokes the system C compiler, to produce an object file
 containing debug information. Cgo parses the DWARF debug information
 for __cgo__N to learn the type of each identifier. (The types also
-distinguish functions from global variables.) If using a standard gcc,
+distinguish functions from global variables.) Cgo reads the constant
-cgo can parse the DWARF debug information for the __cgo_enum__N to
+values from the __cgodebug_* from the object file's data segment.
-learn the identifier's value. The LLVM-based gcc on OS X emits
-incomplete DWARF information for enums; in that case cgo reads the
-constant values from the __cgo_debug_data from the object file's data
-segment.
 
 At this point cgo knows the meaning of each C.xxx well enough to start
 the translation process.
@@ -550,9 +692,12 @@ _cgo_main.c:
 
 	int main() { return 0; }
 	void crosscall2(void(*fn)(void*, int, uintptr_t), void *a, int c, uintptr_t ctxt) { }
-	uintptr_t _cgo_wait_runtime_init_done() { }
+	uintptr_t _cgo_wait_runtime_init_done() { return 0; }
+	void _cgo_release_context(uintptr_t ctxt) { }
+	char* _cgo_topofstack(void) { return (char*)0; }
 	void _cgo_allocate(void *a, int c) { }
 	void _cgo_panic(void *a, int c) { }
+	void _cgo_reginit(void) { }
 
 The extra functions here are stubs to satisfy the references in the C
 code generated for gcc. The build process links this stub, along with

libgo/go/cmd/cgo/gcc.go

@@ -188,21 +188,8 @@ func (p *Package) Translate(f *File) {
 		p.loadDWARF(f, needType)
 	}
 	if p.rewriteCalls(f) {
-		// Add `import _cgo_unsafe "unsafe"` as the first decl
+		// Add `import _cgo_unsafe "unsafe"` after the package statement.
-		// after the package statement.
+		f.Edit.Insert(f.offset(f.AST.Name.End()), "; import _cgo_unsafe \"unsafe\"")
-		imp := &ast.GenDecl{
-			Tok: token.IMPORT,
-			Specs: []ast.Spec{
-				&ast.ImportSpec{
-					Name: ast.NewIdent("_cgo_unsafe"),
-					Path: &ast.BasicLit{
-						Kind:  token.STRING,
-						Value: `"unsafe"`,
-					},
-				},
-			},
-		}
-		f.AST.Decls = append([]ast.Decl{imp}, f.AST.Decls...)
 	}
 	p.rewriteRef(f)
 }
@@ -211,8 +198,8 @@ func (p *Package) Translate(f *File) {
 // in the file f and saves relevant renamings in f.Name[name].Define.
 func (p *Package) loadDefines(f *File) {
 	var b bytes.Buffer
-	b.WriteString(f.Preamble)
 	b.WriteString(builtinProlog)
+	b.WriteString(f.Preamble)
 	stdout := p.gccDefines(b.Bytes())
 
 	for _, line := range strings.Split(stdout, "\n") {
@@ -283,10 +270,6 @@ func (p *Package) guessKinds(f *File) []*Name {
 			if n.IsConst() {
 				continue
 			}
-
-			if isName(n.Define) {
-				n.C = n.Define
-			}
 		}
 
 		// If this is a struct, union, or enum type name, no need to guess the kind.
@@ -315,57 +298,45 @@ func (p *Package) guessKinds(f *File) []*Name {
 	// For each name, we generate these lines, where xxx is the index in toSniff plus one.
 	//
 	//	#line xxx "not-declared"
-	//	void __cgo_f_xxx_1(void) { __typeof__(name) *__cgo_undefined__; }
+	//	void __cgo_f_xxx_1(void) { __typeof__(name) *__cgo_undefined__1; }
 	//	#line xxx "not-type"
-	//	void __cgo_f_xxx_2(void) { name *__cgo_undefined__; }
+	//	void __cgo_f_xxx_2(void) { name *__cgo_undefined__2; }
 	//	#line xxx "not-int-const"
-	//	void __cgo_f_xxx_3(void) { enum { __cgo_undefined__ = (name)*1 }; }
+	//	void __cgo_f_xxx_3(void) { enum { __cgo_undefined__3 = (name)*1 }; }
 	//	#line xxx "not-num-const"
-	//	void __cgo_f_xxx_4(void) { static const double x = (name); }
+	//	void __cgo_f_xxx_4(void) { static const double __cgo_undefined__4 = (name); }
 	//	#line xxx "not-str-lit"
-	//	void __cgo_f_xxx_5(void) { static const char x[] = (name); }
+	//	void __cgo_f_xxx_5(void) { static const char __cgo_undefined__5[] = (name); }
-	//	#line xxx "not-signed-int-const"
-	//	#if 0 < -(name)
-	//	#line xxx "not-signed-int-const"
-	//	#error found unsigned int
-	//	#endif
 	//
 	// If we see an error at not-declared:xxx, the corresponding name is not declared.
 	// If we see an error at not-type:xxx, the corresponding name is a type.
 	// If we see an error at not-int-const:xxx, the corresponding name is not an integer constant.
 	// If we see an error at not-num-const:xxx, the corresponding name is not a number constant.
 	// If we see an error at not-str-lit:xxx, the corresponding name is not a string literal.
-	// If we see an error at not-signed-int-const:xxx, the corresponding name is not a signed integer literal.
 	//
 	// The specific input forms are chosen so that they are valid C syntax regardless of
 	// whether name denotes a type or an expression.
 
 	var b bytes.Buffer
-	b.WriteString(f.Preamble)
 	b.WriteString(builtinProlog)
+	b.WriteString(f.Preamble)
 
 	for i, n := range names {
 		fmt.Fprintf(&b, "#line %d \"not-declared\"\n"+
-			"void __cgo_f_%d_1(void) { __typeof__(%s) *__cgo_undefined__; }\n"+
+			"void __cgo_f_%d_1(void) { __typeof__(%s) *__cgo_undefined__1; }\n"+
 			"#line %d \"not-type\"\n"+
-			"void __cgo_f_%d_2(void) { %s *__cgo_undefined__; }\n"+
+			"void __cgo_f_%d_2(void) { %s *__cgo_undefined__2; }\n"+
 			"#line %d \"not-int-const\"\n"+
-			"void __cgo_f_%d_3(void) { enum { __cgo_undefined__ = (%s)*1 }; }\n"+
+			"void __cgo_f_%d_3(void) { enum { __cgo_undefined__3 = (%s)*1 }; }\n"+
 			"#line %d \"not-num-const\"\n"+
-			"void __cgo_f_%d_4(void) { static const double x = (%s); }\n"+
+			"void __cgo_f_%d_4(void) { static const double __cgo_undefined__4 = (%s); }\n"+
 			"#line %d \"not-str-lit\"\n"+
-			"void __cgo_f_%d_5(void) { static const char s[] = (%s); }\n"+
+			"void __cgo_f_%d_5(void) { static const char __cgo_undefined__5[] = (%s); }\n",
-			"#line %d \"not-signed-int-const\"\n"+
-			"#if 0 < (%s)\n"+
-			"#line %d \"not-signed-int-const\"\n"+
-			"#error found unsigned int\n"+
-			"#endif\n",
 			i+1, i+1, n.C,
 			i+1, i+1, n.C,
 			i+1, i+1, n.C,
 			i+1, i+1, n.C,
 			i+1, i+1, n.C,
-			i+1, n.C, i+1,
 		)
 	}
 	fmt.Fprintf(&b, "#line 1 \"completed\"\n"+
@@ -384,7 +355,6 @@ func (p *Package) guessKinds(f *File) []*Name {
 		notNumConst
 		notStrLiteral
 		notDeclared
-		notSignedIntConst
 	)
 	sawUnmatchedErrors := false
 	for _, line := range strings.Split(stderr, "\n") {
@@ -438,8 +408,6 @@ func (p *Package) guessKinds(f *File) []*Name {
 			sniff[i] |= notNumConst
 		case "not-str-lit":
 			sniff[i] |= notStrLiteral
-		case "not-signed-int-const":
-			sniff[i] |= notSignedIntConst
 		default:
 			if isError {
 				sawUnmatchedErrors = true
@@ -455,22 +423,11 @@ func (p *Package) guessKinds(f *File) []*Name {
 	}
 
 	for i, n := range names {
-		switch sniff[i] &^ notSignedIntConst {
+		switch sniff[i] {
 		default:
-			var tpos token.Pos
+			error_(f.NamePos[n], "could not determine kind of name for C.%s", fixGo(n.Go))
-			for _, ref := range f.Ref {
-				if ref.Name == n {
-					tpos = ref.Pos()
-					break
-				}
-			}
-			error_(tpos, "could not determine kind of name for C.%s", fixGo(n.Go))
 		case notStrLiteral | notType:
-			if sniff[i]&notSignedIntConst != 0 {
-				n.Kind = "uconst"
-			} else {
 			n.Kind = "iconst"
-			}
 		case notIntConst | notStrLiteral | notType:
 			n.Kind = "fconst"
 		case notIntConst | notNumConst | notType:
@@ -510,12 +467,12 @@ func (p *Package) loadDWARF(f *File, names []*Name) {
 	// for each entry in names and then dereference the type we
 	// learn for __cgo__i.
 	var b bytes.Buffer
-	b.WriteString(f.Preamble)
 	b.WriteString(builtinProlog)
+	b.WriteString(f.Preamble)
 	b.WriteString("#line 1 \"cgo-dwarf-inference\"\n")
 	for i, n := range names {
 		fmt.Fprintf(&b, "__typeof__(%s) *__cgo__%d;\n", n.C, i)
-		if n.Kind == "iconst" || n.Kind == "uconst" {
+		if n.Kind == "iconst" {
 			fmt.Fprintf(&b, "enum { __cgo_enum__%d = %s };\n", i, n.C)
 		}
 	}
@@ -524,7 +481,7 @@ func (p *Package) loadDWARF(f *File, names []*Name) {
 	// so we can read them out of the object file.
 	fmt.Fprintf(&b, "long long __cgodebug_ints[] = {\n")
 	for _, n := range names {
-		if n.Kind == "iconst" || n.Kind == "uconst" {
+		if n.Kind == "iconst" {
 			fmt.Fprintf(&b, "\t%s,\n", n.C)
 		} else {
 			fmt.Fprintf(&b, "\t0,\n")
@@ -562,14 +519,6 @@ func (p *Package) loadDWARF(f *File, names []*Name) {
 
 	// Scan DWARF info for top-level TagVariable entries with AttrName __cgo__i.
 	types := make([]dwarf.Type, len(names))
-	nameToIndex := make(map[*Name]int)
-	for i, n := range names {
-		nameToIndex[n] = i
-	}
-	nameToRef := make(map[*Name]*Ref)
-	for _, ref := range f.Ref {
-		nameToRef[ref.Name] = ref
-	}
 	r := d.Reader()
 	for {
 		e, err := r.Next()
@@ -620,10 +569,7 @@ func (p *Package) loadDWARF(f *File, names []*Name) {
 		if types[i] == nil {
 			continue
 		}
-		pos := token.NoPos
+		pos := f.NamePos[n]
-		if ref, ok := nameToRef[n]; ok {
-			pos = ref.Pos()
-		}
 		f, fok := types[i].(*dwarf.FuncType)
 		if n.Kind != "type" && fok {
 			n.Kind = "func"
@@ -633,11 +579,11 @@ func (p *Package) loadDWARF(f *File, names []*Name) {
 			switch n.Kind {
 			case "iconst":
 				if i < len(ints) {
+					if _, ok := types[i].(*dwarf.UintType); ok {
+						n.Const = fmt.Sprintf("%#x", uint64(ints[i]))
+					} else {
 						n.Const = fmt.Sprintf("%#x", ints[i])
 					}
-			case "uconst":
-				if i < len(ints) {
-					n.Const = fmt.Sprintf("%#x", uint64(ints[i]))
 				}
 			case "fconst":
 				if i < len(floats) {
@@ -778,8 +724,9 @@ func (p *Package) rewriteCall(f *File, call *Call, name *Name) bool {
 		stmts = append(stmts, stmt)
 	}
 
+	const cgoMarker = "__cgo__###__marker__"
 	fcall := &ast.CallExpr{
-		Fun:  call.Call.Fun,
+		Fun:  ast.NewIdent(cgoMarker),
 		Args: nargs,
 	}
 	ftype := &ast.FuncType{
@@ -801,14 +748,10 @@ func (p *Package) rewriteCall(f *File, call *Call, name *Name) bool {
 		}
 	}
 
-	// There is a Ref pointing to the old call.Call.Fun.
-	for _, ref := range f.Ref {
-		if ref.Expr == &call.Call.Fun {
-			ref.Expr = &fcall.Fun
-
 	// If this call expects two results, we have to
 	// adjust the results of the function we generated.
-			if ref.Context == "call2" {
+	for _, ref := range f.Ref {
+		if ref.Expr == &call.Call.Fun && ref.Context == ctxCall2 {
 			if ftype.Results == nil {
 				// An explicit void argument
 				// looks odd but it seems to
@@ -827,7 +770,6 @@ func (p *Package) rewriteCall(f *File, call *Call, name *Name) bool {
 				})
 		}
 	}
-	}
 
 	var fbody ast.Stmt
 	if ftype.Results == nil {
@@ -839,14 +781,16 @@ func (p *Package) rewriteCall(f *File, call *Call, name *Name) bool {
 			Results: []ast.Expr{fcall},
 		}
 	}
-	call.Call.Fun = &ast.FuncLit{
+	lit := &ast.FuncLit{
 		Type: ftype,
 		Body: &ast.BlockStmt{
 			List: append(stmts, fbody),
 		},
 	}
-	call.Call.Lparen = token.NoPos
+	text := strings.Replace(gofmt(lit), "\n", ";", -1)
-	call.Call.Rparen = token.NoPos
+	repl := strings.Split(text, cgoMarker)
+	f.Edit.Insert(f.offset(call.Call.Fun.Pos()), repl[0])
+	f.Edit.Insert(f.offset(call.Call.Fun.End()), repl[1])
 
 	return needsUnsafe
 }
@@ -1000,8 +944,8 @@ func (p *Package) checkAddrArgs(f *File, args []ast.Expr, x ast.Expr) []ast.Expr
 // effect is a function call.
 func (p *Package) hasSideEffects(f *File, x ast.Expr) bool {
 	found := false
-	f.walk(x, "expr",
+	f.walk(x, ctxExpr,
-		func(f *File, x interface{}, context string) {
+		func(f *File, x interface{}, context astContext) {
 			switch x.(type) {
 			case *ast.CallExpr:
 				found = true
@@ -1110,7 +1054,17 @@ func (p *Package) rewriteRef(f *File) {
 	// Assign mangled names.
 	for _, n := range f.Name {
 		if n.Kind == "not-type" {
+			if n.Define == "" {
 				n.Kind = "var"
+			} else {
+				n.Kind = "macro"
+				n.FuncType = &FuncType{
+					Result: n.Type,
+					Go: &ast.FuncType{
+						Results: &ast.FieldList{List: []*ast.Field{{Type: n.Type.Go}}},
+					},
+				}
+			}
 		}
 		if n.Mangle == "" {
 			p.mangleName(n)
@@ -1130,10 +1084,10 @@ func (p *Package) rewriteRef(f *File) {
 		}
 		var expr ast.Expr = ast.NewIdent(r.Name.Mangle) // default
 		switch r.Context {
-		case "call", "call2":
+		case ctxCall, ctxCall2:
 			if r.Name.Kind != "func" {
 				if r.Name.Kind == "type" {
-					r.Context = "type"
+					r.Context = ctxType
 					if r.Name.Type == nil {
 						error_(r.Pos(), "invalid conversion to C.%s: undefined C type '%s'", fixGo(r.Name.Go), r.Name.C)
 						break
@@ -1145,7 +1099,7 @@ func (p *Package) rewriteRef(f *File) {
 				break
 			}
 			functions[r.Name.Go] = true
-			if r.Context == "call2" {
+			if r.Context == ctxCall2 {
 				if r.Name.Go == "_CMalloc" {
 					error_(r.Pos(), "no two-result form for C.malloc")
 					break
@@ -1163,8 +1117,9 @@ func (p *Package) rewriteRef(f *File) {
 				r.Name = n
 				break
 			}
-		case "expr":
+		case ctxExpr:
-			if r.Name.Kind == "func" {
+			switch r.Name.Kind {
+			case "func":
 				if builtinDefs[r.Name.C] != "" {
 					error_(r.Pos(), "use of builtin '%s' not in function call", fixGo(r.Name.C))
 				}
@@ -1191,25 +1146,25 @@ func (p *Package) rewriteRef(f *File) {
 					Fun:  &ast.Ident{NamePos: (*r.Expr).Pos(), Name: "_Cgo_ptr"},
 					Args: []ast.Expr{ast.NewIdent(name.Mangle)},
 				}
-			} else if r.Name.Kind == "type" {
+			case "type":
 				// Okay - might be new(T)
 				if r.Name.Type == nil {
 					error_(r.Pos(), "expression C.%s: undefined C type '%s'", fixGo(r.Name.Go), r.Name.C)
 					break
 				}
 				expr = r.Name.Type.Go
-			} else if r.Name.Kind == "var" {
+			case "var":
 				expr = &ast.StarExpr{Star: (*r.Expr).Pos(), X: expr}
+			case "macro":
+				expr = &ast.CallExpr{Fun: expr}
 			}
-
+		case ctxSelector:
-		case "selector":
 			if r.Name.Kind == "var" {
 				expr = &ast.StarExpr{Star: (*r.Expr).Pos(), X: expr}
 			} else {
 				error_(r.Pos(), "only C variables allowed in selector expression %s", fixGo(r.Name.Go))
 			}
-
+		case ctxType:
-		case "type":
 			if r.Name.Kind != "type" {
 				error_(r.Pos(), "expression C.%s used as type", fixGo(r.Name.Go))
 			} else if r.Name.Type == nil {
@@ -1224,6 +1179,7 @@ func (p *Package) rewriteRef(f *File) {
 				error_(r.Pos(), "must call C.%s", fixGo(r.Name.Go))
 			}
 		}
+
 		if *godefs {
 			// Substitute definition for mangled type name.
 			if id, ok := expr.(*ast.Ident); ok {
@@ -1245,7 +1201,17 @@ func (p *Package) rewriteRef(f *File) {
 			expr = &ast.Ident{NamePos: pos, Name: x.Name}
 		}
 
+		// Change AST, because some later processing depends on it,
+		// and also because -godefs mode still prints the AST.
+		old := *r.Expr
 		*r.Expr = expr
+
+		// Record source-level edit for cgo output.
+		repl := gofmt(expr)
+		if r.Name.Kind != "type" {
+			repl = "(" + repl + ")"
+		}
+		f.Edit.Replace(f.offset(old.Pos()), f.offset(old.End()), repl)
 	}
 
 	// Remove functions only used as expressions, so their respective
@@ -1270,7 +1236,7 @@ func (p *Package) gccBaseCmd() []string {
 	if ret := strings.Fields(os.Getenv("GCC")); len(ret) > 0 {
 		return ret
 	}
-	return strings.Fields(defaultCC)
+	return strings.Fields(defaultCC(goos, goarch))
 }
 
 // gccMachine returns the gcc -m flag to use, either "-m32", "-m64" or "-marm".
@@ -2186,6 +2152,12 @@ func (c *typeConv) Type(dtype dwarf.Type, pos token.Pos) *Type {
 		name := c.Ident("_Ctype_" + dt.Name)
 		goIdent[name.Name] = name
 		sub := c.Type(dt.Type, pos)
+		if badPointerTypedef(dt.Name) {
+			// Treat this typedef as a uintptr.
+			s := *sub
+			s.Go = c.uintptr
+			sub = &s
+		}
 		t.Go = name
 		if unionWithPointer[sub.Go] {
 			unionWithPointer[t.Go] = true
@@ -2266,7 +2238,7 @@ func (c *typeConv) Type(dtype dwarf.Type, pos token.Pos) *Type {
 			if ss, ok := dwarfToName[s]; ok {
 				s = ss
 			}
-			s = strings.Join(strings.Split(s, " "), "") // strip spaces
+			s = strings.Replace(s, " ", "", -1)
 			name := c.Ident("_Ctype_" + s)
 			tt := *t
 			typedef[name.Name] = &tt
@@ -2344,6 +2316,17 @@ func (c *typeConv) FuncArg(dtype dwarf.Type, pos token.Pos) *Type {
 			if _, void := base(ptr.Type).(*dwarf.VoidType); void {
 				break
 			}
+			// ...or the typedef is one in which we expect bad pointers.
+			// It will be a uintptr instead of *X.
+			if badPointerTypedef(dt.Name) {
+				break
+			}
+
+			// If we already know the typedef for t just use that.
+			// See issue 19832.
+			if def := typedef[t.Go.(*ast.Ident).Name]; def != nil {
+				break
+			}
 
 			t = c.Type(ptr, pos)
 			if t == nil {
@@ -2500,7 +2483,9 @@ func (c *typeConv) Struct(dt *dwarf.StructType, pos token.Pos) (expr *ast.Struct
 		size := t.Size
 		talign := t.Align
 		if f.BitSize > 0 {
-			if f.BitSize%8 != 0 {
+			switch f.BitSize {
+			case 8, 16, 32, 64:
+			default:
 				continue
 			}
 			size = f.BitSize / 8
@@ -2676,3 +2661,51 @@ func fieldPrefix(fld []*ast.Field) string {
 	}
 	return prefix
 }
+
+// badPointerTypedef reports whether t is a C typedef that should not be considered a pointer in Go.
+// A typedef is bad if C code sometimes stores non-pointers in this type.
+// TODO: Currently our best solution is to find these manually and list them as
+// they come up. A better solution is desired.
+func badPointerTypedef(t string) bool {
+	// The real bad types are CFNumberRef and CFTypeRef.
+	// Sometimes non-pointers are stored in these types.
+	// CFTypeRef is a supertype of those, so it can have bad pointers in it as well.
+	// We return true for the other CF*Ref types just so casting between them is easier.
+	// See comment below for details about the bad pointers.
+	return goos == "darwin" && strings.HasPrefix(t, "CF") && strings.HasSuffix(t, "Ref")
+}
+
+// Comment from Darwin's CFInternal.h
+/*
+// Tagged pointer support
+// Low-bit set means tagged object, next 3 bits (currently)
+// define the tagged object class, next 4 bits are for type
+// information for the specific tagged object class.  Thus,
+// the low byte is for type info, and the rest of a pointer
+// (32 or 64-bit) is for payload, whatever the tagged class.
+//
+// Note that the specific integers used to identify the
+// specific tagged classes can and will change from release
+// to release (that's why this stuff is in CF*Internal*.h),
+// as can the definition of type info vs payload above.
+//
+#if __LP64__
+#define CF_IS_TAGGED_OBJ(PTR)	((uintptr_t)(PTR) & 0x1)
+#define CF_TAGGED_OBJ_TYPE(PTR)	((uintptr_t)(PTR) & 0xF)
+#else
+#define CF_IS_TAGGED_OBJ(PTR)	0
+#define CF_TAGGED_OBJ_TYPE(PTR)	0
+#endif
+
+enum {
+    kCFTaggedObjectID_Invalid = 0,
+    kCFTaggedObjectID_Atom = (0 << 1) + 1,
+    kCFTaggedObjectID_Undefined3 = (1 << 1) + 1,
+    kCFTaggedObjectID_Undefined2 = (2 << 1) + 1,
+    kCFTaggedObjectID_Integer = (3 << 1) + 1,
+    kCFTaggedObjectID_DateTS = (4 << 1) + 1,
+    kCFTaggedObjectID_ManagedObjectID = (5 << 1) + 1, // Core Data
+    kCFTaggedObjectID_Date = (6 << 1) + 1,
+    kCFTaggedObjectID_Undefined7 = (7 << 1) + 1,
+};
+*/

libgo/go/cmd/cgo/main.go

@@ -24,6 +24,9 @@ import (
 	"runtime"
 	"sort"
 	"strings"
+
+	"cmd/internal/edit"
+	"cmd/internal/objabi"
 )
 
 // A Package collects information about the package we're going to write.
@@ -54,6 +57,12 @@ type File struct {
 	Calls    []*Call             // all calls to C.xxx in AST
 	ExpFunc  []*ExpFunc          // exported functions for this file
 	Name     map[string]*Name    // map from Go name to Name
+	NamePos  map[*Name]token.Pos // map from Name to position of the first reference
+	Edit     *edit.Buffer
+}
+
+func (f *File) offset(p token.Pos) int {
+	return fset.Position(p).Offset
 }
 
 func nameKeys(m map[string]*Name) []string {
@@ -75,7 +84,7 @@ type Call struct {
 type Ref struct {
 	Name    *Name
 	Expr    *ast.Expr
-	Context string // "type", "expr", "call", or "call2"
+	Context astContext
 }
 
 func (r *Ref) Pos() token.Pos {
@@ -88,7 +97,7 @@ type Name struct {
 	Mangle   string // name used in generated Go
 	C        string // name used in C
 	Define   string // #define expansion
-	Kind     string // "iconst", "uconst", "fconst", "sconst", "type", "var", "fpvar", "func", "not-type"
+	Kind     string // "iconst", "fconst", "sconst", "type", "var", "fpvar", "func", "macro", "not-type"
 	Type     *Type  // the type of xxx
 	FuncType *FuncType
 	AddError bool
@@ -100,12 +109,12 @@ func (n *Name) IsVar() bool {
 	return n.Kind == "var" || n.Kind == "fpvar"
 }
 
-// IsConst reports whether Kind is either "iconst", "uconst", "fconst" or "sconst"
+// IsConst reports whether Kind is either "iconst", "fconst" or "sconst"
 func (n *Name) IsConst() bool {
 	return strings.HasSuffix(n.Kind, "const")
 }
 
-// A ExpFunc is an exported function, callable from C.
+// An ExpFunc is an exported function, callable from C.
 // Such functions are identified in the Go input file
 // by doc comments containing the line //export ExpName
 type ExpFunc struct {
@@ -214,6 +223,7 @@ var importSyscall = flag.Bool("import_syscall", true, "import syscall in generat
 var goarch, goos string
 
 func main() {
+	objabi.AddVersionFlag() // -V
 	flag.Usage = usage
 	flag.Parse()
 
@@ -294,6 +304,7 @@ func main() {
 		}
 
 		f := new(File)
+		f.Edit = edit.NewBuffer(b)
 		f.ParseGo(input, b)
 		f.DiscardCgoDirectives()
 		fs[i] = f
@@ -314,11 +325,13 @@ func main() {
 		p.Translate(f)
 		for _, cref := range f.Ref {
 			switch cref.Context {
-			case "call", "call2":
+			case ctxCall, ctxCall2:
 				if cref.Name.Kind != "type" {
 					break
 				}
+				old := *cref.Expr
 				*cref.Expr = cref.Name.Type.Go
+				f.Edit.Replace(f.offset(old.Pos()), f.offset(old.End()), gofmt(cref.Name.Type.Go))
 			}
 		}
 		if nerrors > 0 {

libgo/go/cmd/cgo/out.go

@@ -16,6 +16,7 @@ import (
 	"go/token"
 	"io"
 	"os"
+	"path/filepath"
 	"sort"
 	"strings"
 )
@@ -116,7 +117,13 @@ func (p *Package) writeDefs() {
 		// Which is not useful. Moreover we never override source info,
 		// so subsequent source code uses the same source info.
 		// Moreover, empty file name makes compile emit no source debug info at all.
-		noSourceConf.Fprint(fgo2, fset, def.Go)
+		var buf bytes.Buffer
+		noSourceConf.Fprint(&buf, fset, def.Go)
+		if bytes.HasPrefix(buf.Bytes(), []byte("_Ctype_")) {
+			// This typedef is of the form `typedef a b` and should be an alias.
+			fmt.Fprintf(fgo2, "= ")
+		}
+		fmt.Fprintf(fgo2, "%s", buf.Bytes())
 		fmt.Fprintf(fgo2, "\n\n")
 	}
 	if *gccgo {
@@ -424,11 +431,13 @@ func (p *Package) writeDefsFunc(fgo2 io.Writer, n *Name, callsMalloc *bool) {
 	inProlog := builtinDefs[name] != ""
 	cname := fmt.Sprintf("_cgo%s%s", cPrefix, n.Mangle)
 	paramnames := []string(nil)
+	if d.Type.Params != nil {
 		for i, param := range d.Type.Params.List {
 			paramName := fmt.Sprintf("p%d", i)
 			param.Names = []*ast.Ident{ast.NewIdent(paramName)}
 			paramnames = append(paramnames, paramName)
 		}
+	}
 
 	if *gccgo {
 		// Gccgo style hooks.
@@ -526,9 +535,11 @@ func (p *Package) writeDefsFunc(fgo2 io.Writer, n *Name, callsMalloc *bool) {
 		fmt.Fprintf(fgo2, "\tif errno != 0 { r2 = syscall.Errno(errno) }\n")
 	}
 	fmt.Fprintf(fgo2, "\tif _Cgo_always_false {\n")
+	if d.Type.Params != nil {
 		for i := range d.Type.Params.List {
 			fmt.Fprintf(fgo2, "\t\t_Cgo_use(p%d)\n", i)
 		}
+	}
 	fmt.Fprintf(fgo2, "\t}\n")
 	fmt.Fprintf(fgo2, "\treturn\n")
 	fmt.Fprintf(fgo2, "}\n")
@@ -540,7 +551,7 @@ func (p *Package) writeOutput(f *File, srcfile string) {
 	if strings.HasSuffix(base, ".go") {
 		base = base[0 : len(base)-3]
 	}
-	base = strings.Map(slashToUnderscore, base)
+	base = filepath.Base(base)
 	fgo1 := creat(*objDir + base + ".cgo1.go")
 	fgcc := creat(*objDir + base + ".cgo2.c")
 
@@ -549,10 +560,12 @@ func (p *Package) writeOutput(f *File, srcfile string) {
 
 	// Write Go output: Go input with rewrites of C.xxx to _C_xxx.
 	fmt.Fprintf(fgo1, "// Created by cgo - DO NOT EDIT\n\n")
-	conf.Fprint(fgo1, fset, f.AST)
+	fmt.Fprintf(fgo1, "//line %s:1\n", srcfile)
+	fgo1.Write(f.Edit.Bytes())
 
 	// While we process the vars and funcs, also write gcc output.
 	// Gcc output starts with the preamble.
+	fmt.Fprintf(fgcc, "%s\n", builtinProlog)
 	fmt.Fprintf(fgcc, "%s\n", f.Preamble)
 	fmt.Fprintf(fgcc, "%s\n", gccProlog)
 	fmt.Fprintf(fgcc, "%s\n", tsanProlog)
@@ -639,6 +652,9 @@ func (p *Package) writeOutputFunc(fgcc *os.File, n *Name) {
 			fmt.Fprint(fgcc, "(__typeof__(a->r)) ")
 		}
 	}
+	if n.Kind == "macro" {
+		fmt.Fprintf(fgcc, "%s;\n", n.C)
+	} else {
 		fmt.Fprintf(fgcc, "%s(", n.C)
 		for i := range n.FuncType.Params {
 			if i > 0 {
@@ -647,6 +663,7 @@ func (p *Package) writeOutputFunc(fgcc *os.File, n *Name) {
 			fmt.Fprintf(fgcc, "a->p%d", i)
 		}
 		fmt.Fprintf(fgcc, ");\n")
+	}
 	if n.AddError {
 		fmt.Fprintf(fgcc, "\t_cgo_errno = errno;\n")
 	}
@@ -702,6 +719,9 @@ func (p *Package) writeGccgoOutputFunc(fgcc *os.File, n *Name) {
 			fmt.Fprintf(fgcc, "(void*)")
 		}
 	}
+	if n.Kind == "macro" {
+		fmt.Fprintf(fgcc, "%s;\n", n.C)
+	} else {
 		fmt.Fprintf(fgcc, "%s(", n.C)
 		for i := range n.FuncType.Params {
 			if i > 0 {
@@ -710,6 +730,7 @@ func (p *Package) writeGccgoOutputFunc(fgcc *os.File, n *Name) {
 			fmt.Fprintf(fgcc, "p%d", i)
 		}
 		fmt.Fprintf(fgcc, ");\n")
+	}
 	fmt.Fprintf(fgcc, "\t_cgo_tsan_release();\n")
 	if t := n.FuncType.Result; t != nil {
 		fmt.Fprintf(fgcc, "\treturn ")
@@ -1009,7 +1030,7 @@ func (p *Package) writeGccgoExports(fgo2, fm, fgcc, fgcch io.Writer) {
 		default:
 			// Declare a result struct.
 			fmt.Fprintf(fgcch, "\n/* Return type for %s */\n", exp.ExpName)
-			fmt.Fprintf(fgcch, "struct %s_result {\n", exp.ExpName)
+			fmt.Fprintf(fgcch, "struct %s_return {\n", exp.ExpName)
 			forFieldList(fntype.Results,
 				func(i int, aname string, atype ast.Expr) {
 					t := p.cgoType(atype)
@@ -1020,7 +1041,7 @@ func (p *Package) writeGccgoExports(fgo2, fm, fgcc, fgcch io.Writer) {
 					fmt.Fprint(fgcch, "\n")
 				})
 			fmt.Fprintf(fgcch, "};\n")
-			fmt.Fprintf(cdeclBuf, "struct %s_result", exp.ExpName)
+			fmt.Fprintf(cdeclBuf, "struct %s_return", exp.ExpName)
 		}
 
 		cRet := cdeclBuf.String()
@@ -1046,7 +1067,7 @@ func (p *Package) writeGccgoExports(fgo2, fm, fgcc, fgcch io.Writer) {
 			fmt.Fprintf(fgcch, "\n%s", exp.Doc)
 		}
 
-		fmt.Fprintf(fgcch, "extern %s %s %s;\n", cRet, exp.ExpName, cParams)
+		fmt.Fprintf(fgcch, "extern %s %s%s;\n", cRet, exp.ExpName, cParams)
 
 		// We need to use a name that will be exported by the
 		// Go code; otherwise gccgo will make it static and we
@@ -1155,6 +1176,7 @@ func (p *Package) writeExportHeader(fgcch io.Writer) {
 		pkg = p.PackagePath
 	}
 	fmt.Fprintf(fgcch, "/* package %s */\n\n", pkg)
+	fmt.Fprintf(fgcch, "%s\n", builtinExportProlog)
 
 	fmt.Fprintf(fgcch, "/* Start of preamble from import \"C\" comments.  */\n\n")
 	fmt.Fprintf(fgcch, "%s\n", p.Preamble)
@@ -1247,8 +1269,9 @@ func (p *Package) cgoType(e ast.Expr) *Type {
 			// Slice: pointer, len, cap.
 			return &Type{Size: p.PtrSize * 3, Align: p.PtrSize, C: c("GoSlice")}
 		}
+		// Non-slice array types are not supported.
 	case *ast.StructType:
-		// TODO
+		// Not supported.
 	case *ast.FuncType:
 		return &Type{Size: p.PtrSize, Align: p.PtrSize, C: c("void*")}
 	case *ast.InterfaceType:
@@ -1398,7 +1421,7 @@ const builtinProlog = `
 /* Define intgo when compiling with GCC.  */
 typedef ptrdiff_t intgo;
 
-typedef struct { char *p; intgo n; } _GoString_;
+typedef struct { const char *p; intgo n; } _GoString_;
 typedef struct { char *p; intgo n; intgo c; } _GoBytes_;
 _GoString_ GoString(char *p);
 _GoString_ GoStringN(char *p, int l);
@@ -1406,6 +1429,12 @@ _GoBytes_ GoBytes(void *p, int n);
 char *CString(_GoString_);
 void *CBytes(_GoBytes_);
 void *_CMalloc(size_t);
+
+__attribute__ ((unused))
+static size_t _GoStringLen(_GoString_ s) { return s.n; }
+
+__attribute__ ((unused))
+static const char *_GoStringPtr(_GoString_ s) { return s.p; }
 `
 
 const goProlog = `
@@ -1637,6 +1666,27 @@ void localCgoCheckResult(Eface val) {
 }
 `
 
+// builtinExportProlog is a shorter version of builtinProlog,
+// to be put into the _cgo_export.h file.
+// For historical reasons we can't use builtinProlog in _cgo_export.h,
+// because _cgo_export.h defines GoString as a struct while builtinProlog
+// defines it as a function. We don't change this to avoid unnecessarily
+// breaking existing code.
+const builtinExportProlog = `
+#line 1 "cgo-builtin-prolog"
+
+#include <stddef.h> /* for ptrdiff_t below */
+
+#ifndef GO_CGO_EXPORT_PROLOGUE_H
+#define GO_CGO_EXPORT_PROLOGUE_H
+
+typedef ptrdiff_t intgo;
+
+typedef struct { const char *p; intgo n; } _GoString_;
+
+#endif
+`
+
 func (p *Package) gccExportHeaderProlog() string {
 	return strings.Replace(gccExportHeaderProlog, "GOINTBITS", fmt.Sprint(8*p.IntSize), -1)
 }
@@ -1670,7 +1720,7 @@ typedef double _Complex GoComplex128;
 */
 typedef char _check_for_GOINTBITS_bit_pointer_matching_GoInt[sizeof(void*)==GOINTBITS/8 ? 1:-1];
 
-typedef struct { const char *p; GoInt n; } GoString;
+typedef _GoString_ GoString;
 typedef void *GoMap;
 typedef void *GoChan;
 typedef struct { void *t; void *v; } GoInterface;

libgo/go/cmd/go/alldocs.go

@@ -14,12 +14,12 @@
 // The commands are:
 //
 // 	build       compile packages and dependencies
-// 	clean       remove object files
+// 	clean       remove object files and cached files
 // 	doc         show documentation for package or symbol
 // 	env         print Go environment information
 // 	bug         start a bug report
-// 	fix         run go tool fix on packages
+// 	fix         update packages to use new APIs
-// 	fmt         run gofmt on package sources
+// 	fmt         gofmt (reformat) package sources
 // 	generate    generate Go files by processing source
 // 	get         download and install packages and dependencies
 // 	install     compile and install packages and dependencies
@@ -28,7 +28,7 @@
 // 	test        test packages
 // 	tool        run specified go tool
 // 	version     print Go version
-// 	vet         run go tool vet on packages
+// 	vet         report likely mistakes in packages
 //
 // Use "go help [command]" for more information about a command.
 //
@@ -104,15 +104,15 @@
 // 	-x
 // 		print the commands.
 //
-// 	-asmflags 'flag list'
+// 	-asmflags '[pattern=]arg list'
 // 		arguments to pass on each go tool asm invocation.
 // 	-buildmode mode
 // 		build mode to use. See 'go help buildmode' for more.
 // 	-compiler name
 // 		name of compiler to use, as in runtime.Compiler (gccgo or gc).
-// 	-gccgoflags 'arg list'
+// 	-gccgoflags '[pattern=]arg list'
 // 		arguments to pass on each gccgo compiler/linker invocation.
-// 	-gcflags 'arg list'
+// 	-gcflags '[pattern=]arg list'
 // 		arguments to pass on each go tool compile invocation.
 // 	-installsuffix suffix
 // 		a suffix to use in the name of the package installation directory,
@@ -121,7 +121,7 @@
 // 		or, if set explicitly, has _race appended to it. Likewise for the -msan
 // 		flag. Using a -buildmode option that requires non-default compile flags
 // 		has a similar effect.
-// 	-ldflags 'flag list'
+// 	-ldflags '[pattern=]arg list'
 // 		arguments to pass on each go tool link invocation.
 // 	-linkshared
 // 		link against shared libraries previously created with
@@ -139,9 +139,21 @@
 // 		For example, instead of running asm, the go command will run
 // 		'cmd args /path/to/asm <arguments for asm>'.
 //
-// All the flags that take a list of arguments accept a space-separated
+// The -asmflags, -gccgoflags, -gcflags, and -ldflags flags accept a
-// list of strings. To embed spaces in an element in the list, surround
+// space-separated list of arguments to pass to an underlying tool
-// it with either single or double quotes.
+// during the build. To embed spaces in an element in the list, surround
+// it with either single or double quotes. The argument list may be
+// preceded by a package pattern and an equal sign, which restricts
+// the use of that argument list to the building of packages matching
+// that pattern (see 'go help packages' for a description of package
+// patterns). Without a pattern, the argument list applies only to the
+// packages named on the command line. The flags may be repeated
+// with different patterns in order to specify different arguments for
+// different sets of packages. If a package matches patterns given in
+// multiple flags, the latest match on the command line wins.
+// For example, 'go build -gcflags=-S fmt' prints the disassembly
+// only for package fmt, while 'go build -gcflags=all=-S fmt'
+// prints the disassembly for fmt and all its dependencies.
 //
 // For more about specifying packages, see 'go help packages'.
 // For more about where packages and binaries are installed,
@@ -158,11 +170,11 @@
 // See also: go install, go get, go clean.
 //
 //
-// Remove object files
+// Remove object files and cached files
 //
 // Usage:
 //
-// 	go clean [-i] [-r] [-n] [-x] [build flags] [packages]
+// 	go clean [-i] [-r] [-n] [-x] [-cache] [-testcache] [build flags] [packages]
 //
 // Clean removes object files from package source directories.
 // The go command builds most objects in a temporary directory,
@@ -200,6 +212,11 @@
 //
 // The -x flag causes clean to print remove commands as it executes them.
 //
+// The -cache flag causes clean to remove the entire go build cache.
+//
+// The -testcache flag causes clean to expire all test results in the
+// go build cache.
+//
 // For more about build flags, see 'go help build'.
 //
 // For more about specifying packages, see 'go help packages'.
@@ -328,6 +345,8 @@
 // The -json flag prints the environment in JSON format
 // instead of as a shell script.
 //
+// For more about environment variables, see 'go help environment'.
+//
 //
 // Start a bug report
 //
@@ -339,7 +358,7 @@
 // The report includes useful system information.
 //
 //
-// Run go tool fix on packages
+// Update packages to use new APIs
 //
 // Usage:
 //
@@ -355,7 +374,7 @@
 // See also: go fmt, go vet.
 //
 //
-// Run gofmt on package sources
+// Gofmt (reformat) package sources
 //
 // Usage:
 //
@@ -543,10 +562,11 @@
 //
 // Usage:
 //
-// 	go install [build flags] [packages]
+// 	go install [-i] [build flags] [packages]
 //
-// Install compiles and installs the packages named by the import paths,
+// Install compiles and installs the packages named by the import paths.
-// along with their dependencies.
+//
+// The -i flag installs the dependencies of the named packages as well.
 //
 // For more about the build flags, see 'go help build'.
 // For more about specifying packages, see 'go help packages'.
@@ -719,10 +739,10 @@
 //
 // 'Go test' recompiles each package along with any files with names matching
 // the file pattern "*_test.go".
-// Files whose names begin with "_" (including "_test.go") or "." are ignored.
 // These additional files can contain test functions, benchmark functions, and
 // example functions. See 'go help testfunc' for more.
 // Each listed package causes the execution of a separate test binary.
+// Files whose names begin with "_" (including "_test.go") or "." are ignored.
 //
 // Test files that declare a package with the suffix "_test" will be compiled as a
 // separate package, and then linked and run with the main test binary.
@@ -730,11 +750,46 @@
 // The go tool will ignore a directory named "testdata", making it available
 // to hold ancillary data needed by the tests.
 //
-// By default, go test needs no arguments. It compiles and tests the package
+// As part of building a test binary, go test runs go vet on the package
-// with source in the current directory, including tests, and runs the tests.
+// and its test source files to identify significant problems. If go vet
+// finds any problems, go test reports those and does not run the test binary.
+// Only a high-confidence subset of the default go vet checks are used.
+// To disable the running of go vet, use the -vet=off flag.
 //
-// The package is built in a temporary directory so it does not interfere with the
+// Go test runs in two different modes: local directory mode when invoked with
-// non-test installation.
+// no package arguments (for example, 'go test'), and package list mode when
+// invoked with package arguments (for example 'go test math', 'go test ./...',
+// and even 'go test .').
+//
+// In local directory mode, go test compiles and tests the package sources
+// found in the current directory and then runs the resulting test binary.
+// In this mode, caching (discussed below) is disabled. After the package test
+// finishes, go test prints a summary line showing the test status ('ok' or 'FAIL'),
+// package name, and elapsed time.
+//
+// In package list mode, go test compiles and tests each of the packages
+// listed on the command line. If a package test passes, go test prints only
+// the final 'ok' summary line. If a package test fails, go test prints the
+// full test output. If invoked with the -bench or -v flag, go test prints
+// the full output even for passing package tests, in order to display the
+// requested benchmark results or verbose logging.
+//
+// All test output and summary lines are printed to the go command's standard
+// output, even if the test printed them to its own standard error.
+// (The go command's standard error is reserved for printing errors building
+// the tests.)
+//
+// In package list mode, go test also caches successful package test results.
+// If go test has cached a previous test run using the same test binary and
+// the same command line consisting entirely of cacheable test flags
+// (defined as -cpu, -list, -parallel, -run, -short, and -v),
+// go test will redisplay the previous output instead of running the test
+// binary again. In the summary line, go test prints '(cached)' in place of
+// the elapsed time. To disable test caching, use any test flag or argument
+// other than the cacheable flags. The idiomatic way to disable test caching
+// explicitly is to use -count=1. A cached result is treated as executing in
+// no time at all, so a successful package test result will be cached and reused
+// regardless of -timeout setting.
 //
 // In addition to the build flags, the flags handled by 'go test' itself are:
 //
@@ -757,6 +812,10 @@
 // 	    Install packages that are dependencies of the test.
 // 	    Do not run the test.
 //
+// 	-json
+// 	    Convert test output to JSON suitable for automated processing.
+// 	    See 'go doc test2json' for the encoding details.
+//
 // 	-o file
 // 	    Compile the test binary to the named file.
 // 	    The test still runs (unless -c or -i is specified).
@@ -782,7 +841,7 @@
 // The -n flag causes tool to print the command that would be
 // executed but not execute it.
 //
-// For more about each tool command, see 'go tool command -h'.
+// For more about each tool command, see 'go doc cmd/<command>'.
 //
 //
 // Print Go version
@@ -794,7 +853,7 @@
 // Version prints the Go version, as reported by runtime.Version.
 //
 //
-// Run go tool vet on packages
+// Report likely mistakes in packages
 //
 // Usage:
 //
@@ -808,7 +867,9 @@
 // The -n flag prints commands that would be executed.
 // The -x flag prints commands as they are executed.
 //
-// For more about build flags, see 'go help build'.
+// The build flags supported by go vet are those that control package resolution
+// and execution, such as -n, -x, -v, -tags, and -toolexec.
+// For more about these flags, see 'go help build'.
 //
 // See also: go fmt, go fix.
 //
@@ -917,8 +978,10 @@
 // comment, indicating that the package sources are included
 // for documentation only and must not be used to build the
 // package binary. This enables distribution of Go packages in
-// their compiled form alone. See the go/build package documentation
+// their compiled form alone. Even binary-only packages require
-// for more details.
+// accurate import blocks listing required dependencies, so that
+// those dependencies can be supplied when linking the resulting
+// command.
 //
 //
 // GOPATH environment variable
@@ -1096,6 +1159,12 @@
 // 		See https://golang.org/doc/articles/race_detector.html.
 // 	GOROOT
 // 		The root of the go tree.
+// 	GOTMPDIR
+// 		The directory where the go command will write
+// 		temporary source files, packages, and binaries.
+// 	GOCACHE
+// 		The directory where the go command will store
+// 		cached information for reuse in future builds.
 //
 // Environment variables for use with cgo:
 //
@@ -1130,6 +1199,9 @@
 // 	GO386
 // 		For GOARCH=386, the floating point instruction set.
 // 		Valid values are 387, sse2.
+// 	GOMIPS
+// 		For GOARCH=mips{,le}, whether to use floating point instructions.
+// 		Valid values are hardfloat (default), softfloat.
 //
 // Special-purpose environment variables:
 //
@@ -1460,10 +1532,10 @@
 // 			significantly more expensive.
 // 	    Sets -cover.
 //
-// 	-coverpkg pkg1,pkg2,pkg3
+// 	-coverpkg pattern1,pattern2,pattern3
-// 	    Apply coverage analysis in each test to the given list of packages.
+// 	    Apply coverage analysis in each test to packages matching the patterns.
 // 	    The default is for each test to analyze only the package being tested.
-// 	    Packages are specified as import paths.
+// 	    See 'go help packages' for a description of package patterns.
 // 	    Sets -cover.
 //
 // 	-cpu 1,2,4
@@ -1471,6 +1543,9 @@
 // 	    benchmarks should be executed. The default is the current value
 // 	    of GOMAXPROCS.
 //
+// 	-failfast
+// 	    Do not start new tests after the first test failure.
+//
 // 	-list regexp
 // 	    List tests, benchmarks, or examples matching the regular expression.
 // 	    No tests, benchmarks or examples will be run. This will only
@@ -1503,12 +1578,20 @@
 //
 // 	-timeout d
 // 	    If a test binary runs longer than duration d, panic.
+// 	    If d is 0, the timeout is disabled.
 // 	    The default is 10 minutes (10m).
 //
 // 	-v
 // 	    Verbose output: log all tests as they are run. Also print all
 // 	    text from Log and Logf calls even if the test succeeds.
 //
+// 	-vet list
+// 	    Configure the invocation of "go vet" during "go test"
+// 	    to use the comma-separated list of vet checks.
+// 	    If list is empty, "go test" runs "go vet" with a curated list of
+// 	    checks believed to be always worth addressing.
+// 	    If list is "off", "go test" does not run "go vet" at all.
+//
 // The following flags are also recognized by 'go test' and can be used to
 // profile the tests during execution:
 //

libgo/go/cmd/go/go_windows_test.go

@@ -5,12 +5,14 @@
 package main
 
 import (
+	"fmt"
 	"internal/testenv"
 	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"strings"
+	"syscall"
 	"testing"
 )
 
@@ -54,3 +56,82 @@ func TestAbsolutePath(t *testing.T) {
 		t.Fatalf("wrong output found: %v %v", err, string(output))
 	}
 }
+
+func isWindowsXP(t *testing.T) bool {
+	v, err := syscall.GetVersion()
+	if err != nil {
+		t.Fatalf("GetVersion failed: %v", err)
+	}
+	major := byte(v)
+	return major < 6
+}
+
+func runIcacls(t *testing.T, args ...string) string {
+	t.Helper()
+	out, err := exec.Command("icacls", args...).CombinedOutput()
+	if err != nil {
+		t.Fatalf("icacls failed: %v\n%v", err, string(out))
+	}
+	return string(out)
+}
+
+func runGetACL(t *testing.T, path string) string {
+	t.Helper()
+	cmd := fmt.Sprintf(`Get-Acl "%s" | Select -expand AccessToString`, path)
+	out, err := exec.Command("powershell", "-Command", cmd).CombinedOutput()
+	if err != nil {
+		t.Fatalf("Get-Acl failed: %v\n%v", err, string(out))
+	}
+	return string(out)
+}
+
+// For issue 22343: verify that executable file created by "go build" command
+// has discretionary access control list (DACL) set as if the file
+// was created in the destination directory.
+func TestACL(t *testing.T) {
+	if isWindowsXP(t) {
+		t.Skip("Windows XP does not have powershell command")
+	}
+
+	tmpdir, err := ioutil.TempDir("", "TestACL")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpdir)
+
+	newtmpdir := filepath.Join(tmpdir, "tmp")
+	err = os.Mkdir(newtmpdir, 0777)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// When TestACL/tmp directory is created, it will have
+	// the same security attributes as TestACL.
+	// Add Guest account full access to TestACL/tmp - this
+	// will make all files created in TestACL/tmp have different
+	// security attributes to the files created in TestACL.
+	runIcacls(t, newtmpdir,
+		"/grant", "guest:(oi)(ci)f", // add Guest user to have full access
+	)
+
+	src := filepath.Join(tmpdir, "main.go")
+	err = ioutil.WriteFile(src, []byte("package main; func main() { }\n"), 0644)
+	if err != nil {
+		t.Fatal(err)
+	}
+	exe := filepath.Join(tmpdir, "main.exe")
+	cmd := exec.Command(testenv.GoToolPath(t), "build", "-o", exe, src)
+	cmd.Env = append(os.Environ(),
+		"TMP="+newtmpdir,
+		"TEMP="+newtmpdir,
+	)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Fatalf("go command failed: %v\n%v", err, string(out))
+	}
+
+	// exe file is expected to have the same security attributes as the src.
+	if got, expected := runGetACL(t, exe), runGetACL(t, src); got != expected {
+		t.Fatalf("expected Get-Acl output of \n%v\n, got \n%v\n", expected, got)
+	}
+}

libgo/go/cmd/go/internal/base/base.go

@@ -62,8 +62,8 @@ func (c *Command) Name() string {
 }
 
 func (c *Command) Usage() {
-	fmt.Fprintf(os.Stderr, "usage: %s\n\n", c.UsageLine)
+	fmt.Fprintf(os.Stderr, "usage: %s\n", c.UsageLine)
-	fmt.Fprintf(os.Stderr, "%s\n", strings.TrimSpace(c.Long))
+	fmt.Fprintf(os.Stderr, "Run 'go help %s' for details.\n", c.Name())
 	os.Exit(2)
 }
 

libgo/go/cmd/go/internal/base/path.go

@@ -44,28 +44,6 @@ func RelPaths(paths []string) []string {
 	return out
 }
 
-// FilterDotUnderscoreFiles returns a slice containing all elements
-// of path whose base name doesn't begin with "." or "_".
-func FilterDotUnderscoreFiles(path []string) []string {
-	var out []string // lazily initialized
-	for i, p := range path {
-		base := filepath.Base(p)
-		if strings.HasPrefix(base, ".") || strings.HasPrefix(base, "_") {
-			if out == nil {
-				out = append(make([]string, 0, len(path)), path[:i]...)
-			}
-			continue
-		}
-		if out != nil {
-			out = append(out, p)
-		}
-	}
-	if out == nil {
-		return path
-	}
-	return out
-}
-
 // IsTestFile reports whether the source file is a set of tests and should therefore
 // be excluded from coverage analysis.
 func IsTestFile(file string) bool {

libgo/go/cmd/go/internal/base/tool.go

@@ -36,18 +36,9 @@ func Tool(toolName string) string {
 	}
 	// Give a nice message if there is no tool with that name.
 	if _, err := os.Stat(toolPath); err != nil {
-		if isInGoToolsRepo(toolName) {
-			fmt.Fprintf(os.Stderr, "go tool: no such tool %q; to install:\n\tgo get golang.org/x/tools/cmd/%s\n", toolName, toolName)
-		} else {
 		fmt.Fprintf(os.Stderr, "go tool: no such tool %q\n", toolName)
-		}
 		SetExitStatus(2)
 		Exit()
 	}
 	return toolPath
 }
-
-// TODO: Delete.
-func isInGoToolsRepo(toolName string) bool {
-	return false
-}

libgo/go/cmd/go/internal/cache/cache.go

@@ -0,0 +1,453 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package cache implements a build artifact cache.
+package cache
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// An ActionID is a cache action key, the hash of a complete description of a
+// repeatable computation (command line, environment variables,
+// input file contents, executable contents).
+type ActionID [HashSize]byte
+
+// An OutputID is a cache output key, the hash of an output of a computation.
+type OutputID [HashSize]byte
+
+// A Cache is a package cache, backed by a file system directory tree.
+type Cache struct {
+	dir string
+	log *os.File
+	now func() time.Time
+}
+
+// Open opens and returns the cache in the given directory.
+//
+// It is safe for multiple processes on a single machine to use the
+// same cache directory in a local file system simultaneously.
+// They will coordinate using operating system file locks and may
+// duplicate effort but will not corrupt the cache.
+//
+// However, it is NOT safe for multiple processes on different machines
+// to share a cache directory (for example, if the directory were stored
+// in a network file system). File locking is notoriously unreliable in
+// network file systems and may not suffice to protect the cache.
+//
+func Open(dir string) (*Cache, error) {
+	info, err := os.Stat(dir)
+	if err != nil {
+		return nil, err
+	}
+	if !info.IsDir() {
+		return nil, &os.PathError{Op: "open", Path: dir, Err: fmt.Errorf("not a directory")}
+	}
+	for i := 0; i < 256; i++ {
+		name := filepath.Join(dir, fmt.Sprintf("%02x", i))
+		if err := os.MkdirAll(name, 0777); err != nil {
+			return nil, err
+		}
+	}
+	f, err := os.OpenFile(filepath.Join(dir, "log.txt"), os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666)
+	if err != nil {
+		return nil, err
+	}
+	c := &Cache{
+		dir: dir,
+		log: f,
+		now: time.Now,
+	}
+	return c, nil
+}
+
+// fileName returns the name of the file corresponding to the given id.
+func (c *Cache) fileName(id [HashSize]byte, key string) string {
+	return filepath.Join(c.dir, fmt.Sprintf("%02x", id[0]), fmt.Sprintf("%x", id)+"-"+key)
+}
+
+var errMissing = errors.New("cache entry not found")
+
+const (
+	// action entry file is "v1 <hex id> <hex out> <decimal size space-padded to 20 bytes> <unixnano space-padded to 20 bytes>\n"
+	hexSize   = HashSize * 2
+	entrySize = 2 + 1 + hexSize + 1 + hexSize + 1 + 20 + 1 + 20 + 1
+)
+
+// verify controls whether to run the cache in verify mode.
+// In verify mode, the cache always returns errMissing from Get
+// but then double-checks in Put that the data being written
+// exactly matches any existing entry. This provides an easy
+// way to detect program behavior that would have been different
+// had the cache entry been returned from Get.
+//
+// verify is enabled by setting the environment variable
+// GODEBUG=gocacheverify=1.
+var verify = false
+
+func init() { initEnv() }
+
+func initEnv() {
+	verify = false
+	debugHash = false
+	debug := strings.Split(os.Getenv("GODEBUG"), ",")
+	for _, f := range debug {
+		if f == "gocacheverify=1" {
+			verify = true
+		}
+		if f == "gocachehash=1" {
+			debugHash = true
+		}
+	}
+}
+
+// Get looks up the action ID in the cache,
+// returning the corresponding output ID and file size, if any.
+// Note that finding an output ID does not guarantee that the
+// saved file for that output ID is still available.
+func (c *Cache) Get(id ActionID) (Entry, error) {
+	if verify {
+		return Entry{}, errMissing
+	}
+	return c.get(id)
+}
+
+type Entry struct {
+	OutputID OutputID
+	Size     int64
+	Time     time.Time
+}
+
+// get is Get but does not respect verify mode, so that Put can use it.
+func (c *Cache) get(id ActionID) (Entry, error) {
+	missing := func() (Entry, error) {
+		fmt.Fprintf(c.log, "%d miss %x\n", c.now().Unix(), id)
+		return Entry{}, errMissing
+	}
+	f, err := os.Open(c.fileName(id, "a"))
+	if err != nil {
+		return missing()
+	}
+	defer f.Close()
+	entry := make([]byte, entrySize+1) // +1 to detect whether f is too long
+	if n, err := io.ReadFull(f, entry); n != entrySize || err != io.ErrUnexpectedEOF {
+		return missing()
+	}
+	if entry[0] != 'v' || entry[1] != '1' || entry[2] != ' ' || entry[3+hexSize] != ' ' || entry[3+hexSize+1+hexSize] != ' ' || entry[3+hexSize+1+hexSize+1+20] != ' ' || entry[entrySize-1] != '\n' {
+		return missing()
+	}
+	eid, entry := entry[3:3+hexSize], entry[3+hexSize:]
+	eout, entry := entry[1:1+hexSize], entry[1+hexSize:]
+	esize, entry := entry[1:1+20], entry[1+20:]
+	etime, entry := entry[1:1+20], entry[1+20:]
+	var buf [HashSize]byte
+	if _, err := hex.Decode(buf[:], eid); err != nil || buf != id {
+		return missing()
+	}
+	if _, err := hex.Decode(buf[:], eout); err != nil {
+		return missing()
+	}
+	i := 0
+	for i < len(esize) && esize[i] == ' ' {
+		i++
+	}
+	size, err := strconv.ParseInt(string(esize[i:]), 10, 64)
+	if err != nil || size < 0 {
+		return missing()
+	}
+	i = 0
+	for i < len(etime) && etime[i] == ' ' {
+		i++
+	}
+	tm, err := strconv.ParseInt(string(etime[i:]), 10, 64)
+	if err != nil || size < 0 {
+		return missing()
+	}
+
+	fmt.Fprintf(c.log, "%d get %x\n", c.now().Unix(), id)
+
+	c.used(c.fileName(id, "a"))
+
+	return Entry{buf, size, time.Unix(0, tm)}, nil
+}
+
+// GetBytes looks up the action ID in the cache and returns
+// the corresponding output bytes.
+// GetBytes should only be used for data that can be expected to fit in memory.
+func (c *Cache) GetBytes(id ActionID) ([]byte, Entry, error) {
+	entry, err := c.Get(id)
+	if err != nil {
+		return nil, entry, err
+	}
+	data, _ := ioutil.ReadFile(c.OutputFile(entry.OutputID))
+	if sha256.Sum256(data) != entry.OutputID {
+		return nil, entry, errMissing
+	}
+	return data, entry, nil
+}
+
+// OutputFile returns the name of the cache file storing output with the given OutputID.
+func (c *Cache) OutputFile(out OutputID) string {
+	file := c.fileName(out, "d")
+	c.used(file)
+	return file
+}
+
+// Time constants for cache expiration.
+//
+// We set the mtime on a cache file on each use, but at most one per mtimeInterval (1 hour),
+// to avoid causing many unnecessary inode updates. The mtimes therefore
+// roughly reflect "time of last use" but may in fact be older by at most an hour.
+//
+// We scan the cache for entries to delete at most once per trimInterval (1 day).
+//
+// When we do scan the cache, we delete entries that have not been used for
+// at least trimLimit (5 days). Statistics gathered from a month of usage by
+// Go developers found that essentially all reuse of cached entries happened
+// within 5 days of the previous reuse. See golang.org/issue/22990.
+const (
+	mtimeInterval = 1 * time.Hour
+	trimInterval  = 24 * time.Hour
+	trimLimit     = 5 * 24 * time.Hour
+)
+
+// used makes a best-effort attempt to update mtime on file,
+// so that mtime reflects cache access time.
+//
+// Because the reflection only needs to be approximate,
+// and to reduce the amount of disk activity caused by using
+// cache entries, used only updates the mtime if the current
+// mtime is more than an hour old. This heuristic eliminates
+// nearly all of the mtime updates that would otherwise happen,
+// while still keeping the mtimes useful for cache trimming.
+func (c *Cache) used(file string) {
+	info, err := os.Stat(file)
+	if err == nil && c.now().Sub(info.ModTime()) < mtimeInterval {
+		return
+	}
+	os.Chtimes(file, c.now(), c.now())
+}
+
+// Trim removes old cache entries that are likely not to be reused.
+func (c *Cache) Trim() {
+	now := c.now()
+
+	// We maintain in dir/trim.txt the time of the last completed cache trim.
+	// If the cache has been trimmed recently enough, do nothing.
+	// This is the common case.
+	data, _ := ioutil.ReadFile(filepath.Join(c.dir, "trim.txt"))
+	t, err := strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64)
+	if err == nil && now.Sub(time.Unix(t, 0)) < trimInterval {
+		return
+	}
+
+	// Trim each of the 256 subdirectories.
+	// We subtract an additional mtimeInterval
+	// to account for the imprecision of our "last used" mtimes.
+	cutoff := now.Add(-trimLimit - mtimeInterval)
+	for i := 0; i < 256; i++ {
+		subdir := filepath.Join(c.dir, fmt.Sprintf("%02x", i))
+		c.trimSubdir(subdir, cutoff)
+	}
+
+	ioutil.WriteFile(filepath.Join(c.dir, "trim.txt"), []byte(fmt.Sprintf("%d", now.Unix())), 0666)
+}
+
+// trimSubdir trims a single cache subdirectory.
+func (c *Cache) trimSubdir(subdir string, cutoff time.Time) {
+	// Read all directory entries from subdir before removing
+	// any files, in case removing files invalidates the file offset
+	// in the directory scan. Also, ignore error from f.Readdirnames,
+	// because we don't care about reporting the error and we still
+	// want to process any entries found before the error.
+	f, err := os.Open(subdir)
+	if err != nil {
+		return
+	}
+	names, _ := f.Readdirnames(-1)
+	f.Close()
+
+	for _, name := range names {
+		// Remove only cache entries (xxxx-a and xxxx-d).
+		if !strings.HasSuffix(name, "-a") && !strings.HasSuffix(name, "-d") {
+			continue
+		}
+		entry := filepath.Join(subdir, name)
+		info, err := os.Stat(entry)
+		if err == nil && info.ModTime().Before(cutoff) {
+			os.Remove(entry)
+		}
+	}
+}
+
+// putIndexEntry adds an entry to the cache recording that executing the action
+// with the given id produces an output with the given output id (hash) and size.
+func (c *Cache) putIndexEntry(id ActionID, out OutputID, size int64, allowVerify bool) error {
+	// Note: We expect that for one reason or another it may happen
+	// that repeating an action produces a different output hash
+	// (for example, if the output contains a time stamp or temp dir name).
+	// While not ideal, this is also not a correctness problem, so we
+	// don't make a big deal about it. In particular, we leave the action
+	// cache entries writable specifically so that they can be overwritten.
+	//
+	// Setting GODEBUG=gocacheverify=1 does make a big deal:
+	// in verify mode we are double-checking that the cache entries
+	// are entirely reproducible. As just noted, this may be unrealistic
+	// in some cases but the check is also useful for shaking out real bugs.
+	entry := []byte(fmt.Sprintf("v1 %x %x %20d %20d\n", id, out, size, time.Now().UnixNano()))
+	if verify && allowVerify {
+		old, err := c.get(id)
+		if err == nil && (old.OutputID != out || old.Size != size) {
+			// panic to show stack trace, so we can see what code is generating this cache entry.
+			msg := fmt.Sprintf("go: internal cache error: cache verify failed: id=%x changed:<<<\n%s\n>>>\nold: %x %d\nnew: %x %d", id, reverseHash(id), out, size, old.OutputID, old.Size)
+			panic(msg)
+		}
+	}
+	file := c.fileName(id, "a")
+	if err := ioutil.WriteFile(file, entry, 0666); err != nil {
+		os.Remove(file)
+		return err
+	}
+	os.Chtimes(file, c.now(), c.now()) // mainly for tests
+
+	fmt.Fprintf(c.log, "%d put %x %x %d\n", c.now().Unix(), id, out, size)
+	return nil
+}
+
+// Put stores the given output in the cache as the output for the action ID.
+// It may read file twice. The content of file must not change between the two passes.
+func (c *Cache) Put(id ActionID, file io.ReadSeeker) (OutputID, int64, error) {
+	return c.put(id, file, true)
+}
+
+// PutNoVerify is like Put but disables the verify check
+// when GODEBUG=goverifycache=1 is set.
+// It is meant for data that is OK to cache but that we expect to vary slightly from run to run,
+// like test output containing times and the like.
+func (c *Cache) PutNoVerify(id ActionID, file io.ReadSeeker) (OutputID, int64, error) {
+	return c.put(id, file, false)
+}
+
+func (c *Cache) put(id ActionID, file io.ReadSeeker, allowVerify bool) (OutputID, int64, error) {
+	// Compute output ID.
+	h := sha256.New()
+	if _, err := file.Seek(0, 0); err != nil {
+		return OutputID{}, 0, err
+	}
+	size, err := io.Copy(h, file)
+	if err != nil {
+		return OutputID{}, 0, err
+	}
+	var out OutputID
+	h.Sum(out[:0])
+
+	// Copy to cached output file (if not already present).
+	if err := c.copyFile(file, out, size); err != nil {
+		return out, size, err
+	}
+
+	// Add to cache index.
+	return out, size, c.putIndexEntry(id, out, size, allowVerify)
+}
+
+// PutBytes stores the given bytes in the cache as the output for the action ID.
+func (c *Cache) PutBytes(id ActionID, data []byte) error {
+	_, _, err := c.Put(id, bytes.NewReader(data))
+	return err
+}
+
+// copyFile copies file into the cache, expecting it to have the given
+// output ID and size, if that file is not present already.
+func (c *Cache) copyFile(file io.ReadSeeker, out OutputID, size int64) error {
+	name := c.fileName(out, "d")
+	info, err := os.Stat(name)
+	if err == nil && info.Size() == size {
+		// Check hash.
+		if f, err := os.Open(name); err == nil {
+			h := sha256.New()
+			io.Copy(h, f)
+			f.Close()
+			var out2 OutputID
+			h.Sum(out2[:0])
+			if out == out2 {
+				return nil
+			}
+		}
+		// Hash did not match. Fall through and rewrite file.
+	}
+
+	// Copy file to cache directory.
+	mode := os.O_RDWR | os.O_CREATE
+	if err == nil && info.Size() > size { // shouldn't happen but fix in case
+		mode |= os.O_TRUNC
+	}
+	f, err := os.OpenFile(name, mode, 0666)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	if size == 0 {
+		// File now exists with correct size.
+		// Only one possible zero-length file, so contents are OK too.
+		// Early return here makes sure there's a "last byte" for code below.
+		return nil
+	}
+
+	// From here on, if any of the I/O writing the file fails,
+	// we make a best-effort attempt to truncate the file f
+	// before returning, to avoid leaving bad bytes in the file.
+
+	// Copy file to f, but also into h to double-check hash.
+	if _, err := file.Seek(0, 0); err != nil {
+		f.Truncate(0)
+		return err
+	}
+	h := sha256.New()
+	w := io.MultiWriter(f, h)
+	if _, err := io.CopyN(w, file, size-1); err != nil {
+		f.Truncate(0)
+		return err
+	}
+	// Check last byte before writing it; writing it will make the size match
+	// what other processes expect to find and might cause them to start
+	// using the file.
+	buf := make([]byte, 1)
+	if _, err := file.Read(buf); err != nil {
+		f.Truncate(0)
+		return err
+	}
+	h.Write(buf)
+	sum := h.Sum(nil)
+	if !bytes.Equal(sum, out[:]) {
+		f.Truncate(0)
+		return fmt.Errorf("file content changed underfoot")
+	}
+
+	// Commit cache file entry.
+	if _, err := f.Write(buf); err != nil {
+		f.Truncate(0)
+		return err
+	}
+	if err := f.Close(); err != nil {
+		// Data might not have been written,
+		// but file may look like it is the right size.
+		// To be extra careful, remove cached file.
+		os.Remove(name)
+		return err
+	}
+	os.Chtimes(name, c.now(), c.now()) // mainly for tests
+
+	return nil
+}

libgo/go/cmd/go/internal/cache/cache_test.go

@@ -0,0 +1,319 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cache
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+func init() {
+	verify = false // even if GODEBUG is set
+}
+
+func TestBasic(t *testing.T) {
+	dir, err := ioutil.TempDir("", "cachetest-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+	_, err = Open(filepath.Join(dir, "notexist"))
+	if err == nil {
+		t.Fatal(`Open("tmp/notexist") succeeded, want failure`)
+	}
+
+	cdir := filepath.Join(dir, "c1")
+	if err := os.Mkdir(cdir, 0777); err != nil {
+		t.Fatal(err)
+	}
+
+	c1, err := Open(cdir)
+	if err != nil {
+		t.Fatalf("Open(c1) (create): %v", err)
+	}
+	if err := c1.putIndexEntry(dummyID(1), dummyID(12), 13, true); err != nil {
+		t.Fatalf("addIndexEntry: %v", err)
+	}
+	if err := c1.putIndexEntry(dummyID(1), dummyID(2), 3, true); err != nil { // overwrite entry
+		t.Fatalf("addIndexEntry: %v", err)
+	}
+	if entry, err := c1.Get(dummyID(1)); err != nil || entry.OutputID != dummyID(2) || entry.Size != 3 {
+		t.Fatalf("c1.Get(1) = %x, %v, %v, want %x, %v, nil", entry.OutputID, entry.Size, err, dummyID(2), 3)
+	}
+
+	c2, err := Open(cdir)
+	if err != nil {
+		t.Fatalf("Open(c2) (reuse): %v", err)
+	}
+	if entry, err := c2.Get(dummyID(1)); err != nil || entry.OutputID != dummyID(2) || entry.Size != 3 {
+		t.Fatalf("c2.Get(1) = %x, %v, %v, want %x, %v, nil", entry.OutputID, entry.Size, err, dummyID(2), 3)
+	}
+	if err := c2.putIndexEntry(dummyID(2), dummyID(3), 4, true); err != nil {
+		t.Fatalf("addIndexEntry: %v", err)
+	}
+	if entry, err := c1.Get(dummyID(2)); err != nil || entry.OutputID != dummyID(3) || entry.Size != 4 {
+		t.Fatalf("c1.Get(2) = %x, %v, %v, want %x, %v, nil", entry.OutputID, entry.Size, err, dummyID(3), 4)
+	}
+}
+
+func TestGrowth(t *testing.T) {
+	dir, err := ioutil.TempDir("", "cachetest-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	c, err := Open(dir)
+	if err != nil {
+		t.Fatalf("Open: %v", err)
+	}
+
+	n := 10000
+	if testing.Short() {
+		n = 1000
+	}
+
+	for i := 0; i < n; i++ {
+		if err := c.putIndexEntry(dummyID(i), dummyID(i*99), int64(i)*101, true); err != nil {
+			t.Fatalf("addIndexEntry: %v", err)
+		}
+		id := ActionID(dummyID(i))
+		entry, err := c.Get(id)
+		if err != nil {
+			t.Fatalf("Get(%x): %v", id, err)
+		}
+		if entry.OutputID != dummyID(i*99) || entry.Size != int64(i)*101 {
+			t.Errorf("Get(%x) = %x, %d, want %x, %d", id, entry.OutputID, entry.Size, dummyID(i*99), int64(i)*101)
+		}
+	}
+	for i := 0; i < n; i++ {
+		id := ActionID(dummyID(i))
+		entry, err := c.Get(id)
+		if err != nil {
+			t.Fatalf("Get2(%x): %v", id, err)
+		}
+		if entry.OutputID != dummyID(i*99) || entry.Size != int64(i)*101 {
+			t.Errorf("Get2(%x) = %x, %d, want %x, %d", id, entry.OutputID, entry.Size, dummyID(i*99), int64(i)*101)
+		}
+	}
+}
+
+func TestVerifyPanic(t *testing.T) {
+	os.Setenv("GODEBUG", "gocacheverify=1")
+	initEnv()
+	defer func() {
+		os.Unsetenv("GODEBUG")
+		verify = false
+	}()
+
+	if !verify {
+		t.Fatal("initEnv did not set verify")
+	}
+
+	dir, err := ioutil.TempDir("", "cachetest-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	c, err := Open(dir)
+	if err != nil {
+		t.Fatalf("Open: %v", err)
+	}
+
+	id := ActionID(dummyID(1))
+	if err := c.PutBytes(id, []byte("abc")); err != nil {
+		t.Fatal(err)
+	}
+
+	defer func() {
+		if err := recover(); err != nil {
+			t.Log(err)
+			return
+		}
+	}()
+	c.PutBytes(id, []byte("def"))
+	t.Fatal("mismatched Put did not panic in verify mode")
+}
+
+func TestCacheLog(t *testing.T) {
+	dir, err := ioutil.TempDir("", "cachetest-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	c, err := Open(dir)
+	if err != nil {
+		t.Fatalf("Open: %v", err)
+	}
+	c.now = func() time.Time { return time.Unix(1e9, 0) }
+
+	id := ActionID(dummyID(1))
+	c.Get(id)
+	c.PutBytes(id, []byte("abc"))
+	c.Get(id)
+
+	c, err = Open(dir)
+	if err != nil {
+		t.Fatalf("Open #2: %v", err)
+	}
+	c.now = func() time.Time { return time.Unix(1e9+1, 0) }
+	c.Get(id)
+
+	id2 := ActionID(dummyID(2))
+	c.Get(id2)
+	c.PutBytes(id2, []byte("abc"))
+	c.Get(id2)
+	c.Get(id)
+
+	data, err := ioutil.ReadFile(filepath.Join(dir, "log.txt"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := `1000000000 miss 0100000000000000000000000000000000000000000000000000000000000000
+1000000000 put 0100000000000000000000000000000000000000000000000000000000000000 ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad 3
+1000000000 get 0100000000000000000000000000000000000000000000000000000000000000
+1000000001 get 0100000000000000000000000000000000000000000000000000000000000000
+1000000001 miss 0200000000000000000000000000000000000000000000000000000000000000
+1000000001 put 0200000000000000000000000000000000000000000000000000000000000000 ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad 3
+1000000001 get 0200000000000000000000000000000000000000000000000000000000000000
+1000000001 get 0100000000000000000000000000000000000000000000000000000000000000
+`
+	if string(data) != want {
+		t.Fatalf("log:\n%s\nwant:\n%s", string(data), want)
+	}
+}
+
+func dummyID(x int) [HashSize]byte {
+	var out [HashSize]byte
+	binary.LittleEndian.PutUint64(out[:], uint64(x))
+	return out
+}
+
+func TestCacheTrim(t *testing.T) {
+	dir, err := ioutil.TempDir("", "cachetest-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	c, err := Open(dir)
+	if err != nil {
+		t.Fatalf("Open: %v", err)
+	}
+	const start = 1000000000
+	now := int64(start)
+	c.now = func() time.Time { return time.Unix(now, 0) }
+
+	checkTime := func(name string, mtime int64) {
+		t.Helper()
+		file := filepath.Join(c.dir, name[:2], name)
+		info, err := os.Stat(file)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if info.ModTime().Unix() != mtime {
+			t.Fatalf("%s mtime = %d, want %d", name, info.ModTime().Unix(), mtime)
+		}
+	}
+
+	id := ActionID(dummyID(1))
+	c.PutBytes(id, []byte("abc"))
+	entry, _ := c.Get(id)
+	c.PutBytes(ActionID(dummyID(2)), []byte("def"))
+	mtime := now
+	checkTime(fmt.Sprintf("%x-a", id), mtime)
+	checkTime(fmt.Sprintf("%x-d", entry.OutputID), mtime)
+
+	// Get should not change recent mtimes.
+	now = start + 10
+	c.Get(id)
+	checkTime(fmt.Sprintf("%x-a", id), mtime)
+	checkTime(fmt.Sprintf("%x-d", entry.OutputID), mtime)
+
+	// Get should change distant mtimes.
+	now = start + 5000
+	mtime2 := now
+	if _, err := c.Get(id); err != nil {
+		t.Fatal(err)
+	}
+	c.OutputFile(entry.OutputID)
+	checkTime(fmt.Sprintf("%x-a", id), mtime2)
+	checkTime(fmt.Sprintf("%x-d", entry.OutputID), mtime2)
+
+	// Trim should leave everything alone: it's all too new.
+	c.Trim()
+	if _, err := c.Get(id); err != nil {
+		t.Fatal(err)
+	}
+	c.OutputFile(entry.OutputID)
+	data, err := ioutil.ReadFile(filepath.Join(dir, "trim.txt"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	checkTime(fmt.Sprintf("%x-a", dummyID(2)), start)
+
+	// Trim less than a day later should not do any work at all.
+	now = start + 80000
+	c.Trim()
+	if _, err := c.Get(id); err != nil {
+		t.Fatal(err)
+	}
+	c.OutputFile(entry.OutputID)
+	data2, err := ioutil.ReadFile(filepath.Join(dir, "trim.txt"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(data, data2) {
+		t.Fatalf("second trim did work: %q -> %q", data, data2)
+	}
+
+	// Fast forward and do another trim just before the 5 day cutoff.
+	// Note that because of usedQuantum the cutoff is actually 5 days + 1 hour.
+	// We used c.Get(id) just now, so 5 days later it should still be kept.
+	// On the other hand almost a full day has gone by since we wrote dummyID(2)
+	// and we haven't looked at it since, so 5 days later it should be gone.
+	now += 5 * 86400
+	checkTime(fmt.Sprintf("%x-a", dummyID(2)), start)
+	c.Trim()
+	if _, err := c.Get(id); err != nil {
+		t.Fatal(err)
+	}
+	c.OutputFile(entry.OutputID)
+	mtime3 := now
+	if _, err := c.Get(dummyID(2)); err == nil { // haven't done a Get for this since original write above
+		t.Fatalf("Trim did not remove dummyID(2)")
+	}
+
+	// The c.Get(id) refreshed id's mtime again.
+	// Check that another 5 days later it is still not gone,
+	// but check by using checkTime, which doesn't bring mtime forward.
+	now += 5 * 86400
+	c.Trim()
+	checkTime(fmt.Sprintf("%x-a", id), mtime3)
+	checkTime(fmt.Sprintf("%x-d", entry.OutputID), mtime3)
+
+	// Half a day later Trim should still be a no-op, because there was a Trim recently.
+	// Even though the entry for id is now old enough to be trimmed,
+	// it gets a reprieve until the time comes for a new Trim scan.
+	now += 86400 / 2
+	c.Trim()
+	checkTime(fmt.Sprintf("%x-a", id), mtime3)
+	checkTime(fmt.Sprintf("%x-d", entry.OutputID), mtime3)
+
+	// Another half a day later, Trim should actually run, and it should remove id.
+	now += 86400/2 + 1
+	c.Trim()
+	if _, err := c.Get(dummyID(1)); err == nil {
+		t.Fatal("Trim did not remove dummyID(1)")
+	}
+}

libgo/go/cmd/go/internal/cache/default.go

@@ -0,0 +1,100 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cache
+
+import (
+	"cmd/go/internal/base"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"runtime"
+	"sync"
+)
+
+// Default returns the default cache to use, or nil if no cache should be used.
+func Default() *Cache {
+	defaultOnce.Do(initDefaultCache)
+	return defaultCache
+}
+
+var (
+	defaultOnce  sync.Once
+	defaultCache *Cache
+)
+
+// cacheREADME is a message stored in a README in the cache directory.
+// Because the cache lives outside the normal Go trees, we leave the
+// README as a courtesy to explain where it came from.
+const cacheREADME = `This directory holds cached build artifacts from the Go build system.
+Run "go clean -cache" if the directory is getting too large.
+See golang.org to learn more about Go.
+`
+
+// initDefaultCache does the work of finding the default cache
+// the first time Default is called.
+func initDefaultCache() {
+	dir := DefaultDir()
+	if dir == "off" {
+		return
+	}
+	if err := os.MkdirAll(dir, 0777); err != nil {
+		base.Fatalf("initializing cache in $GOCACHE: %s", err)
+	}
+	if _, err := os.Stat(filepath.Join(dir, "README")); err != nil {
+		// Best effort.
+		ioutil.WriteFile(filepath.Join(dir, "README"), []byte(cacheREADME), 0666)
+	}
+
+	c, err := Open(dir)
+	if err != nil {
+		base.Fatalf("initializing cache in $GOCACHE: %s", err)
+	}
+	defaultCache = c
+}
+
+// DefaultDir returns the effective GOCACHE setting.
+// It returns "off" if the cache is disabled.
+func DefaultDir() string {
+	dir := os.Getenv("GOCACHE")
+	if dir != "" {
+		return dir
+	}
+
+	// Compute default location.
+	// TODO(rsc): This code belongs somewhere else,
+	// like maybe ioutil.CacheDir or os.CacheDir.
+	switch runtime.GOOS {
+	case "windows":
+		dir = os.Getenv("LocalAppData")
+
+	case "darwin":
+		dir = os.Getenv("HOME")
+		if dir == "" {
+			return "off"
+		}
+		dir += "/Library/Caches"
+
+	case "plan9":
+		dir = os.Getenv("home")
+		if dir == "" {
+			return "off"
+		}
+		// Plan 9 has no established per-user cache directory,
+		// but $home/lib/xyz is the usual equivalent of $HOME/.xyz on Unix.
+		dir += "/lib/cache"
+
+	default: // Unix
+		// https://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html
+		dir = os.Getenv("XDG_CACHE_HOME")
+		if dir == "" {
+			dir = os.Getenv("HOME")
+			if dir == "" {
+				return "off"
+			}
+			dir += "/.cache"
+		}
+	}
+	return filepath.Join(dir, "go-build")
+}

libgo/go/cmd/go/internal/cache/hash.go

@@ -0,0 +1,174 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cache
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"fmt"
+	"hash"
+	"io"
+	"os"
+	"runtime"
+	"sync"
+)
+
+var debugHash = false // set when GODEBUG=gocachehash=1
+
+// HashSize is the number of bytes in a hash.
+const HashSize = 32
+
+// A Hash provides access to the canonical hash function used to index the cache.
+// The current implementation uses salted SHA256, but clients must not assume this.
+type Hash struct {
+	h    hash.Hash
+	name string        // for debugging
+	buf  *bytes.Buffer // for verify
+}
+
+// hashSalt is a salt string added to the beginning of every hash
+// created by NewHash. Using the Go version makes sure that different
+// versions of the go command (or even different Git commits during
+// work on the development branch) do not address the same cache
+// entries, so that a bug in one version does not affect the execution
+// of other versions. This salt will result in additional ActionID files
+// in the cache, but not additional copies of the large output files,
+// which are still addressed by unsalted SHA256.
+var hashSalt = []byte(runtime.Version())
+
+// Subkey returns an action ID corresponding to mixing a parent
+// action ID with a string description of the subkey.
+func Subkey(parent ActionID, desc string) ActionID {
+	h := sha256.New()
+	h.Write([]byte("subkey:"))
+	h.Write(parent[:])
+	h.Write([]byte(desc))
+	var out ActionID
+	h.Sum(out[:0])
+	if debugHash {
+		fmt.Fprintf(os.Stderr, "HASH subkey %x %q = %x\n", parent, desc, out)
+	}
+	if verify {
+		hashDebug.Lock()
+		hashDebug.m[out] = fmt.Sprintf("subkey %x %q", parent, desc)
+		hashDebug.Unlock()
+	}
+	return out
+}
+
+// NewHash returns a new Hash.
+// The caller is expected to Write data to it and then call Sum.
+func NewHash(name string) *Hash {
+	h := &Hash{h: sha256.New(), name: name}
+	if debugHash {
+		fmt.Fprintf(os.Stderr, "HASH[%s]\n", h.name)
+	}
+	h.Write(hashSalt)
+	if verify {
+		h.buf = new(bytes.Buffer)
+	}
+	return h
+}
+
+// Write writes data to the running hash.
+func (h *Hash) Write(b []byte) (int, error) {
+	if debugHash {
+		fmt.Fprintf(os.Stderr, "HASH[%s]: %q\n", h.name, b)
+	}
+	if h.buf != nil {
+		h.buf.Write(b)
+	}
+	return h.h.Write(b)
+}
+
+// Sum returns the hash of the data written previously.
+func (h *Hash) Sum() [HashSize]byte {
+	var out [HashSize]byte
+	h.h.Sum(out[:0])
+	if debugHash {
+		fmt.Fprintf(os.Stderr, "HASH[%s]: %x\n", h.name, out)
+	}
+	if h.buf != nil {
+		hashDebug.Lock()
+		if hashDebug.m == nil {
+			hashDebug.m = make(map[[HashSize]byte]string)
+		}
+		hashDebug.m[out] = h.buf.String()
+		hashDebug.Unlock()
+	}
+	return out
+}
+
+// In GODEBUG=gocacheverify=1 mode,
+// hashDebug holds the input to every computed hash ID,
+// so that we can work backward from the ID involved in a
+// cache entry mismatch to a description of what should be there.
+var hashDebug struct {
+	sync.Mutex
+	m map[[HashSize]byte]string
+}
+
+// reverseHash returns the input used to compute the hash id.
+func reverseHash(id [HashSize]byte) string {
+	hashDebug.Lock()
+	s := hashDebug.m[id]
+	hashDebug.Unlock()
+	return s
+}
+
+var hashFileCache struct {
+	sync.Mutex
+	m map[string][HashSize]byte
+}
+
+// HashFile returns the hash of the named file.
+// It caches repeated lookups for a given file,
+// and the cache entry for a file can be initialized
+// using SetFileHash.
+// The hash used by FileHash is not the same as
+// the hash used by NewHash.
+func FileHash(file string) ([HashSize]byte, error) {
+	hashFileCache.Lock()
+	out, ok := hashFileCache.m[file]
+	hashFileCache.Unlock()
+
+	if ok {
+		return out, nil
+	}
+
+	h := sha256.New()
+	f, err := os.Open(file)
+	if err != nil {
+		if debugHash {
+			fmt.Fprintf(os.Stderr, "HASH %s: %v\n", file, err)
+		}
+		return [HashSize]byte{}, err
+	}
+	_, err = io.Copy(h, f)
+	f.Close()
+	if err != nil {
+		if debugHash {
+			fmt.Fprintf(os.Stderr, "HASH %s: %v\n", file, err)
+		}
+		return [HashSize]byte{}, err
+	}
+	h.Sum(out[:0])
+	if debugHash {
+		fmt.Fprintf(os.Stderr, "HASH %s: %x\n", file, out)
+	}
+
+	SetFileHash(file, out)
+	return out, nil
+}
+
+// SetFileHash sets the hash returned by FileHash for file.
+func SetFileHash(file string, sum [HashSize]byte) {
+	hashFileCache.Lock()
+	if hashFileCache.m == nil {
+		hashFileCache.m = make(map[string][HashSize]byte)
+	}
+	hashFileCache.m[file] = sum
+	hashFileCache.Unlock()
+}