net/netip: add new IP address package

Co-authored-by: Alex Willmer <[email protected]> (GitHub @moreati)
Co-authored-by: Alexander Yastrebov <[email protected]>
Co-authored-by: David Anderson <[email protected]> (Tailscale CLA)
Co-authored-by: David Crawshaw <[email protected]> (Tailscale CLA)
Co-authored-by: Dmytro Shynkevych <[email protected]> (Tailscale CLA)
Co-authored-by: Elias Naur <[email protected]>
Co-authored-by: Joe Tsai <[email protected]> (Tailscale CLA)
Co-authored-by: Jonathan Yu <[email protected]> (GitHub @jawnsy)
Co-authored-by: Josh Bleecher Snyder <[email protected]> (Tailscale CLA)
Co-authored-by: Maisem Ali <[email protected]> (Tailscale CLA)
Co-authored-by: Manuel Mendez (Go AUTHORS mmendez534@...)
Co-authored-by: Matt Layher <[email protected]>
Co-authored-by: Noah Treuhaft <[email protected]> (GitHub @nwt)
Co-authored-by: Stefan Majer <[email protected]>
Co-authored-by: Terin Stock <[email protected]> (Cloudflare CLA)
Co-authored-by: Tobias Klauser <[email protected]>

Fixes golang#46518

Change-Id: I0041f9e1115d61fa6e95fcf32b01d9faee708712
Reviewed-on: https://go-review.googlesource.com/c/go/+/339309
Run-TryBot: Brad Fitzpatrick <[email protected]>
TryBot-Result: Go Bot <[email protected]>
Reviewed-by: Russ Cox <[email protected]>
Trust: Brad Fitzpatrick <[email protected]>

src/bytes/bytes.go

@@ -21,7 +21,7 @@ func Equal(a, b []byte) bool {
 }
 
 // Compare returns an integer comparing two byte slices lexicographically.
-// The result will be 0 if a==b, -1 if a < b, and +1 if a > b.
+// The result will be 0 if a == b, -1 if a < b, and +1 if a > b.
 // A nil argument is equivalent to an empty slice.
 func Compare(a, b []byte) int {
 	return bytealg.Compare(a, b)

src/crypto/x509/root_darwin.go

@@ -10,11 +10,11 @@ import (
 	"bytes"
 	macOS "crypto/x509/internal/macos"
 	"fmt"
+	"internal/godebug"
 	"os"
-	"strings"
 )
 
-var debugDarwinRoots = strings.Contains(os.Getenv("GODEBUG"), "x509roots=1")
+var debugDarwinRoots = godebug.Get("x509roots") == "1"
 
 func (c *Certificate) systemVerify(opts *VerifyOptions) (chains [][]*Certificate, err error) {
 	return nil, nil

src/go/build/deps_test.go

@@ -173,7 +173,7 @@ var depsRules = `
 	io/fs
 	< embed;
 
-	unicode, fmt !< os, os/signal;
+	unicode, fmt !< net, os, os/signal;
 
 	os/signal, STR
 	< path/filepath
@@ -187,6 +187,8 @@ var depsRules = `
 	OS
 	< golang.org/x/sys/cpu;
 
+	os < internal/godebug;
+
 	# FMT is OS (which includes string routines) plus reflect and fmt.
 	# It does not include package log, which should be avoided in core packages.
 	strconv, unicode
@@ -352,6 +354,13 @@ var depsRules = `
 	  golang.org/x/net/lif,
 	  golang.org/x/net/route;
 
+	os, runtime, strconv, sync, unsafe,
+	internal/godebug
+	< internal/intern;
+
+	internal/bytealg, internal/intern, internal/itoa, math/bits, sort, strconv
+	< net/netip;
+
 	# net is unavoidable when doing any networking,
 	# so large dependencies must be kept out.
 	# This is a long-looking list but most of these
@@ -360,10 +369,12 @@ var depsRules = `
 	golang.org/x/net/dns/dnsmessage,
 	golang.org/x/net/lif,
 	golang.org/x/net/route,
+	internal/godebug,
 	internal/nettrace,
 	internal/poll,
 	internal/singleflight,
 	internal/race,
+	net/netip,
 	os
 	< net;
 
@@ -515,7 +526,8 @@ var depsRules = `
 	FMT, DEBUG, flag, runtime/trace, internal/sysinfo, math/rand
 	< testing;
 
-	FMT, crypto/sha256, encoding/json, go/ast, go/parser, go/token, math/rand, encoding/hex, crypto/sha256
+	FMT, crypto/sha256, encoding/json, go/ast, go/parser, go/token,
+	internal/godebug, math/rand, encoding/hex, crypto/sha256
 	< internal/fuzz;
 
 	internal/fuzz, internal/testlog, runtime/pprof, regexp

src/internal/fuzz/fuzz.go

@@ -12,6 +12,7 @@ import (
 	"crypto/sha256"
 	"errors"
 	"fmt"
+	"internal/godebug"
 	"io"
 	"io/ioutil"
 	"math/bits"
@@ -1063,13 +1064,7 @@ var (
 
 func shouldPrintDebugInfo() bool {
 	debugInfoOnce.Do(func() {
-		debug := strings.Split(os.Getenv("GODEBUG"), ",")
-		for _, f := range debug {
-			if f == "fuzzdebug=1" {
-				debugInfo = true
-				break
-			}
-		}
+		debugInfo = godebug.Get("fuzzdebug") == "1"
 	})
 	return debugInfo
 }

src/internal/godebug/godebug.go

@@ -0,0 +1,34 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package godebug parses the GODEBUG environment variable.
+package godebug
+
+import "os"
+
+// Get returns the value for the provided GODEBUG key.
+func Get(key string) string {
+	return get(os.Getenv("GODEBUG"), key)
+}
+
+// get returns the value part of key=value in s (a GODEBUG value).
+func get(s, key string) string {
+	for i := 0; i < len(s)-len(key)-1; i++ {
+		if i > 0 && s[i-1] != ',' {
+			continue
+		}
+		afterKey := s[i+len(key):]
+		if afterKey[0] != '=' || s[i:i+len(key)] != key {
+			continue
+		}
+		val := afterKey[1:]
+		for i, b := range val {
+			if b == ',' {
+				return val[:i]
+			}
+		}
+		return val
+	}
+	return ""
+}

src/internal/godebug/godebug_test.go

@@ -0,0 +1,34 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package godebug
+
+import "testing"
+
+func TestGet(t *testing.T) {
+	tests := []struct {
+		godebug string
+		key     string
+		want    string
+	}{
+		{"", "", ""},
+		{"", "foo", ""},
+		{"foo=bar", "foo", "bar"},
+		{"foo=bar,after=x", "foo", "bar"},
+		{"before=x,foo=bar,after=x", "foo", "bar"},
+		{"before=x,foo=bar", "foo", "bar"},
+		{",,,foo=bar,,,", "foo", "bar"},
+		{"foodecoy=wrong,foo=bar", "foo", "bar"},
+		{"foo=", "foo", ""},
+		{"foo", "foo", ""},
+		{",foo", "foo", ""},
+		{"foo=bar,baz", "loooooooong", ""},
+	}
+	for _, tt := range tests {
+		got := get(tt.godebug, tt.key)
+		if got != tt.want {
+			t.Errorf("get(%q, %q) = %q; want %q", tt.godebug, tt.key, got, tt.want)
+		}
+	}
+}

src/internal/intern/intern.go

@@ -0,0 +1,178 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package intern lets you make smaller comparable values by boxing
+// a larger comparable value (such as a 16 byte string header) down
+// into a globally unique 8 byte pointer.
+//
+// The globally unique pointers are garbage collected with weak
+// references and finalizers. This package hides that.
+package intern
+
+import (
+	"internal/godebug"
+	"runtime"
+	"sync"
+	"unsafe"
+)
+
+// A Value pointer is the handle to an underlying comparable value.
+// See func Get for how Value pointers may be used.
+type Value struct {
+	_      [0]func() // prevent people from accidentally using value type as comparable
+	cmpVal interface{}
+	// resurrected is guarded by mu (for all instances of Value).
+	// It is set true whenever v is synthesized from a uintptr.
+	resurrected bool
+}
+
+// Get returns the comparable value passed to the Get func
+// that returned v.
+func (v *Value) Get() interface{} { return v.cmpVal }
+
+// key is a key in our global value map.
+// It contains type-specialized fields to avoid allocations
+// when converting common types to empty interfaces.
+type key struct {
+	s      string
+	cmpVal interface{}
+	// isString reports whether key contains a string.
+	// Without it, the zero value of key is ambiguous.
+	isString bool
+}
+
+// keyFor returns a key to use with cmpVal.
+func keyFor(cmpVal interface{}) key {
+	if s, ok := cmpVal.(string); ok {
+		return key{s: s, isString: true}
+	}
+	return key{cmpVal: cmpVal}
+}
+
+// Value returns a *Value built from k.
+func (k key) Value() *Value {
+	if k.isString {
+		return &Value{cmpVal: k.s}
+	}
+	return &Value{cmpVal: k.cmpVal}
+}
+
+var (
+	// mu guards valMap, a weakref map of *Value by underlying value.
+	// It also guards the resurrected field of all *Values.
+	mu      sync.Mutex
+	valMap  = map[key]uintptr{} // to uintptr(*Value)
+	valSafe = safeMap()         // non-nil in safe+leaky mode
+)
+
+// safeMap returns a non-nil map if we're in safe-but-leaky mode,
+// as controlled by GODEBUG=intern=leaky
+func safeMap() map[key]*Value {
+	if godebug.Get("intern") == "leaky" {
+		return map[key]*Value{}
+	}
+	return nil
+}
+
+// Get returns a pointer representing the comparable value cmpVal.
+//
+// The returned pointer will be the same for Get(v) and Get(v2)
+// if and only if v == v2, and can be used as a map key.
+func Get(cmpVal interface{}) *Value {
+	return get(keyFor(cmpVal))
+}
+
+// GetByString is identical to Get, except that it is specialized for strings.
+// This avoids an allocation from putting a string into an interface{}
+// to pass as an argument to Get.
+func GetByString(s string) *Value {
+	return get(key{s: s, isString: true})
+}
+
+// We play unsafe games that violate Go's rules (and assume a non-moving
+// collector). So we quiet Go here.
+// See the comment below Get for more implementation details.
+//go:nocheckptr
+func get(k key) *Value {
+	mu.Lock()
+	defer mu.Unlock()
+
+	var v *Value
+	if valSafe != nil {
+		v = valSafe[k]
+	} else if addr, ok := valMap[k]; ok {
+		v = (*Value)(unsafe.Pointer(addr))
+		v.resurrected = true
+	}
+	if v != nil {
+		return v
+	}
+	v = k.Value()
+	if valSafe != nil {
+		valSafe[k] = v
+	} else {
+		// SetFinalizer before uintptr conversion (theoretical concern;
+		// see https://github.com/go4org/intern/issues/13)
+		runtime.SetFinalizer(v, finalize)
+		valMap[k] = uintptr(unsafe.Pointer(v))
+	}
+	return v
+}
+
+func finalize(v *Value) {
+	mu.Lock()
+	defer mu.Unlock()
+	if v.resurrected {
+		// We lost the race. Somebody resurrected it while we
+		// were about to finalize it. Try again next round.
+		v.resurrected = false
+		runtime.SetFinalizer(v, finalize)
+		return
+	}
+	delete(valMap, keyFor(v.cmpVal))
+}
+
+// Interning is simple if you don't require that unused values be
+// garbage collectable. But we do require that; we don't want to be
+// DOS vector. We do this by using a uintptr to hide the pointer from
+// the garbage collector, and using a finalizer to eliminate the
+// pointer when no other code is using it.
+//
+// The obvious implementation of this is to use a
+// map[interface{}]uintptr-of-*interface{}, and set up a finalizer to
+// delete from the map. Unfortunately, this is racy. Because pointers
+// are being created in violation of Go's unsafety rules, it's
+// possible to create a pointer to a value concurrently with the GC
+// concluding that the value can be collected. There are other races
+// that break the equality invariant as well, but the use-after-free
+// will cause a runtime crash.
+//
+// To make this work, the finalizer needs to know that no references
+// have been unsafely created since the finalizer was set up. To do
+// this, values carry a "resurrected" sentinel, which gets set
+// whenever a pointer is unsafely created. If the finalizer encounters
+// the sentinel, it clears the sentinel and delays collection for one
+// additional GC cycle, by re-installing itself as finalizer. This
+// ensures that the unsafely created pointer is visible to the GC, and
+// will correctly prevent collection.
+//
+// This technique does mean that interned values that get reused take
+// at least 3 GC cycles to fully collect (1 to clear the sentinel, 1
+// to clean up the unsafe map, 1 to be actually deleted).
+//
+// @ianlancetaylor commented in
+// https://github.com/golang/go/issues/41303#issuecomment-717401656
+// that it is possible to implement weak references in terms of
+// finalizers without unsafe. Unfortunately, the approach he outlined
+// does not work here, for two reasons. First, there is no way to
+// construct a strong pointer out of a weak pointer; our map stores
+// weak pointers, but we must return strong pointers to callers.
+// Second, and more fundamentally, we must return not just _a_ strong
+// pointer to callers, but _the same_ strong pointer to callers. In
+// order to return _the same_ strong pointer to callers, we must track
+// it, which is exactly what we cannot do with strong pointers.
+//
+// See https://github.com/inetaf/netaddr/issues/53 for more
+// discussion, and https://github.com/go4org/intern/issues/2 for an
+// illustration of the subtleties at play.