Reload authenticated-emails-file upon update

This change extracts the UserMap class from NewValidator() so that its
LoadAuthenticatedEmailsFile() method can be called concurrently. This method
is called by a goroutine containing a fsnotify.Watcher watching the
authenticated emails file.

Watching isn't forever aborted when the authenticated emails file disappears.
The goroutine will call os.Stat() up to twenty times a second if the file is
persistently missing, but that's the pathological case, not the common one.

The common case is that some editors (including Vim) will perform a
rename-and-replace when updating a file, triggering fsnotify.Rename events,
and the file will temporarily disappear. This watcher goroutine handles that
case.

Also, on some platforms (notably Arch Linux), a remove will be preceded by a
fsnotify.Chmod, causing a race between the upcoming fsnotify.Remove and the
call to UserMap.LoadAuthenticatedEmailsFile(). Hence, we treat fsnotify.Chmod
the same as fsnotify.Remove and fsnotify.Rename. There's no significant
penalty to re-adding a file to the watcher.

Also contains the following small changes from the summary of commits below:

- Minor optimization of email domain search
- Fixed api_test.go on Windows
- Add deferred File.Close() calls where needed
- Log error and return if emails file doesn't parse

These are the original commits from bitly#89 squashed into this one:

0c6f2b6 Refactor validator_test to prepare for more tests
e0c792b Add more test cases to validator_test
a9a9d93 Minor optimization of email domain search
b763ea5 Extract LoadAuthenticatedEmailsFile()
8cdaf7f Introduce synchronized UserMap type
1b84eef Add UserMap methods, locking
af15dcf Reload authenticated-emails-file upon update
6d95548 Make UserMap operations lock-free
        Per:
        - http://stackoverflow.com/questions/21447463/is-assigning-a-pointer-atomic-in-golang
        - https://groups.google.com/forum/#!msg/golang-nuts/ueSvaEKgyLY/ZW_74IC4PekJ
75755d5 Fix tests on Windows
d0eab2e Ignore email file watcher Chmod events
0b9798b Fix watcher on Ubuntu 12.04
3a8251a WaitForReplacement() to retry emails file watch
a57fd29 Add deferred File.Close() calls where needed
        Because correctness: Don't leak file handles anywhere, and prepare for
        future panics and early returns.
52ed3fd Log error and return if emails file doesn't parse
40100d4 Add gopkg.in/fsnotify.v1 dependency to Godeps file
17dfbbc Avoid a race when Remove is preceded by Chmod

Godeps

@@ -2,3 +2,4 @@ github.com/BurntSushi/toml              3883ac1ce943878302255f538fce319d23226223
 github.com/bitly/go-simplejson          3378bdcb5cebedcbf8b5750edee28010f128fe24
 github.com/mreiferson/go-options        ee94b57f2fbf116075426f853e5abbcdfeca8b3d
 github.com/bmizerany/assert             e17e99893cb6509f428e1728281c2ad60a6b31e3
+gopkg.in/fsnotify.v1                    v1.2.0

api/api_test.go

@@ -40,8 +40,8 @@ func TestRequestFailure(t *testing.T) {
 	resp, err := Request(req)
 	assert.Equal(t, (*simplejson.Json)(nil), resp)
 	assert.NotEqual(t, nil, err)
-	if !strings.HasSuffix(err.Error(), "connection refused") {
-		t.Error("expected error when a connection fails")
+	if !strings.Contains(err.Error(), "refused") {
+		t.Error("expected error when a connection fails: ", err)
 	}
 }
 

htpasswd.go

@@ -21,6 +21,7 @@ func NewHtpasswdFromFile(path string) (*HtpasswdFile, error) {
 	if err != nil {
 		return nil, err
 	}
+	defer r.Close()
 	return NewHtpasswd(r)
 }
 

validator.go

@@ -6,43 +6,84 @@ import (
 	"log"
 	"os"
 	"strings"
+	"sync/atomic"
+	"unsafe"
 )
 
-func NewValidator(domains []string, usersFile string) func(string) bool {
-	validUsers := make(map[string]bool)
+type UserMap struct {
+	usersFile string
+	m         unsafe.Pointer
+}
 
+func NewUserMap(usersFile string, onUpdate func()) *UserMap {
+	um := &UserMap{usersFile: usersFile}
+	m := make(map[string]bool)
+	atomic.StorePointer(&um.m, unsafe.Pointer(&m))
 	if usersFile != "" {
 		log.Printf("using authenticated emails file %s", usersFile)
-		r, err := os.Open(usersFile)
-		if err != nil {
-			log.Fatalf("failed opening authenticated-emails-file=%q, %s", usersFile, err)
-		}
-		csv_reader := csv.NewReader(r)
-		csv_reader.Comma = ','
-		csv_reader.Comment = '#'
-		csv_reader.TrimLeadingSpace = true
-		records, err := csv_reader.ReadAll()
-		for _, r := range records {
-			validUsers[strings.ToLower(r[0])] = true
+		started := WatchForUpdates(usersFile, func() {
+			um.LoadAuthenticatedEmailsFile()
+			onUpdate()
+		})
+		if started {
+			log.Printf("watching %s for updates", usersFile)
 		}
+		um.LoadAuthenticatedEmailsFile()
 	}
+	return um
+}
+
+func (um *UserMap) IsValid(email string) (result bool) {
+	m := *(*map[string]bool)(atomic.LoadPointer(&um.m))
+	_, result = m[email]
+	return
+}
+
+func (um *UserMap) LoadAuthenticatedEmailsFile() {
+	r, err := os.Open(um.usersFile)
+	if err != nil {
+		log.Fatalf("failed opening authenticated-emails-file=%q, %s", um.usersFile, err)
+	}
+	defer r.Close()
+	csv_reader := csv.NewReader(r)
+	csv_reader.Comma = ','
+	csv_reader.Comment = '#'
+	csv_reader.TrimLeadingSpace = true
+	records, err := csv_reader.ReadAll()
+	if err != nil {
+		log.Printf("error reading authenticated-emails-file=%q, %s", um.usersFile, err)
+		return
+	}
+	updated := make(map[string]bool)
+	for _, r := range records {
+		updated[strings.ToLower(r[0])] = true
+	}
+	atomic.StorePointer(&um.m, unsafe.Pointer(&updated))
+}
+
+func newValidatorImpl(domains []string, usersFile string,
+	onUpdate func()) func(string) bool {
+	validUsers := NewUserMap(usersFile, onUpdate)
 
 	for i, domain := range domains {
-		domains[i] = strings.ToLower(domain)
+		domains[i] = fmt.Sprintf("@%s", strings.ToLower(domain))
 	}
 
 	validator := func(email string) bool {
 		email = strings.ToLower(email)
 		valid := false
 		for _, domain := range domains {
-			emailSuffix := fmt.Sprintf("@%s", domain)
-			valid = valid || strings.HasSuffix(email, emailSuffix)
+			valid = valid || strings.HasSuffix(email, domain)
 		}
 		if !valid {
-			_, valid = validUsers[email]
+			valid = validUsers.IsValid(email)
 		}
 		log.Printf("validating: is %s valid? %v", email, valid)
 		return valid
 	}
 	return validator
 }
+
+func NewValidator(domains []string, usersFile string) func(string) bool {
+	return newValidatorImpl(domains, usersFile, func() {})
+}

validator_test.go

@@ -7,23 +7,117 @@ import (
 	"testing"
 )
 
-func TestValidatorComparisonsAreCaseInsensitive(t *testing.T) {
-	auth_email_file, err := ioutil.TempFile("", "test_auth_emails_")
+type ValidatorTest struct {
+	auth_email_file *os.File
+}
+
+func NewValidatorTest(t *testing.T) *ValidatorTest {
+	vt := &ValidatorTest{}
+	var err error
+	vt.auth_email_file, err = ioutil.TempFile("", "test_auth_emails_")
 	if err != nil {
 		t.Fatal("failed to create temp file: " + err.Error())
 	}
-	defer os.Remove(auth_email_file.Name())
+	return vt
+}
 
-	auth_email_file.WriteString(
-		strings.Join([]string{"[email protected]"}, "\n"))
-	err = auth_email_file.Close()
-	if err != nil {
-		t.Fatal("failed to close temp file " + auth_email_file.Name() +
-			": " + err.Error())
+func (vt *ValidatorTest) TearDown() {
+	os.Remove(vt.auth_email_file.Name())
+}
+
+// This will close vt.auth_email_file.
+func (vt *ValidatorTest) WriteEmails(t *testing.T, emails []string) {
+	defer vt.auth_email_file.Close()
+	vt.auth_email_file.WriteString(strings.Join(emails, "\n"))
+	if err := vt.auth_email_file.Close(); err != nil {
+		t.Fatal("failed to close temp file " +
+			vt.auth_email_file.Name() + ": " + err.Error())
+	}
+}
+
+func TestValidatorEmpty(t *testing.T) {
+	vt := NewValidatorTest(t)
+	defer vt.TearDown()
+
+	vt.WriteEmails(t, []string(nil))
+	domains := []string(nil)
+	validator := NewValidator(domains, vt.auth_email_file.Name())
+
+	if validator("[email protected]") {
+		t.Error("nothing should validate when the email and " +
+			"domain lists are empty")
+	}
+}
+
+func TestValidatorSingleEmail(t *testing.T) {
+	vt := NewValidatorTest(t)
+	defer vt.TearDown()
+
+	vt.WriteEmails(t, []string{"[email protected]"})
+	domains := []string(nil)
+	validator := NewValidator(domains, vt.auth_email_file.Name())
+
+	if !validator("[email protected]") {
+		t.Error("email should validate")
+	}
+	if validator("[email protected]") {
+		t.Error("email from same domain but not in list " +
+			"should not validate when domain list is empty")
 	}
+}
+
+func TestValidatorSingleDomain(t *testing.T) {
+	vt := NewValidatorTest(t)
+	defer vt.TearDown()
+
+	vt.WriteEmails(t, []string(nil))
+	domains := []string{"example.com"}
+	validator := NewValidator(domains, vt.auth_email_file.Name())
+
+	if !validator("[email protected]") {
+		t.Error("email should validate")
+	}
+	if !validator("[email protected]") {
+		t.Error("email from same domain should validate")
+	}
+}
+
+func TestValidatorMultipleEmailsMultipleDomains(t *testing.T) {
+	vt := NewValidatorTest(t)
+	defer vt.TearDown()
+
+	vt.WriteEmails(t, []string{
+		"[email protected]",
+		"[email protected]",
+	})
+	domains := []string{"example0.com", "example1.com"}
+	validator := NewValidator(domains, vt.auth_email_file.Name())
+
+	if !validator("[email protected]") {
+		t.Error("email from first domain should validate")
+	}
+	if !validator("[email protected]") {
+		t.Error("email from second domain should validate")
+	}
+	if !validator("[email protected]") {
+		t.Error("first email in list should validate")
+	}
+	if !validator("[email protected]") {
+		t.Error("second email in list should validate")
+	}
+	if validator("[email protected]") {
+		t.Error("email not in list that matches no domains " +
+			"should not validate")
+	}
+}
+
+func TestValidatorComparisonsAreCaseInsensitive(t *testing.T) {
+	vt := NewValidatorTest(t)
+	defer vt.TearDown()
 
+	vt.WriteEmails(t, []string{"[email protected]"})
 	domains := []string{"Frobozz.Com"}
-	validator := NewValidator(domains, auth_email_file.Name())
+	validator := NewValidator(domains, vt.auth_email_file.Name())
 
 	if !validator("[email protected]") {
 		t.Error("loaded email addresses are not lower-cased")

validator_watcher_copy_test.go

@@ -0,0 +1,50 @@
+// +build go1.3
+// +build !plan9,!solaris,!windows
+
+// Turns out you can't copy over an existing file on Windows.
+
+package main
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+)
+
+func (vt *ValidatorTest) UpdateEmailFileViaCopyingOver(
+	t *testing.T, emails []string) {
+	orig_file := vt.auth_email_file
+	var err error
+	vt.auth_email_file, err = ioutil.TempFile("", "test_auth_emails_")
+	if err != nil {
+		t.Fatal("failed to create temp file for copy: " + err.Error())
+	}
+	vt.WriteEmails(t, emails)
+	err = os.Rename(vt.auth_email_file.Name(), orig_file.Name())
+	if err != nil {
+		t.Fatal("failed to copy over temp file: " + err.Error())
+	}
+	vt.auth_email_file = orig_file
+}
+
+func TestValidatorOverwriteEmailListViaCopyingOver(t *testing.T) {
+	vt := NewValidatorTest(t)
+	defer vt.TearDown()
+
+	vt.WriteEmails(t, []string{"[email protected]"})
+	domains := []string(nil)
+	updated := make(chan bool)
+	validator := newValidatorImpl(domains, vt.auth_email_file.Name(),
+		func() { updated <- true })
+
+	if !validator("[email protected]") {
+		t.Error("email in list should validate")
+	}
+
+	vt.UpdateEmailFileViaCopyingOver(t, []string{"[email protected]"})
+	<-updated
+
+	if validator("[email protected]") {
+		t.Error("email removed from list should not validate")
+	}
+}