Merge pull request influxdata#460 from influxdata/nc-gdm

Use Go vendoring for deps. Using govend tool.
corewarecjj · Apr 13, 2016 · def367f · def367f
2 parents dc2967a + d8e94d7
commit def367f
Show file tree

Hide file tree

Showing 527 changed files with 200,501 additions and 72 deletions.
diff --git a/.gitignore b/.gitignore
@@ -15,3 +15,9 @@ kapacitor*.zip
 *.test
 /test-logs
 *.prof
+
+# Ignore any built binaries
+/kapacitor
+/kapacitord
+/tickfmt
+/tickdoc
diff --git a/.hooks/pre-commit b/.hooks/pre-commit
@@ -1,13 +1,13 @@
 #!/usr/bin/env bash
 
-fmtcount=`git ls-files | grep '.go$' | xargs gofmt -l 2>&1 | wc -l`
+fmtcount=`git ls-files | grep '.go$' | grep -v '^vendor/' | xargs gofmt -l 2>&1 | wc -l`
 if [ $fmtcount -gt 0 ]; then
     echo "Some files aren't formatted, please run 'go fmt ./...' to format your source code before committing"
     exit 1
 fi
 
 # Due to the way composites work, vet will fail for some of our tests so we ignore it
-vetcount=`go tool vet --composites=false ./ 2>&1  | wc -l`
+vetcount=`go tool vet --composites=false $(go list ./... | grep -v '/vendor/' | sed 's~github.com/influxdata/kapacitor~.~' | grep -v '^\.$') 2>&1  | wc -l`
 if [ $vetcount -gt 0 ]; then
     echo "Some files aren't passing vet heuristics, please run 'go vet ./...' to see the errors it flags and correct your source code before committing"
     exit 1

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -114,11 +114,6 @@ go fmt ./...
 go vet ./...
 ```
 
-To install go vet, run the following command:
-```
-go get golang.org/x/tools/cmd/vet
-```
-
 NOTE: If you have not installed mercurial, the above command will fail.  See [Revision Control Systems](#revision-control-systems) above.
 
 For more information on `go vet`, [read the GoDoc](https://godoc.org/golang.org/x/tools/cmd/vet).
@@ -143,6 +138,28 @@ go get -t ./...
 go test ./...
 ```
 
+Dependencies
+------------
+
+All dependencies should be vendored and locked
+Kapacitor uses the `github.com/govend/govend` utility to vendor dependencies.
+
+The workflow is simple:
+
+```bash
+# Download and lock all deps
+govend -l --prune
+```
+
+To update an existing dependency use:
+
+```bash
+govend -u --prune package 
+```
+
+> NOTE: The use of prune removes uneeded files from dependencies so only the minimum set of files are committed to the repo.
+
+
 Generating Code
 ---------------
 
@@ -159,9 +176,9 @@ go generate ./...
 For the generate command to succeed you will need a few dependencies installed on your system:
 
 * tmpl -- A utility used to generate code from templates. Install via `go get github.com/benbjohnson/tmpl`
-* protoc + protoc-gen-gogo -- A protobuf compiler plus the protoc-gen-gogo extension.
+* protoc + protoc-gen-go -- A protobuf compiler plus the protoc-gen-go extension.
     You need version 3.0.0-beta-2 of protoc.
-    To install the go plugin run `go get github.com/gogo/protobuf/protoc-gen-gogo`
+    To install the go plugin run `go get github.com/golang/protobuf/protoc-gen-go`
 
 NOTE: Since installing dependencies can often be painful we have provided a docker container that comes with all of these dependencies installed.
 See the section below about the build script and docker.

diff --git a/build.py b/build.py
@@ -45,7 +45,7 @@
 DESCRIPTION = "Time series data processing engine"
 
 # SCRIPT START
-go_vet_command = ["go", "tool", "vet", "-composites=false", "./"]
+go_vet_command = ["go", "tool", "vet", "-composites=false"]
 prereqs = [ 'git', 'go' ]
 optional_prereqs = [ 'fpm', 'rpmbuild' ]
 
@@ -122,46 +122,35 @@ def run_generate():
     print "Running generate..."
     run("go get github.com/golang/protobuf/protoc-gen-go")
     run("go get github.com/benbjohnson/tmpl")
-    run("go generate ./...")
-    print "Generate succeeded."
-    return True
+    generate_cmd = ["go", "generate"]
+    generate_cmd.extend(go_list())
+    p = subprocess.Popen(generate_cmd)
+    code = p.wait()
+    if code == 0:
+        print "Generate succeeded."
+        return True
+    else:
+        print "Generate failed."
+        return False
 
-def go_get(branch, update=False, no_stash=False):
+def go_get(branch, update=False, no_uncommitted=False):
+    run("go get github.com/govend/govend")
     get_command = ""
     if update:
-        get_command += "go get -t -u -f -d ./..."
+        get_command += "govend -v -u --prune"
     else:
-        get_command += "go get -t -d ./..."
+        get_command += "govend -v --prune"
 
-    # 'go get' switches to master, so stash what we currently have
-    changes = run("git status --porcelain").strip()
-    if len(changes) > 0:
-        if no_stash:
-            print "There are un-committed changes in your local branch, --no-stash was given, cannot continue"
-            return False
-
-        stash = run("git stash create -a").strip()
-        print "There are un-committed changes in your local branch, stashing them as {}".format(stash)
-        # reset to ensure we don't have any checkout issues
-        run("git reset --hard")
-
-        print "Retrieving Go dependencies (moving to master)..."
-        try:
-            run(get_command, shell=True)
-            sys.stdout.flush()
-        finally:
-            # Unstash even if go get fails so that changes are left in the stash
-            print "Moving back to branch '{}'...".format(branch)
-            run("git checkout {}".format(branch))
-
-            print "Applying previously stashed contents..."
-            run("git stash apply {}".format(stash))
-    else:
-        print "Retrieving Go dependencies..."
-        run(get_command, shell=True)
+    print "Retrieving Go dependencies..."
+    sys.stdout.flush()
+    run(get_command, shell=True)
 
-        print "Moving back to branch '{}'...".format(branch)
-        run("git checkout {}".format(branch))
+    # Check for uncommitted changes if no_uncommitted was given.
+    if no_uncommitted:
+        changes = run("git status --porcelain").strip()
+        if len(changes) > 0:
+            print "There are un-committed changes in your local branch, --no-uncommited was given, cannot continue"
+            return False
 
     return True
 
@@ -345,6 +334,33 @@ def upload_packages(packages, bucket_name=None, nightly=False):
     print ""
     return 0
 
+def go_list(vendor=False, relative=False):
+    """
+    Return a list of packages
+
+    If vendor is False vendor package are not included
+    If relative is True the package prefix defined by PACKAGE_URL is stripped
+    """
+    p = subprocess.Popen(["go", "list", "./..."], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    out, err = p.communicate()
+    packages = out.split('\n')
+    if packages[-1] == '':
+        packages = packages[:-1]
+    if not vendor:
+        non_vendor = []
+        for p in packages:
+            if '/vendor/' not in p:
+                non_vendor.append(p)
+        packages = non_vendor
+    if relative:
+        relative_pkgs = []
+        for p in packages:
+            r = p.replace(PACKAGE_URL, '.')
+            if r != '.':
+                relative_pkgs.append(r)
+        packages = relative_pkgs
+    return packages
+
 def run_tests(race, parallel, timeout, no_vet):
     print "Running tests:"
     print "\tRace: ", race
@@ -353,15 +369,19 @@ def run_tests(race, parallel, timeout, no_vet):
     if timeout is not None:
         print "\tTimeout:", timeout
     sys.stdout.flush()
-    p = subprocess.Popen(["go", "fmt", "./..."], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    fmt_cmd = ["go", "fmt"]
+    fmt_cmd.extend(go_list())
+    p = subprocess.Popen(fmt_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     out, err = p.communicate()
     if len(out) > 0 or len(err) > 0:
         print "Code not formatted. Please use 'go fmt ./...' to fix formatting errors."
         print out
         print err
         return False
     if not no_vet:
-        p = subprocess.Popen(go_vet_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        vet_cmd = list(go_vet_command)
+        vet_cmd.extend(go_list(relative=True))
+        p = subprocess.Popen(vet_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         out, err = p.communicate()
         if len(out) > 0 or len(err) > 0:
             print "Go vet failed. Please run 'go vet ./...' and fix any errors."
@@ -371,15 +391,18 @@ def run_tests(race, parallel, timeout, no_vet):
     else:
         print "Skipping go vet ..."
         sys.stdout.flush()
-    test_command = "go test -v"
+    test_command = ["go","test", "-v"]
     if race:
-        test_command += " -race"
+        test_command.append("-race")
     if parallel is not None:
-        test_command += " -parallel {}".format(parallel)
+        test_command.append("-parallel")
+        test_command.append(str(parallel))
     if timeout is not None:
-        test_command += " -timeout {}".format(timeout)
-    test_command += " ./..."
-    code = os.system(test_command)
+        test_command.append("-timeout")
+        test_command.append(str(timeout))
+    test_command.extend(go_list())
+    p = subprocess.Popen(test_command)
+    code = p.wait()
     if code != 0:
         print "Tests Failed"
         return False
@@ -664,7 +687,7 @@ def main():
     run_get = True
     upload_bucket = None
     generate = False
-    no_stash = False
+    no_uncommitted = False
 
     for arg in sys.argv[1:]:
         if '--outdir' in arg:
@@ -694,8 +717,8 @@ def main():
         elif '--package' in arg:
             # Signifies that packages should be built.
             package = True
-            # If packaging do not allow stashing of local changes
-            no_stash = True
+            # If packaging do not allow uncommitted changes
+            no_uncommitted = True
         elif '--nightly' in arg:
             # Signifies that this is a nightly build.
             nightly = True
@@ -726,10 +749,9 @@ def main():
         elif '--bucket' in arg:
             # The bucket to upload the packages to, relies on boto
             upload_bucket = arg.split("=")[1]
-        elif '--no-stash' in arg:
-            # Do not stash uncommited changes
+        elif '--no-uncommited' in arg:
             # Fail if uncommited changes exist
-            no_stash = True
+            no_uncommitted = True
         elif '--generate' in arg:
             generate = True
         elif '--debug' in arg:
@@ -789,7 +811,7 @@ def main():
             return 1
 
     if run_get:
-        if not go_get(branch, update=update, no_stash=no_stash):
+        if not go_get(branch, update=update, no_uncommitted=no_uncommitted):
             return 1
 
     if test:

diff --git a/test.sh b/test.sh
@@ -129,24 +129,24 @@ fi
 case $ENVIRONMENT_INDEX in
     0)
         # 64 bit tests
-        run_test_docker Dockerfile_build_ubuntu64 test_64bit --debug --test --generate --no-stash
+        run_test_docker Dockerfile_build_ubuntu64 test_64bit --debug --test --generate --no-uncommited
         rc=$?
         ;;
     1)
         # 64 bit race tests
         GORACE="halt_on_error=1"
-        run_test_docker Dockerfile_build_ubuntu64 test_64bit_race --debug --test --generate --no-stash --race
+        run_test_docker Dockerfile_build_ubuntu64 test_64bit_race --debug --test --generate --no-uncommited --race
         rc=$?
         ;;
     2)
         # 32 bit tests
-        run_test_docker Dockerfile_build_ubuntu32 test_32bit --debug --test --generate --no-stash
+        run_test_docker Dockerfile_build_ubuntu32 test_32bit --debug --test --generate --no-uncommited
         rc=$?
         ;;
     3)
         # 64 bit tests on golang go1.6
         GO_CHECKOUT=go1.6rc2
-        run_test_docker Dockerfile_build_ubuntu64_git test_64bit_go1.6 --debug --test --generate --no-stash
+        run_test_docker Dockerfile_build_ubuntu64_git test_64bit_go1.6 --debug --test --generate --no-uncommited
         rc=$?
         ;;
     "save")

diff --git a/tick/cmd/tickdoc/main.go b/tick/cmd/tickdoc/main.go
@@ -42,7 +42,7 @@ import (
 	"text/template"
 	"unicode"
 
-	"github.com/naoina/toml"
+	"github.com/BurntSushi/toml"
 	"github.com/serenize/snaker"
 	"github.com/shurcooL/markdownfmt/markdown"
 )
@@ -150,14 +150,8 @@ func main() {
 	}
 }
 
-func decodeConfig(path string) error {
-	f, err := os.Open(path)
-	if err != nil {
-		return err
-	}
-	dec := toml.NewDecoder(f)
-	err = dec.Decode(&config)
-	if err != nil {
+func decodeConfig(path string) (err error) {
+	if _, err = toml.DecodeFile(path, &config); err != nil {
 		return err
 	}
 	config.headerTemplate, err = template.New("header").Parse(config.PageHeader)

diff --git a/vendor.yml b/vendor.yml
@@ -0,0 +1,51 @@
+vendors:
+- path: github.com/BurntSushi/toml
+  rev: bbd5bb678321a0d6e58f1099321dfa73391c1b6f
+- path: github.com/boltdb/bolt
+  rev: 144418e1475d8bf7abbdc48583500f1a20c62ea7
+- path: github.com/cenkalti/backoff
+  rev: 32cd0c5b3aef12c76ed64aaf678f6c79736be7dc
+- path: github.com/davecgh/go-spew
+  rev: 5215b55f46b2b919f50a1df0eaa5886afe4e3b3d
+- path: github.com/dustin/go-humanize
+  rev: 8929fe90cee4b2cb9deb468b51fb34eba64d1bf0
+- path: github.com/gogo/protobuf
+  rev: 4f262e4b0f3a6cea646e15798109335551e21756
+- path: github.com/golang/protobuf
+  rev: f0a097ddac24fb00e07d2ac17f8671423f3ea47c
+- path: github.com/gorhill/cronexpr
+  rev: f0984319b44273e83de132089ae42b1810f4933b
+- path: github.com/influxdata/influxdb
+  rev: 93a76eecb6d7575225a68235667de68ed906222b
+- path: github.com/influxdata/wlog
+  rev: 7c63b0a71ef8300adc255344d275e10e5c3a71ec
+- path: github.com/influxdb/usage-client
+  rev: 475977e68d79883d9c8d67131c84e4241523f452
+- path: github.com/kimor79/gollectd
+  rev: b5dddb1667dcc1e6355b9305e2c1608a2db6983c
+- path: github.com/mattn/go-runewidth
+  rev: d6bea18f789704b5f83375793155289da36a3c7f
+- path: github.com/pmezard/go-difflib
+  rev: 792786c7400a136282c1664665ae0a8db921c6c2
+- path: github.com/russross/blackfriday
+  rev: b43df972fb5fdf3af8d2e90f38a69d374fe26dd0
+- path: github.com/serenize/snaker
+  rev: 8824b61eca66d308fcb2d515287d3d7a28dba8d6
+- path: github.com/shurcooL/go
+  rev: 07c46ca56e4820cfaf750f74e25bc671dccd2ba4
+- path: github.com/shurcooL/markdownfmt
+  rev: 45e6ea2c4705675a93a32b5f548dbb7997826875
+- path: github.com/shurcooL/sanitized_anchor_name
+  rev: 10ef21a441db47d8b13ebcc5fd2310f636973c77
+- path: github.com/stretchr/testify
+  rev: bcd9e3389dd03b0b668d11f4d462a6af6c2dfd60
+- path: github.com/twinj/uuid
+  rev: 89173bcdda19db0eb88aef1e1cb1cb2505561d31
+- path: golang.org/x/crypto
+  rev: 1777f3ba8c1fed80fcaec3317e3aaa4f627764d2
+- path: golang.org/x/sys
+  rev: 9eef40adf05b951699605195b829612bd7b69952
+- path: gopkg.in/alexcesaro/quotedprintable.v3
+  rev: 2caba252f4dc53eaf6b553000885530023f54623
+- path: gopkg.in/gomail.v2
+  rev: 81ebce5c23dfd25c6c67194b37d3dd3f338c98b1
diff --git a/vendor/github.com/BurntSushi/toml/COMPATIBLE b/vendor/github.com/BurntSushi/toml/COMPATIBLE