Skip to content

Commit

Permalink
remove govend deps (influxdata#632)
Browse files Browse the repository at this point in the history 
add scripts for new subrepo based deps

git subrepo clone https://github.com/influxdata/wlog.git vendor/github.com/influxdata/wlog

subrepo:
  subdir:   "vendor/github.com/influxdata/wlog"
  merged:   "7c63b0a"
upstream:
  origin:   "https://github.com/influxdata/wlog.git"
  branch:   "master"
  commit:   "7c63b0a"
git-subrepo:
  version:  "0.3.0"
  origin:   "???"
  commit:   "???"

git subrepo clone http://github.com/boltdb/bolt.git vendor/github.com/boltdb/bolt

subrepo:
  subdir:   "vendor/github.com/boltdb/bolt"
  merged:   "05e441d"
upstream:
  origin:   "http://github.com/boltdb/bolt.git"
  branch:   "master"
  commit:   "05e441d"
git-subrepo:
  version:  "0.3.0"
  origin:   "???"
  commit:   "???"

git subrepo clone http://github.com/BurntSushi/toml.git vendor/github.com/BurntSushi/toml

subrepo:
  subdir:   "vendor/github.com/BurntSushi/toml"
  merged:   "f0aeabc"
upstream:
  origin:   "http://github.com/BurntSushi/toml.git"
  branch:   "datetimes"
  commit:   "f0aeabc"
git-subrepo:
  version:  "0.3.0"
  origin:   "???"
  commit:   "???"

git subrepo clone http://github.com/cenkalti/backoff.git vendor/github.com/cenkalti/backoff

subrepo:
  subdir:   "vendor/github.com/cenkalti/backoff"
  merged:   "a603017"
upstream:
  origin:   "http://github.com/cenkalti/backoff.git"
  branch:   "master"
  commit:   "a603017"
git-subrepo:
  version:  "0.3.0"
  origin:   "???"
  commit:   "???"

git subrepo clone http://github.com/dustin/go-humanize.git vendor/github.com/dustin/go-humanize

subrepo:
  subdir:   "vendor/github.com/dustin/go-humanize"
  merged:   "499693e"
upstream:
  origin:   "http://github.com/dustin/go-humanize.git"
  branch:   "master"
  commit:   "499693e"
git-subrepo:
  version:  "0.3.0"
  origin:   "???"
  commit:   "???"

git subrepo clone http://github.com/gogo/protobuf.git vendor/github.com/gogo/protobuf

subrepo:
  subdir:   "vendor/github.com/gogo/protobuf"
  merged:   "318371c"
upstream:
  origin:   "http://github.com/gogo/protobuf.git"
  branch:   "master"
  commit:   "318371c"
git-subrepo:
  version:  "0.3.0"
  origin:   "???"
  commit:   "???"

git subrepo clone http://github.com/golang/protobuf.git vendor/github.com/golang/protobuf

subrepo:
  subdir:   "vendor/github.com/golang/protobuf"
  merged:   "8616e8e"
upstream:
  origin:   "http://github.com/golang/protobuf.git"
  branch:   "master"
  commit:   "8616e8e"
git-subrepo:
  version:  "0.3.0"
  origin:   "???"
  commit:   "???"

git subrepo clone http://github.com/gorhill/cronexpr.git vendor/github.com/gorhill/cronexpr

subrepo:
  subdir:   "vendor/github.com/gorhill/cronexpr"
  merged:   "f098431"
upstream:
  origin:   "http://github.com/gorhill/cronexpr.git"
  branch:   "master"
  commit:   "f098431"
git-subrepo:
  version:  "0.3.0"
  origin:   "???"
  commit:   "???"

git subrepo clone http://github.com/influxdata/influxdb.git vendor/github.com/influxdata/influxdb

subrepo:
  subdir:   "vendor/github.com/influxdata/influxdb"
  merged:   "c8e90fa"
upstream:
  origin:   "http://github.com/influxdata/influxdb.git"
  branch:   "master"
  commit:   "c8e90fa"
git-subrepo:
  version:  "0.3.0"
  origin:   "???"
  commit:   "???"

git subrepo clone http://github.com/influxdb/usage-client.git vendor/github.com/influxdb/usage-client

subrepo:
  subdir:   "vendor/github.com/influxdb/usage-client"
  merged:   "475977e"
upstream:
  origin:   "http://github.com/influxdb/usage-client.git"
  branch:   "master"
  commit:   "475977e"
git-subrepo:
  version:  "0.3.0"
  origin:   "???"
  commit:   "???"

git subrepo clone http://github.com/kimor79/gollectd.git vendor/github.com/kimor79/gollectd

subrepo:
  subdir:   "vendor/github.com/kimor79/gollectd"
  merged:   "b5dddb1"
upstream:
  origin:   "http://github.com/kimor79/gollectd.git"
  branch:   "master"
  commit:   "b5dddb1"
git-subrepo:
  version:  "0.3.0"
  origin:   "???"
  commit:   "???"

git subrepo clone http://github.com/mattn/go-runewidth.git vendor/github.com/mattn/go-runewidth

subrepo:
  subdir:   "vendor/github.com/mattn/go-runewidth"
  merged:   "d6bea18"
upstream:
  origin:   "http://github.com/mattn/go-runewidth.git"
  branch:   "master"
  commit:   "d6bea18"
git-subrepo:
  version:  "0.3.0"
  origin:   "???"
  commit:   "???"

git subrepo clone http://github.com/pkg/errors.git vendor/github.com/pkg/errors

subrepo:
  subdir:   "vendor/github.com/pkg/errors"
  merged:   "5776abf"
upstream:
  origin:   "http://github.com/pkg/errors.git"
  branch:   "master"
  commit:   "5776abf"
git-subrepo:
  version:  "0.3.0"
  origin:   "???"
  commit:   "???"

git subrepo clone http://github.com/russross/blackfriday.git vendor/github.com/russross/blackfriday

subrepo:
  subdir:   "vendor/github.com/russross/blackfriday"
  merged:   "1d6b8e9"
upstream:
  origin:   "http://github.com/russross/blackfriday.git"
  branch:   "master"
  commit:   "1d6b8e9"
git-subrepo:
  version:  "0.3.0"
  origin:   "???"
  commit:   "???"

git subrepo clone http://github.com/serenize/snaker.git vendor/github.com/serenize/snaker

subrepo:
  subdir:   "vendor/github.com/serenize/snaker"
  merged:   "8824b61"
upstream:
  origin:   "http://github.com/serenize/snaker.git"
  branch:   "master"
  commit:   "8824b61"
git-subrepo:
  version:  "0.3.0"
  origin:   "???"
  commit:   "???"

git subrepo clone http://github.com/shurcooL/go.git vendor/github.com/shurcooL/go

subrepo:
  subdir:   "vendor/github.com/shurcooL/go"
  merged:   "8b8a7a0"
upstream:
  origin:   "http://github.com/shurcooL/go.git"
  branch:   "master"
  commit:   "8b8a7a0"
git-subrepo:
  version:  "0.3.0"
  origin:   "???"
  commit:   "???"

git subrepo clone http://github.com/shurcooL/markdownfmt.git vendor/github.com/shurcooL/markdownfmt

subrepo:
  subdir:   "vendor/github.com/shurcooL/markdownfmt"
  merged:   "818f50d"
upstream:
  origin:   "http://github.com/shurcooL/markdownfmt.git"
  branch:   "master"
  commit:   "818f50d"
git-subrepo:
  version:  "0.3.0"
  origin:   "???"
  commit:   "???"

git subrepo clone http://github.com/shurcooL/sanitized_anchor_name.git vendor/github.com/shurcooL/sanitized_anchor_name

subrepo:
  subdir:   "vendor/github.com/shurcooL/sanitized_anchor_name"
  merged:   "10ef21a"
upstream:
  origin:   "http://github.com/shurcooL/sanitized_anchor_name.git"
  branch:   "master"
  commit:   "10ef21a"
git-subrepo:
  version:  "0.3.0"
  origin:   "???"
  commit:   "???"

git subrepo clone http://github.com/twinj/uuid.git vendor/github.com/twinj/uuid

subrepo:
  subdir:   "vendor/github.com/twinj/uuid"
  merged:   "efdcd76"
upstream:
  origin:   "http://github.com/twinj/uuid.git"
  branch:   "master"
  commit:   "efdcd76"
git-subrepo:
  version:  "0.3.0"
  origin:   "???"
  commit:   "???"

git subrepo clone http://gopkg.in/gomail.v2.git vendor/gopkg.in/gomail.v2

subrepo:
  subdir:   "vendor/gopkg.in/gomail.v2"
  merged:   "81ebce5"
upstream:
  origin:   "http://gopkg.in/gomail.v2.git"
  branch:   "master"
  commit:   "81ebce5"
git-subrepo:
  version:  "0.3.0"
  origin:   "???"
  commit:   "???"

git subrepo clone --branch=master https://github.com/golang/crypto.git vendor/golang.org/x/crypto

subrepo:
  subdir:   "vendor/golang.org/x/crypto"
  merged:   "77f4136"
upstream:
  origin:   "https://github.com/golang/crypto.git"
  branch:   "master"
  commit:   "77f4136"
git-subrepo:
  version:  "0.3.0"
  origin:   "???"
  commit:   "???"

git subrepo clone https://github.com/benbjohnson/tmpl vendor/github.com/benbjohnson/tmpl

subrepo:
  subdir:   "vendor/github.com/benbjohnson/tmpl"
  merged:   "8e77bc5"
upstream:
  origin:   "https://github.com/benbjohnson/tmpl"
  branch:   "master"
  commit:   "8e77bc5"
git-subrepo:
  version:  "0.3.0"
  origin:   "???"
  commit:   "???"

git subrepo pull vendor/github.com/pkg/errors

subrepo:
  subdir:   "vendor/github.com/pkg/errors"
  merged:   "874c0ec"
upstream:
  origin:   "http://github.com/pkg/errors.git"
  branch:   "master"
  commit:   "874c0ec"
git-subrepo:
  version:  "0.3.0"
  origin:   "???"
  commit:   "???"

git subrepo clone https://github.com/stretchr/testify.git vendor/github.com/stretchr/testify

subrepo:
  subdir:   "vendor/github.com/stretchr/testify"
  merged:   "8d64eb7"
upstream:
  origin:   "https://github.com/stretchr/testify.git"
  branch:   "master"
  commit:   "8d64eb7"
git-subrepo:
  version:  "0.3.0"
  origin:   "???"
  commit:   "???"

git subrepo clone https://github.com/influxdata/usage-client.git vendor/github.com/influxdata/usage-client

subrepo:
  subdir:   "vendor/github.com/influxdata/usage-client"
  merged:   "475977e"
upstream:
  origin:   "https://github.com/influxdata/usage-client.git"
  branch:   "master"
  commit:   "475977e"
git-subrepo:
  version:  "0.3.0"
  origin:   "???"
  commit:   "???"
  • Loading branch information
Nathaniel Cook authored Jun 10, 2016
1 parent 3003b83 commit b8493e4
Show file tree
Hide file tree
Showing 1,617 changed files with 860,520 additions and 109,822 deletions.
55 changes: 45 additions & 10 deletions CONTRIBUTING.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,23 +141,58 @@ go test ./...
Dependencies
------------

All dependencies should be vendored and locked
Kapacitor uses the `github.com/govend/govend` utility to vendor dependencies.
Kapacitor vendors all dependencies.
Kapacitor does not use a dependency manager tool, but rather uses git subrepos to place
dependencies in the vendor directory.
This give complete control over how dependency are managed and keeps the workflow simple.
A few helper scripts are provided to make this process fast and easy.

The workflow is simple:
To manage the subrepo you must first install [git-subrepo](https://github.com/ingydotnet/git-subrepo#installation).

```bash
# Download and lock all deps
govend -l --prune
First list all dependencies, including dependencies of dependencies.

```
./list-deps
```

To update an existing dependency use:
To add a new dependency add a new entry to the `vendor.list` file, of the form:

```bash
govend -u --prune package
```
<package> [branch]
```

The `branch` column is optional.
If `branch` is left empty the default branch will be used.

For example, to add the `github.com/influxdata/foo/bar` dependency add this line to the `vendor.list`.

```
github.com/influxdata/foo https://github.com/influxdata/foo.git
```

Notice that `bar` part of the path was left off since its a subdirectory of the repo.

Commit this change then run:

```
./vendor.sh github.com/influxdata/foo
```

This will add the subrepo for the git repo under `vendor/github.com/influxdata/foo`.

Later to update the dependency use the same command.

```
./vendor.sh github.com/influxdata/foo
```

Or to update all dependencies at once use

```
./vendor.sh
```

> NOTE: The use of prune removes uneeded files from dependencies so only the minimum set of files are committed to the repo.
These scripts are really simple, we may formalize them later but currently simplicity is key.


Generating Code
Expand Down
18 changes: 6 additions & 12 deletions build.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,8 +151,8 @@ def run_generate():
"""Run 'go generate' to rebuild any static assets.
"""
logging.info("Running generate...")
run("go get github.com/golang/protobuf/protoc-gen-go")
run("go get github.com/benbjohnson/tmpl")
run("go install ./vendor/github.com/golang/protobuf/protoc-gen-go")
run("go install ./vendor/github.com/benbjohnson/tmpl")
generate_cmd = ["go", "generate"]
generate_cmd.extend(go_list())
p = subprocess.Popen(generate_cmd)
Expand All @@ -165,17 +165,11 @@ def run_generate():
return False

def go_get(branch, update=False, no_uncommitted=False):
"""Retrieve build dependencies or restore pinned dependencies.
"""
run("go get github.com/govend/govend")
get_command = ""
if update:
get_command += "{}/bin/govend -v -u --prune".format(os.environ.get("GOPATH"))
else:
get_command += "{}/bin/govend -v --prune".format(os.environ.get("GOPATH"))

logging.info("Retrieving Go dependencies...")
run(get_command, shell=True)
Retrieve build dependencies or restore pinned dependencies.
"""
# All Kapacitor deps should be vendored so do not go get them
logging.info("Pretending to retrieve Go dependencies...")

# Check for uncommitted changes if no_uncommitted was given.
if no_uncommitted:
Expand Down
25 changes: 2 additions & 23 deletions cmd/kapacitord/run/server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import (
"runtime"
"runtime/pprof"
"strings"
"time"

"github.com/influxdata/influxdb/influxql"
"github.com/influxdata/influxdb/services/collectd"
Expand Down Expand Up @@ -71,7 +70,7 @@ type Server struct {
ReplayService *replay.Service
InfluxDBService *influxdb.Service

MetaClient *metaclient
MetaClient *kapacitor.NoopMetaClient
QueryExecutor *queryexecutor

Services []Service
Expand Down Expand Up @@ -99,7 +98,7 @@ func NewServer(c *Config, buildInfo *BuildInfo, logService logging.Interface) (*
hostname: c.Hostname,
err: make(chan error),
LogService: logService,
MetaClient: &metaclient{},
MetaClient: &kapacitor.NoopMetaClient{},
QueryExecutor: &queryexecutor{},
Logger: l,
}
Expand Down Expand Up @@ -583,26 +582,6 @@ type tcpaddr struct{ host string }
func (a *tcpaddr) Network() string { return "tcp" }
func (a *tcpaddr) String() string { return a.host }

type metaclient struct{}

func (m *metaclient) WaitForLeader(d time.Duration) error {
return nil
}
func (m *metaclient) CreateDatabase(name string) (*meta.DatabaseInfo, error) {
return nil, nil
}
func (m *metaclient) Database(name string) (*meta.DatabaseInfo, error) {
return &meta.DatabaseInfo{
Name: name,
}, nil
}
func (m *metaclient) Authenticate(username, password string) (ui *meta.UserInfo, err error) {
return nil, errors.New("not authenticated")
}
func (m *metaclient) Users() ([]meta.UserInfo, error) {
return nil, errors.New("no user")
}

type queryexecutor struct{}

func (qe *queryexecutor) Authorize(u *meta.UserInfo, q *influxql.Query, db string) error {
Expand Down
2 changes: 1 addition & 1 deletion integrations/benchmark_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,7 @@ func Bench(b *testing.B, tasksCount, pointCount int, db, rp, measurement, tickSc
config.BindAddress = ":0" // Choose port dynamically
httpdService := httpd.NewService(config, logService.NewLogger("[http] ", log.LstdFlags))

httpdService.Handler.MetaClient = &metaclient{}
httpdService.Handler.MetaClient = &kapacitor.NoopMetaClient{}
err := httpdService.Open()
if err != nil {
b.Fatal(err)
Expand Down
25 changes: 0 additions & 25 deletions integrations/helpers_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ import (

client "github.com/influxdata/influxdb/client/v2"
"github.com/influxdata/influxdb/influxql"
"github.com/influxdata/influxdb/services/meta"
"github.com/influxdata/kapacitor"
"github.com/influxdata/kapacitor/udf"
"github.com/influxdata/wlog"
Expand Down Expand Up @@ -168,27 +167,3 @@ func (d deadman) Threshold() float64 { return d.threshold }
func (d deadman) Id() string { return d.id }
func (d deadman) Message() string { return d.message }
func (d deadman) Global() bool { return d.global }

type metaclient struct{}

func (m *metaclient) WaitForLeader(d time.Duration) error {
return nil
}

func (m *metaclient) CreateDatabase(name string) (*meta.DatabaseInfo, error) {
return nil, nil
}

func (m *metaclient) Database(name string) (*meta.DatabaseInfo, error) {
return &meta.DatabaseInfo{
Name: name,
}, nil
}

func (m *metaclient) Authenticate(username, password string) (ui *meta.UserInfo, err error) {
return nil, errors.New("not authenticated")
}

func (m *metaclient) Users() ([]meta.UserInfo, error) {
return nil, errors.New("no user")
}
46 changes: 46 additions & 0 deletions list-deps
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
#!/bin/bash

# Make sure we are in the dir of the script
DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
cd $DIR

# List all dependent packages and whether they have been vendored.
deps() {
local package packages allDeps paths

# Get the current package
package=$(go list .)
# Get the sub packages excluding vendored packages
packages=$(go list ./... | grep -v "^$package/vendor")
allDeps=$(go list -f '{{ join .Deps "\n"}}' $packages)

for dep in $allDeps
do
# Skip standard lib deps
paths=(${dep//\// })
if ! [[ "${paths[0]}" =~ .*\..* ]]
then
continue
fi
# Skip deps from within current package
if [[ "$dep" =~ ^$package ]]
then
if [[ "$dep" =~ ^$package/vendor ]]
then
# Rewrite vendored deps as normal deps
dep="v ${dep/$package\/vendor\//}"
else
continue
fi
else
dep="n $dep"
fi

echo $dep
done
}


# Deduplicate and sort the output
deps | sort -k 2 -u

37 changes: 37 additions & 0 deletions metaclient.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
package kapacitor

import (
"errors"
"time"

"github.com/influxdata/influxdb/services/meta"
)

type NoopMetaClient struct{}

func (m *NoopMetaClient) WaitForLeader(d time.Duration) error {
return nil
}
func (m *NoopMetaClient) CreateDatabase(name string) (*meta.DatabaseInfo, error) {
return nil, nil
}
func (m *NoopMetaClient) CreateDatabaseWithRetentionPolicy(name string, rpi *meta.RetentionPolicyInfo) (*meta.DatabaseInfo, error) {
return nil, nil
}
func (m *NoopMetaClient) CreateRetentionPolicy(database string, rpi *meta.RetentionPolicyInfo) (*meta.RetentionPolicyInfo, error) {
return nil, nil
}
func (m *NoopMetaClient) Database(name string) *meta.DatabaseInfo {
return &meta.DatabaseInfo{
Name: name,
}
}
func (m *NoopMetaClient) RetentionPolicy(database, name string) (*meta.RetentionPolicyInfo, error) {
return nil, nil
}
func (m *NoopMetaClient) Authenticate(username, password string) (ui *meta.UserInfo, err error) {
return nil, errors.New("not authenticated")
}
func (m *NoopMetaClient) Users() ([]meta.UserInfo, error) {
return nil, errors.New("no user")
}
9 changes: 0 additions & 9 deletions services/httpd/handler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,6 @@ type Handler struct {
Version string

MetaClient interface {
Database(name string) (*meta.DatabaseInfo, error)
Authenticate(username, password string) (ui *meta.UserInfo, err error)
Users() ([]meta.UserInfo, error)
}
Expand Down Expand Up @@ -406,14 +405,6 @@ func (h *Handler) serveWriteLine(w http.ResponseWriter, r *http.Request, body []
return
}

if di, err := h.MetaClient.Database(database); err != nil {
h.writeError(w, influxql.Result{Err: fmt.Errorf("metastore database error: %s", err)}, http.StatusInternalServerError)
return
} else if di == nil {
h.writeError(w, influxql.Result{Err: fmt.Errorf("database not found: %q", database)}, http.StatusNotFound)
return
}

if h.requireAuthentication && user == nil {
h.writeError(w, influxql.Result{Err: fmt.Errorf("user is required to write to database %q", database)}, http.StatusUnauthorized)
return
Expand Down
2 changes: 1 addition & 1 deletion services/reporting/config.go
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package reporting

import (
"github.com/influxdb/usage-client/v1"
client "github.com/influxdata/usage-client/v1"
)

type Config struct {
Expand Down
2 changes: 1 addition & 1 deletion services/reporting/service.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import (
"time"

"github.com/influxdata/kapacitor"
"github.com/influxdb/usage-client/v1"
client "github.com/influxdata/usage-client/v1"
)

const reportingInterval = time.Hour * 12
Expand Down
2 changes: 2 additions & 0 deletions udf/udf.pb.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

24 changes: 24 additions & 0 deletions vendor.list
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
github.com/boltdb/bolt
github.com/BurntSushi/toml
github.com/cenkalti/backoff
github.com/dustin/go-humanize
github.com/gogo/protobuf
github.com/golang/protobuf
github.com/gorhill/cronexpr
github.com/influxdata/influxdb
github.com/influxdata/usage-client
github.com/kimor79/gollectd
github.com/mattn/go-runewidth
github.com/pkg/errors
github.com/russross/blackfriday
github.com/serenize/snaker
github.com/shurcooL/go
github.com/shurcooL/markdownfmt
github.com/shurcooL/sanitized_anchor_name
github.com/twinj/uuid
golang.org/x/crypto master
gopkg.in/gomail.v2
# Generate deps
github.com/benbjohnson/tmpl
# Testing deps
github.com/stretchr/testify
Loading

0 comments on commit b8493e4

Please sign in to comment.