From 04b0fe438f479a866708ce2f3e82b9ba56e7ac64 Mon Sep 17 00:00:00 2001 From: Umputun Date: Mon, 1 May 2023 03:31:39 -0500 Subject: [PATCH] add connection timeout and continue hosts of failure --- README.md | 1 + app/executor/connector.go | 8 +++++--- app/executor/connector_test.go | 9 +++++---- app/executor/remote_test.go | 18 +++++++++--------- app/main.go | 19 +++++++++++-------- app/main_test.go | 2 +- app/runner/runner.go | 15 ++++++++++----- app/runner/runner_test.go | 30 +++++++++++++++--------------- go.mod | 2 ++ go.sum | 5 +++++ vendor/modules.txt | 6 ++++++ 11 files changed, 70 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index 8d7a4577..862a9387 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,7 @@ SimploTask supports the following command-line options: If not specified all the tasks will be executed. - `-d`, `--target=`: Specifies the target name to use for the task execution. The target should be defined in the playbook file and can represent remote hosts, inventory files, or inventory URLs. If not specified the `default` target will be used. User can pass a host name or IP instead of the target name for a quick override. Providing the `-d`, `--target` flag multiple times with different targets sets multiple destination targets or multiple hosts, e.g., `-d prod -d dev` or `-d example1.com -d example2.com`. - `-c`, `--concurrent=`: Sets the number of concurrent hosts to execute tasks. Defaults to `1`, which means hosts will be handled sequentially. +- `ssh-timeout`: Sets the SSH timeout. Defaults to `30s`. - `-f`, `--filter=`: Filter destinations for the specified target. Providing the `-f` flag multiple times with different name, or hosts names or ips/fqdns allow multiple destination hosts from the selected target, e.g., `-f apollo -f h2.example2.com` - `--inventory-file=`: Specifies the inventory file to use for the task execution. Overrides the inventory file defined in the playbook file. diff --git a/app/executor/connector.go b/app/executor/connector.go index 476ab614..16f9790b 100644 --- a/app/executor/connector.go +++ b/app/executor/connector.go @@ -7,6 +7,7 @@ import ( "net" "os" "strings" + "time" "golang.org/x/crypto/ssh" ) @@ -14,11 +15,12 @@ import ( // Connector provides factory methods to create Remote executor. Each executor is connected to a single SSH hostAddr. type Connector struct { privateKey string + timeout time.Duration } // NewConnector creates a new Connector for a given user and private key. -func NewConnector(privateKey string) (res *Connector, err error) { - res = &Connector{privateKey: privateKey} +func NewConnector(privateKey string, timeout time.Duration) (res *Connector, err error) { + res = &Connector{privateKey: privateKey, timeout: timeout} if _, err := os.Stat(privateKey); os.IsNotExist(err) { return nil, fmt.Errorf("private key file %q does not exist", privateKey) } @@ -42,7 +44,7 @@ func (c *Connector) sshClient(ctx context.Context, host, user string) (session * host += ":22" } - dialer := net.Dialer{} + dialer := net.Dialer{Timeout: c.timeout} conn, err := dialer.DialContext(ctx, "tcp", host) if err != nil { return nil, fmt.Errorf("failed to dial: %w", err) diff --git a/app/executor/connector_test.go b/app/executor/connector_test.go index 302c2cab..4bfdea4f 100644 --- a/app/executor/connector_test.go +++ b/app/executor/connector_test.go @@ -3,6 +3,7 @@ package executor import ( "context" "testing" + "time" "github.com/stretchr/testify/require" ) @@ -13,7 +14,7 @@ func TestConnector_Connect(t *testing.T) { defer teardown() t.Run("good connection", func(t *testing.T) { - c, err := NewConnector("testdata/test_ssh_key") + c, err := NewConnector("testdata/test_ssh_key", time.Second*10) require.NoError(t, err) sess, err := c.Connect(ctx, hostAndPort, "h1", "test") require.NoError(t, err) @@ -21,19 +22,19 @@ func TestConnector_Connect(t *testing.T) { }) t.Run("bad user", func(t *testing.T) { - c, err := NewConnector("testdata/test_ssh_key") + c, err := NewConnector("testdata/test_ssh_key", time.Second*10) require.NoError(t, err) _, err = c.Connect(ctx, hostAndPort, "h1", "test33") require.ErrorContains(t, err, "ssh: unable to authenticate") }) t.Run("bad key", func(t *testing.T) { - _, err := NewConnector("testdata/test_ssh_key33") + _, err := NewConnector("testdata/test_ssh_key33", time.Second*10) require.ErrorContains(t, err, "private key file \"testdata/test_ssh_key33\" does not exist", "test") }) t.Run("wrong port", func(t *testing.T) { - c, err := NewConnector("testdata/test_ssh_key") + c, err := NewConnector("testdata/test_ssh_key", time.Second*10) require.NoError(t, err) _, err = c.Connect(ctx, "127.0.0.1:12345", "h1", "test") require.ErrorContains(t, err, "failed to dial: dial tcp 127.0.0.1:12345") diff --git a/app/executor/remote_test.go b/app/executor/remote_test.go index 76c04a37..873940e5 100644 --- a/app/executor/remote_test.go +++ b/app/executor/remote_test.go @@ -20,7 +20,7 @@ func TestExecuter_UploadAndDownload(t *testing.T) { hostAndPort, teardown := startTestContainer(t) defer teardown() - c, err := NewConnector("testdata/test_ssh_key") + c, err := NewConnector("testdata/test_ssh_key", time.Second*10) require.NoError(t, err) sess, err := c.Connect(ctx, hostAndPort, "h1", "test") @@ -49,7 +49,7 @@ func TestExecuter_Upload_FailedNoRemoteDir(t *testing.T) { hostAndPort, teardown := startTestContainer(t) defer teardown() - c, err := NewConnector("testdata/test_ssh_key") + c, err := NewConnector("testdata/test_ssh_key", time.Second*10) require.NoError(t, err) sess, err := c.Connect(ctx, hostAndPort, "h1", "test") require.NoError(t, err) @@ -65,7 +65,7 @@ func TestExecuter_Upload_CantMakeRemoteDir(t *testing.T) { hostAndPort, teardown := startTestContainer(t) defer teardown() - c, err := NewConnector("testdata/test_ssh_key") + c, err := NewConnector("testdata/test_ssh_key", time.Second*10) require.NoError(t, err) sess, err := c.Connect(ctx, hostAndPort, "h1", "test") require.NoError(t, err) @@ -81,7 +81,7 @@ func TestExecuter_Upload_Canceled(t *testing.T) { hostAndPort, teardown := startTestContainer(t) defer teardown() - c, err := NewConnector("testdata/test_ssh_key") + c, err := NewConnector("testdata/test_ssh_key", time.Second*10) require.NoError(t, err) sess, err := c.Connect(ctx, hostAndPort, "h1", "test") require.NoError(t, err) @@ -98,7 +98,7 @@ func TestExecuter_UploadCanceledWithoutMkdir(t *testing.T) { hostAndPort, teardown := startTestContainer(t) defer teardown() - c, err := NewConnector("testdata/test_ssh_key") + c, err := NewConnector("testdata/test_ssh_key", time.Second*10) require.NoError(t, err) sess, err := c.Connect(ctx, hostAndPort, "h1", "test") require.NoError(t, err) @@ -116,7 +116,7 @@ func TestExecuter_ConnectCanceled(t *testing.T) { hostAndPort, teardown := startTestContainer(t) defer teardown() - c, err := NewConnector("testdata/test_ssh_key") + c, err := NewConnector("testdata/test_ssh_key", time.Second*10) require.NoError(t, err) _, err = c.Connect(ctx, hostAndPort, "h1", "test") assert.ErrorContains(t, err, "failed to dial: dial tcp: lookup localhost: i/o timeout") @@ -128,7 +128,7 @@ func TestExecuter_Run(t *testing.T) { hostAndPort, teardown := startTestContainer(t) defer teardown() - c, err := NewConnector("testdata/test_ssh_key") + c, err := NewConnector("testdata/test_ssh_key", time.Second*10) require.NoError(t, err) sess, err := c.Connect(ctx, hostAndPort, "h1", "test") require.NoError(t, err) @@ -169,7 +169,7 @@ func TestExecuter_Sync(t *testing.T) { hostAndPort, teardown := startTestContainer(t) defer teardown() - c, err := NewConnector("testdata/test_ssh_key") + c, err := NewConnector("testdata/test_ssh_key", time.Second*10) require.NoError(t, err) sess, err := c.Connect(ctx, hostAndPort, "h1", "test") require.NoError(t, err) @@ -204,7 +204,7 @@ func TestExecuter_Delete(t *testing.T) { hostAndPort, teardown := startTestContainer(t) defer teardown() - c, err := NewConnector("testdata/test_ssh_key") + c, err := NewConnector("testdata/test_ssh_key", time.Second*10) require.NoError(t, err) sess, err := c.Connect(ctx, hostAndPort, "h1", "test") require.NoError(t, err) diff --git a/app/main.go b/app/main.go index 6dad3951..6410eb4c 100644 --- a/app/main.go +++ b/app/main.go @@ -15,6 +15,7 @@ import ( "github.com/fatih/color" "github.com/go-pkgz/lgr" + "github.com/hashicorp/go-multierror" "github.com/jessevdk/go-flags" "github.com/umputun/simplotask/app/config" @@ -23,10 +24,11 @@ import ( ) type options struct { - PlaybookFile string `short:"p" long:"file" env:"SPOT_FILE" description:"playbook file" default:"spot.yml"` - TaskName string `short:"t" long:"task" description:"task name"` - Targets []string `short:"d" long:"target" description:"target name" default:"default"` - Concurrent int `short:"c" long:"concurrent" description:"concurrent tasks" default:"1"` + PlaybookFile string `short:"p" long:"file" env:"SPOT_FILE" description:"playbook file" default:"spot.yml"` + TaskName string `short:"t" long:"task" description:"task name"` + Targets []string `short:"d" long:"target" description:"target name" default:"default"` + Concurrent int `short:"c" long:"concurrent" description:"concurrent tasks" default:"1"` + SSHTimeout time.Duration `long:"ssh-timeout" description:"ssh timeout" default:"30s"` // target overrides Filter []string `short:"f" long:"filter" description:"filter target hosts"` @@ -72,7 +74,7 @@ func main() { if opts.Dbg { log.Panicf("[ERROR] %v", err) } - fmt.Printf("failed: %v\n", err) + fmt.Printf("failed, %v", err) os.Exit(1) } } @@ -102,7 +104,7 @@ func run(opts options) error { } } - connector, err := executor.NewConnector(sshKey(opts, conf)) + connector, err := executor.NewConnector(sshKey(opts, conf), opts.SSHTimeout) if err != nil { return fmt.Errorf("can't create connector: %w", err) } @@ -116,14 +118,15 @@ func run(opts options) error { Verbose: opts.Verbose, } + errs := new(multierror.Error) if opts.AdHocCmd != "" { // run ad-hoc command r.Verbose = true // always verbose for ad-hoc for _, targetName := range opts.Targets { if err := runTaskForTarget(ctx, r, "ad-hoc", targetName); err != nil { - return err + errs = multierror.Append(errs, err) } } - return nil + return errs.ErrorOrNil() } if opts.TaskName != "" { // run single task diff --git a/app/main_test.go b/app/main_test.go index a9b0d144..baa1e8b0 100644 --- a/app/main_test.go +++ b/app/main_test.go @@ -130,7 +130,7 @@ func Test_runFailed(t *testing.T) { } setupLog(true) err := run(opts) - assert.ErrorContains(t, err, `can't run command "show content"`) + assert.ErrorContains(t, err, `failed command "show content"`) } func Test_runNoConfig(t *testing.T) { diff --git a/app/runner/runner.go b/app/runner/runner.go index 86b7259c..3baadc22 100644 --- a/app/runner/runner.go +++ b/app/runner/runner.go @@ -34,7 +34,7 @@ type Process struct { Only []string } -// Connector is an interface for connecting to a hostAddr, and returning remote executer. +// Connector is an interface for connecting to a host, and returning remote executer. type Connector interface { Connect(ctx context.Context, hostAddr, hostName, user string) (*executor.Remote, error) } @@ -45,7 +45,8 @@ type ProcStats struct { Hosts int } -// Run runs a task for a set of target hosts. Runs in parallel with limited concurrency, each hostAddr is processed in separate goroutine. +// Run runs a task for a set of target hosts. Runs in parallel with limited concurrency, +// each host is processed in separate goroutine. func (p *Process) Run(ctx context.Context, task, target string) (s ProcStats, err error) { tsk, err := p.Config.Task(task) if err != nil { @@ -68,6 +69,10 @@ func (p *Process) Run(ctx context.Context, task, target string) (s ProcStats, er if i == 0 { atomic.AddInt32(&commands, int32(count)) } + if e != nil { + _, errLog := executor.MakeOutAndErrWriters(fmt.Sprintf("%s:%d", host.Host, host.Port), host.Name, p.Verbose) + errLog.Write([]byte(e.Error())) //nolint + } return e }) } @@ -93,7 +98,7 @@ func (p *Process) Run(ctx context.Context, task, target string) (s ProcStats, er return ProcStats{Hosts: len(targetHosts), Commands: int(atomic.LoadInt32(&commands))}, err } -// runTaskOnHost executes all commands of a task on a target hostAddr. hostAddr can be a remote hostAddr or localhost with port. +// runTaskOnHost executes all commands of a task on a target host. hostAddr can be a remote host or localhost with port. func (p *Process) runTaskOnHost(ctx context.Context, tsk *config.Task, hostAddr, hostName, user string) (int, error) { contains := func(list []string, s string) bool { for _, v := range list { @@ -124,7 +129,7 @@ func (p *Process) runTaskOnHost(ctx context.Context, tsk *config.Task, hostAddr, continue } - log.Printf("[INFO] run command %q on hostAddr %s (%s)", cmd.Name, hostAddr, hostName) + log.Printf("[INFO] run command %q on host %q (%s)", cmd.Name, hostAddr, hostName) st := time.Now() params := execCmdParams{cmd: cmd, hostAddr: hostAddr, tsk: tsk, exec: remote} if cmd.Options.Local { @@ -134,7 +139,7 @@ func (p *Process) runTaskOnHost(ctx context.Context, tsk *config.Task, hostAddr, details, err := p.execCommand(ctx, params) if err != nil { if !cmd.Options.IgnoreErrors { - return count, fmt.Errorf("can't run command %q on hostAddr %s (%s): %w", cmd.Name, hostAddr, hostName, err) + return count, fmt.Errorf("failed command %q on host %s (%s): %w", cmd.Name, hostAddr, hostName, err) } fmt.Fprintf(p.ColorWriter.WithHost(hostAddr, hostName), "failed %s%s (%v)", diff --git a/app/runner/runner_test.go b/app/runner/runner_test.go index 82fd41ee..90bbc5ec 100644 --- a/app/runner/runner_test.go +++ b/app/runner/runner_test.go @@ -24,7 +24,7 @@ func TestProcess_Run(t *testing.T) { hostAndPort, teardown := startTestContainer(t) defer teardown() - connector, err := executor.NewConnector("testdata/test_ssh_key") + connector, err := executor.NewConnector("testdata/test_ssh_key", time.Second*10) require.NoError(t, err) conf, err := config.New("testdata/conf.yml", nil) require.NoError(t, err) @@ -46,7 +46,7 @@ func TestProcess_RunOnly(t *testing.T) { hostAndPort, teardown := startTestContainer(t) defer teardown() - connector, err := executor.NewConnector("testdata/test_ssh_key") + connector, err := executor.NewConnector("testdata/test_ssh_key", time.Second*10) require.NoError(t, err) conf, err := config.New("testdata/conf.yml", nil) require.NoError(t, err) @@ -69,7 +69,7 @@ func TestProcess_RunOnlyNoAuto(t *testing.T) { hostAndPort, teardown := startTestContainer(t) defer teardown() - connector, err := executor.NewConnector("testdata/test_ssh_key") + connector, err := executor.NewConnector("testdata/test_ssh_key", time.Second*10) require.NoError(t, err) conf, err := config.New("testdata/conf.yml", nil) require.NoError(t, err) @@ -92,7 +92,7 @@ func TestProcess_RunSkip(t *testing.T) { hostAndPort, teardown := startTestContainer(t) defer teardown() - connector, err := executor.NewConnector("testdata/test_ssh_key") + connector, err := executor.NewConnector("testdata/test_ssh_key", time.Second*10) require.NoError(t, err) conf, err := config.New("testdata/conf.yml", nil) require.NoError(t, err) @@ -116,7 +116,7 @@ func TestProcess_RunVerbose(t *testing.T) { defer teardown() log.SetOutput(io.Discard) - connector, err := executor.NewConnector("testdata/test_ssh_key") + connector, err := executor.NewConnector("testdata/test_ssh_key", time.Second*10) require.NoError(t, err) conf, err := config.New("testdata/conf.yml", nil) require.NoError(t, err) @@ -139,7 +139,7 @@ func TestProcess_RunLocal(t *testing.T) { var buf bytes.Buffer log.SetOutput(&buf) - connector, err := executor.NewConnector("testdata/test_ssh_key") + connector, err := executor.NewConnector("testdata/test_ssh_key", time.Second*10) require.NoError(t, err) conf, err := config.New("testdata/conf-local.yml", nil) require.NoError(t, err) @@ -162,7 +162,7 @@ func TestProcess_RunFailed(t *testing.T) { hostAndPort, teardown := startTestContainer(t) defer teardown() - connector, err := executor.NewConnector("testdata/test_ssh_key") + connector, err := executor.NewConnector("testdata/test_ssh_key", time.Second*10) require.NoError(t, err) conf, err := config.New("testdata/conf.yml", nil) require.NoError(t, err) @@ -174,7 +174,7 @@ func TestProcess_RunFailed(t *testing.T) { ColorWriter: executor.NewColorizedWriter(os.Stdout, "", "", ""), } _, err = p.Run(ctx, "failed_task", hostAndPort) - require.ErrorContains(t, err, `can't run command "bad command" on hostAddr`) + require.ErrorContains(t, err, `failed command "bad command" on host`) } func TestProcess_RunFailed_WithOnError(t *testing.T) { @@ -182,7 +182,7 @@ func TestProcess_RunFailed_WithOnError(t *testing.T) { hostAndPort, teardown := startTestContainer(t) defer teardown() - connector, err := executor.NewConnector("testdata/test_ssh_key") + connector, err := executor.NewConnector("testdata/test_ssh_key", time.Second*10) require.NoError(t, err) conf, err := config.New("testdata/conf.yml", nil) require.NoError(t, err) @@ -199,7 +199,7 @@ func TestProcess_RunFailed_WithOnError(t *testing.T) { log.SetOutput(&buf) _, err = p.Run(ctx, "failed_task_with_onerror", hostAndPort) - require.ErrorContains(t, err, `can't run command "bad command" on hostAddr`) + require.ErrorContains(t, err, `failed command "bad command" on host`) t.Log(buf.String()) require.Contains(t, buf.String(), "onerror called") }) @@ -213,7 +213,7 @@ func TestProcess_RunFailed_WithOnError(t *testing.T) { tsk.OnError = "bad command" p.Config.Tasks[2] = tsk _, err = p.Run(ctx, "failed_task_with_onerror", hostAndPort) - require.ErrorContains(t, err, `can't run command "bad command" on hostAddr`) + require.ErrorContains(t, err, `failed command "bad command" on host`) t.Log(buf.String()) require.NotContains(t, buf.String(), "onerror called") assert.Contains(t, buf.String(), "[WARN]") @@ -226,7 +226,7 @@ func TestProcess_RunFailedErrIgnored(t *testing.T) { hostAndPort, teardown := startTestContainer(t) defer teardown() - connector, err := executor.NewConnector("testdata/test_ssh_key") + connector, err := executor.NewConnector("testdata/test_ssh_key", time.Second*10) require.NoError(t, err) conf, err := config.New("testdata/conf.yml", nil) require.NoError(t, err) @@ -247,7 +247,7 @@ func TestProcess_RunTaskWithWait(t *testing.T) { hostAndPort, teardown := startTestContainer(t) defer teardown() - connector, err := executor.NewConnector("testdata/test_ssh_key") + connector, err := executor.NewConnector("testdata/test_ssh_key", time.Second*10) require.NoError(t, err) conf, err := config.New("testdata/conf.yml", nil) require.NoError(t, err) @@ -375,7 +375,7 @@ func TestProcess_waitPassed(t *testing.T) { hostAndPort, teardown := startTestContainer(t) defer teardown() - connector, err := executor.NewConnector("testdata/test_ssh_key") + connector, err := executor.NewConnector("testdata/test_ssh_key", time.Second*10) require.NoError(t, err) sess, err := connector.Connect(ctx, hostAndPort, "my-hostAddr", "test") require.NoError(t, err) @@ -394,7 +394,7 @@ func TestProcess_waitFailed(t *testing.T) { hostAndPort, teardown := startTestContainer(t) defer teardown() - connector, err := executor.NewConnector("testdata/test_ssh_key") + connector, err := executor.NewConnector("testdata/test_ssh_key", time.Second*10) require.NoError(t, err) sess, err := connector.Connect(ctx, hostAndPort, "my-hostAddr", "test") require.NoError(t, err) diff --git a/go.mod b/go.mod index 1c21df3f..378c59ea 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/go-pkgz/fileutils v0.2.0 github.com/go-pkgz/lgr v0.11.0 github.com/go-pkgz/syncs v1.3.0 + github.com/hashicorp/go-multierror v1.1.1 github.com/jessevdk/go-flags v1.5.0 github.com/pkg/sftp v1.13.5 github.com/stretchr/testify v1.8.2 @@ -29,6 +30,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/uuid v1.3.0 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect github.com/klauspost/compress v1.16.5 // indirect github.com/kr/fs v0.1.0 // indirect github.com/kr/text v0.1.0 // indirect diff --git a/go.sum b/go.sum index 21a0e916..cdd59e0d 100644 --- a/go.sum +++ b/go.sum @@ -39,6 +39,11 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= diff --git a/vendor/modules.txt b/vendor/modules.txt index 47366be2..9ca94e74 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -90,6 +90,12 @@ github.com/golang/protobuf/ptypes/timestamp # github.com/google/uuid v1.3.0 ## explicit github.com/google/uuid +# github.com/hashicorp/errwrap v1.1.0 +## explicit +github.com/hashicorp/errwrap +# github.com/hashicorp/go-multierror v1.1.1 +## explicit; go 1.13 +github.com/hashicorp/go-multierror # github.com/jessevdk/go-flags v1.5.0 ## explicit; go 1.15 github.com/jessevdk/go-flags