Skip to content

Commit

Permalink
Auto-detect Helm repos + support Helm basic auth + fix bugs (argoproj…
Browse files Browse the repository at this point in the history
  • Loading branch information
alexec authored Sep 19, 2019
1 parent 70a97c0 commit 1e5c78e
Show file tree
Hide file tree
Showing 16 changed files with 281 additions and 180 deletions.
33 changes: 21 additions & 12 deletions pkg/apis/application/v1alpha1/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -1005,20 +1005,29 @@ func (repo *Repository) IsLFSEnabled() bool {
return repo.EnableLFS
}

func (m *Repository) HasCredentials() bool {
return m.Username != "" || m.Password != "" || m.SSHPrivateKey != "" || m.InsecureIgnoreHostKey
}

func (m *Repository) CopyCredentialsFrom(source *Repository) {
if source != nil {
m.Username = source.Username
m.Password = source.Password
m.SSHPrivateKey = source.SSHPrivateKey
m.InsecureIgnoreHostKey = source.InsecureIgnoreHostKey
m.Insecure = source.Insecure
m.EnableLFS = source.EnableLFS
m.TLSClientCertData = source.TLSClientCertData
m.TLSClientCertKey = source.TLSClientCertKey
if m.Username == "" {
m.Username = source.Username
}
if m.Password == "" {
m.Password = source.Password
}
if m.SSHPrivateKey == "" {
m.SSHPrivateKey = source.SSHPrivateKey
}
m.InsecureIgnoreHostKey = m.InsecureIgnoreHostKey || source.InsecureIgnoreHostKey
m.Insecure = m.Insecure || source.Insecure
m.EnableLFS = m.EnableLFS || source.EnableLFS
if m.TLSClientCertData == "" {
m.TLSClientCertData = source.TLSClientCertData
}
if m.TLSClientCertKey == "" {
m.TLSClientCertKey = source.TLSClientCertKey
}
if m.TLSClientCAData == "" {
m.TLSClientCAData = source.TLSClientCAData
}
}
}

Expand Down
75 changes: 24 additions & 51 deletions pkg/apis/application/v1alpha1/types_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -405,66 +405,39 @@ func TestAppProjectSpec_DestinationClusters(t *testing.T) {
}
}

func TestRepository_HasCredentials(t *testing.T) {

tests := []struct {
name string
repo Repository
want bool
}{
{
name: "TestHasRepo",
repo: Repository{Repo: "foo"},
want: false,
},
{
name: "TestHasUsername",
repo: Repository{Username: "foo"},
want: true,
},
{
name: "TestHasPassword",
repo: Repository{Password: "foo"},
want: true,
},
{
name: "TestHasSSHPrivateKey",
repo: Repository{SSHPrivateKey: "foo"},
want: true,
},
{
name: "TestHasInsecureHostKey",
repo: Repository{InsecureIgnoreHostKey: true},
want: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := tt.repo.HasCredentials(); got != tt.want {
t.Errorf("Repository.HasCredentials() = %v, want %v", got, tt.want)
}
})
}
}

func TestRepository_CopyCredentialsFrom(t *testing.T) {
tests := []struct {
name string
repo *Repository
source *Repository
want Repository
}{
{"TestNil", nil, Repository{}},
{"TestHasRepo", &Repository{Repo: "foo"}, Repository{}},
{"TestHasUsername", &Repository{Username: "foo"}, Repository{Username: "foo"}},
{"TestHasPassword", &Repository{Password: "foo"}, Repository{Password: "foo"}},
{"TestHasSSHPrivateKey", &Repository{SSHPrivateKey: "foo"}, Repository{SSHPrivateKey: "foo"}},
{"TestHasInsecureHostKey", &Repository{InsecureIgnoreHostKey: true}, Repository{InsecureIgnoreHostKey: true}},
{"Username", &Repository{Username: "foo"}, &Repository{}, Repository{Username: "foo"}},
{"Password", &Repository{Password: "foo"}, &Repository{}, Repository{Password: "foo"}},
{"SSHPrivateKey", &Repository{SSHPrivateKey: "foo"}, &Repository{}, Repository{SSHPrivateKey: "foo"}},
{"InsecureHostKey", &Repository{InsecureIgnoreHostKey: true}, &Repository{}, Repository{InsecureIgnoreHostKey: true}},
{"Insecure", &Repository{Insecure: true}, &Repository{}, Repository{Insecure: true}},
{"EnableLFS", &Repository{EnableLFS: true}, &Repository{}, Repository{EnableLFS: true}},
{"TLSClientCAData", &Repository{TLSClientCAData: "foo"}, &Repository{}, Repository{TLSClientCAData: "foo"}},
{"TLSClientCertData", &Repository{TLSClientCertData: "foo"}, &Repository{}, Repository{TLSClientCertData: "foo"}},
{"TLSClientCertKey", &Repository{TLSClientCertKey: "foo"}, &Repository{}, Repository{TLSClientCertKey: "foo"}},
{"SourceNil", &Repository{}, nil, Repository{}},

{"SourceUsername", &Repository{}, &Repository{Username: "foo"}, Repository{Username: "foo"}},
{"SourcePassword", &Repository{}, &Repository{Password: "foo"}, Repository{Password: "foo"}},
{"SourceSSHPrivateKey", &Repository{}, &Repository{SSHPrivateKey: "foo"}, Repository{SSHPrivateKey: "foo"}},
{"SourceInsecureHostKey", &Repository{}, &Repository{InsecureIgnoreHostKey: true}, Repository{InsecureIgnoreHostKey: true}},
{"SourceInsecure", &Repository{}, &Repository{Insecure: true}, Repository{Insecure: true}},
{"SourceEnableLFS", &Repository{}, &Repository{EnableLFS: true}, Repository{EnableLFS: true}},
{"SourceTLSClientCAData", &Repository{}, &Repository{TLSClientCAData: "foo"}, Repository{TLSClientCAData: "foo"}},
{"SourceTLSClientCertData", &Repository{}, &Repository{TLSClientCertData: "foo"}, Repository{TLSClientCertData: "foo"}},
{"SourceTLSClientCertKey", &Repository{}, &Repository{TLSClientCertKey: "foo"}, Repository{TLSClientCertKey: "foo"}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
repo := Repository{}
repo.CopyCredentialsFrom(tt.source)
assert.Equal(t, tt.want, repo)
r := tt.repo.DeepCopy()
r.CopyCredentialsFrom(tt.source)
assert.Equal(t, tt.want, *r)
})
}
}
Expand Down
21 changes: 19 additions & 2 deletions server/repository/repository.go
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,7 @@ func (s *Server) List(ctx context.Context, q *repositorypkg.RepoQuery) (*appsv1.
}
err = util.RunAllAsync(len(items), func(i int) error {
items[i].ConnectionState = s.getConnectionState(ctx, items[i].Repo)
_ = factory.DetectType(items[i], metrics.NopReporter)
return nil
})
if err != nil {
Expand Down Expand Up @@ -176,6 +177,7 @@ func (s *Server) Create(ctx context.Context, q *repositorypkg.RepoCreateRequest)
return nil, err
}

detectedType := ""
// check we can connect to the repo, copying any existing creds
{
repo := q.Repo.DeepCopy()
Expand All @@ -184,13 +186,19 @@ func (s *Server) Create(ctx context.Context, q *repositorypkg.RepoCreateRequest)
return nil, err
}
repo.CopyCredentialsFrom(creds)
_, err = factory.NewFactory().NewRepo(q.Repo, metrics.NopReporter)
err = factory.DetectType(repo, metrics.NopReporter)
if err != nil {
return nil, err
}
detectedType = repo.Type
_, err = factory.NewFactory().NewRepo(repo, metrics.NopReporter)
if err != nil {
return nil, err
}
}

r := q.Repo
r.Type = detectedType
r.ConnectionState = appsv1.ConnectionState{Status: appsv1.ConnectionStatusSuccessful}
repo, err := s.db.CreateRepository(ctx, r)
if status.Convert(err).Code() == codes.AlreadyExists {
Expand Down Expand Up @@ -256,7 +264,16 @@ func (s *Server) ValidateAccess(ctx context.Context, q *repositorypkg.RepoAccess
TLSClientCertKey: q.TlsClientCertKey,
TLSClientCAData: q.TlsClientCAData,
}
_, err := factory.NewFactory().NewRepo(repo, metrics.NopReporter)
creds, err := s.db.GetRepository(ctx, q.Repo)
if err != nil {
return nil, err
}
repo.CopyCredentialsFrom(creds)
err = factory.DetectType(repo, metrics.NopReporter)
if err != nil {
return nil, err
}
_, err = factory.NewFactory().NewRepo(repo, metrics.NopReporter)
if err != nil {
return nil, err
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ export const ApplicationSummary = (props: {
edit: (formApi: FormApi) => <FormField formApi={formApi} field='spec.source.targetRevision' component={Text}/>,
},
{
title: 'PATH',
title: 'GIT PATH/HELM CHART',
view: app.spec.source.path,
edit: (formApi: FormApi) => <FormField formApi={formApi} field='spec.source.path' component={Text}/>,
},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ export const ApplicationTiles = ({applications, syncApplication, refreshApplicat
<div className='columns small-9'>{app.spec.source.targetRevision || 'latest'}</div>
</div>
<div className='row'>
<div className='columns small-3'>Path:</div>
<div className='columns small-3'>Git Path/Helm Chart:</div>
<div className='columns small-9'>{app.spec.source.path}</div>
</div>
<div className='row'>
Expand Down
11 changes: 5 additions & 6 deletions ui/src/app/settings/components/repos-list/repos-list.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -77,11 +77,9 @@ export class ReposList extends React.Component<RouteComponentProps<any>> {
<div className='argo-table-list__row' key={repo.repo}>
<div className='row'>
<div className='columns small-1'>
<i className={'icon argo-icon-' + (repo.type || 'git')}/>
</div>
<div className='columns small-1'>
{repo.type || 'git'}
<i className={'icon argo-icon-' + (repo.type)}/>
</div>
<div className='columns small-1'>{repo.type}</div>
<div className='columns small-2'>{repo.name}</div>
<div className='columns small-5'>
<Repo url={repo.repo}/>
Expand Down Expand Up @@ -145,7 +143,7 @@ export class ReposList extends React.Component<RouteComponentProps<any>> {
<FormField formApi={formApi} label='Type' field='type' component={FormSelect} componentProps={{options: ['git', 'helm']}}/>
</div>
<div className='argo-form-row'>
<FormField formApi={formApi} label='Name (optional for Git)' field='name' component={Text}/>
<FormField formApi={formApi} label='Name (mandatory for Helm)' field='name' component={Text}/>
</div>
<div className='argo-form-row'>
<FormField formApi={formApi} label='Repository URL' field='url' component={Text}/>
Expand Down Expand Up @@ -186,6 +184,7 @@ export class ReposList extends React.Component<RouteComponentProps<any>> {
<h4>Connect repo using SSH</h4>
<Form onSubmit={(params) => this.connectSSHRepo(params as NewSSHRepoParams)}
getApi={(api) => this.formApiSSH = api}
defaultValues={{type: 'git'}}
validateError={(params: NewSSHRepoParams) => ({
url: !params.url && 'Repo URL is required',
})}>
Expand All @@ -195,7 +194,7 @@ export class ReposList extends React.Component<RouteComponentProps<any>> {
<FormField formApi={formApi} label='Type' field='type' component={FormSelect} componentProps={{options: ['git', 'helm']}}/>
</div>
<div className='argo-form-row'>
<FormField formApi={formApi} label='Name (optional for Git)' field='name' component={Text}/>
<FormField formApi={formApi} label='Name (mandatory for Helm)' field='name' component={Text}/>
</div>
<div className='argo-form-row'>
<FormField formApi={formApi} label='Repository URL' field='url' component={Text}/>
Expand Down
20 changes: 7 additions & 13 deletions util/db/repository.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import (
"hash/fnv"
"strings"

log "github.com/sirupsen/logrus"
"golang.org/x/net/context"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
Expand Down Expand Up @@ -95,18 +94,13 @@ func (db *db) GetRepository(ctx context.Context, repoURL string) (*appsv1.Reposi
}
}

if !repo.HasCredentials() {
index := getRepositoryCredentialIndex(repoCredentials, repoURL)
if index >= 0 {

credential, err := db.credentialsToRepository(repoCredentials[index])

if err != nil {
return nil, err
} else {
log.WithFields(log.Fields{"repoURL": repo.Repo, "credUrl": credential.Repo}).Info("copying credentials")
repo.CopyCredentialsFrom(credential)
}
index = getRepositoryCredentialIndex(repoCredentials, repoURL)
if index >= 0 {
credential, err := db.credentialsToRepository(repoCredentials[index])
if err != nil {
return nil, err
} else {
repo.CopyCredentialsFrom(credential)
}
}

Expand Down
87 changes: 87 additions & 0 deletions util/helm/repo/index.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
package repo

import (
"encoding/base64"
"errors"
"fmt"
"net/http"
"time"

"github.com/patrickmn/go-cache"
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v2"
)

var indexCache = cache.New(5*time.Minute, 5*time.Minute)

type entry struct {
Version string
Created time.Time
}

type index struct {
Entries map[string][]entry
}

func Index(url, username, password string) (*index, error) {

cachedIndex, found := indexCache.Get(url)
if found {
log.WithFields(log.Fields{"url": url}).Debug("index cache hit")
i := cachedIndex.(index)
return &i, nil
}

start := time.Now()

req, err := http.NewRequest("GET", url+"/index.yaml", nil)
if err != nil {
return nil, err
}
if username != "" {
// only basic supported
token := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", username, password)))
req.Header.Add("Authorization", fmt.Sprintf("Basic %s", token))
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, err
}
defer func() { _ = resp.Body.Close() }()

if resp.StatusCode != 200 {
return nil, errors.New("failed to get index: " + resp.Status)
}

index := &index{}
err = yaml.NewDecoder(resp.Body).Decode(index)

log.WithFields(log.Fields{"seconds": time.Since(start).Seconds()}).Info("took to get index")

indexCache.Set(url, *index, cache.DefaultExpiration)

return index, err
}

func (i *index) contains(chart string) bool {
_, ok := i.Entries[chart]
return ok
}

func (i *index) entry(chart, version string) (*entry, error) {
for _, entry := range i.Entries[chart] {
if entry.Version == version {
return &entry, nil
}
}
return nil, fmt.Errorf("unknown chart \"%s/%s\"", chart, version)
}

func (i *index) latest(chart string) (string, error) {
for chartName := range i.Entries {
if chartName == chart {
return i.Entries[chartName][0].Version, nil
}
}
return "", fmt.Errorf("failed to find chart %s", chart)
}
24 changes: 24 additions & 0 deletions util/helm/repo/index_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
package repo

import (
"testing"

"github.com/stretchr/testify/assert"
)

func TestIndex(t *testing.T) {
t.Run("Invalid", func(t *testing.T) {
_, err := Index("", "", "")
assert.Error(t, err)
})
t.Run("Stable", func(t *testing.T) {
index, err := Index("https://kubernetes-charts.storage.googleapis.com", "", "")
assert.NoError(t, err)
assert.NotNil(t, index)
})
t.Run("BasicAuth", func(t *testing.T) {
index, err := Index("https://kubernetes-charts.storage.googleapis.com", "my-username", "my-password")
assert.NoError(t, err)
assert.NotNil(t, index)
})
}
Loading

0 comments on commit 1e5c78e

Please sign in to comment.