From 6ffb1b8ccb805fe6610123a29828ab41252a61b8 Mon Sep 17 00:00:00 2001 From: Chandrasekhar Raman <33996892+chandrasekhar1996@users.noreply.github.com> Date: Tue, 28 Feb 2023 13:56:47 -0800 Subject: [PATCH] Add profileTag support to sia-aws (#2082) Signed-off-by: craman Co-authored-by: craman --- .../aws/options/data/profile_config_tag_key | 4 ++ libs/go/sia/aws/options/options.go | 24 +++++++-- libs/go/sia/aws/options/options_test.go | 30 ++++++++++++ libs/go/sia/host/hostdoc/hostdoc.go | 49 ++++++++++--------- libs/go/sia/host/hostdoc/hostdoc_test.go | 24 +++++++++ libs/go/sia/host/hostdoc/raw/raw.go | 17 ++++--- libs/go/sia/host/ip/ip_test.go | 3 -- provider/aws/sia-ec2/authn.go | 16 ++++-- provider/aws/sia-ec2/cmd/siad/main.go | 4 +- utils/msd-agent/svc/service.go | 9 ++-- utils/msd-agent/svc/service_aws_ec2.go | 2 +- 11 files changed, 133 insertions(+), 49 deletions(-) create mode 100644 libs/go/sia/aws/options/data/profile_config_tag_key diff --git a/libs/go/sia/aws/options/data/profile_config_tag_key b/libs/go/sia/aws/options/data/profile_config_tag_key new file mode 100644 index 00000000000..e3bcc25284c --- /dev/null +++ b/libs/go/sia/aws/options/data/profile_config_tag_key @@ -0,0 +1,4 @@ +{ + "profile": "zts-profile", + "profile_tag": "zts:RestrictTo" +} \ No newline at end of file diff --git a/libs/go/sia/aws/options/options.go b/libs/go/sia/aws/options/options.go index 58b8266dedd..17cf3b0f84d 100644 --- a/libs/go/sia/aws/options/options.go +++ b/libs/go/sia/aws/options/options.go @@ -107,7 +107,8 @@ type Config struct { } type AccessProfileConfig struct { - Profile string `json:"profile,omitempty"` // map of role name to token attributes + Profile string `json:"profile,omitempty"` + ProfileTag string `json:"profile_tag,omitempty"` } // Role contains role details. Attributes are set based on the config values @@ -190,6 +191,7 @@ type Options struct { TokenDir string //Access tokens directory AccessTokens []ac.AccessToken //Access tokens object Profile string //Access profile name + ProfileTag string //Tag associated with access profile roles Threshold float64 //threshold in number of days for cert expiry checks SshThreshold float64 //threshold in number of days for ssh cert expiry checks FileDirectUpdate bool //update key/cert files directly instead of using rename @@ -201,6 +203,11 @@ const ( DefaultThreshold = float64(15) // 15 days ) +func GetInstanceTagValue(metaEndPoint, tagKey string) (string, error) { + tagValue, err := meta.GetData(metaEndPoint, "/latest/meta-data/tags/instance/"+tagKey) + return string(tagValue), err +} + func GetAccountId(metaEndPoint string, useRegionalSTS bool, region string) (string, error) { // first try to get the account from our creds and if // fails we'll fall back to identity document @@ -228,7 +235,8 @@ func InitCredsConfig(roleSuffix, accessProfileSeparator string, useRegionalSTS b Threshold: DefaultThreshold, SshThreshold: DefaultThreshold, }, &AccessProfileConfig{ - Profile: profile, + Profile: profile, + ProfileTag: "", }, nil } @@ -254,7 +262,8 @@ func InitProfileConfig(metaEndPoint, roleSuffix, accessProfileSeparator string) Threshold: DefaultThreshold, SshThreshold: DefaultThreshold, }, &AccessProfileConfig{ - Profile: profile, + Profile: profile, + ProfileTag: "", }, nil } @@ -312,7 +321,8 @@ func InitAccessProfileFileConfig(fileName string) (*AccessProfileConfig, error) } return &AccessProfileConfig{ - Profile: config.Profile, + Profile: config.Profile, + ProfileTag: config.ProfileTag, }, nil } @@ -433,7 +443,8 @@ func InitAccessProfileEnvConfig() (*AccessProfileConfig, error) { } return &AccessProfileConfig{ - Profile: accessProfile, + Profile: accessProfile, + ProfileTag: "", }, nil } @@ -455,6 +466,7 @@ func setOptions(config *Config, account *ConfigAccount, profileConfig *AccessPro ztsRegion := "" dropPrivileges := false profile := "" + profileTag := "" fileDirectUpdate := false tokenDir := fmt.Sprintf("%s/tokens", siaDir) certDir := fmt.Sprintf("%s/certs", siaDir) @@ -622,6 +634,7 @@ func setOptions(config *Config, account *ConfigAccount, profileConfig *AccessPro if profileConfig != nil { profile = profileConfig.Profile + profileTag = profileConfig.ProfileTag } return &Options{ @@ -651,6 +664,7 @@ func setOptions(config *Config, account *ConfigAccount, profileConfig *AccessPro DropPrivileges: dropPrivileges, AccessTokens: accessTokens, Profile: profile, + ProfileTag: profileTag, Threshold: account.Threshold, SshThreshold: account.SshThreshold, FileDirectUpdate: fileDirectUpdate, diff --git a/libs/go/sia/aws/options/options_test.go b/libs/go/sia/aws/options/options_test.go index 1d9476f0689..5942b668d54 100644 --- a/libs/go/sia/aws/options/options_test.go +++ b/libs/go/sia/aws/options/options_test.go @@ -110,6 +110,7 @@ func getAccessProfileConfig(fileName, metaEndPoint string) (*ConfigAccount, *Acc return nil, nil, fmt.Errorf("config non-parsable and unable to determine profile name from profile arn, error: %v", err) } } + return configAccount, profileConfig, nil } @@ -241,6 +242,35 @@ func TestOptionsWithProfileConfig(t *testing.T) { // Make sure profile is correct assert.True(t, opts.Profile == "zts-profile") + assert.True(t, opts.ProfileTag == "") + + // Make sure services are set + assert.True(t, len(opts.Services) == 3) + assert.True(t, opts.Domain == "athenz") + assert.True(t, opts.Name == "athenz.api") + + // Zeroth service should be the one from "service" key, the remaining are from "services" in no particular order + assert.True(t, assertService(opts.Services[0], Service{Name: "api", User: "nobody", Uid: getUid("nobody"), Gid: getUserGid("nobody"), FileMode: 288, Threshold: DefaultThreshold})) + assert.True(t, assertInServices(opts.Services[1:], Service{Name: "ui", User: "root", Uid: 0, Gid: 0, FileMode: 288, Threshold: DefaultThreshold})) + assert.True(t, assertInServices(opts.Services[1:], Service{Name: "yamas", User: "nobody", Uid: getUid("nobody"), Group: "sys", Gid: getGid(t, "sys"), Threshold: DefaultThreshold})) + + assert.Equal(t, DefaultThreshold, opts.SshThreshold) + assert.Equal(t, DefaultThreshold, opts.Threshold) +} + +// TestOptionsWithProfileConfigAndProfileTag test the scenario when profile config file is present anbd has profile tag key +func TestOptionsWithProfileConfigAndProfileTag(t *testing.T) { + _, profileConfig, _ := getAccessProfileConfig("data/profile_config_tag_key", "http://localhost:80") + cfg, cfgAccount, _ := getConfig("data/sia_config", "-service", "http://localhost:80", false, "us-west-2") + opts, e := setOptions(cfg, cfgAccount, profileConfig, "/tmp", "1.0.0") + require.Nilf(t, e, "error should be empty, error: %v", e) + require.NotNil(t, opts, "should be able to get Options") + assert.True(t, opts.RefreshInterval == 1440) + assert.True(t, opts.ZTSRegion == "") + + // Make sure profile is correct + assert.True(t, opts.Profile == "zts-profile") + assert.True(t, opts.ProfileTag == "zts:RestrictTo") // Make sure services are set assert.True(t, len(opts.Services) == 3) diff --git a/libs/go/sia/host/hostdoc/hostdoc.go b/libs/go/sia/host/hostdoc/hostdoc.go index 188365b3ef0..56d6d4b7fd0 100644 --- a/libs/go/sia/host/hostdoc/hostdoc.go +++ b/libs/go/sia/host/hostdoc/hostdoc.go @@ -28,25 +28,27 @@ import ( ) const ( - DOMAIN = "domain" - SERVICE = "service" - SERVICES = "services" - PROFILE = "profile" - PROVIDER = "provider" - IP = "ip" - UUID = "uuid" - ZONE = "zone" + DOMAIN = "domain" + SERVICE = "service" + SERVICES = "services" + PROFILE = "profile" + PROFILE_TAG = "profile_tag" + PROVIDER = "provider" + IP = "ip" + UUID = "uuid" + ZONE = "zone" ) type Doc struct { - Provider provider.Provider - Domain string - Profile string - Services []string - Uuid string - Ip map[string]bool - Zone string - Bytes []byte + Provider provider.Provider + Domain string + Profile string + ProfileTag string + Services []string + Uuid string + Ip map[string]bool + Zone string + Bytes []byte } // NewPlainDoc returns Doc, the provider string from the host_document, and an error @@ -86,13 +88,14 @@ func NewPlainDoc(bytes []byte) (*Doc, string, error) { } return &Doc{ - Domain: d.Domain, - Services: strings.Split(svcs, ","), - Profile: d.Profile, - Uuid: uuid, - Zone: d.Zone, - Ip: ip, - Bytes: bytes, + Domain: d.Domain, + Services: strings.Split(svcs, ","), + Profile: d.Profile, + ProfileTag: d.ProfileTag, + Uuid: uuid, + Zone: d.Zone, + Ip: ip, + Bytes: bytes, }, d.Provider, nil } diff --git a/libs/go/sia/host/hostdoc/hostdoc_test.go b/libs/go/sia/host/hostdoc/hostdoc_test.go index ed2c8ac04c0..796d055e49e 100644 --- a/libs/go/sia/host/hostdoc/hostdoc_test.go +++ b/libs/go/sia/host/hostdoc/hostdoc_test.go @@ -48,6 +48,17 @@ const HOSTDOC_STR_SVCS = ` } ` +const HOSTDOC_STR_PROFILE_TAG_KEY = ` +{ + "domain": "sports", + "service": "soccer", + "profile": "prod", + "profile_tag": "deploy", + "uuid": "3e4c2da84a264d718b218ce58b1b3b8f", + "zone": "west" +} +` + // Todo: Improve the tests here to be able to parse both "service" and "services" func TestNewHostDocParseErr(t *testing.T) { @@ -70,6 +81,19 @@ func TestNewHostDoc(t *testing.T) { a.Equal("west", hostDoc.Zone, "service should match") } +func TestNewHostDocProfileTag(t *testing.T) { + a := assert.New(t) + + hostDoc, _, err := NewPlainDoc([]byte(HOSTDOC_STR_PROFILE_TAG_KEY)) + a.Nil(err) + a.Equal("sports", hostDoc.Domain, "domain should match") + a.Equal("soccer", hostDoc.Services[0], "service should match") + a.Equal("prod", hostDoc.Profile, "profile should match") + a.Equal("3e4c2da8-4a26-4d71-8b21-8ce58b1b3b8f", hostDoc.Uuid, "Uuid should match") + a.Equal("west", hostDoc.Zone, "service should match") + a.Equal("deploy", hostDoc.ProfileTag, "profile tag should match") +} + func TestNewHostDocServices(t *testing.T) { a := assert.New(t) diff --git a/libs/go/sia/host/hostdoc/raw/raw.go b/libs/go/sia/host/hostdoc/raw/raw.go index 26d5d01025a..a0811f7c01e 100644 --- a/libs/go/sia/host/hostdoc/raw/raw.go +++ b/libs/go/sia/host/hostdoc/raw/raw.go @@ -3,12 +3,13 @@ package raw // JsonDoc is used mainly for unmarshalling the raw doc // it is used for backward compatibility to allow both service and services keys type Doc struct { - Provider string `json:"provider,omitempty"` - Domain string `json:"domain"` - Service string `json:"service"` - Services string `json:"services,omitempty"` - Profile string `json:"profile"` - Uuid string `json:"uuid,omitempty"` - Ip []string `json:"ip,omitempty"` - Zone string `json:"zone,omitempty"` + Provider string `json:"provider,omitempty"` + Domain string `json:"domain"` + Service string `json:"service"` + Services string `json:"services,omitempty"` + Profile string `json:"profile"` + ProfileTag string `json:"profile_tag,omitempty"` + Uuid string `json:"uuid,omitempty"` + Ip []string `json:"ip,omitempty"` + Zone string `json:"zone,omitempty"` } diff --git a/libs/go/sia/host/ip/ip_test.go b/libs/go/sia/host/ip/ip_test.go index 2dc9ed3e964..2dd36e751cd 100644 --- a/libs/go/sia/host/ip/ip_test.go +++ b/libs/go/sia/host/ip/ip_test.go @@ -91,9 +91,6 @@ func TestGetIps(t *testing.T) { log.Printf("IPs: %+v", ips) - //ips2, err := hostname.GetIPs() - //log.Printf("IPs from yakl-client hostname package function: %+v", ips2) - // Test independently using 'ifconfig -a' ifIps, err := getIpsFromIfConfig(t) log.Printf("IPs from ifconfig function: %+v", ifIps) diff --git a/provider/aws/sia-ec2/authn.go b/provider/aws/sia-ec2/authn.go index b1d56598cba..392073d4372 100644 --- a/provider/aws/sia-ec2/authn.go +++ b/provider/aws/sia-ec2/authn.go @@ -84,7 +84,7 @@ func GetECSOnEC2TaskId() string { return taskId } -func GetEC2Config(configFile, profileConfigFile, metaEndpoint string, useRegionalSTS bool, region, account string) (*options.Config, *options.ConfigAccount, *options.AccessProfileConfig, error) { +func GetEC2Config(configFile, profileConfigFile, profileTagKey, metaEndpoint string, useRegionalSTS bool, region, account string) (*options.Config, *options.ConfigAccount, *options.AccessProfileConfig, error) { config, configAccount, err := options.InitFileConfig(configFile, metaEndpoint, useRegionalSTS, region, account) if err != nil { log.Printf("Unable to process configuration file '%s': %v\n", configFile, err) @@ -107,7 +107,7 @@ func GetEC2Config(configFile, profileConfigFile, metaEndpoint string, useRegiona } } - profileConfig, err := GetEC2AccessProfile(profileConfigFile, metaEndpoint, useRegionalSTS, region) + profileConfig, err := GetEC2AccessProfile(profileConfigFile, profileTagKey, metaEndpoint, useRegionalSTS, region) if err != nil { log.Printf("Unable to determine profile information: %v\n", err) } @@ -115,7 +115,7 @@ func GetEC2Config(configFile, profileConfigFile, metaEndpoint string, useRegiona return config, configAccount, profileConfig, nil } -func GetEC2AccessProfile(configFile, metaEndpoint string, useRegionalSTS bool, region string) (*options.AccessProfileConfig, error) { +func GetEC2AccessProfile(configFile, profileTagKey, metaEndpoint string, useRegionalSTS bool, region string) (*options.AccessProfileConfig, error) { accessProfileConfig, err := options.InitAccessProfileFileConfig(configFile) if err != nil { log.Printf("Unable to process configuration file '%s': %v\n", configFile, err) @@ -140,5 +140,15 @@ func GetEC2AccessProfile(configFile, metaEndpoint string, useRegionalSTS bool, r } } } + + // If tags is not provided through file then check if value is provided through ec2 instance tags + if accessProfileConfig.ProfileTag == "" && profileTagKey != "" { + log.Printf("Trying to determine profile tag value %v from instance tags\n", profileTagKey) + value, err := options.GetInstanceTagValue(metaEndpoint, profileTagKey) + if err == nil { + accessProfileConfig.ProfileTag = value + } + } + return accessProfileConfig, err } diff --git a/provider/aws/sia-ec2/cmd/siad/main.go b/provider/aws/sia-ec2/cmd/siad/main.go index 740030162db..3e3124c5e13 100644 --- a/provider/aws/sia-ec2/cmd/siad/main.go +++ b/provider/aws/sia-ec2/cmd/siad/main.go @@ -50,7 +50,7 @@ func main() { udsPath := flag.String("uds", "", "uds path") noSysLog := flag.Bool("nosyslog", false, "turn off syslog, log to stdout") accessProfileConf := flag.String("profileconfig", "/etc/sia/profile_config", "The access profile config file") - + accessProfileTagKey := flag.String("profiletagkey", "profile:Tag", "The tag associated with access profile roles") flag.Parse() if *displayVersion { @@ -92,7 +92,7 @@ func main() { log.Fatalf("Unable to extract document details: %v\n", err) } - config, configAccount, accessProfileConfig, err := sia.GetEC2Config(*pConf, *accessProfileConf, *ec2MetaEndPoint, *useRegionalSTS, region, account) + config, configAccount, accessProfileConfig, err := sia.GetEC2Config(*pConf, *accessProfileConf, *accessProfileTagKey, *ec2MetaEndPoint, *useRegionalSTS, region, account) if err != nil { log.Fatalf("Unable to formulate configuration objects, error: %v\n", err) } diff --git a/utils/msd-agent/svc/service.go b/utils/msd-agent/svc/service.go index 5f554d0a793..b93863458c4 100644 --- a/utils/msd-agent/svc/service.go +++ b/utils/msd-agent/svc/service.go @@ -35,10 +35,11 @@ type ServicesData struct { } const ( - PROFILE_CONFIG = "/etc/sia/profile_config" - SIA_CONFIG = "/etc/sia/sia_config" - SIA_DIR = "/var/lib/sia" - HOST_DOCUMENT = "host_document" + PROFILE_CONFIG = "/etc/sia/profile_config" + SIA_CONFIG = "/etc/sia/sia_config" + SIA_DIR = "/var/lib/sia" + HOST_DOCUMENT = "host_document" + PROFILE_TAG_KEY = "profile:Tag" ) var cloudFetcher Fetcher diff --git a/utils/msd-agent/svc/service_aws_ec2.go b/utils/msd-agent/svc/service_aws_ec2.go index f127b4afe16..cdfa7089f31 100644 --- a/utils/msd-agent/svc/service_aws_ec2.go +++ b/utils/msd-agent/svc/service_aws_ec2.go @@ -23,7 +23,7 @@ type EC2Fetcher struct { func (fetcher *EC2Fetcher) Fetch(host MsdHost, accountId string) (ServicesData, error) { - config, configAccount, _, err := sia.GetEC2Config(SIA_CONFIG, PROFILE_CONFIG, Ec2MetaEndPoint, false, "", accountId) + config, configAccount, _, err := sia.GetEC2Config(SIA_CONFIG, PROFILE_CONFIG, PROFILE_TAG_KEY, Ec2MetaEndPoint, false, "", accountId) if err != nil { log.Fatalf("Unable to formulate config, error: %v\n", err) }