Add profileTag support to sia-aws (#2082)

Signed-off-by: craman <[email protected]>
Co-authored-by: craman <[email protected]>

libs/go/sia/aws/options/data/profile_config_tag_key

@@ -0,0 +1,4 @@
+{
+    "profile": "zts-profile",
+    "profile_tag": "zts:RestrictTo"
+}

libs/go/sia/aws/options/options.go

@@ -107,7 +107,8 @@ type Config struct {
 }
 
 type AccessProfileConfig struct {
-	Profile string `json:"profile,omitempty"` // map of role name to token attributes
+	Profile    string `json:"profile,omitempty"`
+	ProfileTag string `json:"profile_tag,omitempty"`
 }
 
 // Role contains role details. Attributes are set based on the config values
@@ -190,6 +191,7 @@ type Options struct {
 	TokenDir           string            //Access tokens directory
 	AccessTokens       []ac.AccessToken  //Access tokens object
 	Profile            string            //Access profile name
+	ProfileTag         string            //Tag associated with access profile roles
 	Threshold          float64           //threshold in number of days for cert expiry checks
 	SshThreshold       float64           //threshold in number of days for ssh cert expiry checks
 	FileDirectUpdate   bool              //update key/cert files directly instead of using rename
@@ -201,6 +203,11 @@ const (
 	DefaultThreshold   = float64(15) // 15 days
 )
 
+func GetInstanceTagValue(metaEndPoint, tagKey string) (string, error) {
+	tagValue, err := meta.GetData(metaEndPoint, "/latest/meta-data/tags/instance/"+tagKey)
+	return string(tagValue), err
+}
+
 func GetAccountId(metaEndPoint string, useRegionalSTS bool, region string) (string, error) {
 	// first try to get the account from our creds and if
 	// fails we'll fall back to identity document
@@ -228,7 +235,8 @@ func InitCredsConfig(roleSuffix, accessProfileSeparator string, useRegionalSTS b
 			Threshold:    DefaultThreshold,
 			SshThreshold: DefaultThreshold,
 		}, &AccessProfileConfig{
-			Profile: profile,
+			Profile:    profile,
+			ProfileTag: "",
 		}, nil
 }
 
@@ -254,7 +262,8 @@ func InitProfileConfig(metaEndPoint, roleSuffix, accessProfileSeparator string)
 			Threshold:    DefaultThreshold,
 			SshThreshold: DefaultThreshold,
 		}, &AccessProfileConfig{
-			Profile: profile,
+			Profile:    profile,
+			ProfileTag: "",
 		}, nil
 }
 
@@ -312,7 +321,8 @@ func InitAccessProfileFileConfig(fileName string) (*AccessProfileConfig, error)
 	}
 
 	return &AccessProfileConfig{
-		Profile: config.Profile,
+		Profile:    config.Profile,
+		ProfileTag: config.ProfileTag,
 	}, nil
 }
 
@@ -433,7 +443,8 @@ func InitAccessProfileEnvConfig() (*AccessProfileConfig, error) {
 	}
 
 	return &AccessProfileConfig{
-		Profile: accessProfile,
+		Profile:    accessProfile,
+		ProfileTag: "",
 	}, nil
 }
 
@@ -455,6 +466,7 @@ func setOptions(config *Config, account *ConfigAccount, profileConfig *AccessPro
 	ztsRegion := ""
 	dropPrivileges := false
 	profile := ""
+	profileTag := ""
 	fileDirectUpdate := false
 	tokenDir := fmt.Sprintf("%s/tokens", siaDir)
 	certDir := fmt.Sprintf("%s/certs", siaDir)
@@ -622,6 +634,7 @@ func setOptions(config *Config, account *ConfigAccount, profileConfig *AccessPro
 
 	if profileConfig != nil {
 		profile = profileConfig.Profile
+		profileTag = profileConfig.ProfileTag
 	}
 
 	return &Options{
@@ -651,6 +664,7 @@ func setOptions(config *Config, account *ConfigAccount, profileConfig *AccessPro
 		DropPrivileges:   dropPrivileges,
 		AccessTokens:     accessTokens,
 		Profile:          profile,
+		ProfileTag:       profileTag,
 		Threshold:        account.Threshold,
 		SshThreshold:     account.SshThreshold,
 		FileDirectUpdate: fileDirectUpdate,

libs/go/sia/aws/options/options_test.go

@@ -110,6 +110,7 @@ func getAccessProfileConfig(fileName, metaEndPoint string) (*ConfigAccount, *Acc
 			return nil, nil, fmt.Errorf("config non-parsable and unable to determine profile name from profile arn, error: %v", err)
 		}
 	}
+
 	return configAccount, profileConfig, nil
 }
 
@@ -241,6 +242,35 @@ func TestOptionsWithProfileConfig(t *testing.T) {
 
 	// Make sure profile is correct
 	assert.True(t, opts.Profile == "zts-profile")
+	assert.True(t, opts.ProfileTag == "")
+
+	// Make sure services are set
+	assert.True(t, len(opts.Services) == 3)
+	assert.True(t, opts.Domain == "athenz")
+	assert.True(t, opts.Name == "athenz.api")
+
+	// Zeroth service should be the one from "service" key, the remaining are from "services" in no particular order
+	assert.True(t, assertService(opts.Services[0], Service{Name: "api", User: "nobody", Uid: getUid("nobody"), Gid: getUserGid("nobody"), FileMode: 288, Threshold: DefaultThreshold}))
+	assert.True(t, assertInServices(opts.Services[1:], Service{Name: "ui", User: "root", Uid: 0, Gid: 0, FileMode: 288, Threshold: DefaultThreshold}))
+	assert.True(t, assertInServices(opts.Services[1:], Service{Name: "yamas", User: "nobody", Uid: getUid("nobody"), Group: "sys", Gid: getGid(t, "sys"), Threshold: DefaultThreshold}))
+
+	assert.Equal(t, DefaultThreshold, opts.SshThreshold)
+	assert.Equal(t, DefaultThreshold, opts.Threshold)
+}
+
+// TestOptionsWithProfileConfigAndProfileTag test the scenario when profile config file is present anbd has profile tag key
+func TestOptionsWithProfileConfigAndProfileTag(t *testing.T) {
+	_, profileConfig, _ := getAccessProfileConfig("data/profile_config_tag_key", "http://localhost:80")
+	cfg, cfgAccount, _ := getConfig("data/sia_config", "-service", "http://localhost:80", false, "us-west-2")
+	opts, e := setOptions(cfg, cfgAccount, profileConfig, "/tmp", "1.0.0")
+	require.Nilf(t, e, "error should be empty, error: %v", e)
+	require.NotNil(t, opts, "should be able to get Options")
+	assert.True(t, opts.RefreshInterval == 1440)
+	assert.True(t, opts.ZTSRegion == "")
+
+	// Make sure profile is correct
+	assert.True(t, opts.Profile == "zts-profile")
+	assert.True(t, opts.ProfileTag == "zts:RestrictTo")
 
 	// Make sure services are set
 	assert.True(t, len(opts.Services) == 3)

libs/go/sia/host/hostdoc/hostdoc.go

@@ -28,25 +28,27 @@ import (
 )
 
 const (
-	DOMAIN   = "domain"
-	SERVICE  = "service"
-	SERVICES = "services"
-	PROFILE  = "profile"
-	PROVIDER = "provider"
-	IP       = "ip"
-	UUID     = "uuid"
-	ZONE     = "zone"
+	DOMAIN      = "domain"
+	SERVICE     = "service"
+	SERVICES    = "services"
+	PROFILE     = "profile"
+	PROFILE_TAG = "profile_tag"
+	PROVIDER    = "provider"
+	IP          = "ip"
+	UUID        = "uuid"
+	ZONE        = "zone"
 )
 
 type Doc struct {
-	Provider provider.Provider
-	Domain   string
-	Profile  string
-	Services []string
-	Uuid     string
-	Ip       map[string]bool
-	Zone     string
-	Bytes    []byte
+	Provider   provider.Provider
+	Domain     string
+	Profile    string
+	ProfileTag string
+	Services   []string
+	Uuid       string
+	Ip         map[string]bool
+	Zone       string
+	Bytes      []byte
 }
 
 // NewPlainDoc returns Doc, the provider string from the host_document, and an error
@@ -86,13 +88,14 @@ func NewPlainDoc(bytes []byte) (*Doc, string, error) {
 	}
 
 	return &Doc{
-		Domain:   d.Domain,
-		Services: strings.Split(svcs, ","),
-		Profile:  d.Profile,
-		Uuid:     uuid,
-		Zone:     d.Zone,
-		Ip:       ip,
-		Bytes:    bytes,
+		Domain:     d.Domain,
+		Services:   strings.Split(svcs, ","),
+		Profile:    d.Profile,
+		ProfileTag: d.ProfileTag,
+		Uuid:       uuid,
+		Zone:       d.Zone,
+		Ip:         ip,
+		Bytes:      bytes,
 	}, d.Provider, nil
 }
 

libs/go/sia/host/hostdoc/hostdoc_test.go

@@ -48,6 +48,17 @@ const HOSTDOC_STR_SVCS = `
 }
 `
 
+const HOSTDOC_STR_PROFILE_TAG_KEY = `
+{
+   "domain": "sports",
+   "service": "soccer",
+   "profile": "prod",
+   "profile_tag": "deploy",
+   "uuid": "3e4c2da84a264d718b218ce58b1b3b8f",
+   "zone": "west"
+}
+`
+
 // Todo: Improve the tests here to be able to parse both "service" and "services"
 
 func TestNewHostDocParseErr(t *testing.T) {
@@ -70,6 +81,19 @@ func TestNewHostDoc(t *testing.T) {
 	a.Equal("west", hostDoc.Zone, "service should match")
 }
 
+func TestNewHostDocProfileTag(t *testing.T) {
+	a := assert.New(t)
+
+	hostDoc, _, err := NewPlainDoc([]byte(HOSTDOC_STR_PROFILE_TAG_KEY))
+	a.Nil(err)
+	a.Equal("sports", hostDoc.Domain, "domain should match")
+	a.Equal("soccer", hostDoc.Services[0], "service should match")
+	a.Equal("prod", hostDoc.Profile, "profile should match")
+	a.Equal("3e4c2da8-4a26-4d71-8b21-8ce58b1b3b8f", hostDoc.Uuid, "Uuid should match")
+	a.Equal("west", hostDoc.Zone, "service should match")
+	a.Equal("deploy", hostDoc.ProfileTag, "profile tag should match")
+}
+
 func TestNewHostDocServices(t *testing.T) {
 	a := assert.New(t)
 

libs/go/sia/host/hostdoc/raw/raw.go

@@ -3,12 +3,13 @@ package raw
 // JsonDoc is used mainly for unmarshalling the raw doc
 // it is used for backward compatibility to allow both service and services keys
 type Doc struct {
-	Provider string   `json:"provider,omitempty"`
-	Domain   string   `json:"domain"`
-	Service  string   `json:"service"`
-	Services string   `json:"services,omitempty"`
-	Profile  string   `json:"profile"`
-	Uuid     string   `json:"uuid,omitempty"`
-	Ip       []string `json:"ip,omitempty"`
-	Zone     string   `json:"zone,omitempty"`
+	Provider   string   `json:"provider,omitempty"`
+	Domain     string   `json:"domain"`
+	Service    string   `json:"service"`
+	Services   string   `json:"services,omitempty"`
+	Profile    string   `json:"profile"`
+	ProfileTag string   `json:"profile_tag,omitempty"`
+	Uuid       string   `json:"uuid,omitempty"`
+	Ip         []string `json:"ip,omitempty"`
+	Zone       string   `json:"zone,omitempty"`
 }

libs/go/sia/host/ip/ip_test.go

@@ -91,9 +91,6 @@ func TestGetIps(t *testing.T) {
 
 	log.Printf("IPs: %+v", ips)
 
-	//ips2, err := hostname.GetIPs()
-	//log.Printf("IPs from yakl-client hostname package function: %+v", ips2)
-
 	// Test independently using 'ifconfig -a'
 	ifIps, err := getIpsFromIfConfig(t)
 	log.Printf("IPs from ifconfig function: %+v", ifIps)

provider/aws/sia-ec2/authn.go

@@ -84,7 +84,7 @@ func GetECSOnEC2TaskId() string {
 	return taskId
 }
 
-func GetEC2Config(configFile, profileConfigFile, metaEndpoint string, useRegionalSTS bool, region, account string) (*options.Config, *options.ConfigAccount, *options.AccessProfileConfig, error) {
+func GetEC2Config(configFile, profileConfigFile, profileTagKey, metaEndpoint string, useRegionalSTS bool, region, account string) (*options.Config, *options.ConfigAccount, *options.AccessProfileConfig, error) {
 	config, configAccount, err := options.InitFileConfig(configFile, metaEndpoint, useRegionalSTS, region, account)
 	if err != nil {
 		log.Printf("Unable to process configuration file '%s': %v\n", configFile, err)
@@ -107,15 +107,15 @@ func GetEC2Config(configFile, profileConfigFile, metaEndpoint string, useRegiona
 		}
 	}
 
-	profileConfig, err := GetEC2AccessProfile(profileConfigFile, metaEndpoint, useRegionalSTS, region)
+	profileConfig, err := GetEC2AccessProfile(profileConfigFile, profileTagKey, metaEndpoint, useRegionalSTS, region)
 	if err != nil {
 		log.Printf("Unable to determine profile information: %v\n", err)
 	}
 
 	return config, configAccount, profileConfig, nil
 }
 
-func GetEC2AccessProfile(configFile, metaEndpoint string, useRegionalSTS bool, region string) (*options.AccessProfileConfig, error) {
+func GetEC2AccessProfile(configFile, profileTagKey, metaEndpoint string, useRegionalSTS bool, region string) (*options.AccessProfileConfig, error) {
 	accessProfileConfig, err := options.InitAccessProfileFileConfig(configFile)
 	if err != nil {
 		log.Printf("Unable to process configuration file '%s': %v\n", configFile, err)
@@ -140,5 +140,15 @@ func GetEC2AccessProfile(configFile, metaEndpoint string, useRegionalSTS bool, r
 			}
 		}
 	}
+
+	// If tags is not provided through file then check if value is provided through ec2 instance tags
+	if accessProfileConfig.ProfileTag == "" && profileTagKey != "" {
+		log.Printf("Trying to determine profile tag value %v from instance tags\n", profileTagKey)
+		value, err := options.GetInstanceTagValue(metaEndpoint, profileTagKey)
+		if err == nil {
+			accessProfileConfig.ProfileTag = value
+		}
+	}
+
 	return accessProfileConfig, err
 }

provider/aws/sia-ec2/cmd/siad/main.go

@@ -50,7 +50,7 @@ func main() {
 	udsPath := flag.String("uds", "", "uds path")
 	noSysLog := flag.Bool("nosyslog", false, "turn off syslog, log to stdout")
 	accessProfileConf := flag.String("profileconfig", "/etc/sia/profile_config", "The access profile config file")
-
+	accessProfileTagKey := flag.String("profiletagkey", "profile:Tag", "The tag associated with access profile roles")
 	flag.Parse()
 
 	if *displayVersion {
@@ -92,7 +92,7 @@ func main() {
 		log.Fatalf("Unable to extract document details: %v\n", err)
 	}
 
-	config, configAccount, accessProfileConfig, err := sia.GetEC2Config(*pConf, *accessProfileConf, *ec2MetaEndPoint, *useRegionalSTS, region, account)
+	config, configAccount, accessProfileConfig, err := sia.GetEC2Config(*pConf, *accessProfileConf, *accessProfileTagKey, *ec2MetaEndPoint, *useRegionalSTS, region, account)
 	if err != nil {
 		log.Fatalf("Unable to formulate configuration objects, error: %v\n", err)
 	}

utils/msd-agent/svc/service.go

@@ -35,10 +35,11 @@ type ServicesData struct {
 }
 
 const (
-	PROFILE_CONFIG = "/etc/sia/profile_config"
-	SIA_CONFIG     = "/etc/sia/sia_config"
-	SIA_DIR        = "/var/lib/sia"
-	HOST_DOCUMENT  = "host_document"
+	PROFILE_CONFIG  = "/etc/sia/profile_config"
+	SIA_CONFIG      = "/etc/sia/sia_config"
+	SIA_DIR         = "/var/lib/sia"
+	HOST_DOCUMENT   = "host_document"
+	PROFILE_TAG_KEY = "profile:Tag"
 )
 
 var cloudFetcher Fetcher

utils/msd-agent/svc/service_aws_ec2.go

@@ -23,7 +23,7 @@ type EC2Fetcher struct {
 
 func (fetcher *EC2Fetcher) Fetch(host MsdHost, accountId string) (ServicesData, error) {
 
-	config, configAccount, _, err := sia.GetEC2Config(SIA_CONFIG, PROFILE_CONFIG, Ec2MetaEndPoint, false, "", accountId)
+	config, configAccount, _, err := sia.GetEC2Config(SIA_CONFIG, PROFILE_CONFIG, PROFILE_TAG_KEY, Ec2MetaEndPoint, false, "", accountId)
 	if err != nil {
 		log.Fatalf("Unable to formulate config, error: %v\n", err)
 	}