Sboag dlaas pull june14 (IBM#97)

* Pointed travis testing to do hostmount minikube

* Debugging permissions error.

* Fix to mkdir problems.

* Fixed Makefile syntax.

* Printing debugging information about pods.

* Printing debugging information about pods.

* Printing debugging information about pods.

* Printing debugging information incl kubectl get pod.

* Enabled debug mode.

* Again.

* Set debug as default.

* tracing from the trainer to lcm

* more debugging

* added lower level logging

* dist: xenial

* Update .travis.yml

* fix typo

* Trying to fix Travis issue.

* Fixed Travis issue.

* Followed Tommy's request and increased resource limits to values from before. Might break CI.

* Parameterized memory values like Tommy requested.

* Attempt to fix CI.

* Removed excessive debug statements and cleaned comments. Probably breaks code.

* DLaaS pull june 14, with security mods

* fixed glide problem

* Added Image.go etc. files, deleted learner_test.go

* temporarily disable framework validation

* FIXME: Disable validation check for bucket until conditionalize for s3fs vs.  option.

* fixed two bugs related to volume mounting

* I think mostly just logging changes

* basic success

* Add FfDL.iml to .gitignore

* removed docker ref to csf_env.properties

* Test for mount_cos before attempting s3 validation

* fixed hostmount by pre-setup of model code in Makefile

* fixed missing import

* log HELM_DEPLOY_DIR, add a bunch of logging for the ci test

* Added create-volumes to jenkins file, more verbose docker build for ui

* Wound back Angular to 6.0.8

* Quiet docker-build-ui docker build

* merged bin/create_static_volumes_config2.sh into bin/create_static_volumes_config.sh

.gitignore

@@ -25,3 +25,4 @@ bin/dind_scripts/config.json
 bin/dind_scripts/go1.10.1.linux-amd64.tar.gz
 bin/dind_scripts/helm-v2.8.2-linux-amd64.tar.gz
 bin/dind_scripts/linux-amd64/
+FfDL.iml

.helmignore

@@ -23,3 +23,8 @@ cli/
 community/
 persistentvol/
 design/
+demos/
+list/
+mybucket/
+persistentvol/
+storage-plugin/

.travis.yml

@@ -43,6 +43,7 @@ script:
   - make $MAKE_ARGS gen-certs
   - make $MAKE_ARGS build
   - make $MAKE_ARGS docker-build
+  - make $MAKE_ARGS create-volumes
   # deploy services
   - make $MAKE_ARGS deploy
   # submit a test job

bin/create_static_volumes_config.sh

@@ -21,3 +21,18 @@ echo
 kubectl create configmap ${CONFIGMAP_NAME} --from-file=PVCs.yaml=<(
     kubectl get pvc --selector type=${volumeType} -o yaml
 )
+
+CONFIGMAP_NAME2=static-volumes-v2
+
+# Delete configmap
+#if kubectl get cm | grep static-volumes &> /dev/null; then kubectl delete configmap ${CONFIGMAP_NAME2}; else echo "No need to delete ${CONFIGMAP_NAME2} since it doesn't exist."; fi
+
+# Create new configmap
+echo
+echo "Using volumes with label type=$volumeType"
+kubectl get pvc --selector type=${volumeType}
+echo
+
+kubectl get pvc --selector type="dlaas-static-volume" -o jsonpath='{"static-volumes-v2:"}{range .items[*]}{"\n  - name: "}{.metadata.name}{"\n    zlabel: "}{.metadata.name}{"\n    status: active\n"}' > PVCs-v2.yaml
+
+kubectl create configmap ${CONFIGMAP_NAME2} --from-file=PVCs-v2.yaml

bin/dind_scripts/experimental_master.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-sudo chown -R ffdlr /home/ffdlr
+sudo chown -R ${USER} /home/${USER}
 chmod +x build_ffdl.sh compile_s3fs.sh create_user.sh import_registry_certificates.sh install_docker.sh install_go.sh install_kubernetes.sh install_nfs.sh install_registry.sh launch_kubernetes.sh launch_registry.sh s3_driver.sh
 echo "This script assumes that you have created a user, e.g. via create_user.sh, and are now logged in as that user."
 
@@ -21,7 +21,7 @@ export DOCKER_REPO_USER=$USER
 export DOCKER_REPO_PASS=7312mInalM4n
 ./launch_registry.sh
 ./launch_kubernetes.sh
-sudo chown -R ffdlr /home/ffdlr/.kube/
+sudo chown -R ${USER} /home/${USER}/.kube/
 ./import_registry_certificates.sh
 ./s3_driver.sh
 

bin/dind_scripts/launch_kubernetes.sh

@@ -1,6 +1,10 @@
 #!/bin/bash
 sudo apt install -y jq
 cd ~
+
+wget https://cdn.rawgit.com/kubernetes-sigs/kubeadm-dind-cluster/master/fixed/dind-cluster-v1.9.sh
+chmod +x dind-cluster-v1.9.sh
+
 sudo ./dind-cluster-v1.9.sh clean
 sudo ./dind-cluster-v1.9.sh up
 # cd ~/go/src/github.com/IBM/FfDL/bin

bin/dind_scripts/s3_driver.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 
 kubectl create secret docker-registry regcred --docker-server=${DOCKER_REPO} --docker-username=${DOCKER_REPO_USER} --docker-password=${DOCKER_REPO_PASS} [email protected]
+docker pull docker.io/ffdl/ibmcloud-object-storage-plugin
 docker tag ibmcloud-object-storage-plugin ${DOCKER_REPO}/ibmcloud-object-storage-plugin
 docker push ${DOCKER_REPO}/ibmcloud-object-storage-plugin
 

bin/escape_for_sed.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+#!/bin/bash
+operating_system=$(uname)
+if [[ "$operating_system" == 'Linux' ]]; then
+    CMD_SED=sed
+elif [[ "$operating_system" == 'Darwin' ]]; then
+    CMD_SED=gsed
+fi
+replacement_string=$(echo "$AWS_URL" | $CMD_SED -r 's/\//\\\//g')
+
+echo $replacement_string

cli/main.go

@@ -107,6 +107,12 @@ func (c *DlaasPlugin) Run(context plugin.PluginContext, args []string) {
 			if er == nil {
 				os.Exit(exitCode)
 			}
+
+			// FIXME QuietPanic does not exist
+			//if err != terminal.QuietPanic {
+			//	fmt.Printf("%v\n", err)
+			//}
+
 			os.Exit(1)
 		}
 	}()

commons/config/config.go

@@ -18,7 +18,6 @@ package config
 
 import (
 	"fmt"
-	"runtime"
 	"io/ioutil"
 	"os"
 	"path"
@@ -30,9 +29,9 @@ import (
 	"google.golang.org/grpc/grpclog"
 
 	log "github.com/sirupsen/logrus"
-
 	"github.com/spf13/viper"
 	v1core "k8s.io/api/core/v1"
+	"runtime"
 )
 
 const (
@@ -150,6 +149,7 @@ const (
 	DlaasResourceLimit          = "resource.limit"
 	DlaasResourceLimitQuerySize = "resource.limit.query.size"
 
+	// FfDL Change: next 5 lines
 	ImagePullPolicy = "image_pull_policy"
 
 	SharedVolumeStorageClassKey = "shared_volume_storage_class"
@@ -171,12 +171,13 @@ func InitViper() {
 
 	viperInitOnce.Do(func() {
 
-		viper.SetEnvPrefix(envPrefix) // will be capitalized automatically
+		viper.SetEnvPrefix(envPrefix) // will be uppercased automatically
 		viper.SetConfigType("yaml")
 
+		// FfDL Change: make image pull policy default
 		viper.SetDefault(ImagePullPolicy, v1core.PullIfNotPresent)
 
-		// Most likely be "standard" in Minikube and "ibmc-s3fs-standard" in DIND, (other value is "default" or "")
+		// FfDL Change:Most likely be "standard" in Minikube and "ibmc-s3fs-standard" in DIND, (other value is "default" or "")
 		viper.SetDefault(SharedVolumeStorageClassKey, "")
 
 		// enable ENV vars and defaults
@@ -192,7 +193,10 @@ func InitViper() {
 		viper.SetDefault(LearnerTagKey, "prod")
 		viper.SetDefault(DataBrokerTagKey, "prod")
 		viper.SetDefault(LearnerRegistryKey, "docker.io/ffdl")
+
+		// FfDL Change: FIXME in DLaaS, this will need to be overridden with "bluemix-cr-ng"
 		viper.SetDefault(LearnerImagePullSecretKey, "regcred")
+		// viper.SetDefault(LearnerImagePullSecretKey, "bluemix-cr-ng")
 
 		// TLS defaults for microservices
 		viper.SetDefault(TLSKey, true)
@@ -288,7 +292,7 @@ func GetFileContents(filename string) string {
 	return contents
 }
 
-// IsTLSEnabled is true if the microservices should all use TLS for communication, otherwise
+// IsTLSEnabled is true if the microservices should all use TLS for communiction, otherwise
 // it is false.
 func IsTLSEnabled() bool {
 	return viper.GetBool(TLSKey)
@@ -427,10 +431,16 @@ func GetDataStoreConfig() map[string]string {
 	if val != "" {
 		m[DomainKey] = val
 	}
-	val = viper.GetString("objectstore." + UsernameKey)
+	val = viper.GetString("objectstore." + RegionKey)
 	if val != "" {
-		m[UsernameKey] = val
+		m[RegionKey] = val
 	}
+	// FfDL Change: This was in the older PR, supposing not needed?
+	//val = viper.GetString("objectstore." + UsernameKey)
+	//if val != "" {
+	//	m[UsernameKey] = val
+	//}
+
 	val = viper.GetString("objectstore." + ProjectKey)
 	if val != "" {
 		m[ProjectKey] = val
@@ -479,7 +489,7 @@ func configKey2EnvVar(key string) string {
 
 // setLogLevel sets the logging level based on the environment
 func setLogLevel() {
-	viper.SetDefault(LogLevelKey, "debug")  // FIXME Should be warn
+	viper.SetDefault(LogLevelKey, "warn")
 
 	env := viper.GetString(EnvKey)
 	if env == "dev" || env == "test" {
@@ -547,32 +557,32 @@ func GetMongoCertLocation() string {
 	return getFileAtLocation("/etc/certs/mongo/mongo.cert") //the file should have been mounted at this path as a part of secrets
 }
 
-//Get LearnerKubeURLKey...
+//GetLearnerKubeURL ...
 func GetLearnerKubeURL() string {
 	return viper.GetString(learnerKubeURLKey)
 }
 
-//Get LearnerKubeCAFileKey...
+//GetLearnerKubeCAFile ...
 func GetLearnerKubeCAFile() string {
 	return viper.GetString(learnerKubeCAFileKey)
 }
 
-//Get LearnerKubetokenKey...
+//GetLearnerKubeToken ...
 func GetLearnerKubeToken() string {
 	return viper.GetString(learnerKubeTokenKey)
 }
 
-//Get LearnerKubeTokenFileKey
+//GetLearnerKubeTokenFile ...
 func GetLearnerKubeTokenFile() string {
 	return viper.GetString(learnerKubeTokenFileKey)
 }
 
-//Get LearnerKubeKeyFileKey
+//GetLearnerKubeKeyFile ...
 func GetLearnerKubeKeyFile() string {
 	return viper.GetString(learnerKubeKeyFileKey)
 }
 
-//Get LearnerKubeCertFileKey
+//GetLearnerKubeCertFile ...
 func GetLearnerKubeCertFile() string {
 	return viper.GetString(learnerKubeCertFileKey)
 }
@@ -626,9 +636,7 @@ func getFileAtLocation(location string) string {
 		log.Debugf("file was found at location %s", location)
 		return location
 	}
-	//log.Debugf("file certificate was missing at location %s", location)
-	//LogStackTrace()
-
+	log.Debugf("file certificate was missing at location %s", location)
 	return "" //empty location means that cert is not required
 }
 

commons/framework/framework.go

@@ -0,0 +1,86 @@
+package framework
+
+import (
+	"encoding/json"
+	"io/ioutil"
+)
+
+//Frameworks All frameworks supported and maintained by dlaas
+type Frameworks struct {
+	Frameworks map[string]*DetailList
+}
+
+//DetailList list of versions for a framework
+type DetailList struct {
+	Versions []*Details
+}
+
+//Details Specific details for a framework version
+type Details struct {
+	Version   string
+	External  bool
+	Build     string
+	PrevBuild string
+}
+
+func readFile(location string) ([]byte, error) {
+	fileData, err := ioutil.ReadFile(location)
+	if err != nil {
+		return []byte(""), err
+	}
+	return fileData, nil
+}
+
+//GetFrameworks returns the frameworks and their versions that are stored in the path to the learnerConfig
+func GetFrameworks(learnerConfigPath string) (Frameworks, error) {
+	var frameworks Frameworks
+	fileData, err := readFile(learnerConfigPath)
+	if err != nil {
+		return frameworks, err
+	}
+	err = json.Unmarshal(fileData, &frameworks)
+	if err != nil {
+		return frameworks, err
+	}
+
+	return frameworks, nil
+}
+
+//GetImageBuildTagForFramework Returns the latest build tag for a specified framework and version
+func GetImageBuildTagForFramework(fwName, fwVersion, learnerConfigPath string) string {
+	frameworks, err := GetFrameworks(learnerConfigPath)
+	if err != nil {
+		return ""
+	}
+
+	frameworkVersions := frameworks.Frameworks[fwName].Versions
+
+	for _, frameworkVersion := range frameworkVersions {
+		if frameworkVersion.Version == fwVersion {
+			return frameworkVersion.Build
+		}
+	}
+
+	return ""
+}
+
+//CheckIfFrameworkExists Checks if the specified framework exists
+func CheckIfFrameworkExists(fwName, fwVersion, learnerConfigPath string) (bool, error) {
+	frameworks, err := GetFrameworks(learnerConfigPath)
+	if err != nil {
+		return false, err
+	}
+
+	frameworkType := frameworks.Frameworks[fwName]
+	if frameworkType == nil {
+		return false, nil
+	}
+	frameworkVersions := frameworkType.Versions
+
+	for _, frameworkVersion := range frameworkVersions {
+		if frameworkVersion.Version == fwVersion {
+			return true, nil
+		}
+	}
+	return false, nil
+}

commons/logger/logger.go

@@ -131,6 +131,7 @@ func FileInfoFindGood() string {
 		if strings.Contains(file, "runtime/extern.go") {
 			continue
 		}
+		// FfDL Change: Not sure if this can be handled in a common way.  Moot point once DLaaS is based on FfDL.
 		if strings.Contains(file, "logger/logger.go") {
 			continue
 		}
@@ -164,6 +165,7 @@ func FileInfoFindGood() string {
 	return fmt.Sprintf("%s:%d %s -", file, line, funcName)
 }
 
+// FfDL Change: Because it's useful
 func LogStackTrace() {
 	pc := make([]uintptr, 30)
 	stackDepth := runtime.Callers(0, pc)

commons/metricsmon/metricspusher.go

@@ -28,6 +28,7 @@ import (
 	"github.com/sony/gobreaker"
 )
 
+//StartMetricsPusher ...
 func StartMetricsPusher(label string, interval time.Duration, url string) chan struct{} {
 	log.Info("Starting code to push out metrics")
 	quit := make(chan struct{})
@@ -74,10 +75,11 @@ func pushMetrics(job string, url string) error {
 	return nil
 }
 
-// Pushes metrics out to statsd server every 30s
+//StartStatsdMetricsPusher ... pushes metrics out to statsd server every 30s
 func StartStatsdMetricsPusher(statsd *statsd.Statsd, pushInterval time.Duration) {
 	log.Info("Starting code to push out metrics via statsd")
 	report := time.NewTicker(pushInterval)
-	// TODO defer report.Stop()
+	//TODO
+	//defer report.Stop()
 	go statsd.SendLoop(report.C, "udp", "statsdexporter:9125")
 }