From c93b76415536cad0556f43e96474e4a883c6f277 Mon Sep 17 00:00:00 2001 From: amitkumardas Date: Wed, 8 Nov 2017 09:47:54 +0530 Subject: [PATCH 1/2] 1. Why is this change necessary ? It has been seen that policy enforcement is required for OpenEBS volumes during its provisioning operations. A volume policy can be related to placements, security/backup, workload characteristics, QOS, etc. Policy Enforcement Workflow: A volume provisioning request to maya is intercepted by volume_endpoint.go file. This is made to pass through various policy enforcement logic. The end point decides on the volume provisioner based on VolumeType. It is then directed to the selected provisioner. The provisioner then decides on the volume orchestrator provider & directs the request to the selected orchestrator. The orchestrator in turn is responsible to convert the volume provisioning request to its own structure and pass it to its APIs. This particular commit does a partial fix of openebs/openebs#409, openebs/openebs#721, openebs/openebs#749, openebs/openebs#751 2. How does this change address the issue ? - Add & Delete volume goes through policy enforcement logic - More properties & setters are introduced in K8s structures to handle state & in turn make them unit-testable - Functions have been introduced to create new instances of certain structures. This will remove the need for fetching instances via registration - Policy related operations contract has been introduced in orchestrator interface - A new, simple but redundant wrapper over K8s client-go operations has been introduced. This avoids modification of existing code while caters to new requirements. This will be used instead of older wrapper in near future. - Typed structures have been introduced instead of map[string]string 3. How to verify this change ? - make should work fine - a set of issues will be created that verifies policy enforcement via unit tests, e2e tests 4. What side effects does this change have ? - Testing is required to un-earth side effects - Lack of unit tests is a concern --- .gitignore | 3 +- cmd/maya-apiserver/app/server/server.go | 8 +- .../app/server/volume_endpoint.go | 39 +- orchprovider/k8s/v1/doc.go | 7 +- orchprovider/k8s/v1/k8s.go | 111 ++++-- orchprovider/k8s/v1/k8s_test.go | 38 +- orchprovider/k8s/v1/util.go | 87 +++-- orchprovider/k8s/v1/util_test.go | 20 +- orchprovider/nomad/v1/api.go | 2 +- orchprovider/nomad/v1/doc.go | 2 +- orchprovider/nomad/v1/helper_funcs.go | 22 +- orchprovider/nomad/v1/nomad_plug.go | 12 +- orchprovider/nomad/v1/util.go | 2 +- orchprovider/nomad/v1/util_test.go | 2 +- orchprovider/orchestration.go | 31 +- pkg/util/util.go | 22 ++ types/v1/defaults.go | 53 +++ types/v1/envs.go | 132 +++++++ types/v1/meta.go | 15 +- types/v1/policies.go | 106 ++++++ types/v1/profile/orchestrator/profile.go | 33 +- types/v1/types.go | 51 ++- types/v1/util.go | 5 +- types/v1/validations.go | 57 +++ volume/policies/v1/doc.go | 18 + volume/policies/v1/policy.go | 94 +++++ volume/policies/v1/policy_jiva.go | 131 +++++++ volume/policies/v1/policy_jiva_k8s.go | 343 ++++++++++++++++++ volume/policies/v1/policy_k8s.go | 178 +++++++++ volume/policies/v1/policy_maya_bot.go | 17 + volume/policies/v1/policy_maya_io.go | 17 + volume/policies/v1/policy_required.go | 133 +++++++ volume/profiles/profiles.go | 85 ++++- 33 files changed, 1717 insertions(+), 159 deletions(-) create mode 100644 types/v1/defaults.go create mode 100644 types/v1/envs.go create mode 100644 types/v1/policies.go create mode 100644 types/v1/validations.go create mode 100644 volume/policies/v1/doc.go create mode 100644 volume/policies/v1/policy.go create mode 100644 volume/policies/v1/policy_jiva.go create mode 100644 volume/policies/v1/policy_jiva_k8s.go create mode 100644 volume/policies/v1/policy_k8s.go create mode 100644 volume/policies/v1/policy_maya_bot.go create mode 100644 volume/policies/v1/policy_maya_io.go create mode 100644 volume/policies/v1/policy_required.go diff --git a/.gitignore b/.gitignore index 9d7675d992..19f67d5722 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ demo/temp/* bin/ nohup.out -.DS_Store \ No newline at end of file +.DS_Store +.testfiles/ diff --git a/cmd/maya-apiserver/app/server/server.go b/cmd/maya-apiserver/app/server/server.go index da08049252..49fd753731 100644 --- a/cmd/maya-apiserver/app/server/server.go +++ b/cmd/maya-apiserver/app/server/server.go @@ -7,8 +7,8 @@ import ( "github.com/openebs/maya/cmd/maya-apiserver/app/config" "github.com/openebs/maya/orchprovider" - "github.com/openebs/maya/orchprovider/k8s/v1" - "github.com/openebs/maya/orchprovider/nomad/v1" + k8sv1 "github.com/openebs/maya/orchprovider/k8s/v1" + nomadv1 "github.com/openebs/maya/orchprovider/nomad/v1" "github.com/openebs/maya/types/v1" "github.com/openebs/maya/volume/provisioners" "github.com/openebs/maya/volume/provisioners/jiva" @@ -90,7 +90,7 @@ func (ms *MayaApiServer) BootstrapPlugins() error { // Below is a callback function that creates a new instance of Kubernetes // orchestration provider func(label v1.NameLabel, name v1.OrchProviderRegistry) (orchprovider.OrchestratorInterface, error) { - return k8s.NewK8sOrchestrator(label, name) + return k8sv1.NewK8sOrchestrator(label, name) }) } @@ -102,7 +102,7 @@ func (ms *MayaApiServer) BootstrapPlugins() error { // Below is a callback function that creates a new instance of Nomad // orchestration provider func(label v1.NameLabel, name v1.OrchProviderRegistry) (orchprovider.OrchestratorInterface, error) { - return nomad.NewNomadOrchestrator(label, name) + return nomadv1.NewNomadOrchestrator(label, name) }) } diff --git a/cmd/maya-apiserver/app/server/volume_endpoint.go b/cmd/maya-apiserver/app/server/volume_endpoint.go index 5a80328afc..01cf8ae6e1 100644 --- a/cmd/maya-apiserver/app/server/volume_endpoint.go +++ b/cmd/maya-apiserver/app/server/volume_endpoint.go @@ -7,6 +7,7 @@ import ( "github.com/golang/glog" "github.com/openebs/maya/types/v1" + policies_v1 "github.com/openebs/maya/volume/policies/v1" "github.com/openebs/maya/volume/provisioners" ) @@ -63,7 +64,7 @@ func (s *HTTPServer) volumeList(resp http.ResponseWriter, req *http.Request) (in vol := &v1.Volume{} // Get the persistent volume provisioner instance - pvp, err := provisioners.GetVolumeProvisioner(vol.Labels) + pvp, err := provisioners.GetVolumeProvisioner(nil) if err != nil { return nil, err } @@ -107,7 +108,7 @@ func (s *HTTPServer) volumeRead(resp http.ResponseWriter, req *http.Request, vol vol.Name = volName // Get persistent volume provisioner instance - pvp, err := provisioners.GetVolumeProvisioner(vol.Labels) + pvp, err := provisioners.GetVolumeProvisioner(nil) if err != nil { return nil, err } @@ -152,8 +153,19 @@ func (s *HTTPServer) volumeDelete(resp http.ResponseWriter, req *http.Request, v vol := &v1.Volume{} vol.Name = volName + // Pass through the policy enforcement logic + policy, err := policies_v1.VolumeDeletePolicy() + if err != nil { + return nil, err + } + + vol, err = policy.Enforce(vol) + if err != nil { + return nil, err + } + // Get the persistent volume provisioner instance - pvp, err := provisioners.GetVolumeProvisioner(vol.Labels) + pvp, err := provisioners.GetVolumeProvisioner(nil) if err != nil { return nil, err } @@ -193,10 +205,10 @@ func (s *HTTPServer) volumeAdd(resp http.ResponseWriter, req *http.Request) (int glog.Infof("Processing Volume add request") - vol := v1.Volume{} + vol := &v1.Volume{} // The yaml/json spec is decoded to vol struct - if err := decodeBody(req, &vol); err != nil { + if err := decodeBody(req, vol); err != nil { return nil, CodedError(400, err.Error()) } @@ -205,14 +217,25 @@ func (s *HTTPServer) volumeAdd(resp http.ResponseWriter, req *http.Request) (int return nil, CodedError(400, fmt.Sprintf("Volume name missing in '%v'", vol)) } + // Pass through the policy enforcement logic + policy, err := policies_v1.VolumeAddPolicy() + if err != nil { + return nil, err + } + + vol, err = policy.Enforce(vol) + if err != nil { + return nil, err + } + // Get persistent volume provisioner instance - pvp, err := provisioners.GetVolumeProvisioner(vol.Labels) + pvp, err := provisioners.GetVolumeProvisioner(nil) if err != nil { return nil, err } // Set the volume provisioner profile to provisioner - _, err = pvp.Profile(&vol) + _, err = pvp.Profile(vol) if err != nil { return nil, err } @@ -224,7 +247,7 @@ func (s *HTTPServer) volumeAdd(resp http.ResponseWriter, req *http.Request) (int // TODO // vol should not be passed again !! - details, err := adder.Add(&vol) + details, err := adder.Add(vol) if err != nil { return nil, err } diff --git a/orchprovider/k8s/v1/doc.go b/orchprovider/k8s/v1/doc.go index 40f6b79f49..340affb29d 100644 --- a/orchprovider/k8s/v1/doc.go +++ b/orchprovider/k8s/v1/doc.go @@ -1,4 +1,3 @@ -// Package k8s enables Kubernetes as the orchestration provider -// that aligns to the interfaces suggested by maya api server's orchprovider -// package. -package k8s +// Package k8s enables Kubernetes as the orchestration provider for openebs +// volumes +package v1 diff --git a/orchprovider/k8s/v1/k8s.go b/orchprovider/k8s/v1/k8s.go index d250921ab3..b2db013806 100644 --- a/orchprovider/k8s/v1/k8s.go +++ b/orchprovider/k8s/v1/k8s.go @@ -1,7 +1,7 @@ // This file registers Kubernetes as an orchestration provider plugin in maya // api server. This orchestration is for persistent volume provisioners which // also are registered in maya api server. -package k8s +package v1 import ( "fmt" @@ -25,18 +25,30 @@ import ( // 2. orchprovider.NetworkPlacements & // 3. orchprovider.StoragePlacements type k8sOrchestrator struct { + // TODO use string datatype // label specified to this orchestrator label v1.NameLabel + // TODO use string datatype // name of the orchestrator as registered in the registry name v1.OrchProviderRegistry + // volume represents the instance of OpenEBS volume that will + // placed in K8s + volume *v1.Volume + + // k8sUtil provides the instance that does the low level + // K8s operations + k8sUtil *k8sUtil + + // TODO Deprecate in favour of k8sUtil // k8sUtlGtr provides the handle to fetch K8sUtilInterface // NOTE: // This will be set at runtime. k8sUtlGtr K8sUtilGetter } +// Deprecate in favour of NewK8sOrchProvider // NewK8sOrchestrator provides a new instance of K8sOrchestrator. func NewK8sOrchestrator(label v1.NameLabel, name v1.OrchProviderRegistry) (orchprovider.OrchestratorInterface, error) { @@ -56,24 +68,46 @@ func NewK8sOrchestrator(label v1.NameLabel, name v1.OrchProviderRegistry) (orchp }, nil } +// NewK8sOrchProvider provides a new instance of K8sOrchestrator. +func NewK8sOrchProvider() (orchprovider.OrchestratorInterface, error) { + return &k8sOrchestrator{ + label: v1.NameLabel("openebs.io/orch-provider"), + name: v1.OrchProviderRegistry("openebs.io/kubernetes"), + }, nil +} + // Label provides the label assigned against this orchestrator. // This is an implementation of the orchprovider.OrchestratorInterface interface. func (k *k8sOrchestrator) Label() string { - // TODO - // Do not typecast. Make it typed - // Ensure this for all orch provider implementors return string(k.label) } // Name provides the name of this orchestrator. // This is an implementation of the orchprovider.OrchestratorInterface interface. func (k *k8sOrchestrator) Name() string { - // TODO - // Do not typecast. Make it typed - // Ensure this for all orch provider implementors return string(k.name) } +// setVolume sets the volume instance +func (k *k8sOrchestrator) setVolume(vol *v1.Volume) error { + if vol == nil { + return fmt.Errorf("Nil volume provided") + } + + k.volume = vol + return nil +} + +// setK8sUtil sets the k8sUtil instance +func (k *k8sOrchestrator) setK8sUtil(k8sUtil *k8sUtil) error { + if k8sUtil == nil { + return fmt.Errorf("Nil k8s util provided") + } + + k.k8sUtil = k8sUtil + return nil +} + // TODO // Deprecate in favour of orchestrator profile // Region is not supported by k8sOrchestrator. @@ -117,23 +151,54 @@ func k8sOrchUtil(k *k8sOrchestrator, volProfile volProfile.VolumeProvisionerProf return k.k8sUtlGtr.GetK8sUtil(volProfile) } -// TODO -// Will it be good if this accepts VolumeProvisionerProfile and updates the -// k8sOrchestrator instance's properties ? -// -// StorageOps provides storage operations instance that deals with all storage -// related functionality by aligning with Kubernetes as the orchestration provider. -// -// NOTE: -// This is an implementation of the orchprovider.OrchestratorInterface interface. -// -// NOTE: -// This is invoked on a per request basis. In other words, every request will -// invoke StorageOps to invoke storage specific operations thereafter. +// StorageOps provides volume operations instance. func (k *k8sOrchestrator) StorageOps() (orchprovider.StorageOps, bool) { return k, true } +// PolicyOps provides a policy operations instance. +// In addition, it is used for various initializations & validations +func (k *k8sOrchestrator) PolicyOps(vol *v1.Volume) (orchprovider.PolicyOps, bool, error) { + err := k.setVolume(vol) + if err != nil { + return nil, true, err + } + + err = k.setK8sUtil(&k8sUtil{ + volume: vol, + }) + if err != nil { + return nil, true, err + } + + return k, true, nil +} + +// FetchPolicies will fetch volume policies based on the volume +func (k *k8sOrchestrator) FetchPolicies() (map[string]string, error) { + kc, supported, err := k.k8sUtil.K8sClientV2() + if err != nil { + return nil, err + } + + if !supported { + return nil, fmt.Errorf("K8s client is not supported") + } + + // fetch k8s Deployment operator + scOps, err := kc.StorageClassOps() + if err != nil { + return nil, err + } + + sc, err := scOps.Get(k.volume.Labels.K8sStorageClass, metav1.GetOptions{}) + if err != nil { + return nil, err + } + + return sc.Parameters, nil +} + // AddStorage will add persistent volume running as containers. In OpenEBS // terms AddStorage will add a VSM. func (k *k8sOrchestrator) AddStorage(volProProfile volProfile.VolumeProvisionerProfile) (*v1.Volume, error) { @@ -821,7 +886,7 @@ func (k *k8sOrchestrator) createReplicaDeployment(volProProfile volProfile.Volum // Considering above scenarios, it might make more sense to have // separate K8s Deployment for each replica. However, // there are dis-advantages in diverging from K8s replica set. - TopologyKey: v1.GetPVPReplicaTopologyKey(vol.Labels), + TopologyKey: v1.GetPVPReplicaTopologyKey(nil), }, }, }, @@ -833,7 +898,7 @@ func (k *k8sOrchestrator) createReplicaDeployment(volProProfile volProfile.Volum Name: vsm + string(v1.ReplicaSuffix) + string(v1.ContainerSuffix), Image: rImg, Command: v1.JivaReplicaCmd, - Args: v1.MakeOrDefJivaReplicaArgs(vol.Labels, clusterIP), + Args: v1.MakeOrDefJivaReplicaArgs(vol, clusterIP), Ports: []k8sApiV1.ContainerPort{ k8sApiV1.ContainerPort{ ContainerPort: v1.DefaultJivaReplicaPort1(), @@ -1100,7 +1165,7 @@ func (k *k8sOrchestrator) getReplicaPods(vsm string, podOps k8sCoreV1.PodInterfa return rp, nil } -// getPods deletes the Pods w.r.t the VSM +// getPods gets the Pods w.r.t the VSM func (k *k8sOrchestrator) getPods(vsm string, volProProfile volProfile.VolumeProvisionerProfile) (*k8sApiV1.PodList, error) { if strings.TrimSpace(vsm) == "" { diff --git a/orchprovider/k8s/v1/k8s_test.go b/orchprovider/k8s/v1/k8s_test.go index 1324a1c85a..83846745d2 100644 --- a/orchprovider/k8s/v1/k8s_test.go +++ b/orchprovider/k8s/v1/k8s_test.go @@ -1,4 +1,4 @@ -package k8s +package v1 import ( "errors" @@ -117,7 +117,9 @@ func TestAddStorage(t *testing.T) { for _, c := range cases { - vol := &v1.Volume{} + vol := &v1.Volume{ + Namespace: "something", + } vol.Name = c.vsmname volP, _ := volProfile.GetDefaultVolProProfile(vol) @@ -176,17 +178,17 @@ func (m *mockK8sOrch) GetK8sUtil(volProfile volProfile.VolumeProvisionerProfile) // mockK8sUtil is instantiated based on a 'Value Based Test' record/row return &mockK8sUtil{ - name: vol.Labels[string(testK8sUtlNameLbl)], - vsmName: vol.Name, - kcSupport: vol.Labels[string(testK8sClientSupportLbl)], - ns: vol.Labels[string(v1.OrchNSLbl)], - injectNSErr: vol.Labels[string(testK8sInjectNSErrLbl)], - inCluster: vol.Labels[string(testK8sInClusterLbl)], - injectInClusterErr: vol.Labels[string(testK8sInjectInClusterErrLbl)], - injectPodErr: vol.Labels[string(testK8sInjectPodErrLbl)], - injectSvcErr: vol.Labels[string(testK8sInjectSvcErrLbl)], - injectVsm: vol.Labels[string(testK8sInjectVSMLbl)], - resultingErr: vol.Labels[string(testK8sErrorLbl)], + //name: vol.Labels[string(testK8sUtlNameLbl)], + vsmName: vol.Name, + //kcSupport: vol.Labels[string(testK8sClientSupportLbl)], + //ns: vol.Labels[string(v1.OrchNSLbl)], + //injectNSErr: vol.Labels[string(testK8sInjectNSErrLbl)], + //inCluster: vol.Labels[string(testK8sInClusterLbl)], + //injectInClusterErr: vol.Labels[string(testK8sInjectInClusterErrLbl)], + //injectPodErr: vol.Labels[string(testK8sInjectPodErrLbl)], + //injectSvcErr: vol.Labels[string(testK8sInjectSvcErrLbl)], + //injectVsm: vol.Labels[string(testK8sInjectVSMLbl)], + //resultingErr: vol.Labels[string(testK8sErrorLbl)], } } @@ -228,6 +230,14 @@ func (m *mockK8sUtil) K8sClient() (K8sClient, bool) { return nil, false } +func (m *mockK8sUtil) K8sClientV2() (K8sClientV2, bool, error) { + if m.kcSupport == "true" { + return nil, true, nil + } else { + return nil, false, nil + } +} + func (m *mockK8sUtil) IsInCluster() (bool, error) { if m.injectInClusterErr != "" { return false, errors.New(m.injectInClusterErr) @@ -575,7 +585,7 @@ type okCreateReplicaPodVolumeProfile struct { // Volume does not return any error func (e *okCreateReplicaPodVolumeProfile) Volume() (*v1.Volume, error) { vol := &v1.Volume{} - vol.Labels = map[string]string{} + //vol.Labels = map[string]string{} return vol, nil } diff --git a/orchprovider/k8s/v1/util.go b/orchprovider/k8s/v1/util.go index 8797a0a6fd..7efa496c30 100644 --- a/orchprovider/k8s/v1/util.go +++ b/orchprovider/k8s/v1/util.go @@ -1,4 +1,4 @@ -package k8s +package v1 import ( "fmt" @@ -30,12 +30,13 @@ type K8sUtilInterface interface { // K8sClient fetches an instance of K8sClients. Will return // false if the util does not support providing K8sClients instance. K8sClient() (K8sClient, bool) + + // K8sClientV2 fetches an instance of K8sClientV2. + K8sClientV2() (K8sClientV2, bool, error) } +// TODO Deprecate in favour of K8sClientV2 // K8sClient is an abstraction to operate on various k8s entities. -// -// NOTE: -// This abstraction makes use of K8s's client-go package. type K8sClient interface { // IsInCluster indicates whether the operation is within cluster or in a // different cluster @@ -58,6 +59,16 @@ type K8sClient interface { // DeploymentOps provides all the CRUD operations associated w.r.t a Deployment DeploymentOps() (k8sExtnsV1Beta1.DeploymentInterface, error) +} + +// K8sClientV2 is an abstraction to operate on various k8s entities. +type K8sClientV2 interface { + // IsInClusterV2 indicates whether the operation is within cluster or in a + // different cluster + IsInClusterV2() (bool, error) + + // NSV2 provides the namespace where operations will be executed + NSV2() (string, error) // StorageClassOps provides all the CRUD & more operations associated // w.r.t a StorageClass @@ -70,6 +81,10 @@ type K8sClient interface { // 2. k8s.K8sClients interface type k8sUtil struct { + // namespace refers to K8s namespace where this operation + // will be performed + namespace string + // inCS refers to the ClientSet capable of communicating // within the current K8s cluster i.e. where this binary is // running @@ -86,8 +101,13 @@ type k8sUtil struct { clientKey string insecure bool + // TODO Deprecate in favour of volume // volProfile has context related information w.r.t k8s volProfile volProfile.VolumeProvisionerProfile + + // volume represents an OpenEBS volume which will be + // placed/updated in K8s + volume *v1.Volume } // This is a plain k8s utility & hence the name @@ -160,10 +180,6 @@ func (k *k8sUtil) IsInCluster() (bool, error) { // Pods is a utility function that provides a instance capable of // executing various k8s pod related operations. func (k *k8sUtil) Pods() (k8sCoreV1.PodInterface, error) { - - // TODO - // use getClientSet instead of below repeated code - // refer to StorageClassOps var cs *kubernetes.Clientset inC, err := k.IsInCluster() @@ -192,10 +208,6 @@ func (k *k8sUtil) Pods() (k8sCoreV1.PodInterface, error) { // Services is a utility function that provides a instance capable of // executing various k8s service related operations. func (k *k8sUtil) Services() (k8sCoreV1.ServiceInterface, error) { - - // TODO - // use getClientSet instead of below repeated code - // refer to StorageClassOps var cs *kubernetes.Clientset inC, err := k.IsInCluster() @@ -224,10 +236,6 @@ func (k *k8sUtil) Services() (k8sCoreV1.ServiceInterface, error) { // Services is a utility function that provides a instance capable of // executing various k8s Deployment related operations. func (k *k8sUtil) DeploymentOps() (k8sExtnsV1Beta1.DeploymentInterface, error) { - - // TODO - // use getClientSet instead of below repeated code - // refer to StorageClassOps var cs *kubernetes.Clientset inC, err := k.IsInCluster() @@ -253,6 +261,45 @@ func (k *k8sUtil) DeploymentOps() (k8sExtnsV1Beta1.DeploymentInterface, error) { return cs.ExtensionsV1beta1().Deployments(ns), nil } +// k8sUtil implements K8sClientV2 interface. Hence it returns +// self +func (k *k8sUtil) K8sClientV2() (K8sClientV2, bool, error) { + + if k.volume == nil { + return nil, true, fmt.Errorf("Volume is not set") + } + + return k, true, nil +} + +// NSV2 provides the namespace where operations will be executed +func (k *k8sUtil) NSV2() (string, error) { + if k.namespace != "" { + return k.namespace, nil + } + + k.namespace = k.volume.Namespace + + // error out if still empty + if k.namespace == "" { + return "", fmt.Errorf("Namespace is empty") + } + + return k.namespace, nil +} + +// InCluster indicates whether the operation is within cluster or in a +// different cluster +func (k *k8sUtil) IsInClusterV2() (bool, error) { + // Which kind of request ? in-cluster or out-of-cluster ? + outCluster := k.volume.Labels.K8sOutCluster + if outCluster == "" { + return true, nil + } + + return false, nil +} + func (k *k8sUtil) StorageClassOps() (storagev1.StorageClassInterface, error) { cs, err := k.getClientSet() @@ -260,12 +307,6 @@ func (k *k8sUtil) StorageClassOps() (storagev1.StorageClassInterface, error) { return nil, err } - // get the required namespace - //ns, err := k.NS() - //if err != nil { - // return nil, err - //} - return cs.StorageV1().StorageClasses(), nil } @@ -285,7 +326,7 @@ func (k *k8sUtil) getClientSet() (*kubernetes.Clientset, error) { } // Else get it fresh for this instance/http request - inC, err := k.IsInCluster() + inC, err := k.IsInClusterV2() if err != nil { return nil, err } diff --git a/orchprovider/k8s/v1/util_test.go b/orchprovider/k8s/v1/util_test.go index 2d7d37ad5a..29bc2bb2d8 100644 --- a/orchprovider/k8s/v1/util_test.go +++ b/orchprovider/k8s/v1/util_test.go @@ -1,4 +1,4 @@ -package k8s +package v1 import ( "fmt" @@ -32,8 +32,10 @@ func TestK8sUtil(t *testing.T) { } // a noop pvc that in turn signals use of defaults - pvc := &v1.Volume{} - pvc.Labels = map[string]string{} + pvc := &v1.Volume{ + Namespace: "default", + } + //pvc.Labels = map[string]string{} volP, _ := volProfile.GetDefaultVolProProfile(pvc) @@ -83,8 +85,10 @@ func TestK8sUtilPods(t *testing.T) { } // a noop pvc that in turn signals use of defaults - pvc := &v1.Volume{} - pvc.Labels = map[string]string{} + pvc := &v1.Volume{ + Namespace: "ok", + } + //pvc.Labels = map[string]string{} volP, _ := volProfile.GetDefaultVolProProfile(pvc) @@ -114,8 +118,10 @@ func TestK8sUtilServices(t *testing.T) { } // a noop pvc that in turn signals use of defaults - pvc := &v1.Volume{} - pvc.Labels = map[string]string{} + pvc := &v1.Volume{ + Namespace: "ok", + } + //pvc.Labels = map[string]string{} volP, _ := volProfile.GetDefaultVolProProfile(pvc) diff --git a/orchprovider/nomad/v1/api.go b/orchprovider/nomad/v1/api.go index b03079db85..c73968fb69 100644 --- a/orchprovider/nomad/v1/api.go +++ b/orchprovider/nomad/v1/api.go @@ -1,7 +1,7 @@ // This file transforms a Nomad scheduler as an orchestration // platform for persistent volume placement. OpenEBS calls this as // placement of storage pod. -package nomad +package v1 import ( "fmt" diff --git a/orchprovider/nomad/v1/doc.go b/orchprovider/nomad/v1/doc.go index 9da55c9fd7..179731fd3b 100644 --- a/orchprovider/nomad/v1/doc.go +++ b/orchprovider/nomad/v1/doc.go @@ -1,3 +1,3 @@ // Package nomad provides Nomad implementation of orchestration provider // that aligns by the interfaces suggested by mayaserver's orchprovider. -package nomad +package v1 diff --git a/orchprovider/nomad/v1/helper_funcs.go b/orchprovider/nomad/v1/helper_funcs.go index 2fed1a3870..d90f1ecbd1 100644 --- a/orchprovider/nomad/v1/helper_funcs.go +++ b/orchprovider/nomad/v1/helper_funcs.go @@ -1,4 +1,4 @@ -package nomad +package v1 import ( "fmt" @@ -37,15 +37,15 @@ func VolToJob(vol *v1.Volume) (*api.Job, error) { return nil, fmt.Errorf("Volume name is missing") } - jivaFEVolSize := v1.GetPVPStorageSize(vol.Labels) + jivaFEVolSize := vol.Capacity jivaBEVolSize := jivaFEVolSize // TODO // ID is same as Name currently // Do we need to think on it ? jobName := helper.StringToPtr(vol.Name) - region := helper.StringToPtr(v1.GetOrchestratorRegion(vol.Labels)) - dc := v1.GetOrchestratorDC(vol.Labels) + region := helper.StringToPtr(v1.GetOrchestratorRegion(nil)) + dc := v1.GetOrchestratorDC(nil) jivaGroupName := "jiva-pod" jivaVolName := vol.Name @@ -58,29 +58,29 @@ func VolToJob(vol *v1.Volume) (*api.Job, error) { feTaskName := "fe" beTaskName := "be" - jivaFeVersion := v1.GetControllerImage(vol.Labels) - jivaNetworkType := v1.GetOrchestratorNetworkType(vol.Labels) + jivaFeVersion := v1.GetControllerImage(nil) + jivaNetworkType := v1.GetOrchestratorNetworkType(nil) - jivaBEPersistentStor := v1.GetPVPPersistentPathOnly(vol.Labels) + jivaBEPersistentStor := v1.GetPVPPersistentPathOnly(nil) - iJivaBECount, err := v1.GetPVPReplicaCountInt(vol.Labels) + iJivaBECount, err := v1.GetPVPReplicaCountInt(nil) if err != nil { return nil, err } - jivaFeIPs, jivaBeIPs, err := v1.GetPVPVSMIPs(vol.Labels) + jivaFeIPs, jivaBeIPs, err := v1.GetPVPVSMIPs(nil) if err != nil { return nil, err } jivaFeIPArr := strings.Split(jivaFeIPs, ",") jivaBeIPArr := strings.Split(jivaBeIPs, ",") - jivaFeSubnet, err := v1.GetOrchestratorNetworkSubnet(vol.Labels) + jivaFeSubnet, err := v1.GetOrchestratorNetworkSubnet(nil) if err != nil { return nil, err } - jivaFeInterface := v1.GetOrchestratorNetworkInterface(vol.Labels) + jivaFeInterface := v1.GetOrchestratorNetworkInterface(nil) // Meta information will be used to: // 1. Persist metadata w.r.t this job diff --git a/orchprovider/nomad/v1/nomad_plug.go b/orchprovider/nomad/v1/nomad_plug.go index 309f89accb..e66ef70805 100644 --- a/orchprovider/nomad/v1/nomad_plug.go +++ b/orchprovider/nomad/v1/nomad_plug.go @@ -2,7 +2,7 @@ // // 1. Generic orchprovider & // 2. Nomad orchprovider -package nomad +package v1 import ( "fmt" @@ -91,6 +91,10 @@ func (n *NomadOrchestrator) Region() string { return n.region } +func (n *NomadOrchestrator) PolicyOps(vol *v1.Volume) (orchprovider.PolicyOps, bool, error) { + return nil, false, nil +} + // StorageOps deals with storage related operations e.g. scheduling, placements, // removal, etc. of persistent volume containers. The low level workings are // delegated to the orchestration provider. @@ -114,7 +118,7 @@ func (n *NomadOrchestrator) ReadStorage(volProProfile volProfile.VolumeProvision return nil, err } - job, err := n.nStorApis.StorageInfo(jobName, vol.Labels) + job, err := n.nStorApis.StorageInfo(jobName, nil) if err != nil { return nil, err } @@ -140,7 +144,7 @@ func (n *NomadOrchestrator) AddStorage(volProProfile volProfile.VolumeProvisione return nil, err } - eval, err := n.nStorApis.CreateStorage(job, vol.Labels) + eval, err := n.nStorApis.CreateStorage(job, nil) if err != nil { return nil, err } @@ -162,7 +166,7 @@ func (n *NomadOrchestrator) DeleteStorage(volProProfile volProfile.VolumeProvisi return false, err } - eval, err := n.nStorApis.DeleteStorage(job, vol.Labels) + eval, err := n.nStorApis.DeleteStorage(job, nil) if err != nil { return false, err diff --git a/orchprovider/nomad/v1/util.go b/orchprovider/nomad/v1/util.go index 221ecc9bff..412cae123f 100644 --- a/orchprovider/nomad/v1/util.go +++ b/orchprovider/nomad/v1/util.go @@ -1,4 +1,4 @@ -package nomad +package v1 import ( "github.com/golang/glog" diff --git a/orchprovider/nomad/v1/util_test.go b/orchprovider/nomad/v1/util_test.go index 5ac995291e..ea86a34e9a 100644 --- a/orchprovider/nomad/v1/util_test.go +++ b/orchprovider/nomad/v1/util_test.go @@ -1,4 +1,4 @@ -package nomad +package v1 // Test the creation of NomadConfig struct // Test if nomadUtil adheres to NomadClients, NomadNetworks & NomadUtilInterface diff --git a/orchprovider/orchestration.go b/orchprovider/orchestration.go index 842fc1c567..d9181cd2cd 100644 --- a/orchprovider/orchestration.go +++ b/orchprovider/orchestration.go @@ -9,12 +9,8 @@ import ( ) // OrchestrationInterface is an interface abstraction of a real orchestrator. -// It represents an abstraction that maya api server expects from its +// It represents an abstraction that serves operations feasible from an // orchestrator. -// -// NOTE: -// OrchestratorInterface should be the only interface that exposes orchestration -// contracts. type OrchestratorInterface interface { // Label assigned against the orchestration provider Label() string @@ -27,11 +23,11 @@ type OrchestratorInterface interface { // StorageOps gets the instance that deals with storage related operations. // Will return false if not supported. - // - // NOTE: - // This is invoked on a per request basis. In other words, every request will - // invoke StorageOps to invoke storage specific operations thereafter. StorageOps() (StorageOps, bool) + + // PolicyOps gets the instance that deals with volume policy related operations. + // Will return false if not supported. + PolicyOps(vol *v1.Volume) (PolicyOps, bool, error) } // StorageOps exposes various storage related operations that deals with @@ -40,24 +36,23 @@ type OrchestratorInterface interface { type StorageOps interface { // AddStorage will add persistent volume running as containers - // - // TODO - // Use VSM as the return type AddStorage(volProProfile volProfile.VolumeProvisionerProfile) (*v1.Volume, error) // DeleteStorage will remove the persistent volume - // - // TODO - // Use VSM as the return type DeleteStorage(volProProfile volProfile.VolumeProvisionerProfile) (bool, error) // ReadStorage will fetch information about the persistent volume - // - // TODO - // Use VSM as the return type ReadStorage(volProProfile volProfile.VolumeProvisionerProfile) (*v1.Volume, error) // ListStorage will list a collection of VSMs in a given context e.g. namespace // if working in a K8s setup, etc. ListStorage(volProProfile volProfile.VolumeProvisionerProfile) (*v1.VolumeList, error) } + +// PolicyOps exposes various volume policy related operations. Volume policies +// influence volume placements, provisioning, backup, etc. decisions. +type PolicyOps interface { + + // FetchPolicies will fetch volume policies based on the passed arguments + FetchPolicies() (map[string]string, error) +} diff --git a/pkg/util/util.go b/pkg/util/util.go index 7650a68e5c..1dbd0e9a52 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -3,6 +3,7 @@ package util import ( "fmt" "os" + "strconv" "strings" "github.com/golang/glog" @@ -44,3 +45,24 @@ func Fatal(msg string) { } os.Exit(1) } + +// StringToInt32 converts a string type to corresponding +// *int32 type +func StringToInt32(val string) (*int32, error) { + n, err := strconv.ParseInt(val, 10, 32) + if err != nil { + return nil, err + } + n32 := int32(n) + return &n32, nil +} + +// StrToInt32 converts a string type to corresponding +// *int32 type +// +// NOTE: +// This swallows the error if any +func StrToInt32(val string) *int32 { + n32, _ := StringToInt32(val) + return n32 +} diff --git a/types/v1/defaults.go b/types/v1/defaults.go new file mode 100644 index 0000000000..0bddd65cb8 --- /dev/null +++ b/types/v1/defaults.go @@ -0,0 +1,53 @@ +/* +Copyright 2017 The OpenEBS Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "github.com/openebs/maya/pkg/util" +) + +// These are a set of defaults +const ( + // DefaultVolumeType contains the default volume type + DefaultVolumeType VolumeType = JivaVolumeType + + // DefaultOrchProvider contains the default orchestrator + DefaultOrchProvider OrchProvider = K8sOrchProvider + + // DefaultNamespace contains the default namespace where + // volume operations will be executed + DefaultNamespace string = "default" + + // DefaultCapacity contains the default volume capacity + DefaultCapacity string = "5G" + + // DefaultJivaControllerImage contains the default jiva controller + // image + DefaultJivaControllerImage string = "openebs/jiva:0.4.0" + + // DefaultJivaReplicaImage contains the default jiva replica image + DefaultJivaReplicaImage string = "openebs/jiva:0.4.0" +) + +var ( + // DefaultJivaReplicas contains the default jiva replica count + DefaultJivaReplicas *int32 = util.StrToInt32("2") + + // DefaultJivaControllers contains the default jiva controller + // count + DefaultJivaControllers *int32 = util.StrToInt32("1") +) diff --git a/types/v1/envs.go b/types/v1/envs.go new file mode 100644 index 0000000000..0062013c9e --- /dev/null +++ b/types/v1/envs.go @@ -0,0 +1,132 @@ +/* +Copyright 2017 The OpenEBS Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "os" + "strings" + + "github.com/openebs/maya/pkg/util" +) + +// ENVKey is a typed variable that holds all environment +// variables +type ENVKey string + +const ( + // VolumeTypeENVK is the ENV key to fetch the volume type + VolumeTypeENVK ENVKey = "OPENEBS_IO_VOLUME_TYPE" + + // OrchProviderENVK is the ENV key to fetch volume's + // orchestration provider + OrchProviderENVK ENVKey = "OPENEBS_IO_ORCH_PROVIDER" + + // K8sStorageClassENVK is the ENV key to fetch volume's storage class + // This is applicable only when K8s is volume's orchestration + // provider + K8sStorageClassENVK ENVKey = "K8S_IO_STORAGE_CLASS" + + // NamespaceENVK is the ENV key to fetch the + // namespace where volume operations will be executed + NamespaceENVK ENVKey = "OPENEBS_IO_NAMESPACE" + + // K8sOutClusterENVK is the ENV key to fetch outside + // K8s cluster information. This K8s cluster is different + // from the current K8s cluster where this binary will + // run. + K8sOutClusterENVK ENVKey = "K8S_IO_OUT_CLUSTER" + + // CapacityENVK is the ENV key to fetch volume's + // capacity value + CapacityENVK ENVKey = "OPENEBS_IO_CAPACITY" + + // JivaReplicasENVK is the ENV key to fetch jiva replica + // count + JivaReplicasENVK ENVKey = "OPENEBS_IO_JIVA_REPLICA_COUNT" + + // JivaControllersENVK is the ENV key to fetch jiva controller + // count + JivaControllersENVK ENVKey = "OPENEBS_IO_JIVA_CONTROLLER_COUNT" + + // JivaReplicaImageENVK is the ENV key to fetch jiva replica + // image + JivaReplicaImageENVK ENVKey = "OPENEBS_IO_JIVA_REPLICA_IMAGE" + + // JivaControllerImageENVK is the ENV key to fetch jiva controller + // image + JivaControllerImageENVK ENVKey = "OPENEBS_IO_JIVA_CONTROLLER_IMAGE" +) + +// VolumeTypeENV will fetch the value of volume type +// from ENV variable if present +func VolumeTypeENV() VolumeType { + val := getEnv(VolumeTypeENVK) + return VolumeType(val) +} + +// OrchProviderENV will fetch the value of volume's orchestrator +// from ENV variable if present +func OrchProviderENV() OrchProvider { + val := getEnv(OrchProviderENVK) + return OrchProvider(val) +} + +func K8sStorageClassENV() string { + val := getEnv(K8sStorageClassENVK) + return val +} + +func NamespaceENV() string { + val := getEnv(NamespaceENVK) + return val +} + +func K8sOutClusterENV() string { + val := getEnv(K8sOutClusterENVK) + return val +} + +func CapacityENV() string { + val := getEnv(CapacityENVK) + return val +} + +func JivaReplicasENV() *int32 { + val := util.StrToInt32(getEnv(JivaReplicasENVK)) + return val +} + +func JivaReplicaImageENV() string { + val := getEnv(JivaReplicaImageENVK) + return val +} + +func JivaControllersENV() *int32 { + val := util.StrToInt32(getEnv(JivaControllersENVK)) + return val +} + +func JivaControllerImageENV() string { + val := getEnv(JivaControllerImageENVK) + return val +} + +// getEnv fetches the environment variable value from the machine's +// environment +func getEnv(envKey ENVKey) string { + return strings.TrimSpace(os.Getenv(string(envKey))) +} diff --git a/types/v1/meta.go b/types/v1/meta.go index f4f81c269c..aeb7c11e92 100644 --- a/types/v1/meta.go +++ b/types/v1/meta.go @@ -174,7 +174,8 @@ type ObjectMeta struct { // and services. // More info: http://kubernetes.io/docs/user-guide/labels // +optional - Labels map[string]string `json:"labels,omitempty" protobuf:"bytes,11,rep,name=labels"` + //Labels map[string]string `json:"labels,omitempty" protobuf:"bytes,11,rep,name=labels"` + Labels Labels `json:"labels,omitempty" protobuf:"bytes,11,rep,name=labels"` // Annotations is an unstructured key value map stored with a resource that may be // set by external tools to store and retrieve arbitrary metadata. They are not @@ -218,6 +219,18 @@ type ObjectMeta struct { ClusterName string `json:"clusterName,omitempty" protobuf:"bytes,15,opt,name=clusterName"` } +type Labels struct { + // OldVolumeLabels contains all the volume policy keys that ensures + // backward compatibility + OldVolumeLabels + + // K8sVolumeLabels contains all the K8s related volume policy keys + K8sVolumeLabels + + // VolumeLabels contains all the volume policy keys + VolumeLabels +} + // Note: // There are two different styles of label selectors used in versioned types: // an older style which is represented as just a string in versioned types, and a diff --git a/types/v1/policies.go b/types/v1/policies.go new file mode 100644 index 0000000000..ce91ef30b5 --- /dev/null +++ b/types/v1/policies.go @@ -0,0 +1,106 @@ +/* +Copyright 2017 The OpenEBS Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +// OldVolumeLabels is a set of labels set against the volume structure +// This is specifically for backward compatibility +type OldVolumeLabels struct { + // CapacityOld contains the volume capacity value + CapacityOld string `json:"volumeprovisioner.mapi.openebs.io/storage-size,omitempty" protobuf:"bytes,1,opt,name=volumeprovisioner.mapi.openebs.io/storage-size"` + + // ReplicaImageOld contains the jiva replica image + ReplicaImageOld string `json:"volumeprovisioner.mapi.openebs.io/replica-image,omitempty" protobuf:"bytes,1,opt,name=volumeprovisioner.mapi.openebs.io/replica-image"` + + // ControllerImageOld contains the jiva controller image + ControllerImageOld string `json:"volumeprovisioner.mapi.openebs.io/controller-image,omitempty" protobuf:"bytes,1,opt,name=volumeprovisioner.mapi.openebs.io/controller-image"` + + // ReplicasOld contains the replica count + // + optional + ReplicasOld *int32 `json:"volumeprovisioner.mapi.openebs.io/replica-count,omitempty" protobuf:"varint,1,opt,name=volumeprovisioner.mapi.openebs.io/replica-count"` + + // ControllersOld contains the controller count + ControllersOld *int32 `json:"volumeprovisioner.mapi.openebs.io/controller-count,omitempty" protobuf:"varint,1,opt,name=volumeprovisioner.mapi.openebs.io/controller-count"` +} + +// K8sVolumeLabels is a typed structure that consists of +// various K8s related info. These are typically used during the +// **registration** phase of volume provisioning (using K8s as its +// orchestration provider). +type K8sVolumeLabels struct { + + // K8sStorageClassEnabled flags if fetching policy from K8s storage + // class is enabled. A value of true implies fetching of volume + // policies from K8s storage class must be undertaken. + // + // NOTE: + // This is an optional setting + K8sStorageClassEnabled bool `json:"k8s.io/storage-class-enabled,omitempty" protobuf:"varint,4,opt,name=k8s.io/storage-class-enabled"` + + // K8sStorageClass contains the name of the K8s storage class + // which will be used during volume operations. A K8s storage + // class will typically have various volume policies set in it. + K8sStorageClass string `json:"k8s.io/storage-class,omitempty" protobuf:"bytes,1,opt,name=k8s.io/storage-class"` + + // K8sOutCluster contains the external K8s cluster information + // where the volume operations will be executed + K8sOutCluster string `json:"k8s.io/out-cluster,omitempty" protobuf:"bytes,1,opt,name=k8s.io/out-cluster"` + + // K8sNamespace contains the K8s namespace where volume operations + // will be executed + K8sNamespace string `json:"k8s.io/namespace,omitempty" protobuf:"bytes,1,opt,name=k8s.io/namespace"` +} + +// VolumeLabels is a typed structure that consists of +// various openebs volume related info. These are typically used +// during the **registration** phase of volume provisioning +type VolumeLabels struct { + // VolumeType contains the openebs volume type + VolumeType VolumeType `json:"openebs.io/volume-type,omitempty" protobuf:"bytes,3,opt,name=openebs.io/volume-type,casttype=VolumeType"` +} + +// VolumeKey is a typed string used to represent openebs +// volume related policy keys. These keys along with their +// values will be fetched from various sources like +// K8s StorageClass, maya.io, bots etc. during volume provisioning. +// The commonality between these different sources are these keys. +type VolumeKey string + +const ( + // CapacityVK is the key to fetch volume capacity + CapacityVK VolumeKey = "openebs.io/capacity" + + // IsK8sServiceVK is the key to fetch a boolean indicating + // if a K8s service is required during volume provisioning + IsK8sServiceVK VolumeKey = "openebs.io/is-k8s-service" + + // K8sTargetKindVK is the key to fetch K8s Kind value. + // It suggests the K8s Kind object a volume is supposed to + // be transformed to. + K8sTargetKindVK VolumeKey = "openebs.io/k8s-target-kind" + + // ReplicaImageVK is the key to fetch the jiva replica image + JivaReplicaImageVK VolumeKey = "openebs.io/jiva-replica-image" + + // JivaControllerImageVK is the key to fetch the jiva controller image + JivaControllerImageVK VolumeKey = "openebs.io/jiva-controller-image" + + // JivaReplicasVK is the key to fetch replica count + JivaReplicasVK VolumeKey = "openebs.io/jiva-replica-count" + + // JivaControllersVK is the key to fetch controller count + JivaControllersVK VolumeKey = "openebs.io/jiva-controller-count" +) diff --git a/types/v1/profile/orchestrator/profile.go b/types/v1/profile/orchestrator/profile.go index e9db2440ce..5b0644a653 100644 --- a/types/v1/profile/orchestrator/profile.go +++ b/types/v1/profile/orchestrator/profile.go @@ -48,21 +48,21 @@ type OrchProviderProfile interface { // TODO // It will decide first based on the provided specifications failing which will // ensure a default profile is returned. -func GetOrchProviderProfile(pvc *v1.Volume) (OrchProviderProfile, error) { - var profileMap map[string]string +func GetOrchProviderProfile(vol *v1.Volume) (OrchProviderProfile, error) { + //var profileMap map[string]string - if pvc != nil && pvc.Labels != nil { - profileMap = pvc.Labels - } else { - profileMap = nil - } + //if pvc != nil && pvc.Labels != nil { + // profileMap = pvc.Labels + //} else { + // profileMap = nil + //} // TODO // This is hard coded to pvcOrchProviderProfile struct // It should be based on inputs/env vars return &pvcOrchProviderProfile{ - pvc: pvc, - profileMap: profileMap, + vol: vol, + profileMap: nil, }, nil } @@ -95,7 +95,7 @@ func GetOrchProviderProfile(pvc *v1.Volume) (OrchProviderProfile, error) { // NOTE: // This is a concrete implementation of orchprovider.VolumeProvisionerProfile type pvcOrchProviderProfile struct { - pvc *v1.Volume + vol *v1.Volume profileMap map[string]string } @@ -133,12 +133,13 @@ func (op *pvcOrchProviderProfile) Name() v1.OrchProviderProfileRegistry { // This method provides a convinient way to access pvc. In other words // orchestration provider profile acts as a wrapper over pvc. func (op *pvcOrchProviderProfile) PVC() (*v1.Volume, error) { - return op.pvc, nil + return op.vol, nil } // NetworkAddr gets the network address in CIDR format func (op *pvcOrchProviderProfile) NetworkAddr() (string, error) { - nAddr := v1.GetOrchestratorNetworkAddr(op.profileMap) + //nAddr := v1.GetOrchestratorNetworkAddr(op.profileMap) + nAddr := v1.GetOrchestratorNetworkAddr(nil) if !nethelper.IsCIDR(nAddr) { return "", fmt.Errorf("Network address not in CIDR format in '%s:%s'", op.Label(), op.Name()) @@ -165,7 +166,10 @@ func (op *pvcOrchProviderProfile) NetworkSubnet() (string, error) { // Get the namespace used at the orchestrator, where the request needs to be // operated on func (op *pvcOrchProviderProfile) NS() (string, error) { - ns := v1.GetOrchestratorNS(op.profileMap) + ns := op.vol.Namespace + if len(ns) == 0 { + return "", fmt.Errorf("Volume namespace is missing") + } return ns, nil } @@ -173,7 +177,8 @@ func (op *pvcOrchProviderProfile) NS() (string, error) { // InCluster indicates if the request to the orchestrator is scoped to the // cluster where this request originated func (op *pvcOrchProviderProfile) InCluster() (bool, error) { - inCluster := v1.GetOrchestratorInCluster(op.profileMap) + //inCluster := v1.GetOrchestratorInCluster(op.profileMap) + inCluster := v1.GetOrchestratorInCluster(nil) return util.CheckTruthy(inCluster), nil } diff --git a/types/v1/types.go b/types/v1/types.go index 10daa0cd9b..2125dd8dc7 100644 --- a/types/v1/types.go +++ b/types/v1/types.go @@ -20,6 +20,21 @@ type Volume struct { // Standard object's metadata ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` + // VolumeType holds the type of this volume + // e.g. Jiva volume type or CStor volume type, etc + VolumeType VolumeType `json:"type,omitempty" protobuf:"bytes,1,opt,name=type,casttype=VolumeType"` + + // OrchProvider holds the container orchestrator that will + // orchestrate OpenEBS volume for its provisioning & other + // requirements + OrchProvider OrchProvider `json:"orchestrator,omitempty" protobuf:"bytes,1,opt,name=orchestrator,casttype=OrchProvider"` + + // Namespace will hold the namespace where this Volume will exist + Namespace string `json:"namespace,omitempty" protobuf:"bytes,1,opt,name=namespace"` + + // Capacity will hold the capacity of this Volume + Capacity string `json:"capacity,omitempty" protobuf:"bytes,1,opt,name=capacity"` + // Specs contains the desired specifications the volume should have. // +optional Specs []VolumeSpec `json:"specs,omitempty" protobuf:"bytes,2,rep,name=specs"` @@ -56,6 +71,9 @@ type VolumeSpec struct { // +optional Replicas *int32 `json:"replicas,omitempty" protobuf:"varint,1,opt,name=replicas"` + // Image represents the container image of this volume + Image string `json:"image,omitempty" protobuf:"bytes,1,opt,name=image"` + // Resources represents the actual resources of the volume Capacity ResourceList // Source represents the location and type of a volume to mount. @@ -69,17 +87,46 @@ type VolumeSpec struct { StorageClassName string `json:"storageClassName,omitempty"` } +// VolumeType defines the OpenEBS volume types that are +// supported by Maya +type VolumeType string + +const ( + // JivaVolumeType represents a jiva volume + JivaVolumeType VolumeType = "jiva" + + // CStorVolumeType represents a cstor volume + //CStorVolumeType VolumeType = "cstor" +) + // VolumeContext defines context of a volume type VolumeContext string const ( // ReplicaVolumeContext represents a volume w.r.t // replica context - ReplicaVolumeContext VolumeContext = "Replica" + ReplicaVolumeContext VolumeContext = "replica" // ControllerVolumeContext represents a volume w.r.t // controller context - ControllerVolumeContext VolumeContext = "Controller" + ControllerVolumeContext VolumeContext = "controller" +) + +// OrchProvider defines the container orchestrators that +// will orchestrate the OpenEBS volumes +type OrchProvider string + +const ( + // K8sOrchProvider represents Kubernetes orchestrator + K8sOrchProvider OrchProvider = "kubernetes" +) + +// K8sKind defines the various K8s Kinds that are understood +// by Maya +type K8sKind string + +const ( + DeploymentKK K8sKind = "deployment" ) // VolumeSource represents the source type of the Openebs volume. diff --git a/types/v1/util.go b/types/v1/util.go index 788304c3c2..be1959f157 100644 --- a/types/v1/util.go +++ b/types/v1/util.go @@ -915,12 +915,13 @@ func DefaultReplicaCount() *int32 { // NOTE: // This utility function does not validate & just returns if not capable of // performing -func MakeOrDefJivaReplicaArgs(profileMap map[string]string, clusterIP string) []string { +func MakeOrDefJivaReplicaArgs(vol *Volume, clusterIP string) []string { if strings.TrimSpace(clusterIP) == "" { return nil } - storSize := GetPVPStorageSize(profileMap) + //storSize := GetPVPStorageSize(profileMap) + storSize := vol.Capacity repArgs := make([]string, len(JivaReplicaArgs)) diff --git a/types/v1/validations.go b/types/v1/validations.go new file mode 100644 index 0000000000..cb9e9f59d5 --- /dev/null +++ b/types/v1/validations.go @@ -0,0 +1,57 @@ +/* +Copyright 2017 The OpenEBS Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +// IsVolumeType verifies if the provided VolumeType is a +// valid volume type or not. +// +// An empty vType is considered as valid whereas an invalid +// valued vType is considered as invalid. +func IsVolumeType(vType VolumeType) bool { + if vType == JivaVolumeType { + return true + } //else if vType == CStorVolumeType { + //return true + //} + + if len(vType) != 0 { + return false + } + + // an empty volume type is valid as it may not have + // been set + return true +} + +// IsOrchProvider verifies if the provided Orchestrator is a +// valid volume orchestrator provider or not. +// +// An empty op is considered as valid whereas an invalid +// valued op is considered as invalid. +func IsOrchProvider(op OrchProvider) bool { + if op == K8sOrchProvider { + return true + } + + if len(op) != 0 { + return false + } + + // an empty volume type is valid as it may not have + // been set + return true +} diff --git a/volume/policies/v1/doc.go b/volume/policies/v1/doc.go new file mode 100644 index 0000000000..0f9c7a518b --- /dev/null +++ b/volume/policies/v1/doc.go @@ -0,0 +1,18 @@ +/* +Copyright 2017 The OpenEBS Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// This package has the implementation logic for policy enforcement +package v1 diff --git a/volume/policies/v1/policy.go b/volume/policies/v1/policy.go new file mode 100644 index 0000000000..42d8be3d5e --- /dev/null +++ b/volume/policies/v1/policy.go @@ -0,0 +1,94 @@ +/* +Copyright 2017 The OpenEBS Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "github.com/openebs/maya/types/v1" +) + +// PolicyInterface provides the contract that can be implemented +// by various volume policy enforcement implementations. +type PolicyInterface interface { + // Enforce the policies on the volume instance + Enforce(volume *v1.Volume) (*v1.Volume, error) +} + +// Policy will enforce policies on an OpenEBS volume. This will be +// the typical instance that will be invoked by volume endpoints. +type Policy struct { + // volume represents the instance against which policies + // will be enforced + volume *v1.Volume + + // policies are a set of policies that will be enforced on + // volume + policies []PolicyInterface +} + +// VolumeAddPolicy provides a policy instance that enforces +// policies during volume provisioning +func VolumeAddPolicy() (*Policy, error) { + // these are the set of policies that will + // be enforced + jkPolicies, err := NewJivaK8sPolicies() + if err != nil { + return nil, err + } + policies := []PolicyInterface{ + &ReqdPolicies{}, + &JivaPolicies{}, + &K8sPolicies{}, + jkPolicies, + } + + return &Policy{ + policies: policies, + }, nil +} + +// VolumeDeletePolicy provides a policy instance that enforces +// policies during volume deletion +func VolumeDeletePolicy() (*Policy, error) { + // these are the set of policies that will + // be enforced + policies := []PolicyInterface{ + &ReqdPolicies{}, + &K8sPolicies{}, + } + + return &Policy{ + policies: policies, + }, nil +} + +func (p *Policy) Enforce(volume *v1.Volume) (*v1.Volume, error) { + p.volume = volume + + // iterate & enforce the policies + // + // NOTE: + // Error in any of these policies will break the chain + for _, pol := range p.policies { + vol, err := pol.Enforce(p.volume) + if err != nil { + return nil, err + } + p.volume = vol + } + + return p.volume, nil +} diff --git a/volume/policies/v1/policy_jiva.go b/volume/policies/v1/policy_jiva.go new file mode 100644 index 0000000000..94d297a359 --- /dev/null +++ b/volume/policies/v1/policy_jiva.go @@ -0,0 +1,131 @@ +/* +Copyright 2017 The OpenEBS Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "fmt" + + "github.com/openebs/maya/types/v1" +) + +// JivaPolicies will enforce policies on an OpenEBS volume. +// +// TIP: +// Read this as Enforce Policies w.r.t Jiva +type JivaPolicies struct { + // volume is the instance on which policies will be enforced + volume *v1.Volume + + // isNoSpecs flags if no volume spec is available + isNoSpecs bool + + // isReplicaSpecAvail flags if replica volume spec + // is available + isReplicaSpecAvail bool + + // isControllerSpecAvail flags if controller volume + // spec is available + isControllerSpecAvail bool +} + +// Enforce will enforce jiva based policies against +// the volume instance +func (p *JivaPolicies) Enforce(volume *v1.Volume) (*v1.Volume, error) { + if volume == nil { + return nil, fmt.Errorf("Nil volume provided for policy enforcement") + } + + // This policy will be executed only if Jiva is the volume type + if volume.VolumeType != v1.JivaVolumeType { + // exit without error + return volume, nil + } + + // set it locally to be used in further operations + p.volume = volume + + // initialize as per jiva volume type + p.init() + + // enforce policies + p.enforce() + + err := p.validate() + if err != nil { + return nil, err + } + + return p.volume, nil +} + +// init intializes volume structure w.r.t jiva volume type +func (p *JivaPolicies) init() { + if len(p.volume.Specs) == 0 { + p.isNoSpecs = true + return + } + + for _, spec := range p.volume.Specs { + if spec.Context == v1.ReplicaVolumeContext { + p.isReplicaSpecAvail = true + } else if spec.Context == v1.ControllerVolumeContext { + p.isControllerSpecAvail = true + } + } + + if !p.isControllerSpecAvail && !p.isReplicaSpecAvail { + p.isNoSpecs = true + } + +} + +// enforce essential policies against the volume +func (p *JivaPolicies) enforce() { + if p.isNoSpecs { + p.volume.Specs = []v1.VolumeSpec{ + v1.VolumeSpec{ + Context: v1.ControllerVolumeContext, + }, + v1.VolumeSpec{ + Context: v1.ReplicaVolumeContext, + }, + } + + return + } + + if !p.isReplicaSpecAvail { + p.volume.Specs = append(p.volume.Specs, v1.VolumeSpec{ + Context: v1.ReplicaVolumeContext, + }) + } + + if !p.isControllerSpecAvail { + p.volume.Specs = append(p.volume.Specs, v1.VolumeSpec{ + Context: v1.ControllerVolumeContext, + }) + } +} + +// validate jiva policies +func (p *JivaPolicies) validate() error { + if len(p.volume.Specs) > 2 { + return fmt.Errorf("Invalid volume specifications were provided") + } + + return nil +} diff --git a/volume/policies/v1/policy_jiva_k8s.go b/volume/policies/v1/policy_jiva_k8s.go new file mode 100644 index 0000000000..02a5b2e69f --- /dev/null +++ b/volume/policies/v1/policy_jiva_k8s.go @@ -0,0 +1,343 @@ +/* +Copyright 2017 The OpenEBS Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "fmt" + + "github.com/openebs/maya/orchprovider" + k8s_v1 "github.com/openebs/maya/orchprovider/k8s/v1" + "github.com/openebs/maya/pkg/util" + "github.com/openebs/maya/types/v1" +) + +// JivaK8sPolicies will enforce policies on an OpenEBS jiva +// volume +// +// TIP: +// Read this as Enforce Jiva based K8s SC Policies +type JivaK8sPolicies struct { + // volume is the instance on which policies will be enforced + volume *v1.Volume + + // orch represents the K8s orchestrator + orch orchprovider.OrchestratorInterface + + // capacity will hold the volume capacity + capacity string + + // repIndex is the index which holds replica volume spec + repIndex int + + // repCount will hold the replica count + repCount *int32 + + // repImage will hold jiva replica image + repImage string + + // ctrlIndex is the index which holds jiva controller spec + ctrlIndex int + + // ctrlCount will hold the jiva controller count + ctrlCount *int32 + + // ctrlImage will hold jiva controller image + ctrlImage string +} + +func NewJivaK8sPolicies() (PolicyInterface, error) { + orch, err := k8s_v1.NewK8sOrchProvider() + if err != nil { + return nil, err + } + + return &JivaK8sPolicies{ + orch: orch, + }, nil +} + +// Enforce will enforce k8s sc based policies against +// the volume instance +func (p *JivaK8sPolicies) Enforce(volume *v1.Volume) (*v1.Volume, error) { + if volume == nil { + return nil, fmt.Errorf("Nil volume provided for policy enforcement") + } + + // This policy will be executed only if this is Jiva volume using K8s as + // its volume orchestrator + if volume.OrchProvider != v1.K8sOrchProvider || volume.VolumeType != v1.JivaVolumeType { + // exit without error + return volume, nil + } + + // set it locally to be used in further operations + p.volume = volume + + // init the policies + err := p.init() + if err != nil { + return nil, err + } + + // enforce the policies against the volume properties + p.enforce() + + // run through validations after enforcement + err = p.validate() + if err != nil { + return nil, err + } + + return p.volume, nil +} + +// init initializes this instance properties from volume +// properties +// +// NOTE: +// The original volume property values should prevail +// over others. Hence, this should be the first invocation +// in the Enforce() method. +func (p *JivaK8sPolicies) init() error { + // direct volume properties prevail over other methods of setting + p.capacity = p.volume.Capacity + + for i, spec := range p.volume.Specs { + if spec.Context == v1.ReplicaVolumeContext { + p.repCount = spec.Replicas + p.repImage = spec.Image + p.repIndex = i + } else if spec.Context == v1.ControllerVolumeContext { + p.ctrlCount = spec.Replicas + p.ctrlImage = spec.Image + p.ctrlIndex = i + } + } + + // init using SC policies + // will merge the un-set properties of this instance + err := p.initWithSC() + if err != nil { + return err + } + + // init using old volume labels + // will merge the un-set properties of this instance + p.initWithOldLabels() + + // init using ENV variables or Defaults + // will merge the un-set properties of this instance + p.initWithENVsAndDefs() + + return nil +} + +// initWithSC fetches the k8s sc policies +// & sets them against this instance's properties +func (p *JivaK8sPolicies) initWithSC() error { + // nothing to do if fetching via storageclass + // is disabled + if !p.volume.Labels.K8sStorageClassEnabled { + return nil + } + + // check if orchestrator is available for operations + // w.r.t K8s StorageClass + if p.orch == nil { + return fmt.Errorf("Nil k8s orchestrator") + } + + // fetch K8s SC based policies + pOrch, supported, err := p.orch.PolicyOps(p.volume) + if err != nil { + return err + } + + if !supported { + return fmt.Errorf("K8s based policy operations is not supported") + } + + // Fetch policies based on storage class name + // + // NOTE: + // StorageClass name would have set previously by + // K8sPolicies against this volume + policies, err := pOrch.FetchPolicies() + if err != nil { + return err + } + + // Marshall these policies against this instance's properties + p.marshall(policies) + + return nil +} + +// marshall extracts the K8s sc based policies to +// corresponding properties of this instance +func (p *JivaK8sPolicies) marshall(policies map[string]string) { + for k, v := range policies { + // volume capacity + if k == string(v1.CapacityVK) && len(p.capacity) == 0 { + p.capacity = v + } + // jiva replica count + if k == string(v1.JivaReplicasVK) && p.repCount == nil { + p.repCount = util.StrToInt32(v) + } + // jiva replica image + if k == string(v1.JivaReplicaImageVK) && len(p.repImage) == 0 { + p.repImage = v + } + // jiva controller count + if k == string(v1.JivaControllersVK) && p.ctrlCount == nil { + p.ctrlCount = util.StrToInt32(v) + } + // jiva controller image + if k == string(v1.JivaControllerImageVK) && len(p.ctrlImage) == 0 { + p.ctrlImage = v + } + } +} + +// initWithOldLabels fetch the volume policies from +// volume's labels property & sets them against this instance's +// properties. +// +// NOTE: +// This is to maintain backward compatibility +func (p *JivaK8sPolicies) initWithOldLabels() { + // volume capacity + if len(p.capacity) == 0 { + p.capacity = p.volume.Labels.CapacityOld + } + // jiva replica count + if p.repCount == nil { + p.repCount = p.volume.Labels.ReplicasOld + } + // jiva replica image + if len(p.repImage) == 0 { + p.repImage = p.volume.Labels.ReplicaImageOld + } + // jiva controller count + if p.ctrlCount == nil { + p.ctrlCount = p.volume.Labels.ControllersOld + } + // jiva controller image + if len(p.ctrlImage) == 0 { + p.ctrlImage = p.volume.Labels.ControllerImageOld + } +} + +// initENVsAndDefs will initialize this instance properties +// using ENV variables or Defaults +func (p *JivaK8sPolicies) initWithENVsAndDefs() { + // possible values for capacity + capVals := []string{ + v1.CapacityENV(), + v1.DefaultCapacity, + } + + // Ensure non-empty value is set + for _, capVal := range capVals { + if len(p.capacity) == 0 { + p.capacity = capVal + } + } + + // possible values for jiva replica count + repCVals := []*int32{ + v1.JivaReplicasENV(), + v1.DefaultJivaReplicas, + } + + // Ensure non-empty value is set + for _, repCVal := range repCVals { + if p.repCount == nil { + p.repCount = repCVal + } + } + + // possible values for jiva replica image + repIVals := []string{ + v1.JivaReplicaImageENV(), + v1.DefaultJivaReplicaImage, + } + + // Ensure non-empty value is set + for _, repIVal := range repIVals { + if len(p.repImage) == 0 { + p.repImage = repIVal + } + } + + // possible values for jiva controller count + ctrlCVals := []*int32{ + v1.JivaControllersENV(), + v1.DefaultJivaControllers, + } + + // Ensure non-empty value is set + for _, ctrlCVal := range ctrlCVals { + if p.ctrlCount == nil { + p.ctrlCount = ctrlCVal + } + } + + // possible values for jiva controller image + ctrlIVals := []string{ + v1.JivaControllerImageENV(), + v1.DefaultJivaControllerImage, + } + + // Ensure non-empty value is set + for _, ctrlIVal := range ctrlIVals { + if len(p.ctrlImage) == 0 { + p.ctrlImage = ctrlIVal + } + } +} + +// enforce essential policies against the volume properties +// from this instance's properties +func (p *JivaK8sPolicies) enforce() { + p.volume.Capacity = p.capacity + + p.volume.Specs[p.repIndex].Replicas = p.repCount + p.volume.Specs[p.repIndex].Image = p.repImage + + p.volume.Specs[p.ctrlIndex].Replicas = p.ctrlCount + p.volume.Specs[p.ctrlIndex].Image = p.ctrlImage +} + +// validate verifies the volume properties that were +// just enforced +func (p *JivaK8sPolicies) validate() error { + if len(p.volume.Capacity) == 0 { + return fmt.Errorf("Nil volume capacity was provided") + } + + if p.volume.Specs[p.repIndex].Replicas == nil { + return fmt.Errorf("Nil or Invalid jiva replica count was provided") + } + + if p.volume.Specs[p.ctrlIndex].Replicas == nil { + return fmt.Errorf("Nil or Invalid jiva controller count was provided") + } + + return nil +} diff --git a/volume/policies/v1/policy_k8s.go b/volume/policies/v1/policy_k8s.go new file mode 100644 index 0000000000..ae6d8a6209 --- /dev/null +++ b/volume/policies/v1/policy_k8s.go @@ -0,0 +1,178 @@ +/* +Copyright 2017 The OpenEBS Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "fmt" + + "github.com/openebs/maya/types/v1" +) + +// K8sPolicies will enforce K8s specific policies on an +// OpenEBS volume. +// +// TIP: +// Read this as Enforce Policies w.r.t K8s +type K8sPolicies struct { + // volume is the instance on which policies will be + // enforced + volume *v1.Volume + + // scEnabled flags if fetching policies from K8s StorageClass + // is enabled + scEnabled bool + + // sc is the K8s StorageClass that will be referred to + // during volume operation + sc string + + // ns is the K8s namespace where volume operation + // will be executed + ns string + + // outCluster is the K8s cluster information that is + // different from current K8s cluster where this volume + // operation is being triggered + outCluster string +} + +// Enforce will enforce k8s based policies against +// the volume instance +func (p *K8sPolicies) Enforce(volume *v1.Volume) (*v1.Volume, error) { + if volume == nil { + return nil, fmt.Errorf("Nil volume provided for policy enforcement") + } + + // This policy will be executed only if K8s is the volume's + // orchestration provider + if volume.OrchProvider != v1.K8sOrchProvider { + // exit without error + return volume, nil + } + + // set it locally to be used in further operations + p.volume = volume + + // initialize as per k8s requirements + p.initSC() + p.initNS() + p.initOutCluster() + + // enforce policies + p.enforce() + + err := p.validate() + if err != nil { + return nil, err + } + + return p.volume, nil +} + +// initSC intializes the storage class +func (p *K8sPolicies) initSC() { + // There is no volume specific property for + // storage class. Hence, Labels' based property + // will prevail over others. + p.scEnabled = p.volume.Labels.K8sStorageClassEnabled + p.sc = p.volume.Labels.K8sStorageClass + + // return if confirmed that fetching via sc is not enabled + if len(p.sc) == 0 && !p.scEnabled { + return + } + + // otherwise enable fetching via sc + p.scEnabled = true + + // possible values for storageclass + scVals := []string{ + v1.K8sStorageClassENV(), + } + + // Ensure non-empty value is set + for _, scval := range scVals { + if len(p.sc) == 0 { + p.sc = scval + } + } +} + +// initSC intializes the storage class +func (p *K8sPolicies) initNS() { + // The volume property will prevail over others + p.ns = p.volume.Namespace + + // possible values for namespace + nsVals := []string{ + p.volume.Labels.K8sNamespace, + v1.NamespaceENV(), + v1.DefaultNamespace, + } + + // Ensure non-empty value is set + for _, nval := range nsVals { + if len(p.ns) == 0 { + p.ns = nval + } + } +} + +// initSC intializes the storage class +func (p *K8sPolicies) initOutCluster() { + // There is no volume specific property for + // out cluster. Hence, Labels' based property + // will prevail over others. + p.outCluster = p.volume.Labels.K8sOutCluster + + // possible values for outcluster + oVals := []string{ + v1.K8sOutClusterENV(), + } + + // Ensure non-empty value is set + for _, oval := range oVals { + if len(p.outCluster) == 0 { + p.outCluster = oval + } + } +} + +// enforce K8s based policies against the volume +func (p *K8sPolicies) enforce() { + // enforce k8s storage class enabled flag + p.volume.Labels.K8sStorageClassEnabled = p.scEnabled + // enforce k8s storage class + p.volume.Labels.K8sStorageClass = p.sc + // enforce volume's namespace + p.volume.Namespace = p.ns + // enforce k8s out cluster info + p.volume.Labels.K8sOutCluster = p.outCluster +} + +// validate verifies the K8s related volume policies +func (p *K8sPolicies) validate() error { + if p.volume.Labels.K8sStorageClassEnabled && len(p.volume.Labels.K8sStorageClass) == 0 { + return fmt.Errorf("K8s storage class cannot be empty") + } + + if len(p.volume.Namespace) == 0 { + return fmt.Errorf("Volume namespace cannot be empty") + } + + return nil +} diff --git a/volume/policies/v1/policy_maya_bot.go b/volume/policies/v1/policy_maya_bot.go new file mode 100644 index 0000000000..6f43726455 --- /dev/null +++ b/volume/policies/v1/policy_maya_bot.go @@ -0,0 +1,17 @@ +/* +Copyright 2017 The OpenEBS Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 diff --git a/volume/policies/v1/policy_maya_io.go b/volume/policies/v1/policy_maya_io.go new file mode 100644 index 0000000000..6f43726455 --- /dev/null +++ b/volume/policies/v1/policy_maya_io.go @@ -0,0 +1,17 @@ +/* +Copyright 2017 The OpenEBS Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 diff --git a/volume/policies/v1/policy_required.go b/volume/policies/v1/policy_required.go new file mode 100644 index 0000000000..29b265e318 --- /dev/null +++ b/volume/policies/v1/policy_required.go @@ -0,0 +1,133 @@ +/* +Copyright 2017 The OpenEBS Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "fmt" + + "github.com/openebs/maya/types/v1" +) + +// ReqdPolicies will enforce policies on an OpenEBS volume. Required +// policies will be extracted from volume's Labels & will be enforced +// into specific properties of the same volume instance. +// +// Required Policies are: +// 1. VolumeType +// 2. OrchProvider +// +// TIP: +// Read this as Enforce Required Policies +type ReqdPolicies struct { + // volume is the instance on which policies will be enforced + volume *v1.Volume + + // volType is an initial property + volType v1.VolumeType + + // orch is an initial property + orch v1.OrchProvider +} + +// Enforce will enforce initial properties against +// the volume instance +func (p *ReqdPolicies) Enforce(volume *v1.Volume) (*v1.Volume, error) { + if volume == nil { + return nil, fmt.Errorf("Nil volume provided for policy enforcement") + } + + p.volume = volume + + // init policies + p.initVolType() + p.initOrchProvider() + + // enforce policies + p.enforce() + + // run through validations + err := p.validate() + if err != nil { + return nil, err + } + + return p.volume, nil +} + +// initVolType initializes the volume type from labels +// ENVs or defaults +func (p *ReqdPolicies) initVolType() { + // set volType from volume property which should + // prevail over other values + p.volType = p.volume.VolumeType + + // possible values for volume type + volTypeVals := []v1.VolumeType{ + p.volume.Labels.VolumeType, + v1.VolumeTypeENV(), + v1.DefaultVolumeType, + } + + // Ensure non-empty value is set + for _, tval := range volTypeVals { + if len(p.volType) == 0 { + p.volType = tval + } + } +} + +// initOrchProvider initializes the orchestrator from labels +// ENVs or defaults +func (p *ReqdPolicies) initOrchProvider() { + // set orch from volume property which should + // prevail over other values + p.orch = p.volume.OrchProvider + + // possible values for orchestrator + orchVals := []v1.OrchProvider{ + v1.OrchProviderENV(), + v1.DefaultOrchProvider, + } + + // Ensure non-empty value is set + for _, oval := range orchVals { + if len(p.orch) == 0 { + p.orch = oval + } + } +} + +// enforce essential policies against the volume +func (p *ReqdPolicies) enforce() { + // Enforce volume type + p.volume.VolumeType = p.volType + // Enforce volume's orchestration provider + p.volume.OrchProvider = p.orch +} + +// validate verifies the required volume policies +func (p *ReqdPolicies) validate() error { + if !v1.IsVolumeType(p.volume.VolumeType) { + return fmt.Errorf("Invalid volume type '%s'", p.volume.VolumeType) + } + + if !v1.IsOrchProvider(p.volume.OrchProvider) { + return fmt.Errorf("Invalid volume orchestrator '%s'", p.volume.OrchProvider) + } + + return nil +} diff --git a/volume/profiles/profiles.go b/volume/profiles/profiles.go index 6292e2c632..416de9f055 100644 --- a/volume/profiles/profiles.go +++ b/volume/profiles/profiles.go @@ -48,7 +48,7 @@ type VolumeProvisionerProfile interface { VSMName() (string, error) // Get the number of controllers - ControllerCount() (int, error) + ControllerCount() (*int32, error) // Gets the controller's image e.g. docker image version. The second return value // indicates if image based replica is supported or not. @@ -157,7 +157,8 @@ func (pp *defVolProProfile) Volume() (*v1.Volume, error) { // e.g. K8s, Nomad, Mesos, Swarm, etc. It can be Docker engine as well. func (pp *defVolProProfile) Orchestrator() (v1.OrchProviderRegistry, bool, error) { // Extract the name of orchestration provider - oName := v1.GetOrchestratorName(pp.vol.Labels) + //oName := v1.GetOrchestratorName(pp.vol.Labels) + oName := v1.GetOrchestratorName(nil) // Get the orchestrator instance return oName, true, nil @@ -209,23 +210,60 @@ func (pp *defVolProProfile) VSMName() (string, error) { } // ControllerCount gets the number of controllers -func (pp *defVolProProfile) ControllerCount() (int, error) { - // Extract the controller count - return v1.GetPVPControllerCountInt(pp.vol.Labels) +func (pp *defVolProProfile) ControllerCount() (*int32, error) { + var rCount *int32 + specs := pp.vol.Specs + + for _, spec := range specs { + if spec.Context == v1.ControllerVolumeContext { + rCount = spec.Replicas + } + } + + if rCount == nil { + return nil, fmt.Errorf("Volume controller count is missing") + } + + return rCount, nil } // ControllerImage gets the controller's image currently its docker image label. func (pp *defVolProProfile) ControllerImage() (string, bool, error) { // Extract the controller image - cImg := v1.GetControllerImage(pp.vol.Labels) + // Extract the replica image + specs := pp.vol.Specs + rImg := "" + + for _, spec := range specs { + if spec.Context == v1.ControllerVolumeContext { + rImg = spec.Image + break + } + } + + if rImg == "" { + return "", true, fmt.Errorf("Volume controller image is missing") + } - return cImg, true, nil + return rImg, true, nil } // ReplicaImage gets the replica's image currently its docker image label. func (pp *defVolProProfile) ReplicaImage() (string, error) { // Extract the replica image - rImg := v1.GetPVPReplicaImage(pp.vol.Labels) + specs := pp.vol.Specs + rImg := "" + + for _, spec := range specs { + if spec.Context == v1.ReplicaVolumeContext { + rImg = spec.Image + break + } + } + + if rImg == "" { + return "", fmt.Errorf("Volume replica image is missing") + } return rImg, nil } @@ -235,7 +273,8 @@ func (pp *defVolProProfile) ReplicaImage() (string, error) { // can return false. func (pp *defVolProProfile) IsControllerNodeTaintTolerations() ([]string, bool, error) { // Extract the node taint toleration for controller - nTTs, err := v1.GetControllerNodeTaintTolerations(pp.vol.Labels) + //nTTs, err := v1.GetControllerNodeTaintTolerations(pp.vol.Labels) + nTTs, err := v1.GetControllerNodeTaintTolerations(nil) if err != nil { return nil, false, err } @@ -256,7 +295,8 @@ func (pp *defVolProProfile) IsControllerNodeTaintTolerations() ([]string, bool, // can return false. func (pp *defVolProProfile) IsReplicaNodeTaintTolerations() ([]string, bool, error) { // Extract the node taint toleration for replica - nTTs, err := v1.GetReplicaNodeTaintTolerations(pp.vol.Labels) + //nTTs, err := v1.GetReplicaNodeTaintTolerations(pp.vol.Labels) + nTTs, err := v1.GetReplicaNodeTaintTolerations(nil) if err != nil { return nil, false, err } @@ -275,10 +315,10 @@ func (pp *defVolProProfile) IsReplicaNodeTaintTolerations() ([]string, bool, err // StorageSize gets the storage size for each persistent volume replica(s) func (pp *defVolProProfile) StorageSize() (string, error) { // Extract the storage size - sSize := v1.GetPVPStorageSize(pp.vol.Labels) + sSize := pp.vol.Capacity - if sSize == "" { - return "", fmt.Errorf("Missing storage size in '%s:%s'", pp.Label(), pp.Name()) + if len(sSize) == 0 { + return "", fmt.Errorf("Volume capacity is missing") } return sSize, nil @@ -286,16 +326,20 @@ func (pp *defVolProProfile) StorageSize() (string, error) { // ReplicaCount get the number of replicas required func (pp *defVolProProfile) ReplicaCount() (*int32, error) { + var rCount *int32 specs := pp.vol.Specs for _, spec := range specs { if spec.Context == v1.ReplicaVolumeContext { - return v1.GetReplicaCount(spec), nil + rCount = spec.Replicas } } - // If you are here, then get from defaults - return v1.GetReplicaCount(v1.VolumeSpec{}), nil + if rCount == nil { + return nil, fmt.Errorf("Volume replica count is missing") + } + + return rCount, nil } // ControllerIPs gets the IP addresses that needs to be assigned against the @@ -305,7 +349,8 @@ func (pp *defVolProProfile) ReplicaCount() (*int32, error) { // There is no default assignment of IPs func (pp *defVolProProfile) ControllerIPs() ([]string, error) { // Extract the controller IPs - cIPs := v1.ControllerIPs(pp.vol.Labels) + //cIPs := v1.ControllerIPs(pp.vol.Labels) + cIPs := v1.ControllerIPs(nil) if cIPs == "" { return nil, nil @@ -327,7 +372,8 @@ func (pp *defVolProProfile) ControllerIPs() ([]string, error) { // There is no default assignment of IPs func (pp *defVolProProfile) ReplicaIPs() ([]string, error) { // Extract the controller IPs - rIPs := v1.ReplicaIPs(pp.vol.Labels) + //rIPs := v1.ReplicaIPs(pp.vol.Labels) + rIPs := v1.ReplicaIPs(nil) if rIPs == "" { return nil, nil @@ -354,7 +400,8 @@ func (pp *defVolProProfile) PersistentPath() (string, error) { } // Extract the persistent path - pPath := v1.GetPVPPersistentPath(pp.vol.Labels, vsm, string(v1.JivaPersistentMountPathDef)) + //pPath := v1.GetPVPPersistentPath(pp.vol.Labels, vsm, string(v1.JivaPersistentMountPathDef)) + pPath := v1.GetPVPPersistentPath(nil, vsm, string(v1.JivaPersistentMountPathDef)) return pPath, nil } From 2a574bacca8c8d8a1302d444a2fa5ce42490dc5a Mon Sep 17 00:00:00 2001 From: amitkumardas Date: Wed, 8 Nov 2017 09:49:30 +0530 Subject: [PATCH 2/2] will fix the issues due to policy enforcement feature --- .../app/server/volume_endpoint.go | 24 ++++++++++++++++++- volume/policies/v1/policy.go | 5 ++-- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/cmd/maya-apiserver/app/server/volume_endpoint.go b/cmd/maya-apiserver/app/server/volume_endpoint.go index 01cf8ae6e1..d44c977ad1 100644 --- a/cmd/maya-apiserver/app/server/volume_endpoint.go +++ b/cmd/maya-apiserver/app/server/volume_endpoint.go @@ -63,6 +63,17 @@ func (s *HTTPServer) volumeList(resp http.ResponseWriter, req *http.Request) (in // Create a Volume vol := &v1.Volume{} + // Pass through the policy enforcement logic + policy, err := policies_v1.VolumeGenericPolicy() + if err != nil { + return nil, err + } + + vol, err = policy.Enforce(vol) + if err != nil { + return nil, err + } + // Get the persistent volume provisioner instance pvp, err := provisioners.GetVolumeProvisioner(nil) if err != nil { @@ -107,6 +118,17 @@ func (s *HTTPServer) volumeRead(resp http.ResponseWriter, req *http.Request, vol vol := &v1.Volume{} vol.Name = volName + // Pass through the policy enforcement logic + policy, err := policies_v1.VolumeGenericPolicy() + if err != nil { + return nil, err + } + + vol, err = policy.Enforce(vol) + if err != nil { + return nil, err + } + // Get persistent volume provisioner instance pvp, err := provisioners.GetVolumeProvisioner(nil) if err != nil { @@ -154,7 +176,7 @@ func (s *HTTPServer) volumeDelete(resp http.ResponseWriter, req *http.Request, v vol.Name = volName // Pass through the policy enforcement logic - policy, err := policies_v1.VolumeDeletePolicy() + policy, err := policies_v1.VolumeGenericPolicy() if err != nil { return nil, err } diff --git a/volume/policies/v1/policy.go b/volume/policies/v1/policy.go index 42d8be3d5e..225d985ddb 100644 --- a/volume/policies/v1/policy.go +++ b/volume/policies/v1/policy.go @@ -61,8 +61,9 @@ func VolumeAddPolicy() (*Policy, error) { } // VolumeDeletePolicy provides a policy instance that enforces -// policies during volume deletion -func VolumeDeletePolicy() (*Policy, error) { +// policies during some of the volume operations other than +// provisioning +func VolumeGenericPolicy() (*Policy, error) { // these are the set of policies that will // be enforced policies := []PolicyInterface{