Merge branch 'master' of github.com:Data-Science-Platform/cluster-bro…

…ccoli into DSP-1547

.gitignore

@@ -8,6 +8,7 @@ target/
 /cluster-broccoli.did
 /nomad.did
 /consul.did
+/couchdb.did
 
 # Instance storage for dev mode
 /instances/

build.sbt

@@ -112,7 +112,7 @@ lazy val root = project
   .settings(
     inThisBuild(
       List(
-        version := "0.9.0-SNAPSHOT",
+        version := "0.9.1-SNAPSHOT",
         scalaVersion := "2.11.11",
         // Enable jcenter for silhouette's dependencies
         resolvers += Resolver.jcenterRepo,

http-api-tests/broccoli-nomad-consul/api-v1-instances-service-status/expected/response-data

@@ -13,18 +13,24 @@
         "id": "cpu",
         "name": "CPU Shares",
         "default": 100,
-        "type": "integer"
+        "type": {
+          "name": "integer"
+        }
       },
       "secret": {
         "id": "secret",
         "name": "A Secret Parameter",
         "default": 123.456,
         "secret": true,
-        "type": "decimal"
+        "type": {
+          "name": "decimal"
+        }
       },
       "id": {
         "id": "id",
-        "type": "string"
+        "type": {
+          "name": "string"
+        }
       }
     },
     "version": "9215ebc3b508fcc0295bffdbfe5695b2"

http-api-tests/broccoli-nomad/api-v1-instances-service-status-unknown/expected/response-data

@@ -13,18 +13,24 @@
         "id": "cpu",
         "name": "CPU Shares",
         "default": 100,
-        "type": "integer"
+        "type": {
+          "name": "integer"
+        }
       },
       "secret": {
         "id": "secret",
         "name": "A Secret Parameter",
         "default": 123.456,
         "secret": true,
-        "type": "decimal"
+        "type": {
+          "name": "decimal"
+        }
       },
       "id": {
         "id": "id",
-        "type": "string"
+        "type": {
+          "name": "string"
+        }
       }
     },
     "version": "9215ebc3b508fcc0295bffdbfe5695b2"

http-api-tests/broccoli-only/api-v1-instances-edit-parameters-200/expected/response-data

@@ -13,18 +13,24 @@
         "id": "cpu",
         "name": "CPU Shares",
         "default": 100,
-        "type": "integer"
+        "type": {
+          "name": "integer"
+        }
       },
       "secret": {
         "id": "secret",
         "name": "A Secret Parameter",
         "default": 123.456,
         "secret": true,
-        "type": "decimal"
+        "type": {
+          "name": "decimal"
+        }
       },
       "id": {
         "id": "id",
-        "type": "string"
+        "type": {
+          "name": "string"
+        }
       }
     },
     "version": "9215ebc3b508fcc0295bffdbfe5695b2"

http-api-tests/broccoli-only/api-v1-instances-edit-template-200/expected/response-data

@@ -15,7 +15,9 @@
     "parameterInfos": {
       "id": {
         "id": "id",
-        "type": "string"
+        "type": {
+          "name": "string"
+        }
       }
     },
     "version": "beb88e5f9ca22d3b2d9fa0e91c2bb255"

http-api-tests/broccoli-only/api-v1-instances-show-after-create/expected/response-data

@@ -13,18 +13,24 @@
         "id": "cpu",
         "name": "CPU Shares",
         "default": 100,
-        "type": "integer"
+        "type": {
+          "name": "integer"
+        }
       },
       "secret": {
         "id": "secret",
         "name": "A Secret Parameter",
         "default": 123.456,
         "secret": true,
-        "type": "decimal"
+        "type": {
+          "name": "decimal"
+        }
       },
       "id": {
         "id": "id",
-        "type": "string"
+        "type": {
+          "name": "string"
+        }
        }
     },
     "version": "9215ebc3b508fcc0295bffdbfe5695b2"

http-api-tests/broccoli-only/api-v1-instances-show-after-edit/expected/response-data

@@ -13,18 +13,24 @@
         "id": "cpu",
         "name": "CPU Shares",
         "default": 100,
-        "type": "integer"
+        "type": {
+          "name": "integer"
+        }
       },
       "secret": {
         "id": "secret",
         "name": "A Secret Parameter",
         "default": 123.456,
         "secret": true,
-        "type": "decimal"
+        "type": {
+          "name": "decimal"
+        }
       },
       "id": {
         "id": "id",
-        "type": "string"
+        "type": {
+          "name": "string"
+        }
       }
     },
     "version": "9215ebc3b508fcc0295bffdbfe5695b2"

http-api-tests/broccoli-only/api-v1-templates-list/expected/response-data

@@ -5,27 +5,41 @@
     "parameters": [
       "URL",
       "enabled",
-      "id"
+      "id",
+      "retries"
     ],
     "parameterInfos": {
       "id": {
         "id": "id",
-        "type": "string"
+        "type": {
+          "name": "string"
+        }
       },
       "enabled": {
         "id": "enabled",
         "default": "true",
-        "type": "raw",
+        "type": {
+          "name": "raw"
+        },
         "orderIndex": 2
       },
       "URL": {
         "id": "URL",
         "default": "localhost:8000",
-        "type": "raw",
+        "type": {
+          "name": "raw"
+        },
         "orderIndex": 1
+      },
+      "retries": {
+        "id": "retries",
+        "type": {
+          "name": "intList",
+          "values": [1, 2, 3, 4, 5]
+        }
       }
     },
-    "version": "149d17a53dba4820b3dff125b35094bf"
+    "version": "68d3dd229390c3171e6a77b836ad2406"
   },
   {
     "id": "http-server",
@@ -40,18 +54,24 @@
         "id": "cpu",
         "name": "CPU Shares",
         "default": 100,
-        "type": "integer"
+        "type": {
+          "name": "integer"
+        }
       },
       "secret": {
         "id": "secret",
         "name": "A Secret Parameter",
         "default": 123.456,
         "secret": true,
-        "type": "decimal"
+        "type": {
+          "name": "decimal"
+        }
       },
       "id": {
         "id": "id",
-        "type": "string"
+        "type": {
+          "name": "string"
+        }
       }
     },
     "version": "9215ebc3b508fcc0295bffdbfe5695b2"
@@ -65,7 +85,9 @@
     "parameterInfos": {
       "id": {
         "id": "id",
-        "type": "string"
+        "type": {
+          "name": "string"
+        }
       }
     },
     "version": "beb88e5f9ca22d3b2d9fa0e91c2bb255"

http-api-tests/broccoli-only/api-v1-templates-show/expected/response-data

@@ -7,7 +7,9 @@
   "parameterInfos": {
     "id": {
       "id": "id",
-      "type": "string"
+      "type": {
+        "name": "string"
+      }
     }
   },
   "version": "beb88e5f9ca22d3b2d9fa0e91c2bb255"

script/instances-0.9.0-to-0.9.1.py

@@ -0,0 +1,45 @@
+import json
+import glob
+import argparse
+import os
+from copy import deepcopy
+
+
+def update_parameter_infos(parameter_infos):
+    updated_parameters = deepcopy(parameter_infos)
+    info_variables = set(parameter_infos.keys())
+    for variable in info_variables:
+        if type(updated_parameters[variable]['type']) == str:
+            old_type = updated_parameters[variable]['type']
+            updated_parameters[variable]['type'] = {"name": old_type}
+        else:
+            print("not changing type for " + variable)
+    return updated_parameters
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("instance_directory")
+
+    opts = parser.parse_args()
+
+    for instance_path in glob.glob(os.path.join(opts.instance_directory, "*.json")):
+        print("Processing %s" % instance_path)
+        instance = json.loads(open(instance_path).read())
+
+        assert "template" in instance
+        template = instance["template"]
+        assert "template" in template
+        assert "parameterInfos" in template
+        parameter_infos = template["parameterInfos"]
+
+        complete_parameter_infos = update_parameter_infos(parameter_infos)
+
+        parameter_infos_changed = False
+        if parameter_infos != complete_parameter_infos:
+            parameter_infos_changed = True
+            template["parameterInfos"] = complete_parameter_infos
+
+        if parameter_infos_changed:
+            print("Overwritting instance: %s" % instance_path)
+            open(instance_path, "w+").write(json.dumps(instance))

script/templates-0.9.0-to-0.9.1.py

@@ -0,0 +1,65 @@
+import os
+import argparse
+import json
+from copy import deepcopy
+from pyhocon import ConfigFactory, HOCONConverter
+
+
+def update_parameters(parameters):
+    updated_parameters = deepcopy(parameters)
+    info_variables = set(updated_parameters.keys())
+    for variable in info_variables:
+        if type(updated_parameters[variable]['type']) == str:
+            old_type = updated_parameters[variable]['type']
+            updated_parameters[variable]['type'] = ConfigFactory.from_dict({"name": old_type})
+        else:
+            print("not changing type for " + variable)
+    return updated_parameters
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("template_directory")
+    parser.add_argument("output_format", choices=["json", "hocon"])
+
+    opts = parser.parse_args()
+
+    for directory_name in os.listdir(os.path.join(opts.template_directory)):
+        if not os.path.isdir(os.path.join(opts.template_directory, directory_name)):
+            print(directory_name + " was not a directory. Skipping.")
+            continue
+
+        directory_path = os.path.join(opts.template_directory, directory_name)
+        template_path = os.path.join(directory_path, "template.json")
+        config_path = os.path.join(directory_path, "template.conf")
+        if not os.path.isfile(template_path):
+            print("%s is not a template directory (template file is missing)" % directory_path)
+            continue
+
+        if not os.path.isfile(config_path):
+            print("%s is not a template directory (config file is missing)" % directory_path)
+            continue
+
+        template = open(template_path).read()
+        config = ConfigFactory.parse_file(config_path)
+        if "parameters" not in config:
+            config["parameters"] = {}
+
+        complete_parameters = update_parameters(config["parameters"])
+        parameters_changed = False
+        if config["parameters"] != complete_parameters:
+            parameters_changed = True
+            config["parameters"] = complete_parameters
+
+        if parameters_changed:
+            print("Overwriting template.conf: %s" % config_path)
+            # We can use the HOCONConverter for dumping the json as well but we have more options to control
+            # the output with the default json writer
+            if opts.output_format == "hocon":
+                open(config_path, "w+").write(HOCONConverter.convert(config, "hocon"))
+            else:
+                open(config_path, "w+").write(
+                    # This is a config tree but json.dumps uses iterators to go over it thinking it is a dict.
+                    # Since configtree already supports these iterators it works correctly for it.
+                    json.dumps(config, indent=2, separators=(',', ': ', ), ensure_ascii=False) + "\n"
+                )

server/src/main/scala/de/frosner/broccoli/auth/Account.scala

@@ -10,7 +10,9 @@ import play.api.libs.json.{Json, OFormat}
   * @param instanceRegex A regex matching instance names this account is allowed to access
   * @param role The role of the user
   */
-final case class Account(name: String, instanceRegex: String, role: Role) extends Identity
+final case class Account(name: String, instanceRegex: String, role: Role) extends Identity {
+  def getOEPrefix: String = name.split("-")(0)
+}
 
 object Account {
   implicit val accountFormat: OFormat[Account] = Json.format[Account]