Initial attempt at security group implementation (untested)

Signed-off-by: Shea Levy <[email protected]>

nix/ec2-security-group.nix

@@ -18,13 +18,19 @@ with pkgs.lib;
       description = "Informational description of the security group";
     };
 
+    region = mkOption {
+      type = types.uniq types.string;
+      description = "Amazon EC2 region.";
+    };
+
     accessKeyId = mkOption {
       type = types.uniq types.string;
       description = "The AWS Access Key ID.";
     };
 
     groupId = mkOption {
-      type = types.uniq types.string;
+      default = null;
+      type = types.uniq (types.nullOr types.string);
       description = "The security group ID. This is set by NixOps.";
     };
 
@@ -40,40 +46,47 @@ with pkgs.lib;
         };
 
         fromPort = mkOption {
+          default = null;
           description = "The bottom of the allowed port range for this rule (TCP/UDP only)";
-          type = types.uniq types.int;
+          type = types.uniq (types.nullOr types.int);
         };
 
         toPort = mkOption {
+          default = null;
           description = "The top of the allowed port range for this rule (TCP/UDP only)";
-          type = types.uniq types.int;
+          type = types.uniq (types.nullOr types.int);
         };
 
         typeNumber = mkOption {
+          default = null;
           description = "ICMP type number (ICMP only, -1 for all)";
-          type = types.uniq types.int;
+          type = types.uniq (types.nullOr types.int);
         };
 
         codeNumber = mkOption {
+          default = null;
           description = "ICMP code number (ICMP only, -1 for all)";
-          type = types.uniq types.int;
+          type = types.uniq (types.nullOr types.int);
         };
 
         sourceGroup = {
-          userId = mkOption {
+          ownerId = mkOption {
+            default = null;
             description = "The AWS account ID that owns the source security group";
-            type = types.uniq types.string;
+            type = types.uniq (types.nullOr types.string);
           };
 
           groupName = mkOption {
+            default = null;
             description = "The name of the source security group (if allowing all instances in a group access instead of an IP range)";
-            type = types.uniq types.string;
+            type = types.uniq (types.nullOr types.string);
           };
         };
 
         sourceIp = mkOption {
+          default = null;
           description = "The source IP range (CIDR notation)";
-          type = types.uniq types.string;
+          type = types.uniq (types.nullOr types.string);
         };
       };
     };

nixops/backends/__init__.py

@@ -361,6 +361,7 @@ def __init__(self):
 import nixops.resources.sqs_queue
 import nixops.resources.s3_bucket
 import nixops.resources.iam_role
+import nixops.resources.ec2_security_group
 
 def create_definition(xml):
     """Create a machine definition object from the given XML representation of the machine's attributes."""
@@ -383,7 +384,8 @@ def create_state(depl, type, name, id):
               nixops.resources.ssh_keypair.SSHKeyPairState,
               nixops.resources.sqs_queue.SQSQueueState,
               nixops.resources.iam_role.IAMRoleState,
-              nixops.resources.s3_bucket.S3BucketState]:
+              nixops.resources.s3_bucket.S3BucketState
+              nixops.resources.ec2_security_group.EC2SecurityGroupState]:
         if type == i.get_type():
             return i(depl, name, id)
     raise nixops.deployment.UnknownBackend("unknown backend type ‘{0}’".format(type))

nixops/backends/ec2.py

@@ -436,9 +436,11 @@ def create_after(self, resources):
         # EC2 instances can require key pairs and IAM roles.  FIXME:
         # only depend on the specific key pair / role needed for this
         # instance.
+        # Ditto for security groups
         return {r for r in resources if
                 isinstance(r, nixops.resources.ec2_keypair.EC2KeyPairState) or
-                isinstance(r, nixops.resources.iam_role.IAMRoleState)}
+                isinstance(r, nixops.resources.iam_role.IAMRoleState) or
+                isinstance(r, nixops.resources.ec2_security_group.EC2SecurityGroupState}
 
 
     def attach_volume(self, device, volume_id):

nixops/deployment.py

@@ -315,6 +315,10 @@ def evaluate(self):
             defn = nixops.resources.s3_bucket.S3BucketDefinition(x)
             self.definitions[defn.name] = defn
 
+        for x in res.find("attr[@name='ec2SecurityGroups']/attrs").findall("attr"):
+            defn = nixops.resources.ec2_security_groups.EC2SecurityGroupDefinition(x)
+            self.definitions[defn.name] = defn
+
 
     def evaluate_option_value(self, machine_name, option_name, xml=False, include_physical=False):
         """Evaluate a single option of a single machine in the deployment specification."""

nixops/resources/__init__.py

@@ -9,6 +9,7 @@ class ResourceDefinition(object):
 
     @classmethod
     def get_type(cls):
+        """A resource type identifier that must match the corresponding ResourceState class"""
         assert False
 
     def __init__(self, xml):
@@ -18,6 +19,7 @@ def __init__(self, xml):
             raise Exception("invalid resource name ‘{0}’".format(self.name))
 
     def show_type(self):
+        """A short description of the type of resource this is"""
         return self.get_type()
 
 
@@ -26,6 +28,7 @@ class ResourceState(object):
 
     @classmethod
     def get_type(cls):
+        """A resource type identifier that must match the corresponding ResourceDefinition clsas"""
         assert False
 
     # Valid values for self.state.  Not all of these make sense for
@@ -80,6 +83,7 @@ def _get_attr(self, name, default=nixops.util.undefined):
             return nixops.util.undefined
 
     def export(self):
+        """Export the resource to move between databases"""
         with self.depl._db:
             c = self.depl._db.cursor()
             c.execute("select name, value from ResourceAttrs where machine = ?", (self.id,))
@@ -89,6 +93,7 @@ def export(self):
             return res
 
     def import_(self, attrs):
+        """Import the resource from another database"""
         with self.depl._db:
             for k, v in attrs.iteritems():
                 if k == 'type': continue
@@ -103,9 +108,11 @@ def import_(self, attrs):
     success = lambda s, m: s.logger.success(m)
 
     def show_type(self):
+        """A short description of the type of resource this is"""
         return self.get_type()
 
     def show_state(self):
+        """A description of the resource's current state"""
         state = self.state
         if state == self.UNKNOWN: return "Unknown"
         elif state == self.MISSING: return "Missing"
@@ -118,16 +125,20 @@ def show_state(self):
         else: raise Exception("machine is in unknown state")
 
     def prefix_definiton(self, attr):
+        """Prefix the resource set with a py2nixable attrpath"""
         raise Exception("not implemented")
 
     def get_physical_spec(self):
+        """py2nixable physical specification of the resource to be fed back into the network"""
         return {}
 
     def get_physical_backup_spec(self, backupid):
+        """py2nixable physical specification of the specified backup"""
         return []
 
     @property
     def resource_id(self):
+        """A unique ID to display for this resource"""
         return None
 
     def create_after(self, resources):
@@ -139,6 +150,7 @@ def create(self, defn, check, allow_reboot, allow_recreate):
         assert False
 
     def after_activation(self, defn):
+        """Actions to be performed after the network is activated"""
         return
 
     def destroy(self, wipe=False):

nixops/resources/ec2_security_group.py

@@ -0,0 +1,194 @@
+# -*- coding: utf-8 -*-
+
+# Automatic provisioning of EC2 security groups.
+
+import boto.ec2.securitygroup
+import nixops.resources
+import nixops.util
+import nixops.ec2_utils
+
+
+class EC2SecurityGroupDefinition(nixops.resources.ResourceDefinition):
+    """Definition of an EC2 security group."""
+
+    @classmethod
+    def get_type(cls):
+        return "ec2-security-group"
+
+    def __init__(self, xml):
+        super(EC2SecurityGroupDefinition, self).__init__(xml)
+        self.security_group_name = xml.find("attrs/attr[@name='name']/string").get("value")
+        self.security_group_description = xml.find("attrs/attr[@name='description']/string").get("value")
+        self.region = xml.find("attrs/attr[@name='region']/string").get("value")
+        self.access_key_id = xml.find("attrs/attr[@name='accessKeyId']/string").get("value")
+        self.rules = []
+        for rule_xml in xml.findall("attrs/attr[@name='rules']/list/attrs"):
+            ip_protocol = rule_xml.find("attrs/attr[@name='protocol']/string").get("value")
+            if ip_protocol == "icmp":
+                from_port = int(rule_xml.find("attrs/attr[@name='typeNumber']/int").get("value"))
+                to_port = int(rule_xml.find("attrs/attr[@name='codeNumber']/int").get("value"))
+            else:
+                from_port = int(rule_xml.find("attrs/attr[@name='fromPort']/int").get("value"))
+                to_port = int(rule_xml.find("attrs/attr[@name='toPort']/int").get("value"))
+            cidr_ip_xml = rule_xml.find("attrs/attr[@name='sourceIp']/string")
+            if not cidr_ip_xml is None:
+                self.rules.append([ ip_protocol, from_port, to_port, cidr_ip_xml.get("value") ])
+            else:
+                group_name = rule_xml.find("attrs/attr[@name='sourceGroup']/attrs/attr[@name='groupName']/string").get("value")
+                owner_id = rule_xml.find("attrs/attr[@name='sourceGroup']/attrs/attr[@name='ownerId']/string").get("value")
+                self.rules.append([ ip_protocol, from_port, to_port, group_name, owner_id ])
+
+
+    def show_type(self):
+        return "{0} [{1}]".format(self.get_type(), self.region)
+
+class EC2SecurityGroupState(nixops.resources.ResourceState):
+    """State of an EC2 security group."""
+
+    region = nixops.util.attr_property("ec2.region", None)
+    security_group_id = nixops.util.attr_property("ec2.securityGroupId", None)
+    security_group_name = nixops.util.attr_property("ec2.securityGroupName", None)
+    security_group_description = nixops.util.attr_property("ec2.securityGroupDescription", None)
+    security_group_rules = nixops.util.attr_property("ec2.securityGroupRules", [], 'json')
+    old_security_groups = nixops.util.attr_property("ec2.oldSecurityGroups", [], 'json')
+    access_key_id = nixops.util.attr_property("ec2.accessKeyId", None)
+
+    @classmethod
+    def get_type(cls):
+        return "ec2-security-group"
+
+    def __init__(self, depl, name, id):
+        super(EC2SecurityGroupState, self).__init__(depl, name, id)
+
+    def show_type(self):
+        s = super(EC2SecurityGroupState, self).show_type()
+        if self.region: s = "{0} [{1}]".format(s, self.region)
+        return s
+
+    def prefix_definiton(self, attr):
+        return {('resources', 'ec2SecurityGroups'): attr}
+
+    def get_physical_spec(self):
+        return {'groupId': self.security_group_id}
+
+    @property
+    def resource_id(self):
+        return self.security_group_name
+
+    def create_after(self, resources):
+        #!!! TODO: Handle dependencies between security groups
+        return {}
+
+    def _connect(self):
+        if self._conn: return
+        self._conn = nixops.ec2_utils.connect(self.region, self.access_key_id)
+
+    def create(self, defn, check, allow_reboot, allow_recreate):
+        # Name or region change means a completely new security group
+        if self.security_group_name and (defn.security_group_name != self.security_group_name or defn.region != self.region):
+            with self.depl._db:
+                self.state = self.UNKNOWN
+                self.old_security_groups = self.old_security_groups + [{'name': self.security_group_name, 'region': self.region}]
+
+        with self.depl._db:
+            self.region = defn.region
+            self.access_key_id = defn.access_key_id
+            self.security_group_name = defn.security_group_name
+            self.security_group_description = defn.security_group_description
+
+        grp = None
+        if check:
+            with self.db:
+                self._connect()
+
+                try:
+                    grp = self._conn.get_all_security_groups([ defn.security_group_name ])[0]
+                    self.state = self.UP
+                    self.security_group_id = grp.id
+                    self.security_group_description = grp.description
+                    rules = []
+                    for rule in grp.rules:
+                        for grant in rule.grants:
+                            if grant.cidr_ip:
+                                new_rule = [ rule.ip_protocol, rule.from_port, rule.to_port, grant.cidr_ip ]
+                            else:
+                                new_rule = [ rule.ip_protocol, rule.from_port, rule.to_port, grant.groupName, grant.owner_id ]
+                            rules.append(new_rule)
+                    self.security_group_rules = rules
+                except boto.exception.EC2ResponseError as e:
+                    if e.error_code == u'InvalidGroup.NotFound':
+                        self.state = self.Missing
+                    else:
+                        raise
+
+        new_rules = set()
+        old_rules = set()
+        for rule in self.security_group_rules:
+            old_rules.add(tuple(rule))
+        for rule in defn.security_group_rules:
+            tupled_rule = tuple(rule)
+            if not tupled_rule in old_rules:
+                new_rules.add(tupled_rule)
+            else:
+                old_rules.remove(tupled_rule)
+
+        if self.state == self.MISSING or self.state == self.UNKNOWN:
+            self._connect()
+            try:
+                self.logger.log("creating EC2 security group `{0}'...".format(self.security_group_name))
+                grp = self._conn.create_security_group(self.security_group_name, self.security_group_description)
+                self.security_group_id = grp.id
+            except boto.exception.EC2ResponseError as e:
+                if self.state != self.UNKNOWN or e.error_code != u'InvalidGroup.Duplicate':
+                    raise
+            self.state = self.STARTING #ugh
+
+        if new_rules:
+            self.logger.log("adding new rules to EC2 security group `{0}'...".format(self.security_group_name))
+            if grp is None:
+                self._connect()
+                grp = self._conn.get_all_security_groups([ self.security_group_name ])[0]
+            for rule in new_rules:
+                if len(rule) == 4:
+                    grp.authorize(ip_protocol=rule[0], from_port=rule[1], to_port=rule[2], cidr_ip=rule[3])
+                else:
+                    src_group = boto.ec2.securitygroup.SecurityGroup(owner_id=rule[4], name=rule[3])
+                    grp.authorize(ip_protocol=rule[0], from_port=rule[1], to_port=rule[2], src_group=src_group)
+
+        if old_rules:
+            self.logger.log("removing old rules from EC2 security group `{0}'...".format(self.security_group_name))
+            if grp is None:
+                self._connect()
+                grp = self._conn.get_all_security_groups([ self.security_group_name ])[0]
+            for rule in old_rules:
+                if len(rule) == 4:
+                    grp.revoke(ip_protocol=rule[0], from_port=rule[1], to_port=rule[2], cidr_ip=rule[3])
+                else:
+                    src_group = boto.ec2.securitygroup.SecurityGroup(owner_id=rule[4], name=rule[3])
+                    grp.revoke(ip_protocol=rule[0], from_port=rule[1], to_port=rule[2], src_group=src_group)
+        self.security_group_rules = defn.security_group_rules
+
+        self.state = self.UP
+
+    def after_activation(self, defn):
+        region = self.region
+        self._connect()
+        conn = self._conn
+        for group in self.old_security_groups:
+            if group['region'] != region:
+                region = group['region']
+                conn = nixops.ec2_utils.connect(region, self.access_key_id)
+            try:
+                conn.delete_security_group(group['name'])
+            except boto.exception.EC2ResponseError as e:
+                if e.error_code != u'InvalidGroup.NotFound':
+                    raise
+        self.old_security_groups = []
+
+    def destroy(self, wipe=False):
+        if self.state == self.UP or self.state == self.STARTING:
+            self.logger.log("deleting EC2 security group `{0}'...".format(self.security_group_name))
+            self._connect()
+            self._conn.delete_security_group(self.security_group_name)
+            self.state = self.MISSING
+        return True