Merge branch 'master' of github.com:Yelp/paasta into mypy_tweaks

debian/changelog

@@ -1,3 +1,19 @@
+paasta-tools (0.68.8) xenial; urgency=medium
+
+  * 0.68.8 tagged with 'make release'
+    Commit: Merge pull request #1623 from Yelp/fix_draining_bug  Always
+    consider num_at_risk_tasks when calculating instance_counts du…
+
+ -- Kyle Anderson <[email protected]>  Mon, 20 Nov 2017 11:01:26 -0800
+
+paasta-tools (0.68.7) xenial; urgency=medium
+
+  * 0.68.7 tagged with 'make release'
+    Commit: Merge pull request #1614 from Yelp/emit-cluster-autoscaler-
+    metrics  Emit metrics from cluster autoscaler
+
+ -- Matthew Bentley <[email protected]>  Mon, 20 Nov 2017 07:48:15 -0800
+
 paasta-tools (0.68.6) xenial; urgency=medium
 
   * 0.68.6 tagged with 'make release'

docs/source/autoscaling.rst

@@ -41,7 +41,7 @@ utilization of your service. If a metrics provider isn't provided, the
 ``"mesos_cpu"`` metrics provider will be used. Within the ``autoscaling``
 attribute, setting a ``decision_policy`` will allow you to specify the logic
 that determines when to autoscale your service. If a decision policy isn't
-provided, the ``"pid"`` decision policy will be used. Specifying a ``setpoint``
+provided, the ``"proportional"`` decision policy will be used. Specifying a ``setpoint``
 allows you to specify a target utilization for your service. The default
 ``setpoint`` is 0.8 (80%). Decision policies and metrics providers have their
 own optional keyword arguments that may be placed into the ``autoscaling``

docs/source/generated/paasta_tools.metrics.rst

@@ -7,6 +7,7 @@ Submodules
 .. toctree::
 
    paasta_tools.metrics.metastatus_lib
+   paasta_tools.metrics.metrics_lib
 
 Module contents
 ---------------

mypy.ini

@@ -35,6 +35,9 @@ disallow_untyped_defs = True
 [mypy-paasta_tools.autoscaling.autoscaling_cluster_lib]
 disallow_untyped_defs = True
 
+[mypy-paasta_tools.metrics.metrics_lib]
+disallow_untyped_defs = True
+
 [mypy-paasta_tools.secret_tools]
 disallow_untyped_defs = True
 

paasta_itests/docker-compose.yml

@@ -11,14 +11,15 @@ services:
 
   mesosmaster:
     build: ../yelp_package/dockerfiles/itest/mesos/
+    image: paasta_itest_mesos:latest
     ports:
       - 5050
     command: 'mesos-master --zk=zk://zookeeper:2181/mesos-testcluster --registry=in_memory --quorum=1 --authenticate --authenticate_slaves --work_dir=/tmp/mesos --credentials=/etc/mesos-secrets'
     depends_on:
       - zookeeper
 
   mesosslave:
-    build: ../yelp_package/dockerfiles/itest/mesos/
+    image: paasta_itest_mesos:latest
     ports:
       - 5051
     environment:
@@ -30,7 +31,7 @@ services:
       - zookeeper
 
   mesosslave2:
-    build: ../yelp_package/dockerfiles/itest/mesos/
+    image: paasta_itest_mesos:latest
     ports:
       - 5051
     environment:
@@ -42,7 +43,7 @@ services:
       - zookeeper
 
   mesosslave3:
-    build: ../yelp_package/dockerfiles/itest/mesos/
+    image: paasta_itest_mesos:latest
     ports:
       - 5051
     environment:
@@ -54,7 +55,7 @@ services:
       - zookeeper
 
   mesosslave4:
-    build: ../yelp_package/dockerfiles/itest/mesos/
+    image: paasta_itest_mesos:latest
     ports:
       - 5051
     environment:
@@ -66,7 +67,7 @@ services:
       - zookeeper
 
   mesosslave5:
-    build: ../yelp_package/dockerfiles/itest/mesos/
+    image: paasta_itest_mesos:latest
     ports:
       - 5051
     environment:
@@ -79,6 +80,7 @@ services:
 
   marathon:
     build: ../yelp_package/dockerfiles/itest/marathon/
+    image: paasta_itest_marathon:latest
     ports:
       - 8080
     environment:
@@ -88,7 +90,7 @@ services:
       - zookeeper
 
   marathon1:
-    build: ../yelp_package/dockerfiles/itest/marathon/
+    image: paasta_itest_marathon:latest
     ports:
       - 8080
     environment:
@@ -98,7 +100,7 @@ services:
       - zookeeper
 
   marathon2:
-    build: ../yelp_package/dockerfiles/itest/marathon/
+    image: paasta_itest_marathon:latest
     ports:
       - 8080
     environment:

paasta_tools/__init__.py

@@ -17,4 +17,4 @@
 # setup phase, the dependencies may not exist on disk yet.
 #
 # Don't bump version manually. See `make release` docs in ./Makefile
-__version__ = '0.68.6'
+__version__ = '0.68.8'

paasta_tools/autoscaling/autoscaling_cluster_lib.py

@@ -45,6 +45,7 @@
 from paasta_tools.mesos_tools import slave_pid_to_ip
 from paasta_tools.mesos_tools import SlaveTaskCount
 from paasta_tools.metrics.metastatus_lib import get_resource_utilization_by_grouping
+from paasta_tools.metrics.metrics_lib import get_metrics_interface
 from paasta_tools.paasta_maintenance import is_safe_to_kill
 from paasta_tools.utils import load_system_paasta_config
 from paasta_tools.utils import Timeout
@@ -125,6 +126,7 @@ def __init__(
         utilization_error: float,
         log_level: str=None,
         draining_enabled: bool=True,
+        enable_metrics: bool=False,
     ) -> None:
         self.resource = resource
         self.pool_settings = pool_settings
@@ -136,13 +138,50 @@ def __init__(
             self.log.setLevel(log_level)
         self.instances: List[Dict] = []
         self.sfr: Optional[Dict[str, Any]] = None
+        self.enable_metrics = enable_metrics
+
+        self.setup_metrics()
 
     @property
     def log(self) -> logging.Logger:
         resource_id = self.resource.get("id", "unknown")
         name = '.'.join([__name__, self.__class__.__name__, resource_id])
         return logging.getLogger(name)
 
+    def setup_metrics(self) -> None:
+        if not self.enable_metrics:
+            return None
+        config = load_system_paasta_config()
+        dims = {
+            'paasta_cluster': config.get_cluster(),
+            'region': self.resource.get('region', 'unknown'),
+            'pool': self.resource.get('pool', 'unknown'),
+            'resource_id': self.resource.get('id', 'unknown'),
+            'resource_type': self.__class__.__name__,
+        }
+        self.metrics = get_metrics_interface('paasta.cluster_autoscaler')
+        self.target_gauge = self.metrics.create_gauge('target_capacity', **dims)
+        self.current_gauge = self.metrics.create_gauge('current_capacity', **dims)
+        self.ideal_gauge = self.metrics.create_gauge('ideal_capacity', **dims)
+        self.max_gauge = self.metrics.create_gauge('max_capacity', **dims)
+        self.min_gauge = self.metrics.create_gauge('min_capacity', **dims)
+        self.mesos_error_gauge = self.metrics.create_gauge('mesos_error', **dims)
+
+    def emit_metrics(
+        self,
+        current: float,
+        target: float,
+        ideal: float,
+    ) -> None:
+        if not self.enable_metrics:
+            return None
+        self.current_gauge.set(current)
+        self.target_gauge.set(target)
+        self.ideal_gauge.set(ideal)
+        self.min_gauge.set(self.resource['min_capacity'])
+        self.max_gauge.set(self.resource['max_capacity'])
+        self.mesos_error_gauge.set(self.utilization_error)
+
     def set_capacity(self, capacity: float) -> Optional[Any]:
         pass
 
@@ -1073,6 +1112,7 @@ def autoscale_local_cluster(
                 log_level=log_level,
                 utilization_error=utilization_errors[(resource['region'], resource['pool'])],
                 draining_enabled=autoscaling_draining_enabled,
+                enable_metrics=True,
             )
             autoscaling_scalers[(resource['region'], resource['pool'])].append(scaler)
         except KeyError:

paasta_tools/deployd/master.py

@@ -17,11 +17,11 @@
 from paasta_tools.deployd.common import rate_limit_instances
 from paasta_tools.deployd.common import ServiceInstance
 from paasta_tools.deployd.leader import PaastaLeaderElection
-from paasta_tools.deployd.metrics import get_metrics_interface
 from paasta_tools.deployd.metrics import QueueMetrics
 from paasta_tools.deployd.workers import PaastaDeployWorker
 from paasta_tools.list_marathon_service_instances import get_service_instances_that_need_bouncing
 from paasta_tools.marathon_tools import DEFAULT_SOA_DIR
+from paasta_tools.metrics.metrics_lib import get_metrics_interface
 from paasta_tools.utils import get_services_for_cluster
 from paasta_tools.utils import load_system_paasta_config
 from paasta_tools.utils import ZookeeperPool
@@ -175,7 +175,7 @@ def watcher_threads_enabled(self):
     def startup(self):
         self.is_leader = True
         self.log.info("This node is elected as leader {}".format(socket.getfqdn()))
-        self.metrics = get_metrics_interface(self.config.get_deployd_metrics_provider())
+        self.metrics = get_metrics_interface('paasta.deployd')
         QueueMetrics(self.inbox, self.bounce_q, self.config.get_cluster(), self.metrics).start()
         self.inbox.start()
         self.log.info("Starting all watcher threads")

paasta_tools/deployd/metrics.py

@@ -1,72 +1,7 @@
-import logging
 import time
 
 from paasta_tools.deployd.common import PaastaThread
 
-log = logging.getLogger(__name__)
-
-try:
-    import yelp_meteorite
-except ImportError:
-    # Sorry to any non-yelpers but you won't
-    # get metrics emitted as our metrics library
-    # is currently not open source
-    yelp_meteorite = None
-
-_metricis_interfaces = {}
-
-
-def register_metrics_interface(name):
-    def outer(func):
-        _metricis_interfaces[name] = func
-        return func
-    return outer
-
-
-def get_metrics_interface(name):
-    return _metricis_interfaces[name]()
-
-
-@register_metrics_interface(None)
-class NoMetrics(object):
-    def create_timer(self, name, **kwargs):
-        return Timer(name)
-
-    def create_gauge(self, name, **kwargs):
-        return Gauge(name)
-
-
-class Timer(object):
-    def __init__(self, name):
-        self.name = name
-
-    def start(self):
-        log.debug("gauge {} start".format(self.name))
-
-    def stop(self):
-        log.debug("gauge {} start".format(self.name))
-
-
-class Gauge(object):
-    def __init__(self, name):
-        self.name = name
-
-    def set(self, value):
-        log.debug("gauge {} set to {}".format(self.name, value))
-
-
-@register_metrics_interface('meteorite')
-class MeteoriteMetrics(object):
-    def __init__(self):
-        if not yelp_meteorite:
-            raise ImportError("yelp_meteorite not imported, please try another deployd_metrics_provider")
-
-    def create_timer(self, name, **kwargs):
-        return yelp_meteorite.create_timer('paasta.deployd.{}'.format(name), kwargs)
-
-    def create_gauge(self, name, **kwargs):
-        return yelp_meteorite.create_gauge('paasta.deployd.{}'.format(name), kwargs)
-
 
 class QueueMetrics(PaastaThread):
     def __init__(self, inbox, bounce_q, cluster, metrics_provider):

paasta_tools/marathon_tools.py

@@ -465,7 +465,7 @@ def copy(self) -> "MarathonServiceConfig":
     def get_autoscaling_params(self) -> AutoscalingParamsDict:
         default_params: AutoscalingParamsDict = {
             'metrics_provider': 'mesos_cpu',
-            'decision_policy': 'pid',
+            'decision_policy': 'proportional',
             'setpoint': 0.8,
         }
         return deep_merge_dictionaries(
@@ -1516,3 +1516,11 @@ def broadcast_log_all_services_running_here(line: str, component: str='monitorin
 
 def broadcast_log_all_services_running_here_from_stdin(component: str='monitoring') -> None:
     broadcast_log_all_services_running_here(sys.stdin.read().strip())
+
+
+def take_up_slack(client: MarathonClient, app: MarathonApp) -> None:
+    slack = max(app.instances - len(app.tasks), 0)
+    if slack > 0:
+        log.info("Scaling %s down from %d to %d instances to remove slack." %
+                 (app.id, app.instances, app.instances - slack))
+        client.scale_app(app_id=app.id, instances=(app.instances - slack), force=True)