Autoscaler hotfix for ray-project#4555. (ray-project#4653)

mbhushan · May 8, 2019 · 0421cba · 0421cba
1 parent 28496c8
commit 0421cba
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 2 deletions.
diff --git a/python/ray/autoscaler/autoscaler.py b/python/ray/autoscaler/autoscaler.py
@@ -158,7 +158,16 @@ def __init__(self):
 
     def update(self, ip, static_resources, dynamic_resources):
         self.static_resources_by_ip[ip] = static_resources
-        self.dynamic_resources_by_ip[ip] = dynamic_resources
+        # We are not guaranteed to have a corresponding dynamic resource for
+        # every static resource because dynamic resources are based on the
+        # available resources in the heartbeat, which does not exist if it is
+        # zero. Thus, we have to update dynamic resources here.
+        dynamic_resources_update = dynamic_resources.copy()
+        for resource_name, capacity in static_resources.items():
+            if resource_name not in dynamic_resources_update:
+                dynamic_resources_update[resource_name] = 0.0
+        self.dynamic_resources_by_ip[ip] = dynamic_resources_update
+
         now = time.time()
         if ip not in self.last_used_time_by_ip or \
                 static_resources != dynamic_resources:

diff --git a/python/ray/monitor.py b/python/ray/monitor.py
@@ -112,7 +112,7 @@ def xray_heartbeat_batch_handler(self, unused_channel, data):
         for j in range(message.BatchLength()):
             heartbeat_message = message.Batch(j)
 
-            num_resources = heartbeat_message.ResourcesAvailableLabelLength()
+            num_resources = heartbeat_message.ResourcesTotalLabelLength()
             static_resources = {}
             dynamic_resources = {}
             for i in range(num_resources):