Skip to content

Commit

Permalink
CPU Time Series. (ray-project#765)
Browse files Browse the repository at this point in the history
Add time series of CPU utilization to web UI.
  • Loading branch information
alanamarzoev authored and robertnishihara committed Jul 26, 2017
1 parent 31329d4 commit 0f0acb8
Showing 1 changed file with 135 additions and 0 deletions.
135 changes: 135 additions & 0 deletions python/ray/WebUI.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -473,6 +473,141 @@
"\n",
"task_completion_time_distribution()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### CPU usage over time plot."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"from bokeh.layouts import gridplot\n",
"from bokeh.plotting import figure, show, helpers\n",
"from bokeh.resources import CDN\n",
"from bokeh.io import output_notebook, push_notebook\n",
"from bokeh.models import Range1d, ColumnDataSource\n",
"import numpy as np\n",
"output_notebook(resources=CDN)\n",
" \n",
"# Parse the client table to determine how many CPUs are available\n",
"num_cpus = 0 \n",
"client_table = ray.global_state.client_table()\n",
"for node_ip, client_list in client_table.items(): \n",
" for client in client_list: \n",
" if \"NumCPUs\" in client: \n",
" num_cpus += client[\"NumCPUs\"]\n",
"\n",
"def compute_utilizations(abs_earliest, abs_latest, num_tasks, tasks, num_buckets):\n",
" # Determine what the earliest and latest tasks are out of the ones that are passed in\n",
" earliest_time = time.time()\n",
" latest_time = 0\n",
" \n",
" if len(tasks) == 0:\n",
" return [], [], []\n",
" \n",
" sum_len = 0\n",
" for task_id, data in tasks.items():\n",
" latest_time = max((latest_time, data[\"store_outputs_end\"]))\n",
" earliest_time = min((earliest_time, data[\"get_arguments_start\"]))\n",
" sum_len += data[\"store_outputs_end\"] - data[\"get_arguments_start\"]\n",
" \n",
" # Add some epsilon to latest_time to ensure that the end time of the last task\n",
" # falls __within__ a bucket, and not on the edge\n",
" latest_time += 1e-6\n",
" \n",
" # Compute average CPU utilization per time bucket by summing cpu-time per bucket\n",
" bucket_time_length = (latest_time - earliest_time) / float(num_buckets)\n",
" cpu_time = [0 for _ in range(num_buckets)]\n",
" \n",
" for data in tasks.values():\n",
" task_start_time = data[\"get_arguments_start\"]\n",
" task_end_time = data[\"store_outputs_end\"]\n",
" \n",
" start_bucket = int((task_start_time - earliest_time) / bucket_time_length)\n",
" end_bucket = int((task_end_time - earliest_time) / bucket_time_length)\n",
" # Walk over each time bucket that this task intersects, adding the amount of\n",
" # time that the task intersects within each bucket\n",
" for bucket_idx in range(start_bucket, end_bucket + 1):\n",
" bucket_start_time = earliest_time + bucket_idx * bucket_time_length\n",
" bucket_end_time = earliest_time + (bucket_idx + 1) * bucket_time_length\n",
" \n",
" task_start_time_within_bucket = max(task_start_time, bucket_start_time)\n",
" task_end_time_within_bucket = min(task_end_time, bucket_end_time)\n",
" task_cpu_time_within_bucket = task_end_time_within_bucket - task_start_time_within_bucket\n",
" \n",
" cpu_time[bucket_idx] += task_cpu_time_within_bucket\n",
" \n",
" # Cpu_utilization is the average cpu utilization of the bucket, which is just\n",
" # cpu_time divided by bucket_time_length\n",
" cpu_utilization = list(map(lambda x: x / float(bucket_time_length), cpu_time))\n",
" \n",
" # Generate histogram bucket edges. Subtract out abs_earliest to get relative time\n",
" all_edges = [earliest_time - abs_earliest + i * bucket_time_length for i in range(num_buckets + 1)]\n",
" # Left edges are all but the rightmost edge, right edges are all but the leftmost edge\n",
" left_edges = all_edges[:-1]\n",
" right_edges = all_edges[1:]\n",
" \n",
" return left_edges, right_edges, cpu_utilization\n",
" \n",
"\n",
"# Update the plot based on the sliders\n",
"def plot_utilization():\n",
" # Create the Bokeh plot\n",
" time_series_fig = figure(title=\"CPU Utilization\",\n",
" tools=[\"save\", \"hover\", \"wheel_zoom\", \"box_zoom\", \"pan\"],\n",
" background_fill_color=\"#FFFFFF\", x_range=[0, 1], y_range=[0, 1])\n",
" \n",
" # Create the data source that the plot will pull from\n",
" time_series_source = ColumnDataSource(data=dict(\n",
" left=[],\n",
" right=[],\n",
" top=[]\n",
" ))\n",
" \n",
" # Plot the rectangles representing the distribution\n",
" time_series_fig.quad(left=\"left\", right=\"right\", top=\"top\", bottom=0,\n",
" source=time_series_source, fill_color=\"#B3B3B3\", line_color=\"#033649\")\n",
" \n",
" # Label the plot axes\n",
" time_series_fig.xaxis.axis_label = \"Time in seconds\"\n",
" time_series_fig.yaxis.axis_label = \"Number of CPUs used\"\n",
" \n",
" handle = show(gridplot(time_series_fig, ncols=1, plot_width=500, plot_height=500, toolbar_location=\"below\"),\n",
" notebook_handle=True)\n",
" \n",
" def update_plot(abs_earliest, abs_latest, abs_num_tasks, tasks):\n",
" num_buckets = 100\n",
" left, right, top = compute_utilizations(abs_earliest, abs_latest, abs_num_tasks, tasks, num_buckets)\n",
" \n",
" time_series_source.data = {\"left\": left, \"right\": right, \"top\": top}\n",
" \n",
" x_range = (max(0, min(left)) if len(left) else 0, max(right) if len(right) else 1)\n",
" y_range = (0, max(top) + 1 if len(top) else 1)\n",
" \n",
" # Define the axis ranges\n",
" x_range = helpers._get_range(x_range)\n",
" time_series_fig.x_range.start = x_range.start\n",
" time_series_fig.x_range.end = x_range.end\n",
" \n",
" y_range = helpers._get_range(y_range)\n",
" time_series_fig.y_range.start = y_range.start\n",
" time_series_fig.y_range.end = num_cpus\n",
" \n",
" # Push the updated data to the notebook\n",
" push_notebook(handle=handle)\n",
" \n",
" get_sliders(update_plot)\n",
"\n",
"plot_utilization()"
]
}
],
"metadata": {
Expand Down

0 comments on commit 0f0acb8

Please sign in to comment.