vmwgfx: integer overflow in vmw_kms_update_layout_ioctl()

There are two issues in vmw_kms_update_layout_ioctl(). First, the for loop forgets to index rects and only checks the first element. Second, there is a potential integer overflow if userspace passes in a large arg->num_outputs. The call to kzalloc() would allocate a small buffer, leading to out-of-bounds read. Reported-by: Haogang Chen <[email protected]> Signed-off-by: Xi Wang <[email protected]> Signed-off-by: Thomas Hellstrom <[email protected]> Signed-off-by: Dave Airlie <[email protected]>
arunthomas · Dec 2, 2011 · bab9efc · bab9efc
1 parent f3a71df
commit bab9efc
Showing 1 changed file with 6 additions and 5 deletions.
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
@@ -1809,7 +1809,8 @@ int vmw_kms_update_layout_ioctl(struct drm_device *dev, void *data,
 	}
 
 	rects_size = arg->num_outputs * sizeof(struct drm_vmw_rect);
-	rects = kzalloc(rects_size, GFP_KERNEL);
+	rects = kcalloc(arg->num_outputs, sizeof(struct drm_vmw_rect),
+			GFP_KERNEL);
 	if (unlikely(!rects)) {
 		ret = -ENOMEM;
 		goto out_unlock;
@@ -1824,10 +1825,10 @@ int vmw_kms_update_layout_ioctl(struct drm_device *dev, void *data,
 	}
 
 	for (i = 0; i < arg->num_outputs; ++i) {
-		if (rects->x < 0 ||
-		    rects->y < 0 ||
-		    rects->x + rects->w > mode_config->max_width ||
-		    rects->y + rects->h > mode_config->max_height) {
+		if (rects[i].x < 0 ||
+		    rects[i].y < 0 ||
+		    rects[i].x + rects[i].w > mode_config->max_width ||
+		    rects[i].y + rects[i].h > mode_config->max_height) {
 			DRM_ERROR("Invalid GUI layout.\n");
 			ret = -EINVAL;
 			goto out_free;