gdrive: download: stream & add progress (iterative#3722)

* wip

* gdrive: add progress

Part of iterative#2865
See iterative#2865 (comment)

* gdrive: move towards next pydrive2 release

- depends on iterative/PyDrive2#30

* update to latest pydrive>=1.4.11

* avoid unneeded API call

* progress: gdrive: ensure proper bar_format

dvc/remote/gdrive.py

@@ -44,17 +44,6 @@ def __init__(self, cred_location):
         super().__init__(message)
 
 
-def _extract(exc, field):
-    from pydrive2.files import ApiRequestError
-
-    assert isinstance(exc, ApiRequestError)
-
-    # https://cloud.google.com/storage/docs/json_api/v1/status-codes#errorformat
-    return (
-        exc.error["errors"][0].get(field, "") if "errors" in exc.error else ""
-    )
-
-
 def _gdrive_retry(func):
     def should_retry(exc):
         from pydrive2.files import ApiRequestError
@@ -68,7 +57,7 @@ def should_retry(exc):
             result = True
 
         if error_code == 403:
-            result = _extract(exc, "reason") in [
+            result = exc.GetField("reason") in [
                 "userRateLimitExceeded",
                 "rateLimitExceeded",
             ]
@@ -396,14 +385,15 @@ def _gdrive_download_file(
         param = {"id": item_id}
         # it does not create a file on the remote
         gdrive_file = self._drive.CreateFile(param)
-        bar_format = (
-            "Downloading {desc:{ncols_desc}.{ncols_desc}}... "
-            + Tqdm.format_sizeof(int(gdrive_file["fileSize"]), "B", 1024)
-        )
+
         with Tqdm(
-            bar_format=bar_format, desc=progress_desc, disable=no_progress_bar
-        ):
-            gdrive_file.GetContentFile(to_file)
+            desc=progress_desc,
+            disable=no_progress_bar,
+            bytes=True,
+            # explicit `bar_format` as `total` will be set by `update_to`
+            bar_format=Tqdm.BAR_FMT_DEFAULT,
+        ) as pbar:
+            gdrive_file.GetContentFile(to_file, callback=pbar.update_to)
 
     @_gdrive_retry
     def _gdrive_delete_file(self, item_id):
@@ -420,7 +410,7 @@ def _gdrive_delete_file(self, item_id):
             if (
                 http_error_code == 403
                 and self._list_params["corpora"] == "drive"
-                and _extract(exc, "location") == "file.permissions"
+                and exc.GetField("location") == "file.permissions"
             ):
                 raise DvcException(
                     "Insufficient permissions to {}. You should have {} "

setup.py

@@ -87,7 +87,7 @@ def run(self):
 # Extra dependencies for remote integrations
 
 gs = ["google-cloud-storage==1.19.0"]
-gdrive = ["pydrive2>=1.4.10"]
+gdrive = ["pydrive2>=1.4.11"]
 s3 = ["boto3>=1.9.201"]
 azure = ["azure-storage-blob==2.1.0"]
 oss = ["oss2==2.6.1"]