Skip to content

Commit

Permalink
[6478] Fix 401 unauthorized for directories in basic auth protected h…
Browse files Browse the repository at this point in the history
…ttp remote (iterative#6479)

* added auth to iter_url and created custom open for http fs

* added default value to auth

* Update dvc/utils/http.py

Co-authored-by: Ruslan Kuprieiev <[email protected]>

* Update dvc/fs/http.py

Co-authored-by: Ruslan Kuprieiev <[email protected]>

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* pr fixes

Co-authored-by: Ruslan Kuprieiev <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
  • Loading branch information
3 people authored Aug 24, 2021
1 parent 13d1276 commit b13aec0
Show file tree
Hide file tree
Showing 3 changed files with 15 additions and 12 deletions.
7 changes: 0 additions & 7 deletions dvc/fs/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
from dvc.progress import Tqdm
from dvc.utils import tmp_fname
from dvc.utils.fs import makedirs, move
from dvc.utils.http import open_url

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -126,12 +125,6 @@ def _check_requires(self, **kwargs):
)

def open(self, path_info, mode: str = "r", encoding: str = None, **kwargs):
if hasattr(self, "_generate_download_url"):
# pylint:disable=no-member
func = self._generate_download_url # type: ignore[attr-defined]
get_url = partial(func, path_info)
return open_url(get_url, mode=mode, encoding=encoding, **kwargs)

raise RemoteActionNotImplemented("open", self.scheme)

def exists(self, path_info) -> bool:
Expand Down
14 changes: 11 additions & 3 deletions dvc/fs/http.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,9 +65,6 @@ def _auth_method(self, url):
self.headers.update({self.custom_auth_header: self.password})
return None

def _generate_download_url(self, path_info):
return path_info.url

@wrap_prop(threading.Lock())
@cached_property
def _session(self):
Expand Down Expand Up @@ -190,6 +187,17 @@ def _upload(
desc=name or to_info.url,
)

def open(self, path_info, mode: str = "r", encoding: str = None, **kwargs):
from dvc.utils.http import open_url

return open_url(
path_info.url,
mode=mode,
encoding=encoding,
auth=self._auth_method(path_info),
**kwargs,
)

@staticmethod
def _content_length(response) -> Optional[int]:
res = response.headers.get("Content-Length")
Expand Down
6 changes: 4 additions & 2 deletions dvc/utils/http.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,15 @@ def open_url(url, mode="r", encoding=None, **iter_opts):


@contextmanager
def iter_url(url, chunk_size=io.DEFAULT_BUFFER_SIZE):
def iter_url(url, chunk_size=io.DEFAULT_BUFFER_SIZE, auth=None):
"""Iterate over chunks requested from url."""
import requests

def request(headers=None):
the_url = url() if callable(url) else url
response = requests.get(the_url, stream=True, headers=headers)
response = requests.get(
the_url, stream=True, headers=headers, auth=auth
)
if response.status_code == 404:
raise FileNotFoundError(f"Can't open {the_url}")
response.raise_for_status()
Expand Down

0 comments on commit b13aec0

Please sign in to comment.