diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index f19993e14c..86af27bf41 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -71,7 +71,7 @@ jobs: uses: actions/cache@v2.1.6 with: path: ${{ steps.pip-cache-dir.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }} + key: ${{ runner.os }}-pip-${{ hashFiles('setup.cfg') }} restore-keys: | ${{ runner.os }}-pip- - name: install pygit2 on Python3.10 in macOS diff --git a/MANIFEST.in b/MANIFEST.in index 5099b94ccc..03d2f17a9a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,2 @@ include fastentrypoints.py include LICENSE -recursive-include requirements *.txt diff --git a/requirements/azure.txt b/requirements/azure.txt deleted file mode 100644 index 12cefc1272..0000000000 --- a/requirements/azure.txt +++ /dev/null @@ -1,3 +0,0 @@ -adlfs==2021.9.1 -azure-identity>=1.4.0 -knack diff --git a/requirements/default.txt b/requirements/default.txt deleted file mode 100644 index a39d60184f..0000000000 --- a/requirements/default.txt +++ /dev/null @@ -1,45 +0,0 @@ -# See https://github.com/pyinstaller/pyinstaller/issues/1945 -ply>=3.9 -colorama>=0.3.9 -configobj>=5.0.6 -gitpython>3 -dulwich>=0.20.23 -pygit2>=1.5.0 -setuptools>=34.0.0 -nanotime>=0.5.2 -pyasn1>=0.4.1 -voluptuous>=0.11.7 -requests>=2.22.0 -# See https://github.com/bdcht/grandalf/issues/26 -grandalf==0.6 -distro>=1.3.0 -appdirs>=1.4.3 -ruamel.yaml>=0.17.11 -toml>=0.10.1 -funcy>=1.14 -pathspec>=0.6.0,<0.9.0 -shortuuid>=0.5.0 -tqdm>=4.45.0,<5 -packaging>=19.0 -zc.lockfile>=1.2.1 -flufl.lock>=3.2,<4 -win-unicode-console>=0.5; sys_platform == 'win32' -networkx>=2.5 -psutil>=5.8.0 -pydot>=1.2.4 -dataclasses>=0.7; python_version < '3.7' -contextvars>=2.1; python_version < '3.7' -importlib-metadata>=1.4; python_version < '3.8' -flatten_dict>=0.4.1,<1 -tabulate>=0.8.7 -pygtrie>=2.3.2 -dpath>=2.0.2,<3 -shtab>=1.3.4,<2 -rich>=10.9.0 -dictdiffer>=0.8.1 -python-benedict>=0.24.2 -pyparsing==2.4.7 -typing_extensions>=3.7.4 -fsspec[http]>=2021.10.0 -aiohttp-retry>=2.4.5 -diskcache>=5.2.1 diff --git a/requirements/gdrive.txt b/requirements/gdrive.txt deleted file mode 100644 index 1eb9811c49..0000000000 --- a/requirements/gdrive.txt +++ /dev/null @@ -1 +0,0 @@ -pydrive2[fsspec]>=1.9.4 diff --git a/requirements/gs.txt b/requirements/gs.txt deleted file mode 100644 index aef3ad5bad..0000000000 --- a/requirements/gs.txt +++ /dev/null @@ -1 +0,0 @@ -gcsfs==2021.10.0 diff --git a/requirements/hdfs.txt b/requirements/hdfs.txt deleted file mode 100644 index fdab464e21..0000000000 --- a/requirements/hdfs.txt +++ /dev/null @@ -1 +0,0 @@ -fsspec[arrow]; python_version < '3.10' diff --git a/requirements/oss.txt b/requirements/oss.txt deleted file mode 100644 index 92e507ee2b..0000000000 --- a/requirements/oss.txt +++ /dev/null @@ -1 +0,0 @@ -ossfs==2021.8.0 diff --git a/requirements/s3.txt b/requirements/s3.txt deleted file mode 100644 index 0d3b4c55ec..0000000000 --- a/requirements/s3.txt +++ /dev/null @@ -1,2 +0,0 @@ -s3fs==2021.10.0 -aiobotocore[boto3]>1.0.1 diff --git a/requirements/ssh.txt b/requirements/ssh.txt deleted file mode 100644 index 5fc1456072..0000000000 --- a/requirements/ssh.txt +++ /dev/null @@ -1 +0,0 @@ -sshfs[bcrypt]>=2021.8.1 diff --git a/requirements/ssh_gssapi.txt b/requirements/ssh_gssapi.txt deleted file mode 100644 index 6345c08961..0000000000 --- a/requirements/ssh_gssapi.txt +++ /dev/null @@ -1 +0,0 @@ -sshfs[gssapi]>=2021.8.1 diff --git a/requirements/terraform.txt b/requirements/terraform.txt deleted file mode 100644 index a7bbfa0e73..0000000000 --- a/requirements/terraform.txt +++ /dev/null @@ -1 +0,0 @@ -tpi[ssh]>=0.0.0 diff --git a/requirements/tests.txt b/requirements/tests.txt deleted file mode 100644 index 6ec36844ef..0000000000 --- a/requirements/tests.txt +++ /dev/null @@ -1,45 +0,0 @@ -wheel==0.37.0 - -# Test requirements -pytest==6.2.5 -pytest-cov==3.0.0 -pytest-xdist==2.4.0 -pytest-mock==3.6.1 -pytest-lazy-fixture==0.6.3 -pytest-docker==0.10.3 - -flaky==3.7.0 -mock==4.0.3 -rangehttpserver==1.2.0 -mock-ssh-server==0.9.1 -wget==3.2 -filelock==3.2.0 -wsgidav==3.1.1 -crc32c==2.2.post0 -xmltodict==0.12.0 -google-compute-engine==2.8.13 -google-cloud-storage==1.42.3 -dvclive==0.3.0 -pywin32>=225; sys_platform == 'win32' and python_version < '3.10' -hdfs==2.6.0 - -# required by collective.checkdocs -Pygments==2.10.0 -collective.checkdocs==0.2 -pydocstyle==6.1.1 -jaraco.windows==5.7.0 - -# pylint requirements -pylint==2.9.6 -# we use this to suppress pytest-related false positives in our tests. -pylint-pytest==1.0.3 -# we use this to suppress some messages in tests, eg: foo/bar naming, -# and, protected method calls in our tests -pylint-plugin-utils==0.6 - -# type-checking -mypy==0.910 -types-requests -types-paramiko -types-tabulate -types-toml diff --git a/requirements/webdav.txt b/requirements/webdav.txt deleted file mode 100644 index e5c5923348..0000000000 --- a/requirements/webdav.txt +++ /dev/null @@ -1 +0,0 @@ -webdav4>=0.9.1 diff --git a/requirements/webhdfs.txt b/requirements/webhdfs.txt deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/setup.cfg b/setup.cfg index 328d148960..eccaa39495 100644 --- a/setup.cfg +++ b/setup.cfg @@ -29,6 +29,130 @@ python_requires = >=3.6 zip_safe = False packages = find: include_package_data = True +install_requires = + # See https://github.com/pyinstaller/pyinstaller/issues/1945 + ply>=3.9 + colorama>=0.3.9 + configobj>=5.0.6 + gitpython>3 + dulwich>=0.20.23 + pygit2>=1.5.0 + setuptools>=34.0.0 + nanotime>=0.5.2 + pyasn1>=0.4.1 + voluptuous>=0.11.7 + requests>=2.22.0 + # See https://github.com/bdcht/grandalf/issues/26 + grandalf==0.6 + distro>=1.3.0 + appdirs>=1.4.3 + ruamel.yaml>=0.17.11 + toml>=0.10.1 + funcy>=1.14 + pathspec>=0.6.0,<0.9.0 + shortuuid>=0.5.0 + tqdm>=4.45.0,<5 + packaging>=19.0 + zc.lockfile>=1.2.1 + flufl.lock>=3.2,<4 + win-unicode-console>=0.5; sys_platform == 'win32' + networkx>=2.5 + psutil>=5.8.0 + pydot>=1.2.4 + dataclasses>=0.7; python_version < '3.7' + contextvars>=2.1; python_version < '3.7' + importlib-metadata>=1.4; python_version < '3.8' + flatten_dict>=0.4.1,<1 + tabulate>=0.8.7 + pygtrie>=2.3.2 + dpath>=2.0.2,<3 + shtab>=1.3.4,<2 + rich>=10.9.0 + dictdiffer>=0.8.1 + python-benedict>=0.24.2 + pyparsing==2.4.7 + typing_extensions>=3.7.4 + fsspec[http]>=2021.10.0 + aiohttp-retry>=2.4.5 + diskcache>=5.2.1 + +[options.extras_require] +# gssapi should not be included in all_remotes, because it doesn't have wheels +# for linux and mac, so it will fail to compile if user doesn't have all the +# requirements, including kerberos itself. Once all the wheels are available, +# we can start shipping it by default. +all = + %(azure)s + %(gdrive)s + %(gs)s + %(hdfs)s + %(oss)s + %(s3)s + %(ssh)s + %(webdav)s +dev = + %(all)s + %(tests)s +# Extra dependencies for remote integrations +azure = + adlfs==2021.9.1 + azure-identity>=1.4.0 + knack +gdrive = pydrive2[fsspec]>=1.9.4 +gs = gcsfs==2021.10.0 +hdfs = fsspec[arrow]; python_version < '3.10' +oss = ossfs==2021.8.0 +s3 = + s3fs==2021.10.0 + aiobotocore[boto3]>1.0.1 +ssh = sshfs[bcrypt]>=2021.8.1 +ssh_gssapi = sshfs[gssapi]>=2021.8.1 +webdav = webdav4>=0.9.1 +# not to break `dvc[webhdfs]` +webhdfs = +terraform = tpi[ssh]>=0.0.0 +tests = + %(terraform)s + wheel==0.37.0 + # Test requirements + pytest==6.2.5 + pytest-cov==3.0.0 + pytest-xdist==2.4.0 + pytest-mock==3.6.1 + pytest-lazy-fixture==0.6.3 + pytest-docker==0.10.3 + flaky==3.7.0 + mock==4.0.3 + rangehttpserver==1.2.0 + mock-ssh-server==0.9.1 + wget==3.2 + filelock==3.2.0 + wsgidav==3.1.1 + crc32c==2.2.post0 + xmltodict==0.12.0 + google-compute-engine==2.8.13 + google-cloud-storage==1.42.3 + dvclive==0.3.0 + pywin32>=225; sys_platform == 'win32' and python_version < '3.10' + hdfs==2.6.0 + # required by collective.checkdocs + Pygments==2.10.0 + collective.checkdocs==0.2 + pydocstyle==6.1.1 + jaraco.windows==5.7.0 + # pylint requirements + pylint==2.9.6 + # we use this to suppress pytest-related false positives in our tests. + pylint-pytest==1.0.3 + # we use this to suppress some messages in tests, eg: foo/bar naming, + # and, protected method calls in our tests + pylint-plugin-utils==0.6 + # type-checking + mypy==0.910 + types-requests==2.25.6 + types-paramiko==0.1.9 + types-tabulate==0.8.2 + types-toml==0.1.5 [options.packages.find] exclude = diff --git a/setup.py b/setup.py index cb676e4396..de93e22fcf 100644 --- a/setup.py +++ b/setup.py @@ -52,30 +52,4 @@ def run(self): _build_py.run(self) -# Extra dependencies for remote integrations -requirements = { - path.stem: path.read_text().strip().splitlines() - for path in Path("requirements").glob("*.txt") -} - -# gssapi should not be included in all_remotes, because it doesn't have wheels -# for linux and mac, so it will fail to compile if user doesn't have all the -# requirements, including kerberos itself. Once all the wheels are available, -# we can start shipping it by default. - -install_requires = requirements.pop("default") -requirements["all"] = [ - req - for key, requirements in requirements.items() - if key not in ("tests", "ssh_gssapi", "terraform") - for req in requirements -] -requirements["tests"] += requirements["terraform"] -requirements["dev"] = requirements["all"] + requirements["tests"] - -setup( - version=version, - install_requires=install_requires, - extras_require=requirements, - cmdclass={"build_py": build_py}, -) +setup(version=version, cmdclass={"build_py": build_py})