move requirements to setup.cfg (iterative#6758)

* move requirements to setup.cfg * hash based on setup.cfg * do not break webhdfs installation
2796gaurav · Oct 8, 2021 · c541534 · c541534
1 parent dd86076
commit c541534
Show file tree

Hide file tree

Showing 17 changed files with 126 additions and 132 deletions.
diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
@@ -71,7 +71,7 @@ jobs:
       uses: actions/[email protected]
       with:
         path: ${{ steps.pip-cache-dir.outputs.dir }}
-        key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
+        key: ${{ runner.os }}-pip-${{ hashFiles('setup.cfg') }}
         restore-keys: |
           ${{ runner.os }}-pip-
     - name: install pygit2 on Python3.10 in macOS

diff --git a/MANIFEST.in b/MANIFEST.in
@@ -1,3 +1,2 @@
 include fastentrypoints.py
 include LICENSE
-recursive-include requirements *.txt
diff --git a/requirements/azure.txt b/requirements/azure.txt
diff --git a/requirements/default.txt b/requirements/default.txt
diff --git a/requirements/gdrive.txt b/requirements/gdrive.txt
diff --git a/requirements/gs.txt b/requirements/gs.txt
diff --git a/requirements/hdfs.txt b/requirements/hdfs.txt
diff --git a/requirements/oss.txt b/requirements/oss.txt
diff --git a/requirements/s3.txt b/requirements/s3.txt
diff --git a/requirements/ssh.txt b/requirements/ssh.txt
diff --git a/requirements/ssh_gssapi.txt b/requirements/ssh_gssapi.txt
diff --git a/requirements/terraform.txt b/requirements/terraform.txt
diff --git a/requirements/tests.txt b/requirements/tests.txt
diff --git a/requirements/webdav.txt b/requirements/webdav.txt
diff --git a/requirements/webhdfs.txt b/requirements/webhdfs.txt
diff --git a/setup.cfg b/setup.cfg
@@ -29,6 +29,130 @@ python_requires = >=3.6
 zip_safe = False
 packages = find:
 include_package_data = True
+install_requires =
+    # See https://github.com/pyinstaller/pyinstaller/issues/1945
+    ply>=3.9
+    colorama>=0.3.9
+    configobj>=5.0.6
+    gitpython>3
+    dulwich>=0.20.23
+    pygit2>=1.5.0
+    setuptools>=34.0.0
+    nanotime>=0.5.2
+    pyasn1>=0.4.1
+    voluptuous>=0.11.7
+    requests>=2.22.0
+    # See https://github.com/bdcht/grandalf/issues/26
+    grandalf==0.6
+    distro>=1.3.0
+    appdirs>=1.4.3
+    ruamel.yaml>=0.17.11
+    toml>=0.10.1
+    funcy>=1.14
+    pathspec>=0.6.0,<0.9.0
+    shortuuid>=0.5.0
+    tqdm>=4.45.0,<5
+    packaging>=19.0
+    zc.lockfile>=1.2.1
+    flufl.lock>=3.2,<4
+    win-unicode-console>=0.5; sys_platform == 'win32'
+    networkx>=2.5
+    psutil>=5.8.0
+    pydot>=1.2.4
+    dataclasses>=0.7; python_version < '3.7'
+    contextvars>=2.1; python_version < '3.7'
+    importlib-metadata>=1.4; python_version < '3.8'
+    flatten_dict>=0.4.1,<1
+    tabulate>=0.8.7
+    pygtrie>=2.3.2
+    dpath>=2.0.2,<3
+    shtab>=1.3.4,<2
+    rich>=10.9.0
+    dictdiffer>=0.8.1
+    python-benedict>=0.24.2
+    pyparsing==2.4.7
+    typing_extensions>=3.7.4
+    fsspec[http]>=2021.10.0
+    aiohttp-retry>=2.4.5
+    diskcache>=5.2.1
+
+[options.extras_require]
+# gssapi should not be included in all_remotes, because it doesn't have wheels
+# for linux and mac, so it will fail to compile if user doesn't have all the
+# requirements, including kerberos itself. Once all the wheels are available,
+# we can start shipping it by default.
+all =
+    %(azure)s
+    %(gdrive)s
+    %(gs)s
+    %(hdfs)s
+    %(oss)s
+    %(s3)s
+    %(ssh)s
+    %(webdav)s
+dev =
+    %(all)s
+    %(tests)s
+# Extra dependencies for remote integrations
+azure =
+    adlfs==2021.9.1
+    azure-identity>=1.4.0
+    knack
+gdrive = pydrive2[fsspec]>=1.9.4
+gs = gcsfs==2021.10.0
+hdfs = fsspec[arrow]; python_version < '3.10'
+oss = ossfs==2021.8.0
+s3 =
+    s3fs==2021.10.0
+    aiobotocore[boto3]>1.0.1
+ssh = sshfs[bcrypt]>=2021.8.1
+ssh_gssapi = sshfs[gssapi]>=2021.8.1
+webdav = webdav4>=0.9.1
+# not to break `dvc[webhdfs]`
+webhdfs =
+terraform = tpi[ssh]>=0.0.0
+tests =
+    %(terraform)s
+    wheel==0.37.0
+    # Test requirements
+    pytest==6.2.5
+    pytest-cov==3.0.0
+    pytest-xdist==2.4.0
+    pytest-mock==3.6.1
+    pytest-lazy-fixture==0.6.3
+    pytest-docker==0.10.3
+    flaky==3.7.0
+    mock==4.0.3
+    rangehttpserver==1.2.0
+    mock-ssh-server==0.9.1
+    wget==3.2
+    filelock==3.2.0
+    wsgidav==3.1.1
+    crc32c==2.2.post0
+    xmltodict==0.12.0
+    google-compute-engine==2.8.13
+    google-cloud-storage==1.42.3
+    dvclive==0.3.0
+    pywin32>=225; sys_platform == 'win32' and python_version < '3.10'
+    hdfs==2.6.0
+    # required by collective.checkdocs
+    Pygments==2.10.0
+    collective.checkdocs==0.2
+    pydocstyle==6.1.1
+    jaraco.windows==5.7.0
+    # pylint requirements
+    pylint==2.9.6
+    # we use this to suppress pytest-related false positives in our tests.
+    pylint-pytest==1.0.3
+    # we use this to suppress some messages in tests, eg: foo/bar naming,
+    # and, protected method calls in our tests
+    pylint-plugin-utils==0.6
+    # type-checking
+    mypy==0.910
+    types-requests==2.25.6
+    types-paramiko==0.1.9
+    types-tabulate==0.8.2
+    types-toml==0.1.5
 
 [options.packages.find]
 exclude =

diff --git a/setup.py b/setup.py
@@ -52,30 +52,4 @@ def run(self):
         _build_py.run(self)
 
 
-# Extra dependencies for remote integrations
-requirements = {
-    path.stem: path.read_text().strip().splitlines()
-    for path in Path("requirements").glob("*.txt")
-}
-
-# gssapi should not be included in all_remotes, because it doesn't have wheels
-# for linux and mac, so it will fail to compile if user doesn't have all the
-# requirements, including kerberos itself. Once all the wheels are available,
-# we can start shipping it by default.
-
-install_requires = requirements.pop("default")
-requirements["all"] = [
-    req
-    for key, requirements in requirements.items()
-    if key not in ("tests", "ssh_gssapi", "terraform")
-    for req in requirements
-]
-requirements["tests"] += requirements["terraform"]
-requirements["dev"] = requirements["all"] + requirements["tests"]
-
-setup(
-    version=version,
-    install_requires=install_requires,
-    extras_require=requirements,
-    cmdclass={"build_py": build_py},
-)
+setup(version=version, cmdclass={"build_py": build_py})