Restructure the extras in setup.py and described them (apache#12548)

Closes: apache#12544

Co-authored-by: Ash Berlin-Taylor <[email protected]>
Co-authored-by: Kaxil Naik <[email protected]>

.pre-commit-config.yaml

@@ -239,7 +239,7 @@ repos:
         files: Dockerfile.*$
         pass_filenames: true
       - id: setup-order
-        name: Checks for an order of dependencies in setup.py
+        name: Check order of dependencies in setup.py
         language: python
         files: ^setup.py$
         pass_filenames: false

CONTRIBUTING.rst

@@ -548,13 +548,13 @@ This is the full list of those extras:
 all_dbs, amazon, apache.atlas, apache.beam, apache.cassandra, apache.druid, apache.hdfs,
 apache.hive, apache.kylin, apache.livy, apache.pig, apache.pinot, apache.spark, apache.sqoop,
 apache.webhdfs, async, atlas, aws, azure, cassandra, celery, cgroups, cloudant, cncf.kubernetes,
-dask, databricks, datadog, devel, devel_hadoop, dingding, discord, doc, docker, druid,
-elasticsearch, exasol, facebook, ftp, gcp, gcp_api, github_enterprise, google, google_auth, grpc,
-hashicorp, hdfs, hive, http, imap, jdbc, jenkins, jira, kerberos, kubernetes, ldap, microsoft.azure,
-microsoft.mssql, microsoft.winrm, mongo, mssql, mysql, odbc, openfaas, opsgenie, oracle, pagerduty,
-papermill, password, pinot, plexus, postgres, presto, qds, qubole, rabbitmq, redis, salesforce,
-samba, segment, sendgrid, sentry, sftp, singularity, slack, snowflake, spark, sqlite, ssh, statsd,
-tableau, vertica, virtualenv, webhdfs, winrm, yandex, yandexcloud, zendesk, all, devel_ci
+dask, databricks, datadog, dingding, discord, docker, druid, elasticsearch, exasol, facebook, ftp,
+gcp, gcp_api, github_enterprise, google, google_auth, grpc, hashicorp, hdfs, hive, http, imap, jdbc,
+jenkins, jira, kerberos, kubernetes, ldap, microsoft.azure, microsoft.mssql, microsoft.winrm, mongo,
+mssql, mysql, odbc, openfaas, opsgenie, oracle, pagerduty, papermill, password, pinot, plexus,
+postgres, presto, qds, qubole, rabbitmq, redis, salesforce, samba, segment, sendgrid, sentry, sftp,
+singularity, slack, snowflake, spark, sqlite, ssh, statsd, tableau, vertica, virtualenv, webhdfs,
+winrm, yandex, yandexcloud, zendesk, all, devel, devel_hadoop, doc, devel_all, devel_ci
 
   .. END EXTRAS HERE
 

INSTALL

@@ -66,13 +66,13 @@ pip install -e . \
 all_dbs, amazon, apache.atlas, apache.beam, apache.cassandra, apache.druid, apache.hdfs,
 apache.hive, apache.kylin, apache.livy, apache.pig, apache.pinot, apache.spark, apache.sqoop,
 apache.webhdfs, async, atlas, aws, azure, cassandra, celery, cgroups, cloudant, cncf.kubernetes,
-dask, databricks, datadog, devel, devel_hadoop, dingding, discord, doc, docker, druid,
-elasticsearch, exasol, facebook, ftp, gcp, gcp_api, github_enterprise, google, google_auth, grpc,
-hashicorp, hdfs, hive, http, imap, jdbc, jenkins, jira, kerberos, kubernetes, ldap, microsoft.azure,
-microsoft.mssql, microsoft.winrm, mongo, mssql, mysql, odbc, openfaas, opsgenie, oracle, pagerduty,
-papermill, password, pinot, plexus, postgres, presto, qds, qubole, rabbitmq, redis, salesforce,
-samba, segment, sendgrid, sentry, sftp, singularity, slack, snowflake, spark, sqlite, ssh, statsd,
-tableau, vertica, virtualenv, webhdfs, winrm, yandex, yandexcloud, zendesk, all, devel_ci
+dask, databricks, datadog, dingding, discord, docker, druid, elasticsearch, exasol, facebook, ftp,
+gcp, gcp_api, github_enterprise, google, google_auth, grpc, hashicorp, hdfs, hive, http, imap, jdbc,
+jenkins, jira, kerberos, kubernetes, ldap, microsoft.azure, microsoft.mssql, microsoft.winrm, mongo,
+mssql, mysql, odbc, openfaas, opsgenie, oracle, pagerduty, papermill, password, pinot, plexus,
+postgres, presto, qds, qubole, rabbitmq, redis, salesforce, samba, segment, sendgrid, sentry, sftp,
+singularity, slack, snowflake, spark, sqlite, ssh, statsd, tableau, vertica, virtualenv, webhdfs,
+winrm, yandex, yandexcloud, zendesk, all, devel, devel_hadoop, doc, devel_all, devel_ci
 
 # END EXTRAS HERE
 

UPDATING.md

@@ -52,6 +52,13 @@ assists users migrating to a new version.
 
 ## Master
 
+### Rename `all` to `devel_all` extra
+
+The `all` extras were reduced to include only user-facing dependencies. This means
+that this extra does not contain development dependencies. If you were relying on
+`all` extra then you should use now `devel_all` or figure out if you need development
+extras at all.
+
 ### `[scheduler] max_threads` config has been renamed to `[scheduler] parsing_processes`
 
 From Airflow 2.0, `max_threads` config under `[scheduler]` section has been renamed to `parsing_processes`.

UPGRADING_TO_2.0.md

@@ -29,10 +29,9 @@ assists users migrating to a new version.
 - [Step 1: Upgrade to Python 3](#step-1-upgrade-to-python-3)
 - [Step 2: Upgrade to Airflow 1.10.13 (a.k.a our "bridge" release)](#step-2-upgrade-to-airflow-11013-aka-our-bridge-release)
 - [Step 3: Set Operators to Backport Providers](#step-3-set-operators-to-backport-providers)
-- [Step 3: Upgrade Airflow DAGs](#step-3-upgrade-airflow-dags)
+- [Step 4: Upgrade Airflow DAGs](#step-4-upgrade-airflow-dags)
   - [Change to undefined variable handling in templates](#change-to-undefined-variable-handling-in-templates)
   - [Changes to the KubernetesPodOperator](#changes-to-the-kubernetespodoperator)
-- [Step 4: Update system configurations](#step-4-update-system-configurations)
   - [Change default value for dag_run_conf_overrides_params](#change-default-value-for-dag_run_conf_overrides_params)
   - [DAG discovery safe mode is now case insensitive](#dag-discovery-safe-mode-is-now-case-insensitive)
   - [Change to Permissions](#change-to-permissions)
@@ -47,6 +46,7 @@ assists users migrating to a new version.
   - [Changes to Exception handling for from DAG callbacks](#changes-to-exception-handling-for-from-dag-callbacks)
   - [Airflow CLI changes in 2.0](#airflow-cli-changes-in-20)
   - [Changes to Airflow Plugins](#changes-to-airflow-plugins)
+  - [Changes to extras names](#changes-to-extras-names)
   - [Support for Airflow 1.10.x releases](#support-for-airflow-110x-releases)
 
 <!-- END doctoc generated TOC please keep comment here to allow auto update -->
@@ -131,7 +131,7 @@ pip install airflow[docker]
 automatically installs the `apache-airflow-providers-docker` package.
 But you can manage/upgrade remove provider packages separately from the airflow core.
 
-## Step 3: Upgrade Airflow DAGs
+## Step 4: Upgrade Airflow DAGs
 
 ### Change to undefined variable handling in templates
 
@@ -245,7 +245,6 @@ Kubernetes secrets into workers.
 
 For a more detailed list of changes to the KubernetesPodOperator API, please read [here](#Changed-Parameters-for-the-KubernetesPodOperator)
 
-## Step 4: Update system configurations
 
 ### Change default value for dag_run_conf_overrides_params
 
@@ -1076,6 +1075,12 @@ class AirflowTestPlugin(AirflowPlugin):
     appbuilder_menu_items = [appbuilder_mitem]
 ```
 
+### Changes to extras names
+
+The `all` extra were reduced to include only user-facing dependencies. This means
+that this extra does not contain development dependencies. If you were using it and
+depending on the development packages then you should use `devel_all`.
+
 ### Support for Airflow 1.10.x releases
 
 As mentioned earlier in Step 2, the 1.10.13 release is intended to be a "bridge release"

dev/provider_packages/README.md

@@ -223,7 +223,7 @@ export BACKPORT_PACKAGES="true"
 ```shell script
 cd /airflow_sources
 
-pip install ".[all]"
+pip install ".[devel_all]"
 
 pip install "apache-airflow==${INSTALL_AIRFLOW_VERSION}"
 

docs/extra-packages-ref.rst

@@ -26,15 +26,19 @@ Here's the list of the :ref:`subpackages <installation:extra_packages>` and what
 +---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
 | subpackage          | install command                                     | enables                                                              |
 +=====================+=====================================================+======================================================================+
-| all                 | ``pip install 'apache-airflow[all]'``               | All Airflow features known to man                                    |
+| all                 | ``pip install 'apache-airflow[all]'``               | All Airflow user facing features (no devel and doc requirements)     |
 +---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
 | all_dbs             | ``pip install 'apache-airflow[all_dbs]'``           | All databases integrations                                           |
 +---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| devel               | ``pip install 'apache-airflow[devel]'``             | Minimum dev tools requirements                                       |
+| devel               | ``pip install 'apache-airflow[devel]'``             | Minimum dev tools requirements (without ``all``)                     |
 +---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| devel_hadoop        | ``pip install 'apache-airflow[devel_hadoop]'``      | Airflow + dependencies on the Hadoop stack                           |
+| devel_hadoop        | ``pip install 'apache-airflow[devel_hadoop]'``      | Same as ``devel`` + dependencies for developing the Hadoop stack     |
 +---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| doc                 | ``pip install 'apache-airflow[doc]'``               | Packages needed to build docs                                        |
+| devel_all           | ``pip install 'apache-airflow[devel_all]'``         | Everything needed for development (``devel_hadoop`` + ``all``)       |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
+| devel_ci            | ``pip install 'apache-airflow[devel_ci]'``          | All dependencies required for CI build.                              |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
+| doc                 | ``pip install 'apache-airflow[doc]'``               | Packages needed to build docs (included in ``devel``)                |
 +---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
 | password            | ``pip install 'apache-airflow[password]'``          | Password authentication for users                                    |
 +---------------------+-----------------------------------------------------+----------------------------------------------------------------------+

scripts/ci/pre_commit/pre_commit_check_setup_extra_packages_ref.py

@@ -45,15 +45,15 @@ def get_file_content(*path_elements: str) -> str:
 def get_extras_from_setup() -> Dict[str, List[str]]:
     """
     Returns an array EXTRAS_REQUIREMENTS with aliases from setup.py file in format:
-    {'package name': ['alias1', 'alias2], ...}
+    {'package name': ['alias1', 'alias2'], ...}
     """
     setup_content = get_file_content(SETUP_PY_FILE)
 
     extras_section_regex = re.compile(r'^EXTRAS_REQUIREMENTS: Dict[^{]+{([^}]+)}', re.MULTILINE)
     extras_section = extras_section_regex.findall(setup_content)[0]
 
     extras_regex = re.compile(
-        rf'^\s+[\"\']({PY_IDENTIFIER})[\"\']:\s*({PY_IDENTIFIER}|\[\])[^#\n]*(#\s*TODO.*)?$', re.MULTILINE
+        rf'^\s*[\"\']({PY_IDENTIFIER})[\"\']:\s*({PY_IDENTIFIER}|\[\])[^#\n]*(#\s*.*)?$', re.MULTILINE
     )
 
     extras_dict: Dict[str, List[str]] = {}
@@ -66,6 +66,15 @@ def get_extras_from_setup() -> Dict[str, List[str]]:
         if not extras_dict.get(package):
             extras_dict[package] = []
         extras_dict[package].append(alias)
+
+    updates_sections_regex = re.compile(r"^EXTRAS_REQUIREMENTS\.update[^{]+{([^}]+)}", re.MULTILINE)
+    updates_sections = updates_sections_regex.findall(setup_content)
+    for update in updates_sections:
+        for extra in extras_regex.findall(update):
+            package = extra[0]
+            if not extras_dict.get(package):
+                extras_dict[package] = [extra[0]]
+
     return extras_dict
 
 
@@ -80,7 +89,6 @@ def get_extras_from_docs() -> List[str]:
     )
     extras = extras_section_regex.findall(docs_content)
 
-    extras = list(filter(lambda entry: entry != 'all', extras))
     return extras
 
 

scripts/in_container/run_prepare_provider_readme.sh

@@ -32,7 +32,7 @@ verify_suffix_versions_for_package_preparation
 echo
 echo "Installing remaining packages from 'all' extras"
 echo
-pip install -e ".[all]" >>"${OUT_FILE_PRINTED_ON_ERROR}" 2>&1
+pip install -e ".[devel_all]" >>"${OUT_FILE_PRINTED_ON_ERROR}" 2>&1
 
 echo > "${OUT_FILE_PRINTED_ON_ERROR}"
 

scripts/in_container/run_test_package_import_all_classes.sh

@@ -46,7 +46,7 @@ else
     echo
     echo "Installing remaining packages from 'all' extras"
     echo
-    pip install ".[all]" >>"${OUT_FILE_PRINTED_ON_ERROR}" 2>&1
+    pip install ".[devel_all]" >>"${OUT_FILE_PRINTED_ON_ERROR}" 2>&1
     echo
     echo "Uninstalling airflow after that"
     echo

setup.py

@@ -589,11 +589,8 @@ def write_version(filename: str = os.path.join(*[my_dir, "airflow", "git_version
     'dask': dask,
     'databricks': databricks,
     'datadog': datadog,
-    'devel': devel_minreq,
-    'devel_hadoop': devel_hadoop,
     'dingding': [],
     'discord': [],
-    'doc': doc,
     'docker': docker,
     'druid': druid,  # TODO: remove this in Airflow 2.1
     'elasticsearch': elasticsearch,
@@ -663,9 +660,10 @@ def write_version(filename: str = os.path.join(*[my_dir, "airflow", "git_version
 
 EXTRAS_PROVIDERS_PACKAGES: Dict[str, Iterable[str]] = {
     'all': list(PROVIDERS_REQUIREMENTS.keys()),
-    # this is not 100% accurate with devel_ci definition, but we really want to have all providers
-    # when devel_ci extra is installed!
+    # this is not 100% accurate with devel_ci and devel_all definition, but we really want
+    # to have all providers when devel_ci extra is installed!
     'devel_ci': list(PROVIDERS_REQUIREMENTS.keys()),
+    'devel_all': list(PROVIDERS_REQUIREMENTS.keys()),
     'all_dbs': [
         "apache.cassandra",
         "apache.druid",
@@ -779,15 +777,24 @@ def write_version(filename: str = os.path.join(*[my_dir, "airflow", "git_version
     'zendesk': ["zendesk"],
 }
 
-
-# Make devel_all contain all providers + extras + unique
-devel_all = list(
+# All "users" extras (no devel extras)
+all_ = list(
     set(
-        devel
-        + [req for req_list in EXTRAS_REQUIREMENTS.values() for req in req_list]
+        [req for req_list in EXTRAS_REQUIREMENTS.values() for req in req_list]
         + [req for req_list in PROVIDERS_REQUIREMENTS.values() for req in req_list]
     )
 )
+EXTRAS_REQUIREMENTS.update(
+    {
+        'all': all_,
+        'devel': devel_minreq,  # includes doc
+        'devel_hadoop': devel_hadoop,  # includes devel_minreq
+        'doc': doc,
+    }
+)
+# This can be simplify to devel_hadoop + all_ due to inclusions
+# but we keep it for explicit sake
+devel_all = list(set(all_ + doc + devel_minreq + devel_hadoop))
 
 PACKAGES_EXCLUDED_FOR_ALL = []
 
@@ -821,6 +828,7 @@ def is_package_excluded(package: str, exclusion_list: List[str]):
     for package in devel_all
     if not is_package_excluded(package=package, exclusion_list=PACKAGES_EXCLUDED_FOR_ALL)
 ]
+
 devel_ci = [
     package
     for package in devel_all
@@ -831,7 +839,7 @@ def is_package_excluded(package: str, exclusion_list: List[str]):
 
 EXTRAS_REQUIREMENTS.update(
     {
-        'all': devel_all,
+        'devel_all': devel_all,
         'devel_ci': devel_ci,
     }
 )