Add missing crypto and s3 extras (apache#12850)

We are missing two extras that were existing in Airflow 1.10 - crypto and s3. They are both deprecated because s3 should be replaced with amazon and crypto is already installed by default by Airflow as 'install_requires', but for backwards compatibility, we add them to 2.0 as well with deprecation comment. Also yandexcloud has been removed (it was not present in 1.10 so no need to deprecate it) and new table with deprecated extras is added to the docs.
NayaraWakewski · Dec 6, 2020 · 3d2ad30 · 3d2ad30
1 parent 537aba7
commit 3d2ad30
Show file tree

Hide file tree

Showing 6 changed files with 77 additions and 19 deletions.
diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
@@ -557,13 +557,14 @@ This is the full list of those extras:
 all_dbs, amazon, apache.atlas, apache.beam, apache.cassandra, apache.druid, apache.hdfs,
 apache.hive, apache.kylin, apache.livy, apache.pig, apache.pinot, apache.spark, apache.sqoop,
 apache.webhdfs, async, atlas, aws, azure, cassandra, celery, cgroups, cloudant, cncf.kubernetes,
-dask, databricks, datadog, dingding, discord, docker, druid, elasticsearch, exasol, facebook, ftp,
-gcp, gcp_api, github_enterprise, google, google_auth, grpc, hashicorp, hdfs, hive, http, imap, jdbc,
-jenkins, jira, kerberos, kubernetes, ldap, microsoft.azure, microsoft.mssql, microsoft.winrm, mongo,
-mssql, mysql, odbc, openfaas, opsgenie, oracle, pagerduty, papermill, password, pinot, plexus,
-postgres, presto, qds, qubole, rabbitmq, redis, salesforce, samba, segment, sendgrid, sentry, sftp,
-singularity, slack, snowflake, spark, sqlite, ssh, statsd, tableau, telegram, vertica, virtualenv,
-webhdfs, winrm, yandex, yandexcloud, zendesk, all, devel, devel_hadoop, doc, devel_all, devel_ci
+crypto, dask, databricks, datadog, dingding, discord, docker, druid, elasticsearch, exasol,
+facebook, ftp, gcp, gcp_api, github_enterprise, google, google_auth, grpc, hashicorp, hdfs, hive,
+http, imap, jdbc, jenkins, jira, kerberos, kubernetes, ldap, microsoft.azure, microsoft.mssql,
+microsoft.winrm, mongo, mssql, mysql, odbc, openfaas, opsgenie, oracle, pagerduty, papermill,
+password, pinot, plexus, postgres, presto, qds, qubole, rabbitmq, redis, s3, salesforce, samba,
+segment, sendgrid, sentry, sftp, singularity, slack, snowflake, spark, sqlite, ssh, statsd, tableau,
+telegram, vertica, virtualenv, webhdfs, winrm, yandex, zendesk, all, devel, devel_hadoop, doc,
+devel_all, devel_ci
 
   .. END EXTRAS HERE
 

diff --git a/INSTALL b/INSTALL
@@ -74,13 +74,14 @@ pip install -e . \
 all_dbs, amazon, apache.atlas, apache.beam, apache.cassandra, apache.druid, apache.hdfs,
 apache.hive, apache.kylin, apache.livy, apache.pig, apache.pinot, apache.spark, apache.sqoop,
 apache.webhdfs, async, atlas, aws, azure, cassandra, celery, cgroups, cloudant, cncf.kubernetes,
-dask, databricks, datadog, dingding, discord, docker, druid, elasticsearch, exasol, facebook, ftp,
-gcp, gcp_api, github_enterprise, google, google_auth, grpc, hashicorp, hdfs, hive, http, imap, jdbc,
-jenkins, jira, kerberos, kubernetes, ldap, microsoft.azure, microsoft.mssql, microsoft.winrm, mongo,
-mssql, mysql, odbc, openfaas, opsgenie, oracle, pagerduty, papermill, password, pinot, plexus,
-postgres, presto, qds, qubole, rabbitmq, redis, salesforce, samba, segment, sendgrid, sentry, sftp,
-singularity, slack, snowflake, spark, sqlite, ssh, statsd, tableau, telegram, vertica, virtualenv,
-webhdfs, winrm, yandex, yandexcloud, zendesk, all, devel, devel_hadoop, doc, devel_all, devel_ci
+crypto, dask, databricks, datadog, dingding, discord, docker, druid, elasticsearch, exasol,
+facebook, ftp, gcp, gcp_api, github_enterprise, google, google_auth, grpc, hashicorp, hdfs, hive,
+http, imap, jdbc, jenkins, jira, kerberos, kubernetes, ldap, microsoft.azure, microsoft.mssql,
+microsoft.winrm, mongo, mssql, mysql, odbc, openfaas, opsgenie, oracle, pagerduty, papermill,
+password, pinot, plexus, postgres, presto, qds, qubole, rabbitmq, redis, s3, salesforce, samba,
+segment, sendgrid, sentry, sftp, singularity, slack, snowflake, spark, sqlite, ssh, statsd, tableau,
+telegram, vertica, virtualenv, webhdfs, winrm, yandex, zendesk, all, devel, devel_hadoop, doc,
+devel_all, devel_ci
 
 # END EXTRAS HERE
 

diff --git a/docs/apache-airflow/extra-packages-ref.rst b/docs/apache-airflow/extra-packages-ref.rst
@@ -132,7 +132,7 @@ Here's the list of the :ref:`subpackages <installation:extra_packages>` and what
 +---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+
 | vertica             | ``pip install 'apache-airflow[vertica]'``           | Vertica hook support as an Airflow backend                                 |
 +---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+
-| yandexcloud         | ``pip install 'apache-airflow[yandexcloud]'``       | Yandex.Cloud hooks and operators                                           |
+| yandex              | ``pip install 'apache-airflow[yandex]'``            | Yandex.cloud hooks and operators                                           |
 +---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+
 | zendesk             | ``pip install 'apache-airflow[zendesk]'``           | Zendesk hooks                                                              |
 +---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+
@@ -232,3 +232,51 @@ Here's the list of the :ref:`subpackages <installation:extra_packages>` and what
 +---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
 | winrm               | ``pip install 'apache-airflow[microsoft.winrm]'``   | WinRM hooks and operators                                            |
 +---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
+
+
+** Deprecated 1.10 Extras **
+
+Those are the extras that have been deprecated in 2.0 and will be removed
+(current plan is to remove them in 2.1):
+
+
++---------------------+-----------------------------+
+| Deprecated extra    | Extra to be used instead    |
++=====================+=============================+
+| atlas               | apache.atlas                |
++---------------------+-----------------------------+
+| aws                 | amazon                      |
++---------------------+-----------------------------+
+| azure               | microsoft.azure             |
++---------------------+-----------------------------+
+| cassandra           | apache.cassandra            |
++---------------------+-----------------------------+
+| crypto              | - *                         |
++---------------------+-----------------------------+
+| druid               | apache.druid                |
++---------------------+-----------------------------+
+| gcp                 | google                      |
++---------------------+-----------------------------+
+| gcp_api             | google                      |
++---------------------+-----------------------------+
+| hdfs                | apache.hdfs                 |
++---------------------+-----------------------------+
+| hive                | apache.hive                 |
++---------------------+-----------------------------+
+| kubernetes          | cncf.kubernetes             |
++---------------------+-----------------------------+
+| mssql               | microsoft.mssql             |
++---------------------+-----------------------------+
+| pinot               | apache.pinot                |
++---------------------+-----------------------------+
+| qds                 | qubole                      |
++---------------------+-----------------------------+
+| s3                  | amazon                      |
++---------------------+-----------------------------+
+| webhdfs             | apache.webhdfs              |
++---------------------+-----------------------------+
+| winrm               | microsoft.winrm             |
++---------------------+-----------------------------+
+
+
+* crypto extra is not needed as cryptography is installed by default in Airflow 2.0
diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt
@@ -547,6 +547,7 @@ cmd
 cmdline
 cmds
 cname
+cncf
 cnf
 cnt
 codebase
@@ -943,6 +944,7 @@ metarouter
 metastore
 mget
 microservice
+microsoft
 middleware
 mik
 milton

diff --git a/scripts/ci/pre_commit/pre_commit_check_setup_extra_packages_ref.py b/scripts/ci/pre_commit/pre_commit_check_setup_extra_packages_ref.py
@@ -60,6 +60,9 @@ def get_extras_from_setup() -> Dict[str, List[str]]:
     for extras in extras_regex.findall(extras_section):
         package = extras[1]
         alias = extras[0]
+        if alias == 'crypto':
+            # Skip crypto package - it is there just for backwards compatibility
+            continue
         # if there are no packages, use the extras alias itself
         if package == '[]':
             package = alias

diff --git a/setup.py b/setup.py
@@ -427,7 +427,7 @@ def write_version(filename: str = os.path.join(*[my_dir, "airflow", "git_version
 winrm = [
     'pywinrm~=0.4',
 ]
-yandexcloud = [
+yandex = [
     'yandexcloud>=0.22.0',
 ]
 zendesk = [
@@ -582,7 +582,7 @@ def write_version(filename: str = os.path.join(*[my_dir, "airflow", "git_version
     "ssh": ssh,
     "telegram": telegram,
     "vertica": vertica,
-    "yandex": yandexcloud,
+    "yandex": yandex,
     "zendesk": zendesk,
 }
 
@@ -611,6 +611,7 @@ def write_version(filename: str = os.path.join(*[my_dir, "airflow", "git_version
     'cgroups': cgroups,
     'cloudant': cloudant,
     'cncf.kubernetes': kubernetes,
+    'crypto': [],  # TODO: remove this in Airflow 2.1
     'dask': dask,
     'databricks': databricks,
     'datadog': datadog,
@@ -660,6 +661,7 @@ def write_version(filename: str = os.path.join(*[my_dir, "airflow", "git_version
     'qubole': qubole,
     'rabbitmq': rabbitmq,
     'redis': redis,
+    's3': amazon,  # TODO: Remove this in Airflow 2.1
     'salesforce': salesforce,
     'samba': samba,
     'segment': segment,
@@ -679,8 +681,7 @@ def write_version(filename: str = os.path.join(*[my_dir, "airflow", "git_version
     'virtualenv': virtualenv,
     'webhdfs': webhdfs,  # TODO: remove this in Airflow 2.1
     'winrm': winrm,  # TODO: remove this in Airflow 2.1
-    'yandex': yandexcloud,  # TODO: remove this in Airflow 2.1
-    'yandexcloud': yandexcloud,
+    'yandex': yandex,
     'zendesk': [],
 }
 
@@ -728,6 +729,7 @@ def write_version(filename: str = os.path.join(*[my_dir, "airflow", "git_version
     'cgroups': [],
     'cloudant': ["cloudant"],
     'cncf.kubernetes': ["cncf.kubernetes"],
+    'crypto': [],  # TODO: Remove this in Airflow 2.1
     'dask': [],
     'databricks': ["databricks"],
     'datadog': ["datadog"],
@@ -780,6 +782,7 @@ def write_version(filename: str = os.path.join(*[my_dir, "airflow", "git_version
     'qubole': ["qubole"],
     'rabbitmq': [],
     'redis': ["redis"],
+    's3': ["amazon"],  # TODO: Remove this in Airflow 2.1
     'salesforce': ["salesforce"],
     'samba': ["samba"],
     'segment': ["segment"],