From 3d2ad30a7259484243ce01708c93f59a3f02a116 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sun, 6 Dec 2020 20:43:45 +0100 Subject: [PATCH] Add missing crypto and s3 extras (#12850) We are missing two extras that were existing in Airflow 1.10 - crypto and s3. They are both deprecated because s3 should be replaced with amazon and crypto is already installed by default by Airflow as 'install_requires', but for backwards compatibility, we add them to 2.0 as well with deprecation comment. Also yandexcloud has been removed (it was not present in 1.10 so no need to deprecate it) and new table with deprecated extras is added to the docs. --- CONTRIBUTING.rst | 15 +++--- INSTALL | 15 +++--- docs/apache-airflow/extra-packages-ref.rst | 50 ++++++++++++++++++- docs/spelling_wordlist.txt | 2 + ...e_commit_check_setup_extra_packages_ref.py | 3 ++ setup.py | 11 ++-- 6 files changed, 77 insertions(+), 19 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index ee673279c813f..6f65efc3e78aa 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -557,13 +557,14 @@ This is the full list of those extras: all_dbs, amazon, apache.atlas, apache.beam, apache.cassandra, apache.druid, apache.hdfs, apache.hive, apache.kylin, apache.livy, apache.pig, apache.pinot, apache.spark, apache.sqoop, apache.webhdfs, async, atlas, aws, azure, cassandra, celery, cgroups, cloudant, cncf.kubernetes, -dask, databricks, datadog, dingding, discord, docker, druid, elasticsearch, exasol, facebook, ftp, -gcp, gcp_api, github_enterprise, google, google_auth, grpc, hashicorp, hdfs, hive, http, imap, jdbc, -jenkins, jira, kerberos, kubernetes, ldap, microsoft.azure, microsoft.mssql, microsoft.winrm, mongo, -mssql, mysql, odbc, openfaas, opsgenie, oracle, pagerduty, papermill, password, pinot, plexus, -postgres, presto, qds, qubole, rabbitmq, redis, salesforce, samba, segment, sendgrid, sentry, sftp, -singularity, slack, snowflake, spark, sqlite, ssh, statsd, tableau, telegram, vertica, virtualenv, -webhdfs, winrm, yandex, yandexcloud, zendesk, all, devel, devel_hadoop, doc, devel_all, devel_ci +crypto, dask, databricks, datadog, dingding, discord, docker, druid, elasticsearch, exasol, +facebook, ftp, gcp, gcp_api, github_enterprise, google, google_auth, grpc, hashicorp, hdfs, hive, +http, imap, jdbc, jenkins, jira, kerberos, kubernetes, ldap, microsoft.azure, microsoft.mssql, +microsoft.winrm, mongo, mssql, mysql, odbc, openfaas, opsgenie, oracle, pagerduty, papermill, +password, pinot, plexus, postgres, presto, qds, qubole, rabbitmq, redis, s3, salesforce, samba, +segment, sendgrid, sentry, sftp, singularity, slack, snowflake, spark, sqlite, ssh, statsd, tableau, +telegram, vertica, virtualenv, webhdfs, winrm, yandex, zendesk, all, devel, devel_hadoop, doc, +devel_all, devel_ci .. END EXTRAS HERE diff --git a/INSTALL b/INSTALL index f02d685ecd251..1fa347d25857f 100644 --- a/INSTALL +++ b/INSTALL @@ -74,13 +74,14 @@ pip install -e . \ all_dbs, amazon, apache.atlas, apache.beam, apache.cassandra, apache.druid, apache.hdfs, apache.hive, apache.kylin, apache.livy, apache.pig, apache.pinot, apache.spark, apache.sqoop, apache.webhdfs, async, atlas, aws, azure, cassandra, celery, cgroups, cloudant, cncf.kubernetes, -dask, databricks, datadog, dingding, discord, docker, druid, elasticsearch, exasol, facebook, ftp, -gcp, gcp_api, github_enterprise, google, google_auth, grpc, hashicorp, hdfs, hive, http, imap, jdbc, -jenkins, jira, kerberos, kubernetes, ldap, microsoft.azure, microsoft.mssql, microsoft.winrm, mongo, -mssql, mysql, odbc, openfaas, opsgenie, oracle, pagerduty, papermill, password, pinot, plexus, -postgres, presto, qds, qubole, rabbitmq, redis, salesforce, samba, segment, sendgrid, sentry, sftp, -singularity, slack, snowflake, spark, sqlite, ssh, statsd, tableau, telegram, vertica, virtualenv, -webhdfs, winrm, yandex, yandexcloud, zendesk, all, devel, devel_hadoop, doc, devel_all, devel_ci +crypto, dask, databricks, datadog, dingding, discord, docker, druid, elasticsearch, exasol, +facebook, ftp, gcp, gcp_api, github_enterprise, google, google_auth, grpc, hashicorp, hdfs, hive, +http, imap, jdbc, jenkins, jira, kerberos, kubernetes, ldap, microsoft.azure, microsoft.mssql, +microsoft.winrm, mongo, mssql, mysql, odbc, openfaas, opsgenie, oracle, pagerduty, papermill, +password, pinot, plexus, postgres, presto, qds, qubole, rabbitmq, redis, s3, salesforce, samba, +segment, sendgrid, sentry, sftp, singularity, slack, snowflake, spark, sqlite, ssh, statsd, tableau, +telegram, vertica, virtualenv, webhdfs, winrm, yandex, zendesk, all, devel, devel_hadoop, doc, +devel_all, devel_ci # END EXTRAS HERE diff --git a/docs/apache-airflow/extra-packages-ref.rst b/docs/apache-airflow/extra-packages-ref.rst index c92f694abc377..b54733259faa5 100644 --- a/docs/apache-airflow/extra-packages-ref.rst +++ b/docs/apache-airflow/extra-packages-ref.rst @@ -132,7 +132,7 @@ Here's the list of the :ref:`subpackages ` and what +---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ | vertica | ``pip install 'apache-airflow[vertica]'`` | Vertica hook support as an Airflow backend | +---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ -| yandexcloud | ``pip install 'apache-airflow[yandexcloud]'`` | Yandex.Cloud hooks and operators | +| yandex | ``pip install 'apache-airflow[yandex]'`` | Yandex.cloud hooks and operators | +---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ | zendesk | ``pip install 'apache-airflow[zendesk]'`` | Zendesk hooks | +---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ @@ -232,3 +232,51 @@ Here's the list of the :ref:`subpackages ` and what +---------------------+-----------------------------------------------------+----------------------------------------------------------------------+ | winrm | ``pip install 'apache-airflow[microsoft.winrm]'`` | WinRM hooks and operators | +---------------------+-----------------------------------------------------+----------------------------------------------------------------------+ + + +** Deprecated 1.10 Extras ** + +Those are the extras that have been deprecated in 2.0 and will be removed +(current plan is to remove them in 2.1): + + ++---------------------+-----------------------------+ +| Deprecated extra | Extra to be used instead | ++=====================+=============================+ +| atlas | apache.atlas | ++---------------------+-----------------------------+ +| aws | amazon | ++---------------------+-----------------------------+ +| azure | microsoft.azure | ++---------------------+-----------------------------+ +| cassandra | apache.cassandra | ++---------------------+-----------------------------+ +| crypto | - * | ++---------------------+-----------------------------+ +| druid | apache.druid | ++---------------------+-----------------------------+ +| gcp | google | ++---------------------+-----------------------------+ +| gcp_api | google | ++---------------------+-----------------------------+ +| hdfs | apache.hdfs | ++---------------------+-----------------------------+ +| hive | apache.hive | ++---------------------+-----------------------------+ +| kubernetes | cncf.kubernetes | ++---------------------+-----------------------------+ +| mssql | microsoft.mssql | ++---------------------+-----------------------------+ +| pinot | apache.pinot | ++---------------------+-----------------------------+ +| qds | qubole | ++---------------------+-----------------------------+ +| s3 | amazon | ++---------------------+-----------------------------+ +| webhdfs | apache.webhdfs | ++---------------------+-----------------------------+ +| winrm | microsoft.winrm | ++---------------------+-----------------------------+ + + +* crypto extra is not needed as cryptography is installed by default in Airflow 2.0 diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 27a278c81566f..adedd09227d45 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -547,6 +547,7 @@ cmd cmdline cmds cname +cncf cnf cnt codebase @@ -943,6 +944,7 @@ metarouter metastore mget microservice +microsoft middleware mik milton diff --git a/scripts/ci/pre_commit/pre_commit_check_setup_extra_packages_ref.py b/scripts/ci/pre_commit/pre_commit_check_setup_extra_packages_ref.py index e0a5a12a6ad42..1878bf1d9c0b8 100755 --- a/scripts/ci/pre_commit/pre_commit_check_setup_extra_packages_ref.py +++ b/scripts/ci/pre_commit/pre_commit_check_setup_extra_packages_ref.py @@ -60,6 +60,9 @@ def get_extras_from_setup() -> Dict[str, List[str]]: for extras in extras_regex.findall(extras_section): package = extras[1] alias = extras[0] + if alias == 'crypto': + # Skip crypto package - it is there just for backwards compatibility + continue # if there are no packages, use the extras alias itself if package == '[]': package = alias diff --git a/setup.py b/setup.py index b5aa93b3d36cc..d4025319d00d4 100644 --- a/setup.py +++ b/setup.py @@ -427,7 +427,7 @@ def write_version(filename: str = os.path.join(*[my_dir, "airflow", "git_version winrm = [ 'pywinrm~=0.4', ] -yandexcloud = [ +yandex = [ 'yandexcloud>=0.22.0', ] zendesk = [ @@ -582,7 +582,7 @@ def write_version(filename: str = os.path.join(*[my_dir, "airflow", "git_version "ssh": ssh, "telegram": telegram, "vertica": vertica, - "yandex": yandexcloud, + "yandex": yandex, "zendesk": zendesk, } @@ -611,6 +611,7 @@ def write_version(filename: str = os.path.join(*[my_dir, "airflow", "git_version 'cgroups': cgroups, 'cloudant': cloudant, 'cncf.kubernetes': kubernetes, + 'crypto': [], # TODO: remove this in Airflow 2.1 'dask': dask, 'databricks': databricks, 'datadog': datadog, @@ -660,6 +661,7 @@ def write_version(filename: str = os.path.join(*[my_dir, "airflow", "git_version 'qubole': qubole, 'rabbitmq': rabbitmq, 'redis': redis, + 's3': amazon, # TODO: Remove this in Airflow 2.1 'salesforce': salesforce, 'samba': samba, 'segment': segment, @@ -679,8 +681,7 @@ def write_version(filename: str = os.path.join(*[my_dir, "airflow", "git_version 'virtualenv': virtualenv, 'webhdfs': webhdfs, # TODO: remove this in Airflow 2.1 'winrm': winrm, # TODO: remove this in Airflow 2.1 - 'yandex': yandexcloud, # TODO: remove this in Airflow 2.1 - 'yandexcloud': yandexcloud, + 'yandex': yandex, 'zendesk': [], } @@ -728,6 +729,7 @@ def write_version(filename: str = os.path.join(*[my_dir, "airflow", "git_version 'cgroups': [], 'cloudant': ["cloudant"], 'cncf.kubernetes': ["cncf.kubernetes"], + 'crypto': [], # TODO: Remove this in Airflow 2.1 'dask': [], 'databricks': ["databricks"], 'datadog': ["datadog"], @@ -780,6 +782,7 @@ def write_version(filename: str = os.path.join(*[my_dir, "airflow", "git_version 'qubole': ["qubole"], 'rabbitmq': [], 'redis': ["redis"], + 's3': ["amazon"], # TODO: Remove this in Airflow 2.1 'salesforce': ["salesforce"], 'samba': ["samba"], 'segment': ["segment"],