diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4473a497b3b7e..ec2d230286aff 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -294,6 +294,13 @@ repos: entry: "\\|\\s*safe" files: \.html$ pass_filenames: true + - id: no-providers-in-core-examples + language: pygrep + name: No providers imports in core example DAGs + description: The core example DAGs have no dependencies other than core Airflow + entry: "^\\s*from airflow\\.providers.*" + pass_filenames: true + files: ^airflow/example_dags/.*\.py$ - id: no-relative-imports language: pygrep name: No relative imports @@ -394,12 +401,6 @@ repos: pass_filenames: false require_serial: true additional_dependencies: ['pyyaml'] - - id: pre-commit-descriptions - name: Check if pre-commits are described - entry: ./scripts/ci/pre_commit/pre_commit_check_pre_commits.sh - language: system - files: ^.pre-commit-config.yaml$|^STATIC_CODE_CHECKS.rst|^breeze-complete$ - require_serial: true - id: sort-in-the-wild name: Sort INTHEWILD.md alphabetically entry: ./scripts/ci/pre_commit/pre_commit_sort_in_the_wild.sh diff --git a/BREEZE.rst b/BREEZE.rst index 633223ce42f6c..43d86d6e4e9bc 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -2007,11 +2007,12 @@ This is the current syntax for `./breeze <./breeze>`_: dont-use-safe-filter end-of-file-fixer fix-encoding-pragma flake8 forbid-tabs helm-lint incorrect-use-of-LoggingMixin insert-license isort language-matters lint-dockerfile lint-openapi markdownlint mermaid mixed-line-ending mypy mypy-helm - no-relative-imports pre-commit-descriptions provide-create-sessions - providers-init-file pydevd pydocstyle pylint pylint-tests python-no-log-warn - pyupgrade restrict-start_date rst-backticks setup-order setup-installation - shellcheck sort-in-the-wild stylelint trailing-whitespace update-breeze-file - update-extras update-local-yml-file update-setup-cfg-file version-sync yamllint + no-providers-in-core-examples no-relative-imports pre-commit-descriptions + provide-create-sessions providers-init-file pydevd pydocstyle pylint pylint-tests + python-no-log-warn pyupgrade restrict-start_date rst-backticks setup-order + setup-installation shellcheck sort-in-the-wild stylelint trailing-whitespace + update-breeze-file update-extras update-local-yml-file update-setup-cfg-file + version-sync yamllint You can pass extra arguments including options to to the pre-commit framework as passed after --. For example: diff --git a/STATIC_CODE_CHECKS.rst b/STATIC_CODE_CHECKS.rst index 582acf6ebb867..e2ab2b0618777 100644 --- a/STATIC_CODE_CHECKS.rst +++ b/STATIC_CODE_CHECKS.rst @@ -88,6 +88,8 @@ require Breeze Docker images to be installed locally: ----------------------------------- ---------------------------------------------------------------- ------------ ``dont-use-safe-filter`` Don't use safe in templates. ----------------------------------- ---------------------------------------------------------------- ------------ +``no-providers-in-core-examples`` Don't use providers imports in core example DAGs +----------------------------------- ---------------------------------------------------------------- ------------ ``no-relative-imports`` Use absolute imports, not relative ----------------------------------- ---------------------------------------------------------------- ------------ ``end-of-file-fixer`` Makes sure that there is an empty line at the end. diff --git a/airflow/example_dags/example_dag_decorator.py b/airflow/example_dags/example_dag_decorator.py index 04c87ac399cd5..bca53749bdf64 100644 --- a/airflow/example_dags/example_dag_decorator.py +++ b/airflow/example_dags/example_dag_decorator.py @@ -17,17 +17,27 @@ # under the License. -import json -from typing import Dict +from typing import Any, Dict + +import requests from airflow.decorators import dag, task +from airflow.models.baseoperator import BaseOperator from airflow.operators.email import EmailOperator -from airflow.providers.http.operators.http import SimpleHttpOperator from airflow.utils.dates import days_ago -DEFAULT_ARGS = { - "owner": "airflow", -} +DEFAULT_ARGS = {"owner": "airflow"} + + +class GetRequestOperator(BaseOperator): + """Custom operator to sand GET request to provided url""" + + def __init__(self, *, url: str, **kwargs): + super().__init__(**kwargs) + self.url = url + + def execute(self, context): + return requests.get(self.url).json() # [START dag_decorator_usage] @@ -39,12 +49,11 @@ def example_dag_decorator(email: str = 'example@example.com'): :param email: Email to send IP to. Defaults to example@example.com. :type email: str """ - # Using default connection as it's set to httpbin.org by default - get_ip = SimpleHttpOperator(task_id='get_ip', endpoint='get', method='GET') + get_ip = GetRequestOperator(task_id='get_ip', url="http://httpbin.org/get") @task(multiple_outputs=True) - def prepare_email(raw_json: str) -> Dict[str, str]: - external_ip = json.loads(raw_json)['origin'] + def prepare_email(raw_json: Dict[str, Any]) -> Dict[str, str]: + external_ip = raw_json['origin'] return { 'subject': f'Server connected from {external_ip}', 'body': f'Seems like today your server executing Airflow is connected from IP {external_ip}
', diff --git a/airflow/operators/email.py b/airflow/operators/email.py index c979eaff2066a..9b403f9efdc8a 100644 --- a/airflow/operators/email.py +++ b/airflow/operators/email.py @@ -47,6 +47,7 @@ class EmailOperator(BaseOperator): """ template_fields = ('to', 'subject', 'html_content') + template_fields_renderers = {"html_content": "html"} template_ext = ('.html',) ui_color = '#e6faf9' diff --git a/airflow/operators/python.py b/airflow/operators/python.py index a9ed07c228ae3..6d254a0f7bc25 100644 --- a/airflow/operators/python.py +++ b/airflow/operators/python.py @@ -69,6 +69,7 @@ class PythonOperator(BaseOperator): """ template_fields = ('templates_dict', 'op_args', 'op_kwargs') + template_fields_renderers = {"templates_dict": "json", "op_args": "py", "op_kwargs": "py"} ui_color = '#ffefeb' # since we won't mutate the arguments, we should just do the shallow copy @@ -145,6 +146,8 @@ class _PythonDecoratedOperator(BaseOperator): """ template_fields = ('op_args', 'op_kwargs') + template_fields_renderers = {"op_args": "py", "op_kwargs": "py"} + ui_color = PythonOperator.ui_color # since we won't mutate the arguments, we should just do the shallow copy diff --git a/airflow/www/utils.py b/airflow/www/utils.py index 6095fb306ce5a..aa57d19c85323 100644 --- a/airflow/www/utils.py +++ b/airflow/www/utils.py @@ -336,6 +336,7 @@ def get_attr_renderer(): 'bash': lambda x: render(x, lexers.BashLexer), 'bash_command': lambda x: render(x, lexers.BashLexer), 'hql': lambda x: render(x, lexers.SqlLexer), + 'html': lambda x: render(x, lexers.HtmlLexer), 'sql': lambda x: render(x, lexers.SqlLexer), 'doc': lambda x: render(x, lexers.TextLexer), 'doc_json': lambda x: render(x, lexers.JsonLexer), diff --git a/breeze-complete b/breeze-complete index e833eafdd1dc2..ab01daddb8fb7 100644 --- a/breeze-complete +++ b/breeze-complete @@ -103,6 +103,7 @@ mermaid mixed-line-ending mypy mypy-helm +no-providers-in-core-examples no-relative-imports pre-commit-descriptions provide-create-sessions