Remove providers imports from core examples (apache#12252)

Core example DAGs should not depend on any non-core dependency
like providers packages.

closes: apache#12247

Co-authored-by: Xiaodong DENG <[email protected]>

.pre-commit-config.yaml

@@ -294,6 +294,13 @@ repos:
         entry: "\\|\\s*safe"
         files: \.html$
         pass_filenames: true
+      - id: no-providers-in-core-examples
+        language: pygrep
+        name: No providers imports in core example DAGs
+        description: The core example DAGs have no dependencies other than core Airflow
+        entry: "^\\s*from airflow\\.providers.*"
+        pass_filenames: true
+        files: ^airflow/example_dags/.*\.py$
       - id: no-relative-imports
         language: pygrep
         name: No relative imports
@@ -394,12 +401,6 @@ repos:
         pass_filenames: false
         require_serial: true
         additional_dependencies: ['pyyaml']
-      - id: pre-commit-descriptions
-        name: Check if pre-commits are described
-        entry: ./scripts/ci/pre_commit/pre_commit_check_pre_commits.sh
-        language: system
-        files: ^.pre-commit-config.yaml$|^STATIC_CODE_CHECKS.rst|^breeze-complete$
-        require_serial: true
       - id: sort-in-the-wild
         name: Sort INTHEWILD.md alphabetically
         entry: ./scripts/ci/pre_commit/pre_commit_sort_in_the_wild.sh

BREEZE.rst

@@ -2007,11 +2007,12 @@ This is the current syntax for  `./breeze <./breeze>`_:
                  dont-use-safe-filter end-of-file-fixer fix-encoding-pragma flake8 forbid-tabs
                  helm-lint incorrect-use-of-LoggingMixin insert-license isort language-matters
                  lint-dockerfile lint-openapi markdownlint mermaid mixed-line-ending mypy mypy-helm
-                 no-relative-imports pre-commit-descriptions provide-create-sessions
-                 providers-init-file pydevd pydocstyle pylint pylint-tests python-no-log-warn
-                 pyupgrade restrict-start_date rst-backticks setup-order setup-installation
-                 shellcheck sort-in-the-wild stylelint trailing-whitespace update-breeze-file
-                 update-extras update-local-yml-file update-setup-cfg-file version-sync yamllint
+                 no-providers-in-core-examples no-relative-imports pre-commit-descriptions
+                 provide-create-sessions providers-init-file pydevd pydocstyle pylint pylint-tests
+                 python-no-log-warn pyupgrade restrict-start_date rst-backticks setup-order
+                 setup-installation shellcheck sort-in-the-wild stylelint trailing-whitespace
+                 update-breeze-file update-extras update-local-yml-file update-setup-cfg-file
+                 version-sync yamllint
 
         You can pass extra arguments including options to to the pre-commit framework as
         <EXTRA_ARGS> passed after --. For example:

STATIC_CODE_CHECKS.rst

@@ -88,6 +88,8 @@ require Breeze Docker images to be installed locally:
 ----------------------------------- ---------------------------------------------------------------- ------------
 ``dont-use-safe-filter``              Don't use safe in templates.
 ----------------------------------- ---------------------------------------------------------------- ------------
+``no-providers-in-core-examples``     Don't use providers imports in core example DAGs
+----------------------------------- ---------------------------------------------------------------- ------------
 ``no-relative-imports``               Use absolute imports, not relative
 ----------------------------------- ---------------------------------------------------------------- ------------
 ``end-of-file-fixer``                 Makes sure that there is an empty line at the end.

airflow/example_dags/example_dag_decorator.py

@@ -17,17 +17,27 @@
 # under the License.
 
 
-import json
-from typing import Dict
+from typing import Any, Dict
+
+import requests
 
 from airflow.decorators import dag, task
+from airflow.models.baseoperator import BaseOperator
 from airflow.operators.email import EmailOperator
-from airflow.providers.http.operators.http import SimpleHttpOperator
 from airflow.utils.dates import days_ago
 
-DEFAULT_ARGS = {
-    "owner": "airflow",
-}
+DEFAULT_ARGS = {"owner": "airflow"}
+
+
+class GetRequestOperator(BaseOperator):
+    """Custom operator to sand GET request to provided url"""
+
+    def __init__(self, *, url: str, **kwargs):
+        super().__init__(**kwargs)
+        self.url = url
+
+    def execute(self, context):
+        return requests.get(self.url).json()
 
 
 # [START dag_decorator_usage]
@@ -39,12 +49,11 @@ def example_dag_decorator(email: str = '[email protected]'):
     :param email: Email to send IP to. Defaults to [email protected].
     :type email: str
     """
-    # Using default connection as it's set to httpbin.org by default
-    get_ip = SimpleHttpOperator(task_id='get_ip', endpoint='get', method='GET')
+    get_ip = GetRequestOperator(task_id='get_ip', url="http://httpbin.org/get")
 
     @task(multiple_outputs=True)
-    def prepare_email(raw_json: str) -> Dict[str, str]:
-        external_ip = json.loads(raw_json)['origin']
+    def prepare_email(raw_json: Dict[str, Any]) -> Dict[str, str]:
+        external_ip = raw_json['origin']
         return {
             'subject': f'Server connected from {external_ip}',
             'body': f'Seems like today your server executing Airflow is connected from IP {external_ip}<br>',

airflow/operators/email.py

@@ -47,6 +47,7 @@ class EmailOperator(BaseOperator):
     """
 
     template_fields = ('to', 'subject', 'html_content')
+    template_fields_renderers = {"html_content": "html"}
     template_ext = ('.html',)
     ui_color = '#e6faf9'
 

airflow/operators/python.py

@@ -69,6 +69,7 @@ class PythonOperator(BaseOperator):
     """
 
     template_fields = ('templates_dict', 'op_args', 'op_kwargs')
+    template_fields_renderers = {"templates_dict": "json", "op_args": "py", "op_kwargs": "py"}
     ui_color = '#ffefeb'
 
     # since we won't mutate the arguments, we should just do the shallow copy
@@ -145,6 +146,8 @@ class _PythonDecoratedOperator(BaseOperator):
     """
 
     template_fields = ('op_args', 'op_kwargs')
+    template_fields_renderers = {"op_args": "py", "op_kwargs": "py"}
+
     ui_color = PythonOperator.ui_color
 
     # since we won't mutate the arguments, we should just do the shallow copy

airflow/www/utils.py

@@ -336,6 +336,7 @@ def get_attr_renderer():
         'bash': lambda x: render(x, lexers.BashLexer),
         'bash_command': lambda x: render(x, lexers.BashLexer),
         'hql': lambda x: render(x, lexers.SqlLexer),
+        'html': lambda x: render(x, lexers.HtmlLexer),
         'sql': lambda x: render(x, lexers.SqlLexer),
         'doc': lambda x: render(x, lexers.TextLexer),
         'doc_json': lambda x: render(x, lexers.JsonLexer),

breeze-complete

@@ -103,6 +103,7 @@ mermaid
 mixed-line-ending
 mypy
 mypy-helm
+no-providers-in-core-examples
 no-relative-imports
 pre-commit-descriptions
 provide-create-sessions