Add Plugins View in web UI (apache#10770)

airflow/cli/commands/plugins_command.py

@@ -49,6 +49,7 @@
     "appbuilder_menu_items",
     "global_operator_extra_links",
     "operator_extra_links",
+    "source",
 ]
 
 

airflow/plugins_manager.py

@@ -60,6 +60,43 @@
 """
 
 
+class AirflowPluginSource:
+    """Class used to define an AirflowPluginSource."""
+
+    def __str__(self):
+        raise NotImplementedError
+
+    def __html__(self):
+        raise NotImplementedError
+
+
+class PluginsDirectorySource(AirflowPluginSource):
+    """Class used to define Plugins loaded from Plugins Directory."""
+
+    def __init__(self, path):
+        self.path = os.path.relpath(path, settings.PLUGINS_FOLDER)
+
+    def __str__(self):
+        return f"$PLUGINS_FOLDER/{self.path}"
+
+    def __html__(self):
+        return f"<em>$PLUGINS_FOLDER/</em>{self.path}"
+
+
+class EntryPointSource(AirflowPluginSource):
+    """Class used to define Plugins loaded from entrypoint."""
+
+    def __init__(self, entrypoint):
+        self.dist = str(entrypoint.dist)
+        self.entrypoint = str(entrypoint)
+
+    def __str__(self):
+        return f"{self.dist}: {self.entrypoint}"
+
+    def __html__(self):
+        return f"<em>{self.dist}:</em> {self.entrypoint}"
+
+
 class AirflowPluginException(Exception):
     """Exception when loading plugin."""
 
@@ -68,6 +105,7 @@ class AirflowPlugin:
     """Class used to define AirflowPlugin."""
 
     name: Optional[str] = None
+    source: Optional[AirflowPluginSource] = None
     operators: List[Any] = []
     sensors: List[Any] = []
     hooks: List[Any] = []
@@ -151,6 +189,7 @@ def load_entrypoint_plugins():
                 plugin_instance = plugin_class()
                 if callable(getattr(plugin_instance, 'on_load', None)):
                     plugin_instance.on_load()
+                    plugin_instance.source = EntryPointSource(entry_point)
                     plugins.append(plugin_instance)
         except Exception as e:  # pylint: disable=broad-except
             log.exception("Failed to import plugin %s", entry_point.name)
@@ -184,6 +223,7 @@ def load_plugins_from_plugin_directory():
 
             for mod_attr_value in (m for m in mod.__dict__.values() if is_valid_plugin(m)):
                 plugin_instance = mod_attr_value()
+                plugin_instance.source = PluginsDirectorySource(file_path)
                 plugins.append(plugin_instance)
 
         except Exception as e:  # pylint: disable=broad-except

airflow/www/extensions/init_views.py

@@ -55,6 +55,7 @@ def init_appbuilder_views(app):
     appbuilder.add_view(views.TaskRescheduleModelView, "Task Reschedules", category="Browse")
     appbuilder.add_view(views.ConfigurationView, "Configurations", category="Admin", category_icon="fa-user")
     appbuilder.add_view(views.ConnectionModelView, "Connections", category="Admin")
+    appbuilder.add_view(views.PluginView, "Plugins", category="Admin")
     appbuilder.add_view(views.PoolModelView, "Pools", category="Admin")
     appbuilder.add_view(views.VariableModelView, "Variables", category="Admin")
     appbuilder.add_view(views.XComModelView, "XComs", category="Admin")

airflow/www/templates/airflow/plugin.html

@@ -0,0 +1,56 @@
+{#
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+#}
+
+{% extends base_template %}
+
+
+{% block title %}
+
+  {{ title }}
+
+{% endblock %}
+
+
+{% block content %}
+
+  <div>
+
+    <h2>{{ title }}</h2>
+
+    {% for plugin in plugins %}
+      <h4>{{ plugin["plugin_no"] }}. {{ plugin["plugin_name"] }}</h4>
+
+      <table class="table table-striped table-bordered">
+        <tr>
+          <th>Attribute</th>
+          <th>Value</th>
+        </tr>
+        {% for attr, value in plugin["attrs"].items() %}
+          <tr>
+            <td>{{ attr }}</td>
+            <td class='code'>{{ value }}</td>
+          </tr>
+        {% endfor %}
+      </table>
+
+    {% endfor %}
+
+  </div>
+
+{% endblock %}

airflow/www/views.py

@@ -49,7 +49,7 @@
 from wtforms import SelectField, validators
 
 import airflow
-from airflow import models, settings
+from airflow import models, plugins_manager, settings
 from airflow.api.common.experimental.mark_tasks import (
     set_dag_run_state_to_failed, set_dag_run_state_to_success,
 )
@@ -2546,6 +2546,58 @@ def prefill_form(self, form, pk):
                 field.data = value
 
 
+class PluginView(AirflowBaseView):
+    """View to show Airflow Plugins"""
+
+    default_view = 'list'
+
+    plugins_attributes_to_dump = [
+        "operators",
+        "sensors",
+        "hooks",
+        "executors",
+        "macros",
+        "admin_views",
+        "flask_blueprints",
+        "menu_links",
+        "appbuilder_views",
+        "appbuilder_menu_items",
+        "global_operator_extra_links",
+        "operator_extra_links",
+        "source",
+    ]
+
+    @expose('/plugin')
+    @has_access
+    def list(self):
+        """List loaded plugins."""
+        plugins_manager.ensure_plugins_loaded()
+        plugins_manager.integrate_dag_plugins()
+        plugins_manager.integrate_executor_plugins()
+        plugins_manager.initialize_extra_operators_links_plugins()
+        plugins_manager.initialize_web_ui_plugins()
+
+        plugins = []
+        for plugin_no, plugin in enumerate(plugins_manager.plugins, 1):
+            plugin_data = {
+                'plugin_no': plugin_no,
+                'plugin_name': plugin.name,
+                'attrs': {},
+            }
+            for attr_name in self.plugins_attributes_to_dump:
+                attr_value = getattr(plugin, attr_name)
+                plugin_data['attrs'][attr_name] = attr_value
+
+            plugins.append(plugin_data)
+
+        title = "Airflow Plugins"
+        return self.render_template(
+            'airflow/plugin.html',
+            plugins=plugins,
+            title=title,
+        )
+
+
 class PoolModelView(AirflowModelView):
     """View to show records from Pool table"""
 

tests/plugins/test_plugins_manager.py

@@ -194,3 +194,32 @@ class AirflowAdminMenuLinksPlugin(AirflowPlugin):
             # assert not logs
             with self.assertRaises(AssertionError), self.assertLogs(plugins_manager.log):
                 plugins_manager.initialize_web_ui_plugins()
+
+
+class TestPluginsDirectorySource(unittest.TestCase):
+    def test_should_return_correct_path_name(self):
+        from airflow import plugins_manager
+
+        source = plugins_manager.PluginsDirectorySource(__file__)
+        self.assertEqual("test_plugins_manager.py", source.path)
+        self.assertEqual("$PLUGINS_FOLDER/test_plugins_manager.py", str(source))
+        self.assertEqual("<em>$PLUGINS_FOLDER/</em>test_plugins_manager.py", source.__html__())
+
+
+class TestEntryPointSource(unittest.TestCase):
+    @mock.patch('airflow.plugins_manager.pkg_resources.iter_entry_points')
+    def test_should_return_correct_source_details(self, mock_ep_plugins):
+        from airflow import plugins_manager
+
+        mock_entrypoint = mock.Mock()
+        mock_entrypoint.name = 'test-entrypoint-plugin'
+        mock_entrypoint.module_name = 'module_name_plugin'
+        mock_entrypoint.dist = 'test-entrypoint-plugin==1.0.0'
+        mock_ep_plugins.return_value = [mock_entrypoint]
+
+        plugins_manager.load_entrypoint_plugins()
+
+        source = plugins_manager.EntryPointSource(mock_entrypoint)
+        self.assertEqual(str(mock_entrypoint), source.entrypoint)
+        self.assertEqual("test-entrypoint-plugin==1.0.0: " + str(mock_entrypoint), str(source))
+        self.assertEqual("<em>test-entrypoint-plugin==1.0.0:</em> " + str(mock_entrypoint), source.__html__())

tests/www/test_views.py

@@ -52,6 +52,7 @@
 from airflow.models.serialized_dag import SerializedDagModel
 from airflow.operators.bash import BashOperator
 from airflow.operators.dummy_operator import DummyOperator
+from airflow.plugins_manager import AirflowPlugin, EntryPointSource, PluginsDirectorySource
 from airflow.security import permissions
 from airflow.ti_deps.dependencies_states import QUEUEABLE_STATES, RUNNABLE_STATES
 from airflow.utils import dates, timezone
@@ -318,6 +319,61 @@ def test_import_variables_success(self):
         self.check_content_in_response('4 variable(s) successfully updated.', resp)
 
 
+class PluginOperator(BaseOperator):
+    pass
+
+
+class EntrypointPlugin(AirflowPlugin):
+    name = 'test-entrypoint-testpluginview'
+
+
+class TestPluginView(TestBase):
+    def test_should_list_plugins_on_page_with_details(self):
+        resp = self.client.get('/plugin')
+        self.check_content_in_response("test_plugin", resp)
+        self.check_content_in_response("Airflow Plugins", resp)
+        self.check_content_in_response("source", resp)
+        self.check_content_in_response("<em>$PLUGINS_FOLDER/</em>test_plugin.py", resp)
+
+    @mock.patch('airflow.plugins_manager.pkg_resources.iter_entry_points')
+    def test_should_list_entrypoint_plugins_on_page_with_details(self, mock_ep_plugins):
+        from airflow.plugins_manager import load_entrypoint_plugins
+
+        mock_entrypoint = mock.Mock()
+        mock_entrypoint.name = 'test-entrypoint-testpluginview'
+        mock_entrypoint.module_name = 'module_name_testpluginview'
+        mock_entrypoint.dist = 'test-entrypoint-testpluginview==1.0.0'
+        mock_entrypoint.load.return_value = EntrypointPlugin
+        mock_ep_plugins.return_value = [mock_entrypoint]
+
+        load_entrypoint_plugins()
+        resp = self.client.get('/plugin')
+
+        self.check_content_in_response("test_plugin", resp)
+        self.check_content_in_response("Airflow Plugins", resp)
+        self.check_content_in_response("source", resp)
+        self.check_content_in_response("<em>test-entrypoint-testpluginview==1.0.0:</em> <Mock id=", resp)
+
+
+class TestPluginsDirectorySource(unittest.TestCase):
+    def test_should_provide_correct_attribute_values(self):
+        source = PluginsDirectorySource("./test_views.py")
+        self.assertEqual("$PLUGINS_FOLDER/../../test_views.py", str(source))
+        self.assertEqual("<em>$PLUGINS_FOLDER/</em>../../test_views.py", source.__html__())
+        self.assertEqual("../../test_views.py", source.path)
+
+
+class TestEntryPointSource(unittest.TestCase):
+    def test_should_provide_correct_attribute_values(self):
+        mock_entrypoint = mock.Mock()
+        mock_entrypoint.dist = 'test-entrypoint-dist==1.0.0'
+        source = EntryPointSource(mock_entrypoint)
+        self.assertEqual("test-entrypoint-dist==1.0.0", source.dist)
+        self.assertEqual(str(mock_entrypoint), source.entrypoint)
+        self.assertEqual("test-entrypoint-dist==1.0.0: " + str(mock_entrypoint), str(source))
+        self.assertEqual("<em>test-entrypoint-dist==1.0.0:</em> " + str(mock_entrypoint), source.__html__())
+
+
 class TestPoolModelView(TestBase):
     def setUp(self):
         super().setUp()
@@ -441,7 +497,7 @@ def prepare_dagruns(self):
             state=State.RUNNING)
 
     def test_index(self):
-        with assert_queries_count(41):
+        with assert_queries_count(42):
             resp = self.client.get('/', follow_redirects=True)
         self.check_content_in_response('DAGs', resp)