Initialize new airflow/ui directory (apache#41846)

* Initialize new airflow/ui directory

configure flask blueprint and render real dags list

Add banner to switch back and forth between UIs

Add readme

basic test setup, chakra not working

Move Flask blueprint, build navbar

Switch to react with working tests

Basic hot refresh

add prettier

do not render the new UI banner on loggin page

add pre-commits to ui

add CI integration

update doc accordingly

redirect to login page on 401+403 errors

update lint command

update lint, lint:fix command

Add codegen to pre-commit

Add ui to breeze

Update build commands

* Fix dev favicon and table strip css bleedover

* breeze and CI fixes

* Fix github actions

* Try fixing selective checks and github actions again

* set pnpm cache-dependency-path

* Fix breeze static checks

* fix ci

* Update CONTRIBUTING.md

---------

Co-authored-by: pierrejeambrun <[email protected]>

.dockerignore

@@ -79,6 +79,7 @@
 airflow/git_version
 # Exclude mode_modules pulled by "yarn" for compilation of www files generated by NPM
 airflow/www/node_modules
+airflow/ui/node_modules
 
 # Exclude link to docs
 airflow/www/static/docs

.github/boring-cyborg.yml

@@ -592,6 +592,7 @@ labelPRBasedOnFilePath:
     - airflow/www/webpack.config.js
     - airflow/www/yarn.lock
     - docs/apache-airflow/ui.rst
+    - airflow/ui/**/*
 
   area:CLI:
     - airflow/cli/**/*.py

.github/workflows/basic-tests.yml

@@ -24,6 +24,10 @@ on:  # yamllint disable-line rule:truthy
         description: "The array of labels (in json form) determining public runners."
         required: true
         type: string
+      run-ui-tests:
+        description: "Whether to run UI tests (true/false)"
+        required: true
+        type: string
       run-www-tests:
         description: "Whether to run WWW tests (true/false)"
         required: true
@@ -83,6 +87,43 @@ jobs:
       - run: python -m pytest -n auto --color=yes
         working-directory: ./dev/breeze/
 
+
+  tests-ui:
+    timeout-minutes: 10
+    name: React UI tests
+    runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
+    if: inputs.run-ui-tests == 'true'
+    steps:
+      - name: "Cleanup repo"
+        shell: bash
+        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - name: "Cleanup docker"
+        run: ./scripts/ci/cleanup_docker.sh
+      - name: Setup pnpm
+        uses: pnpm/[email protected]
+        with:
+          version: 9
+          run_install: false
+      - name: "Setup node"
+        uses: actions/setup-node@v4
+        with:
+          node-version: 21
+          cache: 'pnpm'
+          cache-dependency-path: 'airflow/ui/pnpm-lock.yaml'
+      - name: "Cache eslint"
+        uses: actions/cache@v4
+        with:
+          path: 'airflow/ui/node_modules'
+          key: ${{ runner.os }}-ui-node-modules-${{ hashFiles('airflow/ui/**/pnpm-lock.yaml') }}
+      - run: cd airflow/ui && pnpm install --frozen-lockfile
+      - run: cd airflow/ui && pnpm test
+        env:
+          FORCE_COLOR: 2
+
   tests-www:
     timeout-minutes: 10
     name: React WWW tests

.github/workflows/ci.yml

@@ -89,6 +89,7 @@ jobs:
       skip-provider-tests: ${{ steps.selective-checks.outputs.skip-provider-tests }}
       run-tests: ${{ steps.selective-checks.outputs.run-tests }}
       run-amazon-tests: ${{ steps.selective-checks.outputs.run-amazon-tests }}
+      run-ui-tests: ${{ steps.selective-checks.outputs.run-ui-tests }}
       run-www-tests: ${{ steps.selective-checks.outputs.run-www-tests }}
       run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }}
       basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only }}
@@ -168,6 +169,7 @@ jobs:
     uses: ./.github/workflows/basic-tests.yml
     with:
       runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }}
+      run-ui-tests: ${{needs.build-info.outputs.run-ui-tests}}
       run-www-tests: ${{needs.build-info.outputs.run-www-tests}}
       needs-api-codegen: ${{needs.build-info.outputs.needs-api-codegen}}
       default-python-version: ${{needs.build-info.outputs.default-python-version}}

.gitignore

@@ -165,6 +165,13 @@ node_modules
 npm-debug.log*
 derby.log
 metastore_db
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+.vscode/*
+!.vscode/extensions.json
+/.vite/
 
 # Airflow log files when airflow is run locally
 airflow-*.err

.pre-commit-config.yaml

@@ -67,7 +67,7 @@ repos:
       - id: insert-license
         name: Add license for all CSS/JS/JSX/PUML/TS/TSX files
         files: \.(css|jsx?|puml|tsx?)$
-        exclude: ^\.github/.*$|^.*/.*_vendor/|^airflow/www/static/js/types/api-generated.ts$
+        exclude: ^\.github/.*$|^.*/.*_vendor/|^airflow/www/static/js/types/api-generated.ts$|ui/openapi-gen/
         args:
           - --comment-style
           - "/*!| *| */"
@@ -289,7 +289,7 @@ repos:
         name: Check YAML files with yamllint
         entry: yamllint -c yamllint-config.yml --strict
         types: [yaml]
-        exclude: ^.*airflow\.template\.yaml$|^.*init_git_sync\.template\.yaml$|^.*/.*_vendor/|^chart/(?:templates|files)/.*\.yaml$|openapi/.*\.yaml$|^\.pre-commit-config\.yaml$|^.*/reproducible_build.yaml$
+        exclude: ^.*airflow\.template\.yaml$|^.*init_git_sync\.template\.yaml$|^.*/.*_vendor/|^chart/(?:templates|files)/.*\.yaml$|openapi/.*\.yaml$|^\.pre-commit-config\.yaml$|^.*/reproducible_build.yaml$|^.*pnpm-lock\.yaml$
   - repo: https://github.com/ikamensh/flynt
     rev: '1.0.1'
     hooks:
@@ -314,7 +314,7 @@ repos:
           The word(s) should be in lowercase." && exec codespell "$@"' --
         language: python
         types: [text]
-        exclude: ^.*/.*_vendor/|^airflow/www/static/css/material-icons\.css$|^images/.*$|^RELEASE_NOTES\.txt$|^.*package-lock\.json$|^.*/kinglear\.txt$
+        exclude: ^.*/.*_vendor/|^airflow/www/static/css/material-icons\.css$|^images/.*$|^RELEASE_NOTES\.txt$|^.*package-lock\.json$|^.*/kinglear\.txt$|^.*pnpm-lock\.yaml$
         args:
           - --ignore-words=docs/spelling_wordlist.txt
           - --skip=airflow/providers/*/*.rst,airflow/www/*.log,docs/*/commits.rst,docs/apache-airflow/tutorial/pipeline_example.csv,*.min.js,*.lock,INTHEWILD.md
@@ -580,6 +580,7 @@ repos:
         exclude: >
           (?x)
           ^airflow/api_connexion/openapi/v1.yaml$|
+          ^airflow/ui/openapi-gen/|
           ^airflow/cli/commands/webserver_command.py$|
           ^airflow/cli/commands/internal_api_command.py$|
           ^airflow/config_templates/|
@@ -608,6 +609,7 @@ repos:
           ^airflow/providers/opsgenie/hooks/opsgenie.py$|
           ^airflow/providers/redis/provider.yaml$|
           ^airflow/serialization/serialized_objects.py$|
+          ^airflow/ui/pnpm-lock.yaml$|
           ^airflow/utils/db.py$|
           ^airflow/utils/trigger_rule.py$|
           ^airflow/www/static/css/bootstrap-theme.css$|
@@ -824,6 +826,24 @@ repos:
         files: ^airflow/www/.*\.(css|sass|scss)$
         # Keep dependency versions in sync w/ airflow/www/package.json
         additional_dependencies: ['[email protected]', '[email protected]', '[email protected]']
+      - id: compile-ui-assets
+        name: Compile ui assets (manual)
+        language: node
+        stages: ['manual']
+        types_or: [javascript, ts, tsx]
+        files: ^airflow/ui/
+        entry: ./scripts/ci/pre_commit/compile_ui_assets.py
+        pass_filenames: false
+        additional_dependencies: ['[email protected]']
+      - id: compile-ui-assets-dev
+        name: Compile ui assets in dev mode (manual)
+        language: node
+        stages: ['manual']
+        types_or: [javascript, ts, tsx]
+        files: ^airflow/ui/
+        entry: ./scripts/ci/pre_commit/compile_ui_assets_dev.py
+        pass_filenames: false
+        additional_dependencies: ['[email protected]']
       - id: compile-www-assets
         name: Compile www assets (manual)
         language: node
@@ -1118,9 +1138,17 @@ repos:
         language: node
         'types_or': [javascript, ts, tsx, yaml, css, json]
         files: ^airflow/www/static/(js|css)/|^airflow/api_connexion/openapi/v1\.yaml$
-        entry: ./scripts/ci/pre_commit/www_lint.py
+        entry: ./scripts/ci/pre_commit/lint_www.py
         additional_dependencies: ['[email protected]', "openapi-typescript@>=6.7.4"]
         pass_filenames: false
+      - id: ts-compile-format-lint-ui
+        name: TS types generation / ESLint / Prettier new UI files
+        language: node
+        types_or: [javascript, ts, tsx, yaml, css, json]
+        files: ^airflow/ui/|^airflow/api_connexion/openapi/v1\.yaml$
+        entry: ./scripts/ci/pre_commit/lint_ui.py
+        additional_dependencies: ['[email protected]']
+        pass_filenames: false
       - id: check-tests-unittest-testcase
         name: Check that unit tests do not inherit from unittest.TestCase
         entry: ./scripts/ci/pre_commit/unittest_testcase.py
@@ -1296,8 +1324,7 @@ repos:
         language: python
         entry: ./scripts/ci/pre_commit/migration_reference.py
         pass_filenames: false
-        files: |
-          ^airflow/migrations/versions/.*\.py$|^docs/apache-airflow/migrations-ref\.rst$|^airflow/providers/fab/alembic/versions/.*\.py$
+        files: ^airflow/migrations/versions/.*\.py$|^docs/apache-airflow/migrations-ref\.rst$
         additional_dependencies: ['rich>=12.4.4']
       - id: update-er-diagram
         name: Update ER diagram

.rat-excludes

@@ -166,3 +166,4 @@ version.txt
 
 # Front end generated files
 api-generated.ts
+openapi-gen

airflow/ui/.prettierignore

@@ -0,0 +1,5 @@
+.mypy_cache/
+templates/**/*.html
+dist/
+*.md
+*.yaml

airflow/ui/.prettierrc

@@ -0,0 +1,11 @@
+{
+  "overrides": [
+    {
+      "files": "*.json",
+      "options": {
+        "tabWidth": 2
+      }
+    }
+  ],
+  "trailingComma": "es5"
+}

airflow/ui/CONTRIBUTING.md

@@ -0,0 +1,70 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+ -->
+
+# Contributing to the UI
+
+## Getting Started
+
+1. Install `pnpm`. Check their [docs](https://pnpm.io/installation)
+
+2. Run `breeze start-airflow --dev-mode` will install and run the local dev server to allow for hot-reloading during development.
+
+3. Click on the banner in the old UI to redirect you to the new ui or go to `localhost:28080/ui`
+
+## CLI Commands
+
+- `pnpm install`
+- `pnpm lint` lint check
+- `pnpm test` run ui tests
+- `pnpm build` build the project to appear in `ui/dist`, this is what the webserver will usually look for
+- `pnpm dev` run the vite dev server to enable hot reloading for local development. You also need the `dev_mode` environment variable enabled on the webserver.
+
+## Learn
+
+If you're new to modern frontend development or parts of our stack, you may want to check out these resources to understand our codebase:
+
+- Typescript is an extension of javascript to add type-checking to our app. Files ending in `.ts` or `.tsx` will be type-checked. Check out the [handbook](https://www.typescriptlang.org/docs/handbook/typescript-in-5-minutes-func.html) for an introduction or feel free to keep this [cheatsheet](https://github.com/typescript-cheatsheets/react) open while developing.
+
+- React powers our entire app so it would be valuable to learn JSX, the html-in-js templates React utilizes. Files that contain JSX will end in `.tsx` instead of `.ts`. Check out their official [tutorial](https://reactjs.org/tutorial/tutorial.html#overview) for a basic overview.
+
+- Chakra-UI is our component library and theming system. You'll notice we have no traditional css nor html tags. This is all handled in Chakra with importing standard components like `<Box>` or `<Text>` that are styled globally in `src/theme.ts` file and then by passing styles as component props. Check out their [docs](https://chakra-ui.com/docs/getting-started) to see all the included components and hooks.
+
+- Testing is done with React Testing Library. We follow their idea of "The more your tests resemble the way your software is used,
+  the more confidence they can give you." Keep their [cheatsheet](https://testing-library.com/docs/react-testing-library/cheatsheet) open when writing tests
+
+- We use Vite as our app framework for running a dev server and build commands. Check out their [docs](https://vitejs.dev/guide/) if you need to customize it.
+
+- State management is handled with [Context](https://reactjs.org/docs/context.html) and [react-query](https://tanstack.com/query/latest/docs/framework/react/overview). Context is used for App-level state that doesn't change often (authentication, dark/light mode). React Query handles all the state and side effects (loading, error, caching, etc) of async data from the API.
+
+## Project Structure
+
+- `/openapi` autogenerated types and queries based on the public REST API openapi spec. Do not manually edit. To regenerate use:
+
+```console
+pnpm run codegen
+```
+
+- `/src/assets` static assets for the UI
+- `/src/components` shared components across the UI
+- `/dist` build files
+- TODO: build out project structure more
+
+## Find open issues
+
+TODO

airflow/ui/README.md

@@ -0,0 +1,69 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+ -->
+
+# React + TypeScript + Vite
+
+This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules.
+
+Currently, two official plugins are available:
+
+- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react/README.md) uses [Babel](https://babeljs.io/) for Fast Refresh
+- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react-swc) uses [SWC](https://swc.rs/) for Fast Refresh
+
+## Expanding the ESLint configuration
+
+If you are developing a production application, we recommend updating the configuration to enable type aware lint rules:
+
+- Configure the top-level `parserOptions` property like this:
+
+```js
+export default tseslint.config({
+  languageOptions: {
+    // other options...
+    parserOptions: {
+      project: ['./tsconfig.node.json', './tsconfig.app.json'],
+      tsconfigRootDir: import.meta.dirname,
+    },
+  },
+})
+```
+
+- Replace `tseslint.configs.recommended` to `tseslint.configs.recommendedTypeChecked` or `tseslint.configs.strictTypeChecked`
+- Optionally add `...tseslint.configs.stylisticTypeChecked`
+- Install [eslint-plugin-react](https://github.com/jsx-eslint/eslint-plugin-react) and update the config:
+
+```js
+// eslint.config.js
+import react from 'eslint-plugin-react'
+
+export default tseslint.config({
+  // Set the react version
+  settings: { react: { version: '18.3' } },
+  plugins: {
+    // Add the react plugin
+    react,
+  },
+  rules: {
+    // other rules...
+    // Enable its recommended rules
+    ...react.configs.recommended.rules,
+    ...react.configs['jsx-runtime'].rules,
+  },
+})
+```

airflow/ui/dev/index.html

@@ -0,0 +1,26 @@
+<!-- Entry html file when the Vite dev server is running -->
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <link
+      rel="icon"
+      type="image/png"
+      href="http://localhost:5173/public/pin_32.png"
+    />
+    <script type="module" src="http://localhost:5173/@vite/client"></script>
+    <script type="module">
+      import RefreshRuntime from "http://localhost:5173/@react-refresh";
+      RefreshRuntime.injectIntoGlobalHook(window);
+      window.$RefreshReg$ = () => {};
+      window.$RefreshSig$ = () => (type) => type;
+      window.__vite_plugin_react_preamble_installed__ = true;
+    </script>
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Airflow 3.0</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="http://localhost:5173/src/main.tsx"></script>
+  </body>
+</html>