Migration of all DAG details to existing grid view dag details panel (a…

…pache#31690) * dag details added in grid view * dag details data integrated and all links added * deleted old dag_details and all refs * reverted view.py and bug resolution on object parsing * resolved bug while combining dag and dagDetails * failing tests fixed * Update airflow/www/static/js/dag/details/Dag.tsx Co-authored-by: Brent Bovenzi <[email protected]> * Update airflow/www/static/js/components/ViewScheduleInterval.tsx Co-authored-by: Brent Bovenzi <[email protected]> * code clean and static checks fixed --------- Co-authored-by: Brent Bovenzi <[email protected]>
eladkal · Jun 6, 2023 · 02d94c5 · 02d94c5
1 parent 6f8cd65
commit 02d94c5
Show file tree

Hide file tree

Showing 7 changed files with 267 additions and 13 deletions.
diff --git a/airflow/www/static/js/api/index.ts b/airflow/www/static/js/api/index.ts
@@ -42,6 +42,7 @@ import useUpstreamDatasetEvents from "./useUpstreamDatasetEvents";
 import useTaskInstance from "./useTaskInstance";
 import useDag from "./useDag";
 import useDagCode from "./useDagCode";
+import useDagDetails from "./useDagDetails";
 import useHealth from "./useHealth";
 import usePools from "./usePools";
 import useDags from "./useDags";
@@ -59,6 +60,7 @@ export {
   useClearTask,
   useDag,
   useDagCode,
+  useDagDetails,
   useDagRuns,
   useDags,
   useDataset,

diff --git a/airflow/www/static/js/api/useDagDetails.ts b/airflow/www/static/js/api/useDagDetails.ts
@@ -0,0 +1,53 @@
+/*!
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import axios, { AxiosResponse } from "axios";
+import { useQuery } from "react-query";
+
+import { getMetaValue } from "src/utils";
+import type { API } from "src/types";
+import type { DAG, DAGDetail } from "src/types/api-generated";
+import useDag from "./useDag";
+
+const dagDetailsApiUrl = getMetaValue("dag_details_api");
+
+const combineResults = (
+  dagData: DAG,
+  dagDetailsData: DAGDetail
+): Omit<DAGDetail, "defaultView"> => ({ ...dagData, ...dagDetailsData });
+
+const useDagDetails = () => {
+  const { data: dagData } = useDag();
+  const dagDetailsResult = useQuery(
+    ["dagDetailsQuery"],
+    () => axios.get<AxiosResponse, API.DAGDetail>(dagDetailsApiUrl),
+    {
+      enabled: !!dagData,
+    }
+  );
+  return {
+    ...dagDetailsResult,
+    data: combineResults(
+      dagData || {},
+      dagDetailsResult.data ? dagDetailsResult.data : {}
+    ),
+  };
+};
+
+export default useDagDetails;
diff --git a/airflow/www/static/js/components/ViewScheduleInterval.tsx b/airflow/www/static/js/components/ViewScheduleInterval.tsx
@@ -0,0 +1,61 @@
+/*!
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import React from "react";
+import { Flex, Text } from "@chakra-ui/react";
+
+interface Props {
+  data: Record<string, number | null>;
+}
+
+const ViewScheduleInterval = ({ data }: Props) => {
+  const genericTimeDeltaUnits = [
+    "days",
+    "day",
+    "seconds",
+    "microseconds",
+    "years",
+    "year",
+    "months",
+    "month",
+    "leapdays",
+    "hours",
+    "hour",
+    "minutes",
+    "minute",
+    "second",
+    "microsecond",
+  ];
+
+  return (
+    <Flex flexWrap="wrap">
+      {!!data &&
+        genericTimeDeltaUnits.map(
+          (unit) =>
+            !!data[unit] && (
+              <Text mr={1} key={unit}>
+                {data[unit]} {unit}
+              </Text>
+            )
+        )}
+    </Flex>
+  );
+};
+
+export default ViewScheduleInterval;
diff --git a/airflow/www/static/js/dag/details/Dag.tsx b/airflow/www/static/js/dag/details/Dag.tsx
@@ -29,23 +29,32 @@ import {
   Heading,
   Text,
   Box,
+  Badge,
+  Code,
 } from "@chakra-ui/react";
-import { mean } from "lodash";
+import { mean, omit } from "lodash";
 
 import { getDuration, formatDuration } from "src/datetime_utils";
 import {
+  appendSearchParams,
   finalStatesMap,
   getMetaValue,
   getTaskSummary,
+  toSentenceCase,
   useOffsetTop,
 } from "src/utils";
-import { useGridData } from "src/api";
+import { useGridData, useDagDetails } from "src/api";
 import Time from "src/components/Time";
+import ViewScheduleInterval from "src/components/ViewScheduleInterval";
 import type { TaskState } from "src/types";
 
+import type { DAG, DAGDetail } from "src/types/api-generated";
+import URLSearchParamsWrapper from "src/utils/URLSearchParamWrapper";
 import { SimpleStatus } from "../StatusBox";
 
-const dagDetailsUrl = getMetaValue("dag_details_url");
+const dagId = getMetaValue("dag_id");
+const tagIndexUrl = getMetaValue("tag_index_url");
+const taskInstancesUrl = getMetaValue("task_instances_list_url");
 
 const Dag = () => {
   const {
@@ -54,9 +63,32 @@ const Dag = () => {
   const detailsRef = useRef<HTMLDivElement>(null);
   const offsetTop = useOffsetTop(detailsRef);
 
+  const { data: dagDetailsData, isLoading: isLoadingDagDetails } =
+    useDagDetails();
+
+  // fields to exclude from "dagDetailsData" since handled seprately or not required
+  const dagDataExcludeFields = [
+    "defaultView",
+    "fileToken",
+    "scheduleInterval",
+    "tags",
+    "owners",
+    "params",
+  ];
+
+  const listParams = new URLSearchParamsWrapper({
+    _flt_3_dag_id: dagId,
+  });
+
+  const getRedirectUri = (state: string): string => {
+    listParams.set("_flt_3_state", state);
+    return appendSearchParams(taskInstancesUrl, listParams);
+  };
+
   const taskSummary = getTaskSummary({ task: groups });
   const numMap = finalStatesMap();
   const durations: number[] = [];
+
   dagRuns.forEach((dagRun) => {
     durations.push(getDuration(dagRun.startDate, dagRun.endDate));
     const stateKey = dagRun.state == null ? "no_status" : dagRun.state;
@@ -72,8 +104,15 @@ const Dag = () => {
         <Tr key={val}>
           <Td>
             <Flex alignItems="center">
-              <SimpleStatus state={val as TaskState} mr={2} />
-              <Text>Total {val}</Text>
+              <SimpleStatus state={val as TaskState} />
+              <Link
+                href={getRedirectUri(val)}
+                title={`View all ${val} DAGS`}
+                color="blue"
+                ml="5px"
+              >
+                Total {val}
+              </Link>
             </Flex>
           </Td>
           <Td>{key}</Td>
@@ -89,16 +128,35 @@ const Dag = () => {
   const firstStart = dagRuns[0]?.startDate;
   const lastStart = dagRuns[dagRuns.length - 1]?.startDate;
 
+  // parse value for each key if date or not
+  const parseStringData = (value: string) =>
+    Number.isNaN(Date.parse(value)) ? value : <Time dateTime={value} />;
+
+  // render dag and dag_details data
+  const renderDagDetailsData = (
+    data: DAG | DAGDetail,
+    excludekeys: Array<string>
+  ) => (
+    <>
+      {Object.entries(data).map(
+        ([key, value]) =>
+          !excludekeys.includes(key) && (
+            <Tr key={key}>
+              <Td>{toSentenceCase(key)}</Td>
+              <Td>{parseStringData(String(value))}</Td>
+            </Tr>
+          )
+      )}
+    </>
+  );
+
   return (
     <Box
       height="100%"
       maxHeight={`calc(100% - ${offsetTop}px)`}
       ref={detailsRef}
       overflowY="auto"
     >
-      <Button as={Link} variant="ghost" colorScheme="blue" href={dagDetailsUrl}>
-        More Details
-      </Button>
       <Table variant="striped">
         <Tbody>
           {durations.length > 0 && (
@@ -169,6 +227,80 @@ const Dag = () => {
               <Td>{value}</Td>
             </Tr>
           ))}
+          {!isLoadingDagDetails && !!dagDetailsData && (
+            <>
+              <Tr borderBottomWidth={2} borderBottomColor="gray.300">
+                <Td>
+                  <Heading size="sm">DAG Details</Heading>
+                </Td>
+                <Td />
+              </Tr>
+              {renderDagDetailsData(dagDetailsData, dagDataExcludeFields)}
+              <Tr>
+                <Td>Owners</Td>
+                <Td>
+                  <Flex flexWrap="wrap">
+                    {dagDetailsData.owners?.map((owner) => (
+                      <Badge key={owner} colorScheme="blue">
+                        {owner}
+                      </Badge>
+                    ))}
+                  </Flex>
+                </Td>
+              </Tr>
+              <Tr>
+                <Td>Tags</Td>
+                <Td>
+                  {!!dagDetailsData.tags && dagDetailsData.tags?.length > 0 ? (
+                    <Flex flexWrap="wrap">
+                      {dagDetailsData.tags?.map((tag) => (
+                        <Button
+                          key={tag.name}
+                          as={Link}
+                          colorScheme="teal"
+                          size="xs"
+                          href={tagIndexUrl.replace(
+                            "_TAG_NAME_",
+                            tag?.name || ""
+                          )}
+                          mr={3}
+                        >
+                          {tag.name}
+                        </Button>
+                      ))}
+                    </Flex>
+                  ) : (
+                    "No tags"
+                  )}
+                </Td>
+              </Tr>
+              <Tr>
+                <Td>Schedule interval</Td>
+                <Td>
+                  {dagDetailsData.scheduleInterval?.type ===
+                  "CronExpression" ? (
+                    <Text>{dagDetailsData.scheduleInterval?.value}</Text>
+                  ) : (
+                    // for TimeDelta and RelativeDelta
+                    <ViewScheduleInterval
+                      data={omit(dagDetailsData.scheduleInterval, [
+                        "type",
+                        "value",
+                      ])}
+                    />
+                  )}
+                </Td>
+              </Tr>
+              <Tr>
+                <Td>Params</Td>
+                <Td>
+                  <Code width="100%">
+                    <pre>{JSON.stringify(dagDetailsData.params, null, 2)}</pre>
+                  </Code>
+                </Td>
+              </Tr>
+            </>
+          )}
         </Tbody>
       </Table>
     </Box>

diff --git a/airflow/www/static/js/utils/index.ts b/airflow/www/static/js/utils/index.ts
@@ -177,6 +177,14 @@ const getStatusBackgroundColor = (color: string, hasNote: boolean) =>
     ? `linear-gradient(-135deg, ${Color(color).hex()}60 5px, ${color} 0);`
     : color;
 
+const toSentenceCase = (camelCase: string): string => {
+  if (camelCase) {
+    const result = camelCase.replace(/([A-Z])/g, " $1");
+    return result[0].toUpperCase() + result.substring(1).toLowerCase();
+  }
+  return "";
+};
+
 export {
   hoverDelay,
   finalStatesMap,
@@ -188,4 +196,5 @@ export {
   getDagRunLabel,
   getStatusBackgroundColor,
   useOffsetTop,
+  toSentenceCase,
 };
diff --git a/airflow/www/templates/airflow/dag.html b/airflow/www/templates/airflow/dag.html
@@ -60,14 +60,14 @@
   <meta name="grid_url" content="{{ url_for('Airflow.grid', dag_id=dag.dag_id) }}">
   <meta name="datasets_url" content="{{ url_for('Airflow.datasets') }}">
   <meta name="grid_url_no_root" content="{{ url_for('Airflow.grid', dag_id=dag.dag_id, num_runs=num_runs_arg, base_date=base_date_arg) }}">
-  <meta name="dag_details_url" content="{{ url_for('Airflow.dag_details', dag_id=dag.dag_id) }}">
   <meta name="graph_url" content="{{ url_for('Airflow.graph', dag_id=dag.dag_id, root=root) }}">
   <meta name="task_url" content="{{ url_for('Airflow.task', dag_id=dag.dag_id) }}">
   <meta name="log_url" content="{{ url_for('Airflow.log', dag_id=dag.dag_id) }}">
   <meta name="xcom_url" content="{{ url_for('Airflow.xcom', dag_id=dag.dag_id) }}">
   <meta name="rendered_templates_url" content="{{ url_for('Airflow.rendered_templates', dag_id=dag.dag_id) }}">
   <meta name="rendered_k8s_url" content="{{ url_for('Airflow.rendered_k8s', dag_id=dag.dag_id) }}">
   <meta name="task_instances_list_url" content="{{ url_for('TaskInstanceModelView.list') }}">
+  <meta name="tag_index_url" content="{{ url_for('Airflow.index', tags='_TAG_NAME_') }}">
   <meta name="mapped_instances_api" content="{{ url_for('/api/v1.airflow_api_connexion_endpoints_task_instance_endpoint_get_mapped_task_instances', dag_id=dag.dag_id, dag_run_id='_DAG_RUN_ID_', task_id='_TASK_ID_') }}">
   <meta name="task_log_api" content="{{ url_for('/api/v1.airflow_api_connexion_endpoints_log_endpoint_get_log', dag_id=dag.dag_id, dag_run_id='_DAG_RUN_ID_', task_id='_TASK_ID_', task_try_number='-1') }}">
   <meta name="upstream_dataset_events_api" content="{{ url_for('/api/v1.airflow_api_connexion_endpoints_dag_run_endpoint_get_upstream_dataset_events', dag_id=dag.dag_id, dag_run_id='_DAG_RUN_ID_') }}">
@@ -77,6 +77,7 @@
   <meta name="set_dag_run_note" content="{{ url_for('/api/v1.airflow_api_connexion_endpoints_dag_run_endpoint_set_dag_run_note', dag_id=dag.dag_id, dag_run_id='_DAG_RUN_ID_') }}">
   <meta name="dag_api" content="{{ url_for('/api/v1.airflow_api_connexion_endpoints_dag_endpoint_get_dag', dag_id=dag.dag_id) }}">
   <meta name="dag_source_api" content="{{ url_for('/api/v1.airflow_api_connexion_endpoints_dag_source_endpoint_get_dag_source', file_token='_FILE_TOKEN_') }}">
+  <meta name="dag_details_api" content="{{ url_for('/api/v1.airflow_api_connexion_endpoints_dag_endpoint_get_dag_details', dag_id=dag.dag_id) }}">
 
   <!-- End Urls -->
   <meta name="is_paused" content="{{ dag_is_paused }}">

diff --git a/airflow/www/templates/airflow/dags.html b/airflow/www/templates/airflow/dags.html
@@ -387,10 +387,6 @@ <h2>{{ page_title }}</h2>
                       <span class="material-icons" aria-hidden="true">code</span>
                       Code
                     </a>
-                    <a href="{{ url_for('Airflow.dag_details', dag_id=dag.dag_id) }}" class="dags-table-more__link">
-                      <span class="material-icons" aria-hidden="true">details</span>
-                      Details
-                    </a>
                     <a href="{{ url_for('Airflow.gantt', dag_id=dag.dag_id) }}" class="dags-table-more__link">
                       <span class="material-icons" aria-hidden="true">vertical_distribute</span>
                       Gantt