fix(web_console): configure KIBANA_HOST with .env (bytedance#303)

web_console/.eslintrc

@@ -8,7 +8,14 @@
   "rules": {
     "max-len": [
       "error",
-      120
+      {
+        "code": 120,
+        "ignoreComments": true,
+        "ignoreUrls": true,
+        "ignoreStrings": true,
+        "ignoreTemplateLiterals": true,
+        "ignoreRegExpLiterals": true
+      }
     ],
     "camelcase": "off",
     "consistent-return": "off",
@@ -38,6 +45,7 @@
     "jsx-a11y/anchor-is-valid": "off",
     "jsx-a11y/click-events-have-key-events": "off",
     "jsx-a11y/no-static-element-interactions": "off",
+    "jsx-a11y/iframe-has-title": "off",
     "react/state-in-constructor": "off",
     "react/jsx-filename-extension": [
       "error",

web_console/README.md

@@ -26,6 +26,8 @@ the web console of [Fedlearner][fedlearner].
   * use `server.config.js` to config environment variables
   * `K8S_HOST`: the hostname of Kubernetes api server
   * `K8S_PORT`: the port of Kubernetes api server
+  * use `.env.development` to maintain browser environment config for development
+  * use `.env.production` to maintain browser environment config for production
 - Contribution
   * use [Conventional Commits][conventionalcommits] for commit message
   * code coverage **must be** greater than `80%`

web_console/pages/job/charts/[id].jsx

@@ -1,49 +1,51 @@
-import React from 'react';
+import React, { useMemo } from 'react';
 import useSWR from 'swr';
 import { useRouter } from 'next/router';
-import { Text, Card, Grid } from '@zeit-ui/react'
-
+import css from 'styled-jsx/css';
+import { Text, Card, Grid } from '@zeit-ui/react';
 import { fetcher } from '../../../libs/http';
 import Layout from '../../../components/Layout';
-import css from 'styled-jsx/css';
-
-const { converted2Urls } = require('../../../utils/kibana')
+import getJobDashboardUrls from '../../../utils/kibana';
 
-function useStyle () {
+function useStyle() {
   return css`
     iframe {
       border: none;
       width: 100%;
       height: 80vh;
     }
-  `
+  `;
 }
 
-export default function charts (props) {
-  const router = useRouter()
-  const styles = useStyle()
-  const { id } = router.query
-  const { data: jobData } = useSWR(id ? `job/${id}` : null, fetcher);
-  const job = jobData ? jobData.data : null;
-  const urls = job
-  ? converted2Urls(
-    job.localdata.job_type, job.localdata.created_at,
-    new Date().toISOString(), job.localdata.name
-  )
-  : []
+export default function Chart(props) {
+  const router = useRouter();
+  const styles = useStyle();
+  const { id } = router.query;
+  const { data, isValidating } = useSWR(id ? `job/${id}` : null, fetcher);
+  const job = data?.data?.localdata;
+  const urls = useMemo(() => {
+    if (job) return getJobDashboardUrls(job);
+    return [];
+  }, [job]);
+
   return (
     <Layout theme={props.theme} toggleTheme={props.toggleTheme}>
-      <Text h2>{job?.localdata.name || 'Loading...'}</Text>
-      <Grid.Container gap={2}>
-        {urls.map(url =>
-          <Grid sm={24} md={24}>
-            <Card>
-              <iframe src={url}/>
-            </Card>
-          </Grid>
-        )}
-      </Grid.Container>
+      {isValidating && <Text h2>Loading...</Text>}
+      {urls.length > 0
+        ? (
+          <Grid.Container gap={2}>
+            {urls.map((url) => (
+              <Grid sm={24} md={24}>
+                <Card>
+                  <iframe src={url} />
+                </Card>
+              </Grid>
+            ))}
+          </Grid.Container>
+        )
+        : <Text h2>No metrics yet :(</Text>}
+
       <style jsx>{styles}</style>
     </Layout>
-  )
-}
+  );
+}

web_console/utils/kibana.js

@@ -1,94 +1,95 @@
-const { metrics_list } = require('./metrics_list');
+const dayjs = require('dayjs');
 const getConfig = require('./get_confg');
 
+const JOB_METRICS = {
+  data_join: [],
+  psi_data_join: [],
+  tree_model: [],
+  nn_model: [
+    {
+      query: 'name%20:%22auc%22',
+      mode: 'avg',
+      title: 'auc',
+    },
+    {
+      query: 'name%20:%22loss%22',
+      mode: 'avg',
+      title: 'loss',
+    },
+    {
+      query: 'name%20:%22receive_timer%22',
+      mode: 'avg',
+      title: 'receive spend',
+    },
+    {
+      query: 'name%20:%22iter_spend%22',
+      mode: 'avg',
+      title: 'per session run spend',
+    },
+    {
+      query: 'name%20:%22resend_counter%22',
+      mode: 'sum',
+      title: 'count of resend',
+    },
+    {
+      query: 'name%20:%22send_counter%22',
+      mode: 'sum',
+      title: 'count of send',
+    },
+    {
+      query: 'name%20:%22reconnect_counter%22',
+      mode: 'sum',
+      title: 'count of reconnect',
+    },
+    {
+      query: 'name%20:%22load_data_block_counter%22',
+      mode: 'sum',
+      title: 'count of load data block',
+    },
+    {
+      query: 'name%20:%22load_data_block_fail_counter%22',
+      mode: 'sum',
+      title: 'count of fail to load data block',
+    },
+  ],
+};
+
 const config = getConfig({
-    KIBANA_HOST: process.env.KIBANA_HOST,
-    KIBANA_PORT: process.env.KIBANA_PORT,
+  KIBANA_HOST: process.env.NEXT_PUBLIC_KIBANA_HOST,
+  KIBANA_PORT: process.env.NEXT_PUBLIC_KIBANA_PORT,
 });
 
-function getCommonUrl(start_time, end_time, application_id){
-    let url = `https://${config.KIBANA_HOST}:${config.KIBANA_PORT}/fedlearner/app/kibana#` +
-    "/visualize/edit/95e50f80-dd3a-11ea-a472-39251c5aaace?_g=(filter" +
-    "s:!(),refreshInterval:(pause:!t,value:0),time:(from:'" + start_time +
-    "',to:'" + end_time + "'))&_a=(filters:!(('$state':(store:appState)," +
-    "meta:(alias:!n,disabled:!f,index:'73340100-dd33-11ea-a472-39251c" +
-    "5aaace',key:tags.application_id,negate:!f,params:(query:" +
-    application_id +"),type:phrase),query:(match_phrase:(tags.applica" +
-    "tion_id:" + application_id + ")))),linked:!f,";
-    return url;
+function getBaseUrl(application_id, from, to) {
+  return `https://${config.KIBANA_HOST}:${config.KIBANA_PORT}/fedlearner/app/kibana#/visualize/edit/95e50f80-dd3a-11ea-a472-39251c5aaace?_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:'${from}',to:'${to}'))&_a=(filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'73340100-dd33-11ea-a472-39251c5aaace',key:tags.application_id,negate:!f,params:(query:${application_id}),type:phrase),query:(match_phrase:(tags.application_id:${application_id})))),linked:!f,`;
 }
 
 /**
  * used for job creation at server-side
  *
- * @param {Object} start_time - Data's start time like "2020-08-19T09:42:04.402Z"
- * @param {Object} end_time - Data's end time like "2020-08-19T09:42:40.368Z"
- * @param {Object} application_id - Job's application id
- * @param {Object} query - Query method
- * @param {Object} mode - Data integration method
- * @param {Object} title - The title of chart
- * @return {Object} - A chart url
+ * @param {string} application_id - name of Kubernetes application
+ * @param {string} from - date string in ISO format
+ * @param {string} to - date string in ISO format
+ * @param {string} query - metrics target
+ * @param {string} mode - computing function of ElasticSearch
+ * @param {string} title - metrics title
+ * @return {string} - a Kibana dashboard url
  */
-function converted2Url(start_time, end_time, application_id, query, mode, title) {
-    let url = getCommonUrl(start_time, end_time, application_id);
-    url = url + "query:(language:kuery,query:'" + query +
-        "'),uiState:(),vis:(aggs:!((enabled:!t,id:'1',params:(field:value)" +
-        ",schema:metric,type:" + mode + "),(enabled:!t,id:'2',params:" +
-        "(drop_partials:!f,extended_bounds:(),field:date_time,interval" +
-        ":auto,min_doc_count:1,scaleMetricValues:!f,timeRange:(from:'" +
-        start_time + "',to:'" + end_time + "'),useNormalizedEsInterval:" +
-        "!t),schema:segment,type:date_histogram),(enabled:!t,id:'3'," +
-        "params:(field:name.keyword,missingBucket:!f,missingBucketLa" +
-        "bel:Missing,order:desc,orderBy:'1',otherBucket:!f,otherBuck" +
-        "etLabel:Other,size:5),schema:group,type:terms)),params:(add" +
-        "Legend:!t,addTimeMarker:!f,addTooltip:!t,categoryAxes:!((id" +
-        ":CategoryAxis-1,labels:(filter:!t,show:!t,truncate:100),pos" +
-        "ition:bottom,scale:(type:linear),show:!t,style:(),title:()," +
-        "type:category)),dimensions:(series:!((accessor:1,aggType:te" +
-        "rms,format:(id:terms,params:(id:string,missingBucketLabel:M" +
-        "issing,otherBucketLabel:Other,parsedUrl:(basePath:%2Ffedlea" +
-        "rner,origin:'http:%2F%2Fad-kibana.bytedance.net',pathname:%" +
-        "2Ffedlearner%2Fapp%2Fkibana))),label:'name.keyword:%20Desce" +
-        "nding',params:())),x:(accessor:0,aggType:date_histogram,for" +
-        "mat:(id:date,params:(pattern:'HH:mm:ss')),label:'date_time%" +
-        "20per%20second',params:(bounds:(max:'" + end_time + "',min:'" +
-        start_time + "'),date:!t,format:'HH:mm:ss',interval:PT1S,interv" +
-        "alESUnit:s,intervalESValue:1)),y:!((accessor:2,aggType:avg," +
-        "format:(id:number,params:(parsedUrl:(basePath:%2Ffedlearner" +
-        ",origin:'http:%2F%2Fad-kibana.bytedance.net',pathname:%2Ffe" +
-        "dlearner%2Fapp%2Fkibana))),label:'Average%20value',params:(" +
-        ")))),grid:(categoryLines:!f),labels:(),legendPosition:right" +
-        ",seriesParams:!((data:(id:'1',label:'" + mode + "%20of%20va" +
-        "lue'),drawLinesBetweenPoints:!t,interpolate:linear,lineWidt" +
-        "h:2,mode:normal,show:!t,showCircles:!t,type:line,valueAxis:" +
-        "ValueAxis-1)),thresholdLine:(color:%23E7664C,show:!f,style:" +
-        "full,value:10,width:1),times:!(),type:line,valueAxes:!((id:" +
-        "ValueAxis-1,labels:(filter:!f,rotate:0,show:!t,truncate:100" +
-        "),name:LeftAxis-1,position:left,scale:(mode:normal,type:lin" +
-        "ear),show:!t,style:(),title:(text:'" + mode + "%20of%20valu" +
-        "e'),type:value))),title:" + title + ",type:line))";
-    return url;
+function getDashboardUrl(application_id, from, to, query, mode, title) {
+  const baseUrl = getBaseUrl(application_id, from, to);
+  return `${baseUrl}query:(language:kuery,query:'${query}'),uiState:(),vis:(aggs:!((enabled:!t,id:'1',params:(field:value),schema:metric,type:${mode}),(enabled:!t,id:'2',params:(drop_partials:!f,extended_bounds:(),field:date_time,interval:auto,min_doc_count:1,scaleMetricValues:!f,timeRange:(from:'${from}',to:'${to}'),useNormalizedEsInterval:!t),schema:segment,type:date_histogram),(enabled:!t,id:'3',params:(field:name.keyword,missingBucket:!f,missingBucketLabel:Missing,order:desc,orderBy:'1',otherBucket:!f,otherBucketLabel:Other,size:5),schema:group,type:terms)),params:(addLegend:!t,addTimeMarker:!f,addTooltip:!t,categoryAxes:!((id:CategoryAxis-1,labels:(filter:!t,show:!t,truncate:100),position:bottom,scale:(type:linear),show:!t,style:(),title:(),type:category)),dimensions:(series:!((accessor:1,aggType:terms,format:(id:terms,params:(id:string,missingBucketLabel:Missing,otherBucketLabel:Other,parsedUrl:(basePath:%2Ffedlearner,origin:'https:%2F%2F${config.KIBANA_HOST}',pathname:%2Ffedlearner%2Fapp%2Fkibana))),label:'name.keyword:%20Descending',params:())),x:(accessor:0,aggType:date_histogram,format:(id:date,params:(pattern:'HH:mm:ss')),label:'date_time%20per%20second',params:(bounds:(max:'${to}',min:'${from}'),date:!t,format:'HH:mm:ss',interval:PT1S,intervalESUnit:s,intervalESValue:1)),y:!((accessor:2,aggType:avg,format:(id:number,params:(parsedUrl:(basePath:%2Ffedlearner,origin:'https:%2F%2F${config.KIBANA_HOST}',pathname:%2Ffedlearner%2Fapp%2Fkibana))),label:'Average%20value',params:()))),grid:(categoryLines:!f),labels:(),legendPosition:right,seriesParams:!((data:(id:'1',label:'${mode}%20of%20value'),drawLinesBetweenPoints:!t,interpolate:linear,lineWidth:2,mode:normal,show:!t,showCircles:!t,type:line,valueAxis:ValueAxis-1)),thresholdLine:(color:%23E7664C,show:!f,style:full,value:10,width:1),times:!(),type:line,valueAxes:!((id:ValueAxis-1,labels:(filter:!f,rotate:0,show:!t,truncate:100),name:LeftAxis-1,position:left,scale:(mode:normal,type:linear),show:!t,style:(),title:(text:'${mode}%20of%20value'),type:value))),title:${title},type:line))`;
 }
 
 /**
- * used for job creation at server-side
+ * get Kibana dashboard urls of a job
  *
- * @param {Object} job_type - Federation job's type like data_join and so on
- * @param {Object} start_time - Data's start time like "2020-08-19T09:42:04.402Z"
- * @param {Object} end_time - Data's end time like "2020-08-19T09:42:40.368Z"
- * @param {Object} application_id - Job's application id
- * @return {Object} - list of metrics chart urls
+ * @param {Object} job - a job instance
+ * @return {string[]} - a list of Kibana dashboard url
  */
-function converted2Urls(job_type, start_time, end_time, application_id) {
-    let urls = [];
-    for(let i = 0; i < metrics_list[job_type].length; ++i){
-        let url = converted2Url(start_time, end_time, application_id,
-                                metrics_list[job_type][i]["query"],
-                                metrics_list[job_type][i]["mode"],
-                                metrics_list[job_type][i]["title"]);
-        urls.push(url);
-    }
-    return urls;
+function getJobDashboardUrls(job) {
+  const { name, job_type, created_at } = job;
+  const from = dayjs(created_at).toISOString();
+  const to = dayjs().toISOString();
+  return JOB_METRICS[job_type].map(({ query, mode, title }) => getDashboardUrl(name, from, to, query, mode, title));
 }
 
-module.exports = { converted2Urls }
+module.exports = getJobDashboardUrls;