rename output dir for data join (bytedance#156)

zhangqixun · Jul 21, 2020 · bd19d14 · bd19d14
1 parent 1bd5885
commit bd19d14
Show file tree

Hide file tree

Showing 5 changed files with 17 additions and 14 deletions.
diff --git a/deploy/scripts/data_join/run_data_join_master.sh b/deploy/scripts/data_join/run_data_join_master.sh
@@ -20,7 +20,7 @@ export CUDA_VISIBLE_DEVICES=
 source /app/deploy/scripts/hdfs_common.sh || true
 
 python -m fedlearner.data_join.cmd.prepare_launch_data_join_cli \
-    --data_source_name=$DATA_SOURCE_NAME \
+    --data_source_name=$APPLICATION_ID \
     --partition_num=$PARTITION_NUM \
     --start_time=$START_TIME \
     --end_time=$END_TIME \
@@ -38,4 +38,4 @@ python -m fedlearner.data_join.cmd.data_join_master_service \
     --etcd_addrs=$ETCD_ADDR \
     --etcd_base_dir=$ETCD_BASE_DIR \
     --listen_port=50051 \
-    --data_source_name=$DATA_SOURCE_NAME
+    --data_source_name=$APPLICATION_ID
diff --git a/web_console/tests/fixtures/test_data_join.yaml b/web_console/tests/fixtures/test_data_join.yaml
@@ -28,7 +28,7 @@ spec:
                   fieldRef:
                     fieldPath: metadata.name
               - name: ROLE
-                value: "Follower"
+                value: "follower"
               - name: APPLICATION_ID
                 value: test_data_join
               - name: OUTPUT_BASE_DIR
@@ -43,8 +43,6 @@ spec:
                 value: fedlearner-stack-ingress-nginx-controller.default.svc.cluster.local:80
               - name: EGRESS_HOST
                 value: external.name
-              - name: DATA_SOURCE_NAME
-                value: test_data_join
               - name: PARTITION_NUM
                 value: "2"
               - name: START_TIME
@@ -98,7 +96,7 @@ spec:
                   fieldRef:
                     fieldPath: metadata.name
               - name: ROLE
-                value: "Follower"
+                value: "follower"
               - name: APPLICATION_ID
                 value: test_data_join
               - name: OUTPUT_BASE_DIR

diff --git a/web_console/tests/fixtures/test_data_portal.yaml b/web_console/tests/fixtures/test_data_portal.yaml
@@ -42,7 +42,7 @@ spec:
               - name: INPUT_BASE_DIR
                 value: /data/criteo_data/100wexamples
               - name: OUTPUT_BASE_DIR
-                value: /data/data_portal_output/test_data_portal
+                value: /data/data_portal_output
               - name: RAW_DATA_PUBLISH_DIR
                 value: portal_publish_dir/test_data_portal
               - name: DATA_PORTAL_TYPE

diff --git a/web_console/tests/utils/job_builder.test.js b/web_console/tests/utils/job_builder.test.js
@@ -236,7 +236,7 @@ describe('portalGenerateYaml', () => {
       name: 'test_data_portal',
       output_partition_num: 2,
       data_portal_type: 'Streaming',
-      input: 'criteo_data/100wexamples',
+      input: '/data/criteo_data/100wexamples',
       output: '/data/portal_output', // TODO DEPRECATED for for shilei
       context: {
         file_wildcard: '*.rd',
@@ -421,7 +421,6 @@ describe('DataJoinGenerateYaml', () => {
                 spec: {
                   containers: [{
                     env: [
-                      { name: 'DATA_SOURCE_NAME', value: 'test_data_join' },
                       { name: 'PARTITION_NUM', value: '2' },
                       { name: 'START_TIME', value: '0' },
                       { name: 'END_TIME', value: '999999999999' },

diff --git a/web_console/utils/job_builder.js b/web_console/utils/job_builder.js
@@ -130,6 +130,15 @@ function generateYaml(federation, job, job_params, ticket) {
   yaml = mergeJson(yaml, ticket.public_params);
   yaml = mergeJson(yaml, ticket.private_params);
 
+  let output_base_dir;
+  if (job.job_type == 'data_join' || job.job_type == 'psi_data_join') {
+    output_base_dir = joinPath(
+      k8s_settings.storage_root_path, 'data_source', job.name);
+  } else {
+    output_base_dir = joinPath(
+      k8s_settings.storage_root_path, 'job_output', job.name);
+  }
+
   const replica_specs = yaml.spec.flReplicaSpecs;
   for (const key in replica_specs) {
     let base_spec = {
@@ -142,10 +151,7 @@ function generateYaml(federation, job, job_params, ticket) {
               { name: 'POD_NAME', valueFrom: { fieldRef: { fieldPath: 'metadata.name' } } },
               { name: 'ROLE', value: ticket.role },
               { name: 'APPLICATION_ID', value: job.name },
-              {
-                name: 'OUTPUT_BASE_DIR',
-                value: joinPath(k8s_settings.storage_root_path, 'job_output', job.name)
-              },
+              { name: 'OUTPUT_BASE_DIR', value: output_base_dir },
             ],
             imagePullPolicy: 'IfNotPresent',
             name: 'tensorflow',
@@ -206,11 +212,11 @@ function portalGenerateYaml(federation, raw_data) {
     },
   };
   yaml = mergeJson(yaml, k8s_settings.global_job_spec);
-
   yaml = mergeJson(yaml, raw_data.context.yaml_spec);
 
   let master_spec = yaml.spec.flReplicaSpecs.Master;
   master_spec = mergeJson(master_spec, k8s_settings.global_replica_spec);
+
   master_spec = mergeJson(master_spec, {
     pair: false,
     replicas: 1,