more stable code

ahmadMuhammadGd · Sep 14, 2024 · b1908aa · b1908aa
1 parent 9d26dcc
commit b1908aa
Showing 1 changed file with 6 additions and 9 deletions.
diff --git a/shell-scripts/run.sh b/shell-scripts/run.sh
@@ -11,25 +11,22 @@
 
 
 # echo -e "AIRFLOW_UID=$(id -u)" > ./airflow/.env
-
-docker compose up minio mc nessie -d &&
-docker compose -f ./airflow/airflow-compose.yaml up airflow-init -d  &&
+set -e
+docker compose up minio mc nessie -d
+docker compose -f ./airflow/airflow-compose.yaml up airflow-init -d
 docker compose -f ./airflow/airflow-compose.yaml up -d  
 
 sh ./shell-scripts/setup/minio-setup.sh
 
 docker compose up spark -d
 
-sh ./shell-scripts/setup/airflow-setup.sh
-
-docker exec spark sh /spark-container/ThriftServer-Iceberg-Nessie.sh \
-    > bash-logs/1-thrift-server.log
-
 docker exec spark spark-submit /spark-container/spark/jobs/init_project.py \
     > bash-logs/0-init_job.log
 
+docker exec spark sh /spark-container/ThriftServer-Iceberg-Nessie.sh \
+    > bash-logs/1-thrift-server.log 2>&1 &
 
-
+sh ./shell-scripts/setup/airflow-setup.sh
 # docker exec spark spark-submit /spark-container/spark/jobs/ingest.py -o queued/sampled_data_2.csv -t 2024-09-03 16:35:47.837268 \\
 #     > bash-logs/1-ingest.log