Tidy up for ensemble runs.

Author: ClaraDraper-NOAA

settings_cycle_template

@@ -2,6 +2,8 @@
 
 # experiment name
 export exp_name=
+STARTDATE=
+ENDDATE=
 
 # temporary directory where experiment is run from
 export WORKDIR=/scratch2/BMC/gsienkf/${USER}/workdir/
@@ -12,37 +14,35 @@ export OUTDIR=/scratch2/BMC/gsienkf/${USER}/cycle_land/${exp_name}/
 
 # for LETKF, this is size of ensemble.
 # for LETKF-OI pseudo ensemble, or non-ensemble runs use 1 
-ensemble_size=
+export ensemble_size=
 
 # length of each forecast
-FCSTHR=
+export FCSTHR=
 
 #forcing options: gdas, gswp3, gefs_ens
-atmos_forc=
+export atmos_forc=
 
 #FV3 resolution
 export RES=
 export TPATH=
 export TSTUB= # file stub for orography files in $TPATH
               # oro_C${RES} for atm only, oro_C${RES}.mx100 for atm/ocean.
-export TTYPE="mx100" # orography file type. Options: "mx100" for coupled atm/sea/ice model, "" for uncoupled (atm or land only)
 
 # number of cycles to submit in a single job
 # on hera in the debug queue 20 is a good number.
-dates_per_job=20
+export cycles_per_job=20
 
 # directory with initial conditions
 # can find some here:/scratch2/BMC/gsienkf/Clara.Draper/DA_test_cases/land-offline_workflow/offline_ICS/single
-ICSDIR=/scratch2/BMC/gsienkf/Clara.Draper/DA_test_cases/land-offline_workflow/offline_ICS/test_ICS/
+export ICSDIR=/scratch2/BMC/gsienkf/Clara.Draper/DA_test_cases/land-offline_workflow/offline_ICS/test_ICS/
 
 # namelist for do_landDA.sh
-# set to empty to not call do_landDA.sh
-DA_config= 
-#export DA_config="" # do not call.
+# set to "openloop" to not call do_landDA.sh
+export DA_config=
 
 # if want different DA at different times, list here. 
-DA_config00=${DA_config} 
-DA_config06=${DA_config} 
-DA_config12=${DA_config} 
-DA_config18=${DA_config} 
+export DA_config00=${DA_config} 
+export DA_config06=${DA_config} 
+export DA_config12=${DA_config} 
+export DA_config18=${DA_config} 
 

settings_cycle_test

@@ -30,16 +30,15 @@ export TSTUB="oro_C96.mx100" # file stub for orography files in $TPATH
 
 # number of cycles to submit in a single job
 # on hera in the debug queue 20 is a good number.
-export dates_per_job=1
+export cycles_per_job=1
 
 # directory with initial conditions
 # can find some here:/scratch2/BMC/gsienkf/Clara.Draper/DA_test_cases/land-offline_workflow/offline_ICS/single
 export ICSDIR=/scratch2/BMC/gsienkf/Clara.Draper/DA_test_cases/land-offline_workflow/offline_ICS/test_ICS/
 
 # namelist for do_landDA.sh
-# set to empty to not call do_landDA.sh
+# set to "openloop" to not call do_landDA.sh
 export DA_config="settings_DA_test"
-#export DA_config="" # do not call.
 
 # if want different DA at different times, list here. 
 export DA_config00=${DA_config} 

submit_cycle.sh

@@ -17,7 +17,7 @@ source $analdate
 THISDATE=$STARTDATE
 date_count=0
 
-while [ $date_count -lt $dates_per_job ]; do
+while [ $date_count -lt $cycles_per_job ]; do
 
     if [ $THISDATE -ge $ENDDATE ]; then 
         echo "All done, at date ${THISDATE}"  >> $logfile
@@ -132,10 +132,10 @@ while [ $date_count -lt $dates_per_job ]; do
 
         if [ $ensemble_size == 1 ]; then 
             mem_ens="mem000" 
-            NN=""
+            NNN="000"
         else 
             mem_ens="mem`printf %03i $n_ens`"
-            NN="`printf %02i $n_ens`" # forcing ensemble number
+            NNN="`printf %03i $n_ens`" 
         fi 
 
         MEM_WORKDIR=${WORKDIR}/${mem_ens}
@@ -180,20 +180,41 @@ while [ $date_count -lt $dates_per_job ]; do
         sed -i -e "s/XXFREQ/${FREQ}/g" ufs-land.namelist
         sed -i -e "s/XXRDD/${RDD}/g" ufs-land.namelist
         sed -i -e "s/XXRHH/${RHH}/g" ufs-land.namelist
-        sed -i -e "s/XXMEM/${NN}/g" ufs-land.namelist
+        sed -i -e "s/XXMEM/${NNN}/g" ufs-land.namelist
 
         # submit model
         echo '************************************************'
-        echo 'calling model' 
+        echo "calling model for member ${NNN}"
         echo $MEM_WORKDIR
-        $LSMexec
+        srun -n 1 $LSMexec 
+        #PID=$!
+        #$LSMexec
 
         # no error codes on exit from model, check for restart below instead
         #    if [[ $? != 0 ]]; then
         #        echo "model failed"
         #        exit 
         #    fi
 
+        n_ens=$((n_ens+1))
+    done # n_ens < ensemble_size
+
+    # wait for model(s) to complete.
+    wait 
+
+    # check model ouput
+    n_ens=1
+    while [ $n_ens -le $ensemble_size ]; do
+
+        if [ $ensemble_size == 1 ]; then 
+            mem_ens="mem000" 
+        else 
+            mem_ens="mem`printf %03i $n_ens`"
+        fi 
+
+        MEM_WORKDIR=${WORKDIR}/${mem_ens}
+        MEM_MODL_OUTDIR=${OUTDIR}/${mem_ens}
+
         if [[ -e ${MEM_WORKDIR}/ufs_land_restart.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.nc ]]; then 
            cp ${MEM_WORKDIR}/ufs_land_restart.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.nc ${MEM_MODL_OUTDIR}/restarts/vector/ufs_land_restart_back.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.nc
         else 
@@ -209,7 +230,7 @@ while [ $date_count -lt $dates_per_job ]; do
     THISDATE=$NEXTDATE
     date_count=$((date_count+1))
 
-done #  date_count -lt dates_per_job
+done #  date_count -lt cycles_per_job
 
 
 ############################