forked from StormSurgeLive/asgs
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathasgs_main.sh
executable file
·3279 lines (3196 loc) · 162 KB
/
asgs_main.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/bin/bash
# set -x
#trap read debug
#----------------------------------------------------------------
# asgs_main.sh: This is the main driver script for the ADCIRC Surge Guidance
# System (ASGS). It performs configuration tasks via config.sh, then enters a
# loop which is executed once per advisory cycle.
#----------------------------------------------------------------
# Copyright(C) 2006--2019 Jason Fleming
# Copyright(C) 2006--2007, 2019 Brett Estrade
#
# This file is part of the ADCIRC Surge Guidance System (ASGS).
#
# The ASGS is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# ASGS is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with the ASGS. If not, see <http://www.gnu.org/licenses/>.
#----------------------------------------------------------------
#
#####################################################################
# B E G I N F U N C T I O N S
#####################################################################
#
echoHelp()
{ clear
echo "@@@ Help @@@"
echo "Usage:"
echo " bash %$0 [-s /full/path/to/statefile] [-c /fullpath/of/asgs_config.sh] -e environment"
echo
echo "Options:"
echo "-c : set location of configuration file"
echo "-e (environment): set the computer that the ASGS is running on"
echo "-s : start from a previous statefile (used when started by cron)"
echo "-h : show help"
exit;
}
# reads/rereads+rebuilds derived variables
readConfig()
{
# Initialize variables accessed from ASGS config parameters to reasonable values
source ${SCRIPTDIR}/config/config_defaults.sh
# Initialize model parameters to appropriate values
source ${SCRIPTDIR}/config/model_defaults.sh
# HPC environment defaults (using the functions in platforms.sh)
env_dispatch ${HPCENVSHORT}
# set email addresses etc according to the Operator
source ${SCRIPTDIR}/config/operator_defaults.sh
# set default output file formats and frequencies
source ${SCRIPTDIR}/config/io_defaults.sh
# set default values related to forcing URLs etc
source ${SCRIPTDIR}/config/forcing_defaults.sh
# pick up config parameters, set by the Operator, that differ from the defaults
source ${CONFIG}
# maintain backward compatibility with old config files
if [[ $ENSEMBLESIZE != "null" ]]; then
SCENARIOPACKAGESIZE=$ENSEMBLESIZE
fi
#
RUNARCHIVEBASE=$SCRATCHDIR
}
#
# subroutine to check for the existence of required files that have
# been specified in config.sh
checkFileExistence()
{ FPATH=$1
FTYPE=$2
FNAME=$3
THIS="asgs_main.sh>checkFileExistence()"
if [[ -z $FNAME ]]; then
RMQMessage "EXIT" "$CURRENT_EVENT" "$THIS" "FAIL" "The $FTYPE was not specified in the configuration file. When it is specified, the ASGS will look for it in the path ${FPATH}."
fatal "$THIS: The $FTYPE was not specified in the configuration file. When it is specified, the ASGS will look for it in the path ${FPATH}."
fi
success=no
if [ $FNAME ]; then
if [ -e "${FPATH}/${FNAME}" ]; then
logMessage "$THIS: The $FTYPE '${FPATH}/${FNAME}' was found."
success=yes
else
# if this is a mesh or nodal attributes file, attempt to download and uncompress it
if [[ $FTYPE = "ADCIRC mesh file" ]]; then
logMessage "Downloading $FTYPE from ${MESHURL}/${FNAME}.xz."
curl --version >> $SYSLOG
curl ${MESHURL}/${FNAME}.xz > ${FPATH}/${FNAME}.xz 2> errmsg || warn "$THIS: Failed to download mesh from ${MESHURL}/${FNAME}.xz to ${FPATH}/${FNAME}.xz: `cat errmsg`."
fi
if [[ $FTYPE = "ADCIRC nodal attributes (fort.13) file" ]]; then
logMessage "Attempting to download $FTYPE from ${NODALATTRIBUTESURL}/${FNAME}.xz."
curl ${NODALATTRIBUTESURL}/${FNAME}.xz > ${FPATH}/${FNAME}.xz 2> errmsg || warn "$THIS: Failed to download nodal attributes file from ${NODALATTRIBUTESURL}/${FNAME}.xz to ${FPATH}/${FNAME}.xz: `cat errmsg`."
fi
if [[ $FTYPE = "ADCIRC static water level offset data file" ]]; then
logMessage "Attempting to download $FTYPE from ${OFFSETURL}/${FNAME}.xz."
curl ${OFFSETURL}/${FNAME}.xz > ${FPATH}/${FNAME}.xz 2> errmsg || warn "$THIS: Failed to download file from ${OFFSETURL}/${FNAME}.xz to ${FPATH}/${FNAME}.xz: `cat errmsg`."
fi
logMessage "THIS: Uncompressing ${FPATH}/${FNAME}.xz."
xz -d ${FPATH}/${FNAME}.xz 2> errmsg 2>&1 || warn "$THIS: Failed to uncompress ${FPATH}/${FNAME}.xz : `cat errmsg`."
[[ -e ${FPATH}/${FNAME} ]] && success=yes || success=no
fi
fi
if [[ $success = no ]]; then
RMQMessage "EXIT" "$CURRENT_EVENT" "$THIS" "FAIL" "The $FTYPE '${FPATH}/${FNAME}' does not exist."
fatal "$THIS: The $FTYPE '${FPATH}/${FNAME}' does not exist."
fi
}
#
# compare the modification times of the input files with the archive of
# subdomain files to avoid using a stale archive
# @jasonfleming: 20180814: moved the storage of the prepped archive
# from the
checkArchiveFreshness()
{ PREPPEDARCHIVE=$1
HINDCASTARCHIVE=$2
GRIDFILE=$3
CONTROLTEMPLATE=$4
ELEVSTATIONS=$5
VELSTATIONS=$6
METSTATIONS=$7
NAFILE=$8
SCRATCHDIR=$9
THIS="asgs_main.sh>checkArchiveFreshness()"
RMQMessage "INFO" "$CURRENT_EVENT" "$THIS" "$CURRENT_STATE" "Checking to see if the archive of preprocessed subdomain files is up to date."
logMessage "$THIS: Checking to see if the archive of preprocessed subdomain files is up to date."
for archiveFile in $PREPPEDARCHIVE $HINDCASTARCHIVE; do
if [ ! -e $RUNARCHIVEBASE/$archiveFile ]; then
logMessage "$THIS: The subdomain archive file $SCRATCHDIR/$archiveFile does not exist."
continue
fi
for inputFile in $GRIDFILE $CONTROLTEMPLATE $ELEVSTATIONS $VELSTATIONS $METSTATIONS $NAFILE; do
if [ ! -e $INPUTDIR/$inputFile ]; then
RMQMessage "WARN" "$CURRENT_EVENT" "$THIS" "WARN" "The input file $INPUTDIR/$inputFile does not exist."
warn "$THIS: The input file $INPUTDIR/$inputFile does not exist."
continue
fi
# see if the archiveFile is older than inputFile
if [ $SCRATCHDIR/$archiveFile -ot $INPUTDIR/$inputFile ]; then
logMessage "$THIS: A change in the input files has been detected. The archive file $archiveFile is older than the last modification time of the input file ${inputFile}. The archive file is therefore stale and will be deleted. A fresh one will automatically be created the next time adcprep is run."
rm $SCRATCHDIR/$archiveFile 2>> $SYSLOG
break
fi
done
done
}
#
# subroutine to check for the existence of required directories
# that have been specified in config.sh
checkDirExistence()
{ DIR=$1
TYPE=$2
THIS="asgs_main.sh>checkDirExistence()"
if [[ -z $DIR ]]; then
RMQMessage "EXIT" "$CURRENT_EVENT" "$THIS" "FAIL" "The $TYPE was not specified in the configuration file."
fatal "$THIS: The $TYPE was not specified in the configuration file."
fi
if [[ -e $DIR ]] ; then
logMessage "$THIS: The $TYPE '$DIR' was found."
else
RMQMessage "EXIT" "$CURRENT_EVENT" "$THIS" "FAIL" "The $TYPE $DIR does not exist."
fatal "$THIS: The $TYPE '$DIR' does not exist."
fi
}
#
# subroutine to check for the existence and nonzero length of the
# hotstart file
# the subroutine assumes that the hotstart file is named fort.$LUN or
# fort.$LUN.nc depending on the format and expects it to be provided with
# the full path if it is not in the current directory.
checkHotstart()
{
FROMDIR=$1
HOTSTARTFORMAT=$2
LUN=$3
#
THIS="asgs_main.sh>checkHotstart()"
HOTSTARTFILE=''
# set name and specific file location based on format (netcdf or binary)
if [[ $HOTSTARTFORMAT = netcdf ]]; then
HOTSTARTFILE=$FROMDIR/fort.$LUN.nc
else
HOTSTARTFILE=$FROMDIR/PE0000/fort.$LUN
fi
# check for existence of hotstart file
if [ ! -e $HOTSTARTFILE ]; then
RMQMessage "EXIT" "$CURRENT_EVENT" "$THIS" "FAIL" "The hotstart file '$HOTSTARTFILE' was not found. The preceding simulation run must have failed to produce it."
fatal "$THIS: The hotstart file '$HOTSTARTFILE' was not found. The preceding simulation run must have failed to produce it."
# if it exists, check size to be sure its nonzero
else
hotstartSize=`stat -c %s $HOTSTARTFILE`
if [ $hotstartSize == "0" ]; then
RMQMessage "EXIT" "$CURRENT_EVENT" "$THIS" "FAIL" "The hotstart file $HOTSTARTFILE is of zero length. The preceding simulation run must have failed to produce it."
fatal "$THIS: The hotstart file '$HOTSTARTFILE' is of zero length. The preceding simulation run must have failed to produce it properly."
else
RMQMessage "INFO" "$CURRENT_EVENT" "$THIS" "$CURRENT_STATE" "The hotstart file '$HOTSTARTFILE' was found and it contains $hotstartSize bytes."
logMessage "$THIS: The hotstart file '$HOTSTARTFILE' was found and it contains $hotstartSize bytes."
# check time in hotstart file to be sure it can be found and that
# it is nonzero
# jgf20170131: hstime reports errors to stderr so we must capture
# that with backticks and tee to the log file
HSTIME=''
if [[ $HOTSTARTFORMAT = netcdf ]]; then
HSTIME=`$ADCIRCDIR/hstime -f $HOTSTARTFILE -n 2>&1 | tee --append ${SYSLOG}`
else
HSTIME=`$ADCIRCDIR/hstime -f $HOTSTARTFILE 2>&1 | tee --append ${SYSLOG}`
fi
failureOccurred=$?
errorOccurred=`expr index "$HSTIME" ERROR`
if [[ $failureOccurred != 0 || $errorOccurred != 0 ]]; then
RMQMessage "EXIT" "$CURRENT_EVENT" "$THIS>$ENSTORM" "FAIL "Hstime failed: $HSTIME""
fatal "$THIS: The hstime utility could not read the ADCIRC time from the file '$HOTSTARTFILE'. The output from hstime was as follows: '$HSTIME'."
else
if float_cond '$HSTIME == 0.0'; then
THIS="asgs_main.sh>checkHotstart()"
fatal "$THIS: The time in the hotstart file '$HOTSTARTFILE' is zero. The preceding simulation run must have failed to produce a proper hotstart file."
fi
fi
RMQMessage "INFO" "$CURRENT_EVENT" "$THIS>$ENSTORM" "$CURRENT_STATE" "$ENSTORM: The time in the hotstart file is '$HSTIME' seconds."
fi
fi
}
#
# Evaluate a floating point number conditional expression.
# From http://www.linuxjournal.com/content/floating-point-math-bash
function float_cond()
{
THIS="asgs_main.sh>float_cond()"
local cond=0
if [[ $# -gt 0 ]]; then
cond=$(echo "$*" | bc -q 2>/dev/null)
if [[ -z "$cond" ]]; then cond=0; fi
if [[ "$cond" != 0 && "$cond" != 1 ]]; then cond=0; fi
fi
local stat=$((cond == 0))
return $stat
}
#
# Retrieve and build ADCIRC(+SWAN) executables. This function will set
# the value of ADCIRCDIR if ADCIRCBUILD = "dynamic".
get_adcirc()
{
ADCIRCDIR=$1 # this value may be changed in this function
DEBUG=$2
SWAN=$3
NETCDF=$4
NETCDF4=$5
NETCDF4_COMPRESSION=$6
XDMF=$7
SOURCEURL=$8
AUTOUPDATE=$9
EXEBASEPATH=${10}
SCRIPTDIR=${11}
SWANMACROSINC=${12}
ADCOPTIONS="${13}"
SYSLOG=${14}
#
# If the path to the ADCIRC executables is hard coded, just verify
# their existence and return.
if [[ $ADCIRCBUILD = static ]]; then
for executable in adcirc padcirc padcswan adcprep hstime aswip; do
if [[ ! -e $ADCIRCDIR/$executable ]]; then
warn "Could not find the executable file $ADCIRCDIR/${executable}."
return 1
fi
done
# leave the value of ADCIRCDIR as-is
logMessage "All ADCIRC(+SWAN) executable files were found successfully."
return 0
fi
#
# Set the name of the properties file that describes the executables that
# we're about to generate.
PROPERTIES=executables.properties
#
logMessage "Checking for suitable ADCIRC(+SWAN) executables."
#
# Check to see if we already have executables in a directory that
# matches the specification.
#
# Start by generating the path that is specified by this combination of
# parameters. Assume the SOURCEURL is an http URL for an svn repository;
# extract the end of the path for use in naming the directory where the
# executables will be compiled.
EXEPATH=`basename $SOURCEURL`
#
# Add the first letter of each of the arguments to the path name
EXEPATH="${EXEPATH}_D${DEBUG:0:1}S${SWAN:0:1}N${NETCDF:0:1}N4${NETCDF4:0:1}N4C${NETCDF4_COMPRESSION:0:1}X${XDMF:0:1}"
#
# Prepend the base executables path to the path we've constructed.
EXEPATH=$EXEBASEPATH/$EXEPATH
#
# Check for existence of executables.
EXEFOUND=t
for executable in adcirc padcirc padcswan adcprep hstime aswip; do
if [[ ! -e $EXEPATH/work/$executable ]]; then
EXEFOUND=f
fi
done
#
# If we found all the executables, and we aren't supposed to try to
# update and recompile them, then we're done.
if [[ $EXEFOUND = t && $AUTOUPDATE = off ]]; then
logMessage "Existing ADCIRC(+SWAN) executables were successfully found."
ADCIRCDIR=$EXEPATH/work # <-- setting the value of ADCIRCDIR
return 0
fi
#
logMessage "ADCIRC(+SWAN) executables were not found. They will be (re)built."
#
# Check the code out of svn and into the specified directory.
# TODO: Deal with svn username/password.
if [[ ! -d $EXEPATH ]]; then
mkdir -p $EXEPATH 2>> $SYSLOG
fi
cd $EXEPATH 2>> $SYSLOG
#
# Check the source code out of the repository. TODO: Enable other sources
# of source code, e.g., a tar.gz file on the local file system.
logMessage "Retrieving source code."
# TODO: Figure out how/when to 'svn update' source code already in place if
# AUTOUPDATE is on.
svn checkout $SOURCEURL . >> build.log 2>> $SYSLOG
mv build.log $EXEPATH/work 2>> $SYSLOG
#
# Now build the ADCIRC and ADCIRC+SWAN executables.
cd $EXEPATH/work 2>> $SYSLOG
#
# Write properties file to record what this script is attempting to do
# and make it easy to look in the executables directory to see how the
# code was compiled.
echo "DEBUG : $DEBUG" > $PROPERTIES
echo "SWAN : $SWAN" >> $PROPERTIES
echo "NETCDF : $NETCDF" >> $PROPERTIES
echo "NETCDF4 : $NETCDF4" >> $PROPERTIES
echo "NETCDF4_COMPRESSION : $NETCDF4_COMPRESSION" >> $PROPERTIES
echo "XDMF : $XDMF" >> $PROPERTIES
echo "SOURCEURL : $SOURCEURL" >> $PROPERTIES
echo "AUTOUPDATE : $AUTOUPDATE" >> $PROPERTIES
echo "EXEBASEPATH : $EXEBASEPATH" >> $PROPERTIES
echo "SCRIPTDIR : $SCRIPTDIR" >> $PROPERTIES
echo "SWANMACROSINC : $SWANMACROSINC" >> $PROPERTIES
echo "ADCOPTIONS : $ADCOPTIONS" >> $PROPERTIES
echo "SYSLOG : $SYSLOG" >> $PROPERTIES
#
# Set the correct SWAN compiler flags for this HPC platform.
cp ../swan/$SWANMACROSINC ../swan/macros.inc 2>> $SYSLOG
#
# Build the executables using the settings listed in the platforms.sh file.
logMessage "Building executables."
for executable in adcirc padcirc padcswan adcprep hstime aswip; do
logMessage "Building ${executable}."
MAKECMDLINE="make $executable $ADCOPTIONS DEBUG=$DEBUG SWAN=$SWAN NETCDF=$NETCDF NETCDF4=$NETCDF4 NETCDF4_COMPRESSION=$NETCDF4_COMPRESSION XDMF=$XDMF"
echo "MAKECMDLINE is $MAKECMDLINE" >> build.log 2>> $SYSLOG
$MAKECMDLINE >> build.log 2>&1
if [[ $? == 0 ]]; then
logMessage "Successfully built ${executable}."
else
warn "Failed to build $EXEPATH/work/${executable}."
return 1
fi
done
# All executables were built successfully; set the value of ADCIRCDIR.
ADCIRCDIR=$EXEPATH/work
}
#
#
# subroutine to run adcprep, using a pre-prepped archive of fort.13,
# fort.14 and fort.18 files
prep()
{ ADVISDIR=$1 # directory containing the now/forecast runs for this cycle
INPUTDIR=$2 # directory where grid and nodal attribute files are found
ENSTORM=$3 # scenario name (nowcast, storm1, storm5, etc)
START=$4 # coldstart or hotstart
FROMDIR=$5 # directory containing files to hotstart this run from
HPCENVSHORT=$6 # machine to run on (jade, desktop, queenbee, etc)
NCPU=$7 # number of CPUs to request in parallel jobs
PREPPEDARCHIVE=$8 # preprocessed fort.13 and fort.14 package
GRIDFILE=$9 # fulldomain grid
ACCOUNT=${10} # account to charge time to
OUTPUTOPTIONS="${11}" # contains list of args for appending files
HOTSTARTCOMP=${12} # fulldomain or subdomain
WALLTIME=${13} # HH:MM:SS format
HOTSTARTFORMAT=${14} # "binary" or "netcdf"
MINMAX=${15} # "continuous" or "reset"
HOTSWAN=${16} # "yes" or "no" to reinitialize SWAN only
NAFILE=${17} # full domain nodal attributes file
#
THIS="asgs_main.sh>prep()"
debugMessage "top of prep() has the following values: RUNDIR=$RUNDIR ADVISDIR=$ADVISDIR ENSTORM=$ENSTORM NOTIFYSCRIPT=${OUTPUTDIR}/${NOTIFY_SCRIPT} HPCENV=$HPCENV STORMNAME=$STORMNAME YEAR=$YEAR STORMDIR=$STORMDIR ADVISORY=$ADVISORY LASTADVISORYNUM=$LASTADVISORYNUM STATEFILE=$STATEFILE GRIDFILE=$GRIDFILE EMAILNOTIFY=$EMAILNOTIFY JOBFAILEDLIST=${JOB_FAILED_LIST} ARCHIVEBASE=$ARCHIVEBASE ARCHIVEDIR=$ARCHIVEDIR"
HPCENVSHORT=$6 # machine to run on (jade, desktop, queenbee, etc)
NCPU=$7 # number of CPUs to request in parallel jobs
PREPPEDARCHIVE=$8 # preprocessed fort.13 and fort.14 package
GRIDFILE=$9 # fulldomain grid
ACCOUNT=${10} # account to charge time to
OUTPUTOPTIONS="${11}" # contains list of args for appending files
HOTSTARTCOMP=${12} # fulldomain or subdomain
WALLTIME=${13} # HH:MM:SS format
HOTSTARTFORMAT=${14} # "binary" or "netcdf"
MINMAX=${15} # "continuous" or "reset"
HOTSWAN=${16} # "yes" or "no" to reinitialize SWAN only
NAFILE=${17} # full domain nodal attributes file, must be last in the
DATETIME=`date +'%Y-%h-%d-T%H:%M:%S%z'`
echo "time.adcprep.start : ${DATETIME}" >> ${STORMDIR}/run.properties
# set the name of the archive of preprocessed input files
PREPPED=$PREPPEDARCHIVE
if [[ $START = coldstart ]]; then
PREPPED=$HINDCASTARCHIVE
fi
# determine if there is an archive of preprocessed input files
HAVEARCHIVE=yes
if [[ ! -e ${SCRATCHDIR}/${PREPPED} ]]; then
HAVEARCHIVE=no
fi
# create directory to run in
if [ ! -d $ADVISDIR/$ENSTORM ]; then
mkdir $ADVISDIR/$ENSTORM 2>> ${SYSLOG}
fi
cd $ADVISDIR/$ENSTORM 2>> ${SYSLOG}
logMessage "Linking to full domain input files."
# symbolically link grid
if [ ! -e $ADVISDIR/$ENSTORM/fort.14 ]; then
ln -s $INPUTDIR/$GRIDFILE $ADVISDIR/$ENSTORM/fort.14 2>> ${SYSLOG}
fi
# symbolically link nodal attributes
if [ ! -e $ADVISDIR/$ENSTORM/fort.13 ]; then
if [[ ! -z $NAFILE && $NAFILE != null ]]; then
ln -s $INPUTDIR/$NAFILE $ADVISDIR/$ENSTORM/fort.13 2>> ${SYSLOG}
fi
fi
if [ $START = coldstart ]; then
# if we have variable river flux, link the fort.20 and fort.88 files
if [[ $VARFLUX = on || $VARFLUX = default ]]; then
# jgf20110525: For now, just copy a static file to this location
# and adcprep it. TODO: When real time flux data become available,
# grab those instead of relying on a static file.
ln -s ${INPUTDIR}/${HINDCASTRIVERFLUX} ./fort.20
# run adcprep to decompose the river elevation init (fort.88) file
ln -s ${INPUTDIR}/${RIVERINIT} ./fort.88
fi
else
# hotstart
#
# copy in the swaninit file which contains the name of the swan
# control file (conventionally named fort.26 when used with ADCIRC)
if [[ $WAVES = on ]]; then
cp $INPUTDIR/swaninit.template $ADVISDIR/$ENSTORM/swaninit 2>> ${SYSLOG}
fi
# jgfdebug: TODO: FIXME: Hardcoded the time varying weirs input file
if [ -e $INPUTDIR/time-bonnet.in ]; then
logMessage "$ENSTORM: $THIS: Copying $INPUTDIR/time-bonnet.in to $ADVISDIR/$ENSTORM."
cp $INPUTDIR/time-bonnet.in $ADVISDIR/$ENSTORM 2>> ${SYSLOG}
fi
logMessage "$ENSTORM: $THIS: Copying existing output files to this directory."
if [[ $MINMAX = continuous ]]; then
# copy max and min files so that the max values will be
# preserved across hotstarts
# @jasonfleming: Applying netcdf fix from @mattbilskie
for file in maxele.63 maxwvel.63 minpr.63 maxrs.63 maxvel.63 elemaxdry.63 nodeflag.63 rising.63 tinun.63 maxele.63.nc maxinundepth.63.nc maxrs.63.nc maxvel.63.nc maxwvel.63.nc minpr.63.nc elemaxdry.63.nc nodeflag.63.nc rising.63.nc tinun.63.nc swan_*_max.*; do
if [ -e $FROMDIR/$file ]; then
logMessage "$ENSTORM: $THIS: Copying $FROMDIR/$file to $ADVISDIR/$ENSTORM/$file so that its values will be preserved across the hotstart."
cp $FROMDIR/$file $ADVISDIR/$ENSTORM/$file 2>> ${SYSLOG}
fi
done
else
logMessage "$ENSTORM: $THIS: MINMAX was set to '$MINMAX' in the ASGS config file; as a result, the maxele.63 etc files will not be from the previous run to the current run. ADCIRC will start the record of max and min values anew."
fi
# copy existing fulldomain files if they are supposed to be appended
for file in fort.61 fort.62 fort.63 fort.64 fort.71 fort.72 fort.73 fort.74; do
matcharg=--${file/./}append
if [[ $file = "fort.71" || $file = "fort.72" ]]; then
matcharg="--fort7172append"
elif [[ $file = "fort.73" || $file = "fort.74" ]]; then
matcharg="--fort7374append"
fi
# check the output options to see if the file is being appended
for arg in $OUTPUTOPTIONS ; do
if [[ $matcharg = $arg ]]; then
# the file is being appended; check to see if it is in netcdf
# format, nd if so, use the netcdf name
netCDFArg=${matcharg/append/netcdf/}
for outputArg in $OUTPUTOPTIONS ; do
if [[ $outputArg = $netCDFArg ]]; then
file=${file/./}.nc
fi
done
if [ -e $FROMDIR/$file ]; then
logMessage "$ENSTORM: $THIS: Copying $FROMDIR/$file to $ADVISDIR/$ENSTORM/$file so that it will be appended during the upcoming run."
cp $FROMDIR/$file $ADVISDIR/$ENSTORM/$file 2>> ${SYSLOG}
fi
fi
done
done
# bring in hotstart file(s)
if [[ $QUEUESYS = serial ]]; then
if [[ $HOTSTARTFORMAT = netcdf ]]; then
# copy netcdf file so we overwrite the one that adcprep created
cp --remove-destination $FROMDIR/fort.67.nc $ADVISDIR/$ENSTORM/fort.68.nc >> $SYSLOG 2>&1
else
cp $FROMDIR/fort.67 $ADVISDIR/$ENSTORM/fort.68 >> $SYSLOG 2>&1
fi
fi
# swan hotstart file
if [[ $WAVES = on && $HOTSWAN = on ]]; then
cp $FROMDIR/swan.67 $ADVISDIR/$ENSTORM/swan.68 >> $SYSLOG 2>&1
fi
fi
#
#
# R E T U R N N O W
# I F T H I S I S A S E R I A L R U N
#
# adcprep is not required if the job is to run in serial
if [[ $QUEUESYS = "serial" ]]; then
return
fi
#
# C O N T I N U E W I T H A D C P R E P
# F O R P A R A L L E L R U N
#
TIMESTAMP=`date +%d%b%Y:%H:%M:%S`
DATETIME=`date +'%Y-%h-%d-T%H:%M:%S%z'`
echo "time.adcprep.start : ${DATETIME}" >> ${STORMDIR}/run.properties
# set the name of the archive of preprocessed input files
PREPPED=$PREPPEDARCHIVE
if [[ $START = coldstart ]]; then
PREPPED=$HINDCASTARCHIVE
fi
# determine if there is an archive of preprocessed input files
HAVEARCHIVE=yes
if [[ ! -e ${SCRATCHDIR}/${PREPPED} ]]; then
HAVEARCHIVE=no
fi
if [[ $HAVEARCHIVE = yes ]]; then
# copy in the files that have already been preprocessed
logMessage "$ENSTORM: $THIS: Copying input files that have already been decomposed."
cp ${SCRATCHDIR}/${PREPPED} . 2>> ${SYSLOG}
gunzip -f ${PREPPED} 2>> ${SYSLOG}
# untar the uncompressed archive
UNCOMPRESSEDARCHIVE=${PREPPED%.gz}
# extract the archive redirecting stdout (list of files extracted
# by tar) to scenario.log and any error messages to both scenario.log
# and syslog with a time stamp
tar xvf $UNCOMPRESSEDARCHIVE >> scenario.log 2> >(awk -v this='asgs_main.sh>prep' -v level=ERROR -f $SCRIPTDIR/monitoring/timestamp.awk | tee -a ${SYSLOG})
logMessage "$ENSTORM: $THIS: Removing $UNCOMPRESSEDARCHIVE"
rm $UNCOMPRESSEDARCHIVE 2>> ${SYSLOG}
fi
#
# this is a P A R A L L E L C O L D S T A R T
if [ $START = coldstart ]; then
# now run adcprep to decompose the files
if [[ $HAVEARCHIVE = no ]]; then
logMessage "$ENSTORM: $THIS: Running adcprep to partition the mesh for $NCPU compute processors."
prepFile partmesh $NCPU $ACCOUNT $WALLTIME
THIS="asgs_main.sh>prep()"
logMessage "$ENSTORM: $THIS: Running adcprep to prepare all files."
prepFile prepall $NCPU $ACCOUNT $WALLTIME
THIS="asgs_main.sh>prep()"
else
logMessage "$ENSTORM: $THIS: Running adcprep to prepare new fort.15 file."
prepFile prep15 $NCPU $ACCOUNT $WALLTIME
THIS="asgs_main.sh>prep()"
if [[ $VARFLUX = on || $VARFLUX = default ]]; then
logMessage "$ENSTORM: $THIS: Running adcprep to prepare new fort.20 file."
prepFile prep20 $NCPU $ACCOUNT $WALLTIME
THIS="asgs_main.sh>prep()"
logMessage "$ENSTORM: $THIS: Running adcprep to prepare fort.88 file."
prepFile prep88 $NCPU $ACCOUNT $WALLTIME
THIS="asgs_main.sh>prep()"
fi
fi
else
# this is a P A R A L L E L H O T S T A R T
#
# run adcprep to decompose the new files
if [[ $HAVEARCHIVE = no ]]; then
logMessage "$ENSTORM: $THIS: Running adcprep to partition the mesh for $NCPU compute processors."
prepFile partmesh $NCPU $ACCOUNT $WALLTIME
THIS="asgs_main.sh>prep()"
logMessage "$ENSTORM: $THIS: Running adcprep to prepare all files."
prepFile prepall $NCPU $ACCOUNT $WALLTIME
THIS="asgs_main.sh>prep()"
else
logMessage "$ENSTORM: $THIS: Running adcprep to prepare new fort.15 file."
prepFile prep15 $NCPU $ACCOUNT $WALLTIME
THIS="asgs_main.sh>prep()"
if [[ $VARFLUX = on || $VARFLUX = default ]]; then
logMessage "$ENSTORM: $THIS: Running adcprep to prepare new fort.20 file."
prepFile prep20 $NCPU $ACCOUNT $WALLTIME
THIS="asgs_main.sh>prep()"
fi
if [[ $WAVES = on ]]; then
PE=0
format="%04d"
while [[ $PE -lt $NCPU ]]; do
PESTRING=`printf "$format" $PE`
ln -s $ADVISDIR/$ENSTORM/fort.26 $ADVISDIR/$ENSTORM/PE${PESTRING}/fort.26 2>> ${SYSLOG}
PE=`expr $PE + 1`
done
fi
fi
# bring in hotstart file(s)
if [[ $HOTSTARTCOMP = fulldomain ]]; then
if [[ $HOTSTARTFORMAT = netcdf ]]; then
# copy netcdf file so we overwrite the one that adcprep created
cp --remove-destination $FROMDIR/fort.67.nc $ADVISDIR/$ENSTORM/fort.68.nc >> $SYSLOG 2>&1
mv fort.68.nc fort.68.nc.orig
nccopy -d0 fort.68.nc.orig fort.68.nc
else
ln -s $FROMDIR/PE0000/fort.67 $ADVISDIR/$ENSTORM/fort.68 >> $SYSLOG 2>&1
fi
fi
# jgfdebug
if [[ $HOTSTARTCOMP = subdomain ]]; then
logMessage "$ENSTORM: $THIS: Starting copy of subdomain hotstart files."
# copy the subdomain hotstart files over
# subdomain hotstart files are always binary formatted
PE=0
format="%04d"
while [ $PE -lt $NCPU ]; do
PESTRING=`printf "$format" $PE`
if [[ $HOTSTARTCOMP = subdomain ]]; then
cp $FROMDIR/PE${PESTRING}/fort.67 $ADVISDIR/$ENSTORM/PE${PESTRING}/fort.68 2>> ${SYSLOG}
fi
PE=`expr $PE + 1`
done
logMessage "$ENSTORM: $THIS: Completed copy of subdomain hotstart files."
fi
#
# H O T S T A R T I N G S W A N
#
# Globalizing and localizing the SWAN hotstart files can take
# a significant amount of time and must be done in serial. If the
# number of subdomains for this run is the same as the number of
# subdomains used in the run used as the source of SWAN hotstart files,
# then try to use the SWAN subdomain hotstart files directly.
if [[ $WAVES = on && $HOTSWAN = on ]]; then
logMessage "$ENSTORM: $THIS: Preparing SWAN hotstart file."
swanHotstartOK=no
# if archiving of the hotstart source run has started but is not
# complete, wait until it is complete so that we don't
# accidentally ingest partially complete tar files or partially
# globalized fulldomain swan hotstart files or start to copy
# subdomain swan hotstart files that are being deleted by an
# archiving script
logMessage "$ENSTORM: $THIS: Detecting time that SWAN hotstart archiving process started in ${FROMDIR}."
swanArchiveStart=`sed -n 's/[ ^]*$//;s/time.archive.start\s*:\s*//p' $FROMDIR/run.properties`
if [[ ! -z $swanArchiveStart ]]; then
# archiving process has started
logMessage "$ENSTORM: $THIS: The archiving process for the hotstart source run started at $swanArchiveStart."
waitMinutes=0 # number of minutes waiting for the archiving process to complete
waitMinutesMax=60 # max number of minutes to wait for upstream archiving process to finish
while [[ $waitMinutes -lt $waitMinutesMax ]]; do
# wait until it is finished or has errored out
logMessage "$ENSTORM: $THIS: Detecting finish or error condition for archiving SWAN hotstart files in ${FROMDIR}."
swanArchiveFinish=`sed -n 's/[ ^]*$//;s/time.archive.finish\s*:\s*//p' $FROMDIR/run.properties`
swanArchiveError=`sed -n 's/[ ^]*$//;s/time.archive.error\s*:\s*//p' $FROMDIR/run.properties`
if [[ ! -z $swanArchiveFinish || ! -z $swanArchiveError ]]; then
logMessage "$ENSTORM: $THIS: The archiving process for the hotstart source run has finished."
break
else
sleep 60
waitMinutes=`expr $waitMinutes + 1`
fi
done
if [[ $waitMinutes -ge 60 ]]; then
warn "$ENSTORM: $THIS: The archiving process for the hotstart source run did not finish within $watiMinutesMax minutes. Attempting to collect SWAN hotstart files anyway."
fi
else
# FIXME: how to handle this situation?
warn "$ENSTORM: $THIS: The SWAN hotstart archiving process has not started in ${FROMDIR}."
fi
logMessage "$ENSTORM: $THIS: Detecting number of subdomains for SWAN hotstart files in ${FROMDIR}."
hotSubdomains=`sed -n 's/[ ^]*$//;s/hpc.job.padcswan.ncpu\s*:\s*//p' $FROMDIR/run.properties`
logMessage "hotSubdomains is $hotSubdomains ; NCPU is $NCPU ; FROMDIR is $FROMDIR"
if [[ $hotSubdomains = $NCPU ]]; then
logMessage "$ENSTORM: $THIS: The number of subdomains is the same as hotstart source; subdomain SWAN hotstart files will be copied directly."
# subdomain swan hotstart files
if [[ -e $FROMDIR/PE0000/swan.67 ]]; then
logMessage "$ENSTORM: $THIS: Starting copy of subdomain swan hotstart files."
# copy the subdomain hotstart files over
# subdomain hotstart files are always binary formatted
PE=0
format="%04d"
while [ $PE -lt $NCPU ]; do
PESTRING=`printf "$format" $PE`
cp $FROMDIR/PE${PESTRING}/swan.67 $ADVISDIR/$ENSTORM/PE${PESTRING}/swan.68 2>> ${SYSLOG}
PE=`expr $PE + 1`
done
logMessage "$ENSTORM: $THIS: Completed copy of subdomain hotstart files."
swanHotstartOK=yes
fi
# subdomain SWAN hotstart files in a tar archive
if [[ $swanHotstartOK = no ]]; then
logMessage "$ENSTORM: $THIS: Could not copy subdomain SWAN hotstart files directly."
for suffix in tar tar.gz tar.bz2 ; do
logMessage "$ENSTORM: $THIS: Looking for ${FROMDIR}/swan.67.${suffix}."
if [[ -e $FROMDIR/swan.67.${suffix} ]]; then
logMessage "$ENSTORM: $THIS: Found $FROMDIR/swan.67.${suffix}."
cp $FROMDIR/swan.67.${suffix} ./swan.68.${suffix} 2>> $SYSLOG
scenarioMessage "$THIS: Untarring SWAN hotstart files:"
case $suffix in
tar)
tar xvf swan.68.${suffix} >> scenario.log 2> >(awk -v this='asgs_main.sh>prep' -v level=ERROR -f $SCRIPTDIR/monitoring/timestamp.awk | tee -a ${SYSLOG})
if [[ $? == 0 ]]; then swanHotstartOK=yes ; fi
;;
tar.gz)
tar xvzf swan.68.${suffix} >> scenario.log 2> >(awk -v this='asgs_main.sh>prep' -v level=ERROR -f $SCRIPTDIR/monitoring/timestamp.awk | tee -a ${SYSLOG})
if [[ $? == 0 ]]; then swanHotstartOK=yes ; fi
;;
tar.bz2)
tar xvjf swan.68.${suffix} >> scenario.log 2> >(awk -v this='asgs_main.sh>prep' -v level=ERROR -f $SCRIPTDIR/monitoring/timestamp.awk | tee -a ${SYSLOG})
if [[ $? == 0 ]]; then swanHotstartOK=yes ; fi
;;
*)
warn "$ENSTORM: $THIS: SWAN hotstart file archive $FROMDIR/swan.67.${suffix} unrecognized."
;;
esac
for dir in `ls -d PE*`; do
mv $dir/swan.67 $dir/swan.68 2>> $SYSLOG
done
rm swan.68.${suffix} 2>> $SYSLOG
break
fi
done
fi
if [[ $swanHotstartOK = no ]]; then
logMessage "$ENSTORM: $THIS: Failed to obtain subdomain SWAN hotstart files."
fi
else
logMessage "$ENSTORM: $THIS: The number of subdomains is different from the hotstart source; a fulldomain SWAN hotstart file will be decomposed to the subdomains."
fi
#
# if copying subdomain SWAN hotstart files did not work
# or is not appropriate because the number of subdomains in
# this run is different from the hotstart source, try to
# decompose a fulldomain SWAN hotstart file
if [[ $swanHotstartOK = no ]]; then
logMessage "$ENSTORM: $THIS: Decomposing fulldomain SWAN hotstart file."
# fulldomain swan hotstart file or archive of subdomain
# swan hotstart files
if [[ -e $FROMDIR/swan.67 ]]; then
cp $FROMDIR/swan.67 ./swan.68 2>> $SYSLOG
elif [[ -e $FROMDIR/swan.67.gz ]]; then
cp $FROMDIR/swan.67.gz ./swan.68.gz 2>> $SYSLOG
gunzip swan.68.gz 2>> $SYSLOG
elif [[ -e $FROMDIR/swan.67.bz2 ]]; then
cp $FROMDIR/swan.67.bz2 ./swan.68.bz2 2>> $SYSLOG
bunzip2 swan.68.bz2 2>> $SYSLOG
fi
if [[ -e swan.68 ]]; then
logMessage "$ENSTORM: $THIS: Starting decomposition of fulldomain swan hotstart file to subdomains."
${ADCIRCDIR}/../swan/unhcat.exe <<EOF 2>> ${SYSLOG}
2
swan.68
F
EOF
if [[ $? == 0 ]]; then swanHotstartOK=yes ; fi
fi
if [[ $swanHotstartOK = yes ]]; then
logMessage "$ENSTORM: $THIS: Completed decomposition of fulldomain swan hotstart file."
else
error "$ENSTORM: $THIS: Failed to obtain any swan hotstart file."
fi
fi
fi
if [[ $WAVES = off ]]; then
logMessage "$ENSTORM: $THIS: SWAN coupling is not active."
fi
if [[ $WAVES = on && $HOTSWAN = off ]]; then
logMessage "$ENSTORM: $THIS: SWAN coupling is active but SWAN hotstart files are not available in $FROMDIR. SWAN will be cold started."
fi
fi
# if we don't have an archive of our preprocessed files, create
# one so that we don't have to do another prepall
if [[ $HAVEARCHIVE = no ]]; then
logMessage "$ENSTORM: $THIS: Creating an archive of preprocessed files and saving to ${SCRATCHDIR}/${PREPPED} to avoid having to run prepall again."
FILELIST='partmesh.txt PE*/fort.14 PE*/fort.18'
if [[ ! -z $NAFILE && $NAFILE != null ]]; then
FILELIST='partmesh.txt PE*/fort.14 PE*/fort.18 PE*/fort.13'
fi
tar czf ${INPUTDIR}/${PREPPED} ${FILELIST} 2>> ${SYSLOG}
# check status of tar operation; if it failed, delete the file
# it attempted to make and alert the operator
if [[ $? != 0 ]]; then
warn "$ENSTORM: $THIS: The construction of a tar archive of the preprocessed input files has failed."
rm ${SCRATCHDIR}/${PREPPED} 2>> ${SYSLOG} 2>&1
fi
fi
DATETIME=`date +'%Y-%h-%d-T%H:%M:%S%z'`
echo "time.adcprep.finish : $DATETIME" >> ${STORMDIR}/run.properties
debugMessage "bottom of prep() has the following values: RUNDIR=$RUNDIR ADVISDIR=$ADVISDIR ENSTORM=$ENSTORM NOTIFYSCRIPT=${OUTPUTDIR}/${NOTIFY_SCRIPT} HPCENV=$HPCENV STORMNAME=$STORMNAME YEAR=$YEAR STORMDIR=$STORMDIR ADVISORY=$ADVISORY LASTADVISORYNUM=$LASTADVISORYNUM STATEFILE=$STATEFILE GRIDFILE=$GRIDFILE EMAILNOTIFY=$EMAILNOTIFY JOBFAILEDLIST=${JOB_FAILED_LIST} ARCHIVEBASE=$ARCHIVEBASE ARCHIVEDIR=$ARCHIVEDIR"
}
#
# function to run adcprep in a platform dependent way to decompose
# the fort.15, fort.20, or fort.88 file
prepFile()
{ JOBTYPE=$1
NCPU=$2
ACCOUNT=$3
WALLTIME=$4
THIS="asgs_main.sh>prepFile()"
CURRENT_STATE="WAIT"
echo "hpc.job.${JOBTYPE}.for.ncpu : $NCPU" >> $ADVISDIR/$ENSTORM/run.properties
echo "hpc.job.${JOBTYPE}.limit.walltime : $ADCPREPWALLTIME" >> $ADVISDIR/$ENSTORM/run.properties
echo "hpc.job.${JOBTYPE}.account : $ACCOUNT" >> $ADVISDIR/$ENSTORM/run.properties
JOBENVSTRING="( "
for string in ${JOBENV[*]}; do
JOBENVSTRING="$JOBENVSTRING $string"
done
JOBENVSTRING="$JOBENVSTRING )"
echo "hpc.job.${JOBTYPE}.jobenv : $JOBENVSTRING" >> $STORMDIR/run.properties
echo "hpc.job.${JOBTYPE}.path.jobenvdir : $JOBENVDIR" >> $STORMDIR/run.properties
echo "hpc.job.${JOBTYPE}.file.qscripttemplate : $QSCRIPTTEMPLATE" >> $ADVISDIR/$ENSTORM/run.properties
echo "hpc.job.${JOBTYPE}.parallelism : serial" >> $STORMDIR/run.properties
echo "hpc.job.${JOBTYPE}.serqueue : $SERQUEUE" >> $STORMDIR/run.properties
echo "hpc.job.${JOBTYPE}.serialmodules : $SERIALMODULES" >> $STORMDIR/run.properties
# FIXME: there is a hack in qscript.pl to change this to 20 for the priority queue on
# queenbee and supermic per LONI/LSU requirements (idiosyncracy on those platforms)
echo "hpc.job.${JOBTYPE}.ppn : 1" >> $STORMDIR/run.properties
if [[ $QUEUESYS = "SLURM" ]]; then
echo "hpc.slurm.job.${JOBTYPE}.reservation : $RESERVATION" >> $STORMDIR/run.properties
echo "hpc.slurm.job.${JOBTYPE}.constraint : $CONSTRAINT" >> $STORMDIR/run.properties
echo "hpc.slurm.job.${JOBTYPE}.qos : $QOS" >> $STORMDIR/run.properties
fi
#
# start log redirect processes for centralized logging
initCentralizedScenarioLogging
#
case $QUEUESYS in
"SLURM" | "PBS" | "SGE" )
queuesyslc=`echo $QUEUESYS | tr '[:upper:]' '[:lower:]'`
RMQMessage "INFO" "$CURRENT_EVENT" "$THIS>$ENSTORM" "$CURRENT_STATE" "Preparing queue script for adcprep.${JOBTYPE}.${queuesyslc}."
scenarioMessage "$ENSTORM: $THIS: Preparing queue script for adcprep with the following: perl $SCRIPTDIR/$QSCRIPTGEN --jobtype $JOBTYPE"
perl $SCRIPTDIR/$QSCRIPTGEN --jobtype $JOBTYPE >> scenario.log 2> >(awk -v this=$QSCRIPTGEN -v level=ERROR -f $SCRIPTDIR/monitoring/timestamp.awk | tee -a ${SYSLOG} | tee -a $CYCLELOG | tee -a scenario.log )
# submit adcprep job, check to make sure queue script submission
# succeeded, and if not, retry
while [ true ]; do
DATETIME=`date +'%Y-%h-%d-T%H:%M:%S%z'`
echo "time.hpc.job.${JOBTYPE}.submit : $DATETIME" >> run.properties
# submit job , capture stdout from sbatch and direct it
# to scenario.log; capture stderr and send to all logs
$SUBMITSTRING ${JOBTYPE}.${queuesyslc} >> scenario.log 2> >(awk -v this='asgs_main.sh>prep' -v level=ERROR -f $SCRIPTDIR/monitoring/timestamp.awk | tee -a ${SYSLOG} | tee -a $CYCLELOG | tee -a scenario.log )
if [[ $? = 0 ]]; then
break # job submission command returned a "success" status
else
warn "$ENSTORM: $THIS: $SUBMITSTRING ${JOBTYPE}.${queuesyslc} failed; will retry in 60 seconds."
sleep 60
fi
done
CURRENT_STATE="WAIT"
monitorJobs $QUEUESYS ${JOBTYPE} ${ENSTORM} $WALLTIME
THIS="asgs_main.sh>prepFile()"
CURRENT_STATE="WAIT"
RMQMessage "INFO" "$CURRENT_EVENT" "$THIS>$ENSTORM" "$CURRENT_STATE" "Finished adcprepping file ($JOBTYPE)."
logMessage "$ENSTORM: $THIS: Finished adcprepping file ($JOBTYPE)."
;;
*)
logMessage "Submitting job with $ADCIRCDIR/adcprep --np $NCPU --${JOBTYPE} >> $ADVISDIR/$ENSTORM/scenario.log 2>&1"
$ADCIRCDIR/adcprep --np $NCPU --${JOBTYPE} --strict-boundaries >> $ADVISDIR/$ENSTORM/scenario.log 2>&1
# check to see if adcprep completed successfully
if [[ $? != 0 ]]; then
error "$ENSTORM: $THIS: The adcprep ${JOBTYPE} job failed. See the file $ADVISDIR/$ENSTORM/scenario.log for details."
echo "$ENSTORM: $THIS: The adcprep ${JOBTYPE} job failed. See the file $ADVISDIR/$ENSTORM/scenario.log for details." >> jobFailed
fi
;;
esac
}
#
# subroutine that calls an external script over and over until it
# pulls down a new advisory (then it returns)
downloadCycloneData()
{ STORM=$1
YEAR=$2
RUNDIR=$3
SCRIPTDIR=$4
OLDADVISDIR=$5
TRIGGER=$6
ADVISORY=$7
FTPSITE=$8
RSSSITE=$9
FDIR=${10}
HDIR=${11}
STATEFILE=${12}
#
THIS="asgs_main.sh>downloadCycloneData()"
APPLOGFILE=$RUNDIR/get_atcf.pl.log
# activity_indicator "Checking remote site for new advisory..." &
logMessage "$THIS: Checking remote site for new advisory..." $APPLOGFILE
# pid=$!; trap "stop_activity_indicator ${pid}; exit" EXIT
cd $RUNDIR 2>> ${SYSLOG}
newAdvisory=false
newAdvisoryNum=null
forecastFileName=al${STORM}${YEAR}.fst
hindcastFileName=bal${STORM}${YEAR}.dat
# check to see if we have a leftover forecast.properties file from
# a previous advisory laying around here in our run directory, and if
# so, delete it
if [[ -e forecast.properties ]]; then
rm forecast.properties 2>> ${SYSLOG}
fi
OPTIONS="--storm $STORM --year $YEAR --ftpsite $FTPSITE --fdir $FDIR --hdir $HDIR --rsssite $RSSSITE --trigger $TRIGGER --adv $ADVISORY"
logMessage "$THIS: Options for get_atcf.pl are as follows : $OPTIONS" $APPLOGFILE
if [ "$START" = coldstart ]; then
RMQMessage "INFO" "$CURRENT_EVENT" "$THIS" "$CURRENT_STATE" "Downloading initial hindcast/forecast."
logMessage "$THIS: Downloading initial hindcast/forecast."
else
RMQMessage "INFO" "$CURRENT_EVENT" "$THIS" "$CURRENT_STATE" "Checking remote site for new advisory..."
logMessage "$THIS: Checking remote site for new advisory..."
fi
while [ $newAdvisory = false ]; do
if [[ $TRIGGER != "atcf" ]]; then
appMessage "perl $SCRIPTDIR/get_atcf.pl $OPTIONS" $APPLOGFILE
newAdvisoryNum=`perl $SCRIPTDIR/get_atcf.pl $OPTIONS 2>> $SYSLOG`
fi
# check to see if we have a new one, and if so, determine the
# new advisory number correctly
case $TRIGGER in
"atcf")
# if the forecast is already in ATCF format, then simply copy it
# to the run directory
cp $HDIR/$hindcastFileName . 2>> ${SYSLOG}
cp $FDIR/$forecastFileName . 2>> ${SYSLOG}
linkTarget=`readlink $FDIR/$forecastFileName`
# assume the advisory number is the first two characters in the
# symbolic link target of the forecast file name
newAdvisoryNum=${linkTarget:0:2}
if [[ $newAdvisoryNum -gt $ADVISORY ]]; then
newAdvisory="true"
else
newAdvisory="false"
fi
;;
"ftp")
if [ $START = hotstart ]; then
if ! diff $OLDADVISDIR/$forecastFileName ./$forecastFileName > /dev/null 2>> ${SYSLOG}; then
# forecasts from NHC ftp site do not have advisory number
newAdvisoryNum=`printf "%02d" $[$ADVISORY + 1]`
newAdvisory="true"
fi
fi
;;
"rss" | "rssembedded" )
# if there was a new advisory, the get_atcf.pl script
# would have returned the advisory number in stdout
if [[ ! -z $newAdvisoryNum && $newAdvisoryNum != null ]]; then
newAdvisory="true"
if [ -e $forecastFileName ]; then
mv $forecastFileName $forecastFileName.ftp 2>> $SYSLOG
fi
fi
;;
*)
fatal "$THIS: Invalid 'TRIGGER' type: '$TRIGGER'; must be ftp, rss, rssembedded, or atcf."
;;
esac
if [ $START = coldstart ]; then
if [ $TRIGGER = ftp ]; then
newAdvisoryNum=$ADVISORY
fi
newAdvisory="true"
fi
if [[ $newAdvisory = false ]]; then
sleep 60 # we are hotstarting, the advisory is same as last one
fi
done
RMQMessage "INFO" "$CURRENT_EVENT" "$THIS" "$CURRENT_STATE" "New forecast detected."
logMessage "$THIS: New forecast detected." $APPLOGFILE
cp -f $STATEFILE ${STATEFILE}.old 2>> ${SYSLOG}
sed 's/ADVISORY=.*/ADVISORY='$newAdvisoryNum'/' $STATEFILE > ${STATEFILE}.new 2>> ${SYSLOG} 2>&1
logMessage "$ENSTORM: $THIS: The new advisory number is ${newAdvisoryNum}." $APPLOGFILE
cp -f ${STATEFILE}.new $STATEFILE 2>> ${SYSLOG}
if [[ $TRIGGER = rss || $TRIGGER = rssembedded ]]; then
perl ${SCRIPTDIR}/nhc_advisory_bot.pl --input ${forecastFileName}.html --output $forecastFileName --metadata forecast.properties >> ${SYSLOG} 2>&1
fi
if [[ $FTPSITE = filesystem ]]; then
cp $HDIR/$hindcastFileName $hindcastFileName 2>> ${SYSLOG}
fi
}
#
# subroutine that polls an external ftp site for background meteorology data,
# converts it to OWI format (reprojecting the data if necessary), makes
# symbolic links to it, and returns.
downloadBackgroundMet()
{
RUNDIR=$1
SCRIPTDIR=$2
BACKSITE=$3