Skip to content

Commit

Permalink
add logging for azure transfer
Browse files Browse the repository at this point in the history
  • Loading branch information
Brandi Cantarel committed Sep 29, 2020
1 parent f06318f commit d50dd15
Show file tree
Hide file tree
Showing 6 changed files with 9 additions and 6 deletions.
4 changes: 3 additions & 1 deletion dnanexus/init_dnanexus.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,8 @@ fi
procbase="$prodir/$seqrunid";
outnf="$procbase/analysis"
year="20${seqrunid:0:2}"
mkdir -p /archive/PHG/PHG_Clinical/toarchive/backups/${year}
mkdir -p /archive/PHG/PHG_Clinical/toarchive/transfer_logs/${year}/${seqrunid}/
echo "*****DONE Setting Variables******"


Expand Down Expand Up @@ -82,7 +84,7 @@ then
rsync -avz ${seqdatadir}/Reports ${seqdatadir}/Stats [email protected]:/swnas/qc_nuclia/demultiplexing/$seqrunid/
rsync -avz --prune-empty-dirs --include "*/" --include="*.fastq.gz" --include="*.csv" --exclude="*" $seqdatadir /archive/PHG/PHG_Clinical/toarchive/seqruns/$year
source ${baseDir}/azure_credentials
az storage blob upload-batch -d seqruns -s /archive/PHG/PHG_Clinical/toarchive/seqruns/${year}/${seqrunid} --destination-path ${year}/${seqrunid}
az storage blob upload-batch -d seqruns -s /archive/PHG/PHG_Clinical/toarchive/seqruns/${year}/${seqrunid} --destination-path ${year}/${seqrunid} > /archive/PHG/PHG_Clinical/toarchive/transfer_logs/${year}/${seqrunid}/seqruns2azure.log
fi

echo "*****Done Demultiplexing******"
Expand Down
2 changes: 1 addition & 1 deletion scripts/finalize_case.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -301,7 +301,7 @@ then
mv $casedir $wkdir/toarchive/caseDirs/${caseID}
rsync -avz $wkdir/cases/${caseID} [email protected]:/swnas/cases
source ${baseDir}/../azure_credentials
az storage blob upload-batch -d cases -s $wkdir/cases/${caseID} --destination-path ${year}/${caseID}
az storage blob upload-batch -d cases -s $wkdir/cases/${caseID} --destination-path ${year}/${caseID} > /archive/PHG/PHG_Clinical/toarchive/transfer_logs/${year}/${seqrunid}/${caseID}.log
else
rsync -avz $wkdir/cases/${caseID} [email protected]:/swnas/cases/answer_devel
fi
Expand Down
2 changes: 1 addition & 1 deletion scripts/rnaonly.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,4 +92,4 @@ mv /archive/PHG/PHG_Clinical/casesTemp/${myarray[1]} /archive/PHG/PHG_Clinical/c
rsync -avz /archive/PHG/PHG_Clinical/cases/${myarray[1]} [email protected]:/swnas/cases

cd /archive/PHG/PHG_Clinical/cases
az storage blob upload-batch -d cases -s ${myarray[1]} --destination-path ${myarray[1]}
az storage blob upload-batch -d cases -s /archive/PHG/PHG_Clinical/cases/${myarray[1]} --destination-path ${year}/${myarray[1]} > /archive/PHG/PHG_Clinical/toarchive/transfer_logs/${year}/${seqrunid}/${caseID}.log
2 changes: 1 addition & 1 deletion scripts/syncCase2Azure.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ if [[ -z $testing ]]
then
rsync -avz $wkdir/cases/${subject} [email protected]:/swnas/cases
source ${baseDir}/../azure_credentials
az storage blob upload-batch -d seqruns -s $wkdir/cases/${subject} --destination-path ${year}/${subject}
az storage blob upload-batch -d seqruns -s $wkdir/cases/${subject} --destination-path ${year}/${subject} > /archive/PHG/PHG_Clinical/toarchive/transfer_logs/${year}/${seqrunid}/${caseID}.log
else
rsync -avz $wkdir/casesTemp/${subject} [email protected]:/swnas/cases/answer_devel
fi
2 changes: 1 addition & 1 deletion scripts/syncase_rnaonly.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,4 +92,4 @@ mv /archive/PHG/PHG_Clinical/casesTemp/${myarray[1]} /archive/PHG/PHG_Clinical/c
rsync -avz /archive/PHG/PHG_Clinical/cases/${myarray[1]} [email protected]:/swnas/cases

cd /archive/PHG/PHG_Clinical/cases
az storage blob upload-batch -d cases -s ${myarray[1]} --destination-path ${myarray[1]}
az storage blob upload-batch -d cases -s /archive/PHG/PHG_Clinical/cases/${myarray[1]} --destination-path ${year}/${myarray[1]} > /archive/PHG/PHG_Clinical/toarchive/transfer_logs/${year}/${seqrunid}/${caseID}.log
3 changes: 2 additions & 1 deletion utsw_biohpc/init_workflows.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ outnf="$procbase/analysis"
workdir="$procbase/work"
year="20${seqrunid:0:2}"
mkdir -p /archive/PHG/PHG_Clinical/toarchive/backups/${year}
/archive/PHG/PHG_Clinical/toarchive/transfer_logs/${year}/${seqrunid}/
echo "*****DONE Setting Variables******"


Expand Down Expand Up @@ -96,7 +97,7 @@ then
rsync -avz ${seqdatadir}/Reports ${seqdatadir}/Stats [email protected]:/swnas/qc_nuclia/demultiplexing/$seqrunid/
rsync -avz --prune-empty-dirs --include "*/" --include="*.fastq.gz" --include="*.csv" --exclude="*" $seqdatadir /archive/PHG/PHG_Clinical/toarchive/seqruns/$year
source ${baseDir}/azure_credentials
az storage blob upload-batch -d seqruns -s /archive/PHG/PHG_Clinical/toarchive/seqruns/${year}/${seqrunid} --destination-path ${year}/${seqrunid}
az storage blob upload-batch -d seqruns -s /archive/PHG/PHG_Clinical/toarchive/seqruns/${year}/${seqrunid} --destination-path ${year}/${seqrunid} > /archive/PHG/PHG_Clinical/toarchive/transfer_logs/${year}/${seqrunid}/seqruns2azure.log
fi

echo "*****Done Demultiplexing******"
Expand Down

0 comments on commit d50dd15

Please sign in to comment.