Update PEcAn.remote calls throughout PEcAn

Makefile

@@ -1,6 +1,6 @@
 NCPUS ?= 1
 
-BASE := logger utils db settings visualization qaqc
+BASE := logger utils db settings visualization qaqc remote
 
 MODELS := biocro clm45 dalec ed fates gday jules linkages \
 				lpjguess maat maespa preles sipnet
@@ -56,6 +56,7 @@ $(call depends,base/db): .install/base/logger .install/base/utils
 $(call depends,base/settings): .install/base/logger .install/base/utils .install/base/db
 $(call depends,base/visualization): .install/base/logger .install/base/db
 $(call depends,base/qaqc): .install/base/logger
+$(call depends,base/remote): .install/base/logger .install/base/utils
 $(call depends,modules/data.atmosphere): .install/base/logger .install/base/utils
 $(call depends,modules/data.land): .install/base/logger .install/base/db .install/base/utils
 $(call depends,modules/meta.analysis): .install/base/logger .install/base/utils .install/base/db

base/all/DESCRIPTION

@@ -27,7 +27,8 @@ Depends:
     PEcAn.assim.batch,
     PEcAn.emulator,
     PEcAn.priors,
-    PEcAn.benchmark
+    PEcAn.benchmark,
+    PEcAn.remote
 Suggests:
     PEcAn.ed,
     PEcAn.sipnet,

base/db/R/query.file.path.R

@@ -18,7 +18,7 @@ query.file.path <- function(input.id, host_name, con){
   )
   path <- file.path(dbfile$file_path,dbfile$file_name)
   cmd <- paste0("file.exists( '",path,"')")
-  PEcAn.utils::remote.execute.R(script = cmd, host = machine.host, verbose=TRUE)
+  PEcAn.remote::remote.execute.R(script = cmd, host = machine.host, verbose=TRUE)
   #   Check - to be determined later
   #   if(file.exists(path)){
   #     return(path)

base/remote/R/check_model_run.R

@@ -14,4 +14,4 @@ check_model_run <- function(out, stop.on.error = TRUE) {
       PEcAn.logger::logger.error(msg)
     }
   }
-}
+}

base/remote/R/remote.copy.update.R

@@ -11,7 +11,7 @@
 #' @export
 remote.copy.update <- function(input_id, remote_dir, remote_file_name = NULL, host, con){
 
-  remote.execute.cmd(host, "mkdir", c("-p", remote_dir))
+  PEcAn.remote::remote.execute.cmd(host, "mkdir", c("-p", remote_dir))
 
   local_file_record <- db.query(paste("SELECT * from dbfiles where container_id =", input_id), con)
 

base/remote/R/start.model.runs.R

@@ -70,8 +70,8 @@ start.model.runs <- function(settings, write = TRUE, stop.on.error = TRUE) {
 
     # if running on a remote cluster, create folders and copy any data to remote host
     if (!is_local) {
-      remote.execute.cmd(settings$host, "mkdir", c("-p", file.path(settings$host$outdir, run_id_string)))
-      remote.copy.to(settings$host, file.path(settings$rundir, run_id_string), settings$host$rundir, delete = TRUE)
+      PEcAn.remote::remote.execute.cmd(settings$host, "mkdir", c("-p", file.path(settings$host$outdir, run_id_string)))
+      PEcAn.remote::remote.copy.to(settings$host, file.path(settings$rundir, run_id_string), settings$host$rundir, delete = TRUE)
     }
 
     # check to see if we use the model launcer
@@ -102,7 +102,7 @@ start.model.runs <- function(settings, write = TRUE, stop.on.error = TRUE) {
 
         if (!is_local) {
           # copy data back to local
-          remote.copy.from(settings$host, file.path(settings$host$outdir, run_id_string), settings$modeloutdir)
+          PEcAn.remote::remote.copy.from(settings$host, file.path(settings$host$outdir, run_id_string), settings$modeloutdir)
         }
 
         # write finished time to database
@@ -120,7 +120,7 @@ start.model.runs <- function(settings, write = TRUE, stop.on.error = TRUE) {
 
     if (!is_local) {
       # copy launcher and joblist
-      remote.copy.to(settings$host, file.path(settings$rundir,
+      PEcAn.remote::remote.copy.to(settings$host, file.path(settings$rundir,
                                               format(firstrun, scientific = FALSE)), settings$host$rundir, delete = TRUE)
     }
 
@@ -168,7 +168,7 @@ start.model.runs <- function(settings, write = TRUE, stop.on.error = TRUE) {
 
         # Copy data back to local
         if (!is_local) {
-          remote.copy.from(host = settings$host,
+          PEcAn.remote::remote.copy.from(host = settings$host,
                            src = file.path(settings$host$outdir, run_id_string),
                            dst = settings$modeloutdir)
         }

base/utils/R/convert.input.R

@@ -72,14 +72,14 @@ convert.input <- function(input.id, outfolder, formatname, mimetype, site.id, st
       ## Do overwrite if set to TRUE
       if(overwrite){
         # collect files to flag for deletion
-        files.to.delete <- remote.execute.R( paste0("list.files('",
+        files.to.delete <- PEcAn.remote::remote.execute.R( paste0("list.files('",
                                                     existing.dbfile[["file_path"]],
                                                     "', full.names=TRUE)"),
                                              host, user = NA, verbose = TRUE,R = Rbinary, scratchdir = outfolder)
 
         file.deletion.commands <- .get.file.deletion.commands(files.to.delete)
 
-        remote.execute.R( file.deletion.commands$move.to.tmp,
+        PEcAn.remote::remote.execute.R( file.deletion.commands$move.to.tmp,
                           host, user = NA, 
                           verbose = TRUE,R = Rbinary, scratchdir = outfolder)
 
@@ -88,12 +88,12 @@ convert.input <- function(input.id, outfolder, formatname, mimetype, site.id, st
         successful <- FALSE
         on.exit(if (exists("successful") && successful) {
                 PEcAn.logger::logger.info("Conversion successful, with overwrite=TRUE. Deleting old files.")
-                remote.execute.R( file.deletion.commands$delete.tmp, 
+                PEcAn.remote::remote.execute.R( file.deletion.commands$delete.tmp, 
                                   host, user = NA, 
                                   verbose = TRUE,  R = Rbinary, scratchdir = outfolder )
         } else {
                 PEcAn.logger::logger.info("Conversion failed. Replacing old files.")
-                remote.execute.R( file.deletion.commands$replace.from.tmp, 
+                PEcAn.remote::remote.execute.R( file.deletion.commands$replace.from.tmp, 
                                   host, user = NA, 
                                   verbose = TRUE, R = Rbinary, scratchdir = outfolder )
         }
@@ -169,29 +169,29 @@ convert.input <- function(input.id, outfolder, formatname, mimetype, site.id, st
       if (overwrite) {
         # collect files to flag for deletion
 
-        files.to.delete <- remote.execute.R( paste0("list.files('",
+        files.to.delete <- PEcAn.remote::remote.execute.R( paste0("list.files('",
                                                     existing.dbfile[["file_path"]],
                                                     "', full.names=TRUE)"),
                                              host, user = NA, verbose = TRUE,R = Rbinary, scratchdir = outfolder)
 
         file.deletion.commands <- .get.file.deletion.commands(files.to.delete)
 
-        remote.execute.R( file.deletion.commands$move.to.tmp,
+        PEcAn.remote::remote.execute.R( file.deletion.commands$move.to.tmp,
                           host, user = NA, 
                           verbose = TRUE,R = Rbinary, scratchdir = outfolder)
 
         # Schedule files to be replaced or deleted on exiting the function
         successful <- FALSE
         on.exit(if (exists("successful") && successful) {
           PEcAn.logger::logger.info("Conversion successful, with overwrite=TRUE. Deleting old files.")
-          remote.execute.R( file.deletion.commands$delete.tmp,
+          PEcAn.remote::remote.execute.R( file.deletion.commands$delete.tmp,
                             host, user = NA, 
                             verbose = TRUE,  R = Rbinary, scratchdir = outfolder )
 
         } else {
 
           PEcAn.logger::logger.info("Conversion failed. Replacing old files.")
-          remote.execute.R( file.deletion.commands$replace.from.tmp,
+          PEcAn.remote::remote.execute.R( file.deletion.commands$replace.from.tmp,
                             host, user = NA,
                             verbose = TRUE, R = Rbinary, scratchdir = outfolder )
         } 
@@ -403,7 +403,7 @@ convert.input <- function(input.id, outfolder, formatname, mimetype, site.id, st
     cmdFcn <- paste0(pkg, "::", fcn, "(", arg.string, ")")
     PEcAn.logger::logger.debug(paste0("convert.input executing the following function:\n", cmdFcn))
 
-    result <- remote.execute.R(script = cmdFcn, host, user = NA, verbose = TRUE, R = Rbinary, scratchdir = outfolder)
+    result <- PEcAn.remote::remote.execute.R(script = cmdFcn, host, user = NA, verbose = TRUE, R = Rbinary, scratchdir = outfolder)
   }
 
   PEcAn.logger::logger.info("RESULTS: Convert.Input")

base/utils/R/utils.R

@@ -523,21 +523,6 @@ as.sequence <- function(x, na.rm = TRUE) {
 #--------------------------------------------------------------------------------------------------#
 
 
-#--------------------------------------------------------------------------------------------------#
-##' Test ssh access
-##'
-##' Test to determine if access to a remote server is available.
-##' Can be used to exclude / include tests or to prevent / identify access errors
-##' @title Test Remote
-##' @param host
-##' @return logical - TRUE if remote connection is available
-##' @author Rob Kooper
-test.remote <- function(host) {
-  return(try(remote.execute.cmd(host, "/bin/true")) == 0)
-} # test.remote
-#--------------------------------------------------------------------------------------------------#
-
-
 #--------------------------------------------------------------------------------------------------#
 ##' Create a temporary settings file
 ##'
@@ -786,4 +771,4 @@ retry.func <- function(expr, isError=function(x) "try-error" %in% class(x), maxE
 
 ####################################################################################################
 ### EOF.  End of R script file.
-####################################################################################################
+####################################################################################################

base/utils/inst/LBNL_remote_test.R

@@ -34,22 +34,22 @@ settings <- list(host=host)
 
 
 ## test remote.copy.to
-PEcAn.utils::remote.copy.to(host,"favicon.ico","~/favicon.ico")
+PEcAn.remote::remote.copy.to(host,"favicon.ico","~/favicon.ico")
 
 ## test remote.execute.cmd
-foo <- PEcAn.utils::remote.execute.cmd(host,"pwd")
+foo <- PEcAn.remote::remote.execute.cmd(host,"pwd")
 print(foo)
-PEcAn.utils::remote.execute.cmd(host,"mv",c("/global/home/users/dietze/favicon.ico","/global/home/users/dietze/favicon.jpg"))
+PEcAn.remote::remote.execute.cmd(host,"mv",c("/global/home/users/dietze/favicon.ico","/global/home/users/dietze/favicon.jpg"))
 
 ## test remote.copy.from
-PEcAn.utils::remote.copy.from(host,"~/favicon.jpg","favicon.jpg")
+PEcAn.remote::remote.copy.from(host,"~/favicon.jpg","favicon.jpg")
 
 ## test remote.execute.R
-b <- PEcAn.utils::remote.execute.R(script = "return(1)",host = host,R=R,verbose=TRUE,scratchdir="/global/scratch/dietze/")
+b <- PEcAn.remote::remote.execute.R(script = "return(1)",host = host,R=R,verbose=TRUE,scratchdir="/global/scratch/dietze/")
 
-c <- PEcAn.utils::remote.execute.R(script = "return(require(PEcAn.data.atmosphere))",host = host,R=R,verbose=TRUE,scratchdir="/global/scratch/dietze/")
+c <- PEcAn.remote::remote.execute.R(script = "return(require(PEcAn.data.atmosphere))",host = host,R=R,verbose=TRUE,scratchdir="/global/scratch/dietze/")
 
-d <- PEcAn.utils::remote.execute.R(script = "return(.libPaths())",host = host,R=R,verbose=TRUE,scratchdir="/global/scratch/dietze/")
+d <- PEcAn.remote::remote.execute.R(script = "return(.libPaths())",host = host,R=R,verbose=TRUE,scratchdir="/global/scratch/dietze/")
 
 
 ## kill tunnels

documentation/tutorials/ParameterAssimilation/PDA.Rmd

@@ -255,7 +255,7 @@ The PDA you ran has automatically produced an updated XML file (`pecan.pda***.xm
   PEcAn.settings::write.settings(settings, outputfile=paste0('pecan.pda', postPDA.settings$assim.batch$ensemble.id,'.xml'))
 
  # Start ecosystem model runs, this one takes awhile...
-  start.model.runs(postPDA.settings, postPDA.settings$database$bety$write)
+  PEcAn.remote::start.model.runs(postPDA.settings, postPDA.settings$database$bety$write)
 
  # Get results of model runs
   get.results(postPDA.settings)

models/biocro/inst/workflow.R

@@ -28,7 +28,7 @@ run.meta.analysis(settings$pfts, settings$meta.analysis$iter, settings$meta.anal
 
 run.write.configs(model)  # Calls model specific write.configs e.g. write.config.ed.R
 ## load met data
-start.model.runs(model)  # Start ecosystem model runs
+PEcAn.remote::start.model.runs(model)  # Start ecosystem model runs
 
 get.results(settings)  # Get results of model runs
 

models/biocro/vignettes/C4grass_sa_vd.Rmd

@@ -18,7 +18,7 @@ run.meta.analysis(pfts = settings$pfts, iterations = settings$meta.analysis$iter
 
 run.write.configs(settings = settings, write = FALSE) # Calls model specific write.configs e.g. write.config.ed.R
 ## load met data
-start.model.runs(settings = settings, write = FALSE)         # Start ecosystem model runs
+PEcAn.remote::start.model.runs(settings = settings, write = FALSE)         # Start ecosystem model runs
 
 get.results(settings)         # Get results of model runs
 

models/biocro/vignettes/workflow.R

@@ -26,7 +26,7 @@ run.meta.analysis(settings$pfts, settings$meta.analysis$iter, settings$meta.anal
 ## @knitr , echo=FALSE,warning=FALSE,cache=TRUE
 run.write.configs(model)  # Calls model specific write.configs e.g. write.config.ed.R
 ## load met data
-start.model.runs(model)  # Start ecosystem model runs
+PEcAn.remote::start.model.runs(model)  # Start ecosystem model runs
 read.outputs(settings$model$type, settings)
 # read.outputs(model, settings) #, variables = 'StemBiom')
 

models/biocro/vignettes/workflow.Rmd

@@ -36,7 +36,7 @@ run.write.configs(settings, settings$database$bety$write)
 
 ```{r, echo=FALSE,warning=FALSE,cache=TRUE}
 ## load met data
-start.model.runs(settings, settings$database$bety$write)         # Start ecosystem model runs
+PEcAn.remote::start.model.runs(settings, settings$database$bety$write)         # Start ecosystem model runs
 ```
 
 ```{r, echo=FALSE,warning=FALSE,cache=TRUE}

models/ed/R/write.configs.ed.R

@@ -308,7 +308,7 @@ remove.config.ED2 <- function(main.outdir = settings$outdir, settings) {
   if (!settings$host$name == "localhost") {
     ## Remove model run congfig and log files on remote host
     remote_ls <- function(path, pattern) {
-      remote.execute.cmd(host = settings$host, cmd = "ls", args = file.path(path, pattern))
+      PEcAn.remote::remote.execute.cmd(host = settings$host, cmd = "ls", args = file.path(path, pattern))
     }
     config <- remote_ls(settings$host$rundir, "c.*")
     ed2in <- remote_ls(settings$host$rundir, "ED2INc.*")
@@ -317,7 +317,7 @@ remove.config.ED2 <- function(main.outdir = settings$outdir, settings) {
 
     if (length(config) > 0 | length(ed2in) > 0) {
       todelete <- c(config, ed2in[-grep("log", ed2in)], output)  ## Keep log files
-      remote.execute.cmd(settings$host, "rm", c("-f", todelete))
+      PEcAn.remote::remote.execute.cmd(settings$host, "rm", c("-f", todelete))
     }
   }
 } # remove.config.ED2

models/jules/R/write.config.JULES.R

@@ -104,7 +104,7 @@ write.config.JULES <- function(defaults, trait.values, settings, run.id) {
     rmt.cmd <- paste0("PEcAn.JULES::detect.timestep(met.dir='",
                       met.dir,"', met.regexp='",met.regexp,
                       "', start_date= '",start_date,"')")
-    dt <- PEcAn.utils::remote.execute.R(script=rmt.cmd,host=settings$host,verbose=TRUE)
+    dt <- PEcAn.remote::remote.execute.R(script=rmt.cmd,host=settings$host,verbose=TRUE)
   }
   ## -------------------- END DETECT TIMESTEP --------------------
 
@@ -132,7 +132,7 @@ write.config.JULES <- function(defaults, trait.values, settings, run.id) {
                         ",  resample = ",settings$spin$resample,
                         ",  run_start_date = '",start_date,
                         "')")
-      start_date <- PEcAn.utils::remote.execute.R(script=rmt.cmd,host=settings$host,verbose=TRUE)
+      start_date <- PEcAn.remote::remote.execute.R(script=rmt.cmd,host=settings$host,verbose=TRUE)
     }
   }  ## end spin
 

models/maat/inst/simple_workflow.R

@@ -48,7 +48,7 @@ if (!file.exists(file.path(settings$rundir, "runs.txt")) | settings$meta.analysi
 if (!file.exists(file.path(settings$rundir, "runs.txt"))) {
   PEcAn.logger::logger.severe("No ensemble or sensitivity analysis specified in pecan.xml, work is done.")
 } else {
-  start.model.runs(settings, settings$database$bety$write)
+  PEcAn.remote::start.model.runs(settings, settings$database$bety$write)
 }
 #--------------------------------------------------------------------------------------------------#
 

modules/assim.batch/R/pda.bayesian.tools.R

@@ -133,7 +133,7 @@ pda.bayesian.tools <- function(settings, params.id = NULL, param.names = NULL, p
                                                                                                              now, sep = "."))
 
     ## Start model run
-    start.model.runs(settings, settings$database$bety$write)
+    PEcAn.remote::start.model.runs(settings, settings$database$bety$write)
 
     ## Read model outputs
     align.return <- pda.get.model.output(settings, run.id, bety, inputs)

modules/assim.batch/R/pda.emulator.R

@@ -239,7 +239,7 @@ pda.emulator <- function(settings, external.data = NULL, external.priors = NULL,
       save(list = ls(all.names = TRUE),envir=environment(),file=pda.restart.file)
 
       ## start model runs
-      start.model.runs(settings, settings$database$bety$write)
+      PEcAn.remote::start.model.runs(settings, settings$database$bety$write)
 
       ## Retrieve model outputs and error statistics
       model.out <- list()

modules/assim.batch/R/pda.mcmc.R

@@ -204,7 +204,7 @@ pda.mcmc <- function(settings, params.id = NULL, param.names = NULL, prior.id =
                                  run.names = paste0("MCMC_chain.", chain, "_iteration.", i, "_variable.", j))
 
           ## Start model run
-          start.model.runs(settings, settings$database$bety$write)
+          PEcAn.remote::start.model.runs(settings, settings$database$bety$write)
 
           ## Read model outputs
           align.return <- pda.get.model.output(settings, run.id, bety, inputs)

modules/assim.batch/R/pda.mcmc.bs.R

@@ -204,7 +204,7 @@ pda.mcmc.bs <- function(settings, params.id = NULL, param.names = NULL, prior.id
                                run.names = paste0("MCMC_chain.", chain, "_iteration.", i))
 
         ## Start model run
-        start.model.runs(settings, settings$database$bety$write)
+        PEcAn.remote::start.model.runs(settings, settings$database$bety$write)
 
         ## Read model outputs
         align.return <- pda.get.model.output(settings, run.id, bety, inputs)

modules/assim.sequential/R/sda.enkf.R

@@ -208,7 +208,7 @@ sda.enkf <- function(settings, obs.mean, obs.cov, IC = NULL, Q = NULL, adjustmen
       append = FALSE)
 
   ## start model runs
-  start.model.runs(settings, settings$database$bety$write)
+  PEcAn.remote::start.model.runs(settings, settings$database$bety$write)
   save(list = ls(envir = environment(), all.names = TRUE), 
        file = file.path(outdir, "sda.initial.runs.Rdata"), envir = environment())
 
@@ -905,7 +905,7 @@ sda.enkf <- function(settings, obs.mean, obs.cov, IC = NULL, Q = NULL, adjustmen
       ### Run model                                                         ###
       ###-------------------------------------------------------------------### 
       print(paste("Running Model for Year", as.Date(obs.times[t]) + 1))
-      start.model.runs(settings, settings$database$bety$write)
+      PEcAn.remote::start.model.runs(settings, settings$database$bety$write)
     }
 
     ###-------------------------------------------------------------------###

modules/data.land/R/fia2ED.R

@@ -338,10 +338,10 @@ fia.to.psscss <- function(settings,
     css.file.remote  <- file.path(out.dir.remote, paste0(prefix.psscss, ".css"))
     site.file.remote <- file.path(out.dir.remote, paste0(prefix.site, ".site"))
 
-    remote.execute.cmd(settings$host, "mkdir", c("-p", out.dir.remote))
-    remote.copy.to(settings$host, pss.file.local, pss.file.remote)
-    remote.copy.to(settings$host, css.file.local, css.file.remote)
-    remote.copy.to(settings$host, site.file.local, site.file.remote)
+    PEcAn.remote::remote.execute.cmd(settings$host, "mkdir", c("-p", out.dir.remote))
+    PEcAn.remote::remote.copy.to(settings$host, pss.file.local, pss.file.remote)
+    PEcAn.remote::remote.copy.to(settings$host, css.file.local, css.file.remote)
+    PEcAn.remote::remote.copy.to(settings$host, site.file.local, site.file.remote)
     files <- c(pss.file.remote, css.file.remote, site.file.remote)
   }