From 90c031535ec24755f5c46cc6c00fb35f0e226a47 Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Tue, 1 Oct 2024 16:02:25 +0200 Subject: [PATCH 1/2] Add possibility to set env variables for nextflow --- .../webapp/components/admin/app/settings/settings.js | 3 +++ .../components/admin/app/settings/settings.stache | 12 ++++++++++-- src/main/java/cloudgene/mapred/jobs/Environment.java | 1 + .../mapred/plugins/nextflow/NextflowBinary.java | 9 +++++++++ .../mapred/plugins/nextflow/NextflowPlugin.java | 11 +++++++++++ .../mapred/plugins/nextflow/NextflowStep.java | 4 ++++ 6 files changed, 38 insertions(+), 2 deletions(-) diff --git a/src/main/html/webapp/components/admin/app/settings/settings.js b/src/main/html/webapp/components/admin/app/settings/settings.js index 853cf669..26b47b94 100644 --- a/src/main/html/webapp/components/admin/app/settings/settings.js +++ b/src/main/html/webapp/components/admin/app/settings/settings.js @@ -35,10 +35,13 @@ export default Control.extend({ var nextflowProfile = $('#nextflow-profile').val(); var nextflowConfig = $('#nextflow-config').val(); var nextflowWork = $('#nextflow-work').val(); + var nextflowEnv = $('#nextflow-env').val(); + this.application.attr('config').attr('nextflow.profile', nextflowProfile); this.application.attr('config').attr('nextflow.config', nextflowConfig); this.application.attr('config').attr('nextflow.work', nextflowWork); + this.application.attr('config').attr('nextflow.env', nextflowEnv); this.application.save(function (data) { bootbox.alert("Application settings updated."); }, diff --git a/src/main/html/webapp/components/admin/app/settings/settings.stache b/src/main/html/webapp/components/admin/app/settings/settings.stache index 7f354225..cebca1d3 100644 --- a/src/main/html/webapp/components/admin/app/settings/settings.stache +++ b/src/main/html/webapp/components/admin/app/settings/settings.stache @@ -14,16 +14,24 @@

Custom Configuration:
- All hardware specific parameters can be set here. If you use the AWSBatch profile this is the place to set your credentials

+ All app-specific parameters can be set here.


+ +

Custom Environment:
+All app-specific environment variables can be set here.

+ + + +
+
diff --git a/src/main/java/cloudgene/mapred/jobs/Environment.java b/src/main/java/cloudgene/mapred/jobs/Environment.java index ad50bbba..86a60ced 100644 --- a/src/main/java/cloudgene/mapred/jobs/Environment.java +++ b/src/main/java/cloudgene/mapred/jobs/Environment.java @@ -40,6 +40,7 @@ public Environment addContext(CloudgeneContext context) { add("JOB_PRIORITY", context.getJob().getPriority() + ""); add("JOB_SUBMITTED_ON", context.getJob().getSubmittedOn() + ""); } + add("JOB_LOCATION", context.getLocalTemp()); add("USER_NAME", context.getUser().getUsername()); add("USER_EMAIL", context.getUser().getMail()); add("USER_FULL_NAME", context.getUser().getFullName()); diff --git a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowBinary.java b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowBinary.java index b32aca2e..5f8dcb0e 100644 --- a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowBinary.java +++ b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowBinary.java @@ -37,6 +37,8 @@ public class NextflowBinary { private String name; + private File envScript; + public static NextflowBinary build(Settings settings) { String binary = new BinaryFinder("nextflow").settings(settings, "nextflow", "home").env("NEXTFLOW_HOME") .envPath().path("/usr/local/bin").find(); @@ -125,10 +127,17 @@ public void setName(String name) { this.name = name; } + public void setEnvScript(File envScript) { + this.envScript = envScript; + } + public List buildCommand() { List nextflow = new Vector(); nextflow.add("PATH=$PATH:/usr/local/bin"); + if (envScript != null && envScript.exists()) { + nextflow.add("source " + envScript.getAbsolutePath() + ";"); + } nextflow.add(getBinary()); nextflow.add("-log"); diff --git a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowPlugin.java b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowPlugin.java index d4c279d6..c2f8c0df 100644 --- a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowPlugin.java +++ b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowPlugin.java @@ -21,6 +21,7 @@ public class NextflowPlugin implements IPlugin { public static final String ID = "nextflow"; public static final String NEXTFLOW_CONFIG = "nextflow.config"; public static final String NEXTFLOW_YAML = "nextflow.yaml"; + public static final String NEXTFLOW_ENV = "nextflow.env"; private Settings settings; @@ -82,6 +83,10 @@ public void updateConfig(WdlApp app, Map config) throws IOExcept writer.write(properties); writer.close(); + String nextflowEnv = FileUtil.path(appFolder, NEXTFLOW_ENV); + String contentEnv = config.get("nextflow.env"); + StringBuffer contentNextflowEnv = new StringBuffer(contentEnv == null ? "" : contentEnv); + FileUtil.writeStringBufferToFile(nextflowEnv, contentNextflowEnv); } @Override @@ -114,6 +119,12 @@ public Map getConfig(WdlApp app) throws IOException { config.put("nextflow.work", properties.get("work").toString()); } + String nextflowEnv = FileUtil.path(appFolder, NEXTFLOW_ENV); + if (new File(nextflowEnv).exists()) { + String content = FileUtil.readFileAsString(nextflowEnv); + config.put("nextflow.env", content); + } + return config; } diff --git a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java index 4a587247..4d0db173 100644 --- a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java +++ b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java @@ -105,6 +105,10 @@ public boolean run(WdlStep step, CloudgeneContext context) { String appConfig = FileUtil.path(appFolder, "nextflow.config"); nextflow.addConfig(appConfig); + String appEnv = FileUtil.path(appFolder, "nextflow.env"); + nextflow.setEnvScript(new File(appEnv)); + + // set work directory IWorkspace workspace = job.getWorkspace(); From 54dc3cbfad004fddf78f316e07b9458529c13613 Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Tue, 1 Oct 2024 16:33:22 +0200 Subject: [PATCH 2/2] Add possibility to set global env variables --- .../admin/settings/nextflow/nextflow.js | 3 +- .../admin/settings/nextflow/nextflow.stache | 16 ++++++++-- .../plugins/nextflow/NextflowBinary.java | 12 ++++---- .../plugins/nextflow/NextflowPlugin.java | 3 ++ .../mapred/plugins/nextflow/NextflowStep.java | 5 +++- .../controller/ServerAdminController.java | 5 ++-- .../responses/NextflowConfigResponse.java | 30 ++++++++++++++----- .../mapred/server/services/ServerService.java | 6 ++++ 8 files changed, 60 insertions(+), 20 deletions(-) diff --git a/src/main/html/webapp/components/admin/settings/nextflow/nextflow.js b/src/main/html/webapp/components/admin/settings/nextflow/nextflow.js index c59be201..0c62d1cf 100644 --- a/src/main/html/webapp/components/admin/settings/nextflow/nextflow.js +++ b/src/main/html/webapp/components/admin/settings/nextflow/nextflow.js @@ -26,7 +26,8 @@ export default Control.extend({ 'submit': function (form, event) { event.preventDefault(); - this.nextflowConfig.attr('content', $(form).find("[name='content']").val()); + this.nextflowConfig.attr('config', $(form).find("[name='config']").val()); + this.nextflowConfig.attr('env', $(form).find("[name='env']").val()); this.nextflowConfig.save(function (data) { bootbox.alert("Nextflow configuration updated."); }, function (response) { diff --git a/src/main/html/webapp/components/admin/settings/nextflow/nextflow.stache b/src/main/html/webapp/components/admin/settings/nextflow/nextflow.stache index 3ee1dafd..705f62a6 100644 --- a/src/main/html/webapp/components/admin/settings/nextflow/nextflow.stache +++ b/src/main/html/webapp/components/admin/settings/nextflow/nextflow.stache @@ -4,15 +4,25 @@
-
All global parameters can be set here.

+ +
- + + +
+

Custom Environment:
+ All global environment variables can be set here.

+
+
    {{#nextflowConfig.variables}} diff --git a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowBinary.java b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowBinary.java index 5f8dcb0e..0405d23c 100644 --- a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowBinary.java +++ b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowBinary.java @@ -37,7 +37,7 @@ public class NextflowBinary { private String name; - private File envScript; + private List envScripts = new Vector<>(); public static NextflowBinary build(Settings settings) { String binary = new BinaryFinder("nextflow").settings(settings, "nextflow", "home").env("NEXTFLOW_HOME") @@ -127,16 +127,18 @@ public void setName(String name) { this.name = name; } - public void setEnvScript(File envScript) { - this.envScript = envScript; + public void addEnvScript(File envScript) { + this.envScripts.add(envScript); } public List buildCommand() { List nextflow = new Vector(); nextflow.add("PATH=$PATH:/usr/local/bin"); - if (envScript != null && envScript.exists()) { - nextflow.add("source " + envScript.getAbsolutePath() + ";"); + for (File envScript: envScripts) { + if (envScript != null && envScript.exists()) { + nextflow.add("source " + envScript.getAbsolutePath() + ";"); + } } nextflow.add(getBinary()); diff --git a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowPlugin.java b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowPlugin.java index c2f8c0df..dd295e08 100644 --- a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowPlugin.java +++ b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowPlugin.java @@ -133,4 +133,7 @@ public String getNextflowConfig() { return FileUtil.path(Configuration.getConfigDirectory(), "nextflow.config"); } + public String getNextflowEnv() { + return FileUtil.path(Configuration.getConfigDirectory(), "nextflow.env"); + } } diff --git a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java index 4d0db173..6db0eaf6 100644 --- a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java +++ b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java @@ -105,8 +105,11 @@ public boolean run(WdlStep step, CloudgeneContext context) { String appConfig = FileUtil.path(appFolder, "nextflow.config"); nextflow.addConfig(appConfig); + String globalEnv = plugin.getNextflowEnv(); + nextflow.addEnvScript(new File(globalEnv)); + String appEnv = FileUtil.path(appFolder, "nextflow.env"); - nextflow.setEnvScript(new File(appEnv)); + nextflow.addEnvScript(new File(appEnv)); // set work directory diff --git a/src/main/java/cloudgene/mapred/server/controller/ServerAdminController.java b/src/main/java/cloudgene/mapred/server/controller/ServerAdminController.java index 1490ff63..a6a052dd 100644 --- a/src/main/java/cloudgene/mapred/server/controller/ServerAdminController.java +++ b/src/main/java/cloudgene/mapred/server/controller/ServerAdminController.java @@ -125,9 +125,10 @@ public NextflowConfigResponse getNextflowConfig() { } @Post("/nextflow/config/update") - public NextflowConfigResponse updateNextflowConfig(String content) { + public NextflowConfigResponse updateNextflowConfig(String config, String env) { - serverService.updateNextflowConfig(content); + serverService.updateNextflowConfig(config); + serverService.updateNextflowEnv(env); return NextflowConfigResponse.build(application.getSettings()); diff --git a/src/main/java/cloudgene/mapred/server/responses/NextflowConfigResponse.java b/src/main/java/cloudgene/mapred/server/responses/NextflowConfigResponse.java index 1bdb8e98..116f8675 100644 --- a/src/main/java/cloudgene/mapred/server/responses/NextflowConfigResponse.java +++ b/src/main/java/cloudgene/mapred/server/responses/NextflowConfigResponse.java @@ -17,7 +17,9 @@ @JsonClassDescription public class NextflowConfigResponse { - private String content = ""; + private String config = ""; + + private String env = ""; private List variables = new Vector(); @@ -28,21 +30,33 @@ public static NextflowConfigResponse build(Settings settings) { NextflowConfigResponse response = new NextflowConfigResponse(); String configFilename = plugin.getNextflowConfig(); File configFile = new File(configFilename); - if (!configFile.exists()) { - return response; + if (configFile.exists()) { + response.setConfig(FileUtil.readFileAsString(configFilename)); } - response.setContent(FileUtil.readFileAsString(configFilename)); + String envFilename = plugin.getNextflowEnv(); + File envFile = new File(envFilename); + if (envFile.exists()) { + response.setEnv(FileUtil.readFileAsString(envFilename)); + } response.setVariables(settings.buildEnvironment().toList()); return response; } - public void setContent(String content) { - this.content = content; + public void setConfig(String config) { + this.config = config; + } + + public String getConfig() { + return config; + } + + public void setEnv(String env) { + this.env = env; } - public String getContent() { - return content; + public String getEnv() { + return env; } public void setVariables(List variables) { diff --git a/src/main/java/cloudgene/mapred/server/services/ServerService.java b/src/main/java/cloudgene/mapred/server/services/ServerService.java index f2f1af57..405dc77b 100644 --- a/src/main/java/cloudgene/mapred/server/services/ServerService.java +++ b/src/main/java/cloudgene/mapred/server/services/ServerService.java @@ -253,4 +253,10 @@ public void updateNextflowConfig(String content) { FileUtil.writeStringBufferToFile(filename, new StringBuffer(content)); } + public void updateNextflowEnv(String content) { + NextflowPlugin plugin = (NextflowPlugin) PluginManager.getInstance().getPlugin(NextflowPlugin.ID); + String filename = plugin.getNextflowEnv(); + FileUtil.writeStringBufferToFile(filename, new StringBuffer(content)); + } + }