Merge pull request #27 from evotools/v1.8.5

V1.8.5
evotools · Feb 3, 2025 · 4efbed1 · 4efbed1
2 parents f208a3a + 714d26a
commit 4efbed1
Show file tree

Hide file tree

Showing 34 changed files with 950 additions and 2,034 deletions.
diff --git a/.github/workflows/CI-blat.yml b/.github/workflows/CI-blat.yml
@@ -41,8 +41,8 @@ jobs:
 
       - name: Install mamba
         run: |
-          wget -nv -O Mambaforge.sh https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-$(uname)-$(uname -m).sh
-          bash Mambaforge.sh -b -p ./mambaforge
+          wget -nv -O Miniforge.sh https://github.com/conda-forge/miniforge/releases/download/24.11.3-0/Miniforge3-24.11.3-0-$(uname)-$(uname -m).sh
+          bash Miniforge.sh -b -p ./mambaforge
           ./mambaforge/bin/mamba init
           source ./mambaforge/etc/profile.d/conda.sh 
 

diff --git a/.github/workflows/CI-gsa.yml b/.github/workflows/CI-gsa.yml
@@ -41,8 +41,8 @@ jobs:
 
       - name: Install mamba
         run: |
-          wget -nv -O Mambaforge.sh https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-$(uname)-$(uname -m).sh
-          bash Mambaforge.sh -b -p ./mambaforge
+          wget -nv -O Miniforge.sh https://github.com/conda-forge/miniforge/releases/download/24.11.3-0/Miniforge3-24.11.3-0-$(uname)-$(uname -m).sh
+          bash Miniforge.sh -b -p ./mambaforge
           ./mambaforge/bin/mamba init
           source ./mambaforge/etc/profile.d/conda.sh 
 

diff --git a/.github/workflows/CI-lastz.yml b/.github/workflows/CI-lastz.yml
@@ -41,8 +41,8 @@ jobs:
 
       - name: Install mamba
         run: |
-          wget -nv -O Mambaforge.sh https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-$(uname)-$(uname -m).sh
-          bash Mambaforge.sh -b -p ./mambaforge
+          wget -nv -O Miniforge.sh https://github.com/conda-forge/miniforge/releases/download/24.11.3-0/Miniforge3-24.11.3-0-$(uname)-$(uname -m).sh
+          bash Miniforge.sh -b -p ./mambaforge
           ./mambaforge/bin/mamba init
           source ./mambaforge/etc/profile.d/conda.sh 
 

diff --git a/.github/workflows/CI-mm2.yml b/.github/workflows/CI-mm2.yml
@@ -41,8 +41,8 @@ jobs:
 
       - name: Install mamba
         run: |
-          wget -nv -O Mambaforge.sh https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-$(uname)-$(uname -m).sh
-          bash Mambaforge.sh -b -p ./mambaforge
+          wget -nv -O Miniforge.sh https://github.com/conda-forge/miniforge/releases/download/24.11.3-0/Miniforge3-24.11.3-0-$(uname)-$(uname -m).sh
+          bash Miniforge.sh -b -p ./mambaforge
           ./mambaforge/bin/mamba init
           source ./mambaforge/etc/profile.d/conda.sh 
 

diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml
@@ -20,8 +20,8 @@ jobs:
     #    echo $CONDA/bin >> $GITHUB_PATH
     - name: Install mamba
       run: |
-        wget -nv -O Mambaforge.sh https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-$(uname)-$(uname -m).sh
-        bash Mambaforge.sh -b -p ./mambaforge
+        wget -nv -O Miniforge.sh https://github.com/conda-forge/miniforge/releases/download/24.11.3-0/Miniforge3-24.11.3-0-$(uname)-$(uname -m).sh
+        bash Miniforge.sh -b -p ./mambaforge
         ./mambaforge/bin/mamba init
         source ./mambaforge/etc/profile.d/conda.sh 
     - name: Create generic environment

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,4 +1,7 @@
 # Changelog
+## [v1.8.5]
+- Upkeep release.
+
 ## [v1.8.4]
 - Fix broken anaconda environment creation due to wrong repository order 
 - Better schema file

diff --git a/conf/base.config b/conf/base.config
@@ -1,40 +1,165 @@
-
 process {   
-  cpus = { check_max( 1 * task.attempt, 'cpus' ) }
-  memory = { check_max( 6.GB * task.attempt, 'memory' ) }
-  time = { check_max( 4.h * task.attempt, 'time' ) }
+  cpus = { 1 * task.attempt }
+  memory = { 6.GB * task.attempt }
+  time = { 4.h * task.attempt }
 
   errorStrategy = { task.attempt <= 2 ? "retry" : "ignore" }   
   maxRetries = 2 
   // Base configuration for the processes 
   withLabel: small{
-    cpus = { check_max( 1, 'cpus' ) }
-    memory = { check_max( 4.GB * task.attempt, 'memory' ) }
-    time = { check_max( 6.h * task.attempt, 'time' ) }
+    cpus =  {
+      def baseCpu = 1
+      if (params.max_cpus){
+        baseCpu.compareTo(params.max_cpus as int) == 1 ? params.max_cpus as int  : baseMem
+      } else {
+        baseCpu
+      }
+    }
+    memory = {
+      def baseMem = 4.GB * task.attempt
+      if (params.max_memory){
+        baseMem.compareTo(params.max_memory as nextflow.util.MemoryUnit) == 1 ? params.max_memory as nextflow.util.MemoryUnit : baseMem
+      } else {
+        baseMem
+      }
+    }
+    time = {
+      def baseTime = 6.h * task.attempt
+      if (params.max_time){
+        baseTime.compareTo(params.max_time as nextflow.util.Duration) == 1 ? params.max_time as nextflow.util.Duration : baseTime
+      } else {
+        baseTime
+      }
+    }
   }
   withLabel: medium{
-    cpus = { check_max( 1, 'cpus' ) }
-    memory = { check_max( 7.GB * task.attempt, 'memory' ) }
-    time = { check_max( 12.h * task.attempt, 'time' ) }
+    cpus =  {
+      def baseCpu = 1
+      if (params.max_cpus){
+        baseCpu.compareTo(params.max_cpus as int) == 1 ? params.max_cpus as int  : baseMem
+      } else {
+        baseCpu
+      }
+    }
+    memory = {
+      def baseMem = 7.GB * task.attempt
+      if (params.max_memory){
+        baseMem.compareTo(params.max_memory as nextflow.util.MemoryUnit) == 1 ? params.max_memory as nextflow.util.MemoryUnit : baseMem
+      } else {
+        baseMem
+      }
+    }
+    time = {
+      def baseTime = 12.h * task.attempt
+      if (params.max_time){
+        baseTime.compareTo(params.max_time as nextflow.util.Duration) == 1 ? params.max_time as nextflow.util.Duration : baseTime
+      } else {
+        baseTime
+      }
+    }
   }
   withLabel: large{
-    cpus = { check_max( 1, 'cpus' ) }
-    memory = { check_max( 12.GB * task.attempt, 'memory' ) }
-    time = { check_max( 23.h * task.attempt, 'time' ) }
+    cpus =  {
+      def baseCpu = 1
+      if (params.max_cpus){
+        baseCpu.compareTo(params.max_cpus as int) == 1 ? params.max_cpus as int  : baseMem
+      } else {
+        baseCpu
+      }
+    }
+    memory = {
+      def baseMem = 12.GB * task.attempt
+      if (params.max_memory){
+        baseMem.compareTo(params.max_memory as nextflow.util.MemoryUnit) == 1 ? params.max_memory as nextflow.util.MemoryUnit : baseMem
+      } else {
+        baseMem
+      }
+    }
+    time = {
+      def baseTime = 23.h * task.attempt
+      if (params.max_time){
+        baseTime.compareTo(params.max_time as nextflow.util.Duration) == 1 ? params.max_time as nextflow.util.Duration : baseTime
+      } else {
+        baseTime
+      }
+    }
   }
   withLabel: small_multi{
-    cpus = { check_max( 2 * task.attempt, 'cpus' ) }
-    memory = { check_max( 8.GB * task.attempt, 'memory' ) }
-    time = { check_max( 4.h * task.attempt, 'time' ) }
+    cpus =  {
+      def baseCpu = 2 * task.attempt
+      if (params.max_cpus){
+        baseCpu.compareTo(params.max_cpus as int) == 1 ? params.max_cpus as int  : baseMem
+      } else {
+        baseCpu
+      }
+    }
+    memory = {
+      def baseMem = 8.GB * task.attempt
+      if (params.max_memory){
+        baseMem.compareTo(params.max_memory as nextflow.util.MemoryUnit) == 1 ? params.max_memory as nextflow.util.MemoryUnit : baseMem
+      } else {
+        baseMem
+      }
+    }
+    time = {
+      def baseTime = 4.h * task.attempt
+      if (params.max_time){
+        baseTime.compareTo(params.max_time as nextflow.util.Duration) == 1 ? params.max_time as nextflow.util.Duration : baseTime
+      } else {
+        baseTime
+      }
+    }
   }
   withLabel: gsalign{
-    cpus = { check_max(params.gsalign_threads * task.attempt, 'cpus') }
-    memory = { check_max( 6.GB * params.minimap2_threads * task.attempt, 'memory' ) }
-    time = { check_max( 12.h * task.attempt, 'time' ) }
+    cpus =  {
+      def baseCpu = params.gsalign_threads ? params.gsalign_threads as int * task.attempt : 1 * task.attempt
+      if (params.max_cpus){
+        baseCpu.compareTo(params.max_cpus as int) == 1 ? params.max_cpus as int : baseMem
+      } else {
+        baseCpu
+      }
+    }
+    memory = {
+      def baseMem = 6.GB * task.attempt
+      if (params.max_memory){
+        baseMem.compareTo(params.max_memory as nextflow.util.MemoryUnit) == 1 ? params.max_memory as nextflow.util.MemoryUnit : baseMem
+      } else {
+        baseMem
+      }
+    }
+    time = {
+      def baseTime = 12.h * task.attempt
+      if (params.max_time){
+        baseTime.compareTo(params.max_time as nextflow.util.Duration) == 1 ? params.max_time as nextflow.util.Duration : baseTime
+      } else {
+        baseTime
+      }
+    }
   }
   withLabel: minimap2{
-    cpus = { check_max(params.minimap2_threads * task.attempt, 'cpus') }
-    memory = { check_max( 8.GB * params.minimap2_threads * task.attempt, 'memory' ) }
-    time = { check_max( 12.h * task.attempt, 'time' ) }
+    cpus =  {
+      def baseCpu = params.minimap2_threads ? params.minimap2_threads as int * task.attempt : 1 * task.attempt
+      if (params.max_cpus){
+        baseCpu.compareTo(params.max_cpus as int) == 1 ? params.max_cpus as int  : baseMem
+      } else {
+        baseCpu
+      }
+    }
+    memory = {
+      def baseMem = 8.GB * task.attempt
+      if (params.max_memory){
+        baseMem.compareTo(params.max_memory as nextflow.util.MemoryUnit) == 1 ? params.max_memory as nextflow.util.MemoryUnit : baseMem
+      } else {
+        baseMem
+      }
+    }
+    time = {
+      def baseTime = 12.h * task.attempt
+      if (params.max_time){
+        baseTime.compareTo(params.max_time as nextflow.util.Duration) == 1 ? params.max_time as nextflow.util.Duration : baseTime
+      } else {
+        baseTime
+      }
+    }
   }
 }
diff --git a/conf/eddie.config b/conf/eddie.config
@@ -4,22 +4,32 @@
  * ------------------------------------------------------
  */
  //Profile config names for nf-core/configs
- params {
+params {
+  // iGenomes reference base
+  saveReference = true
+  igenomes_base = '/exports/igmm/eddie/BioinformaticsResources/igenomes'
   config_profile_description = 'University of Edinburgh (eddie) cluster profile using anaconda tweaked by nf-core/configs.'
   config_profile_contact = 'Andrea Talenti (@RenzoTale88)'
   config_profile_url = 'https://www.ed.ac.uk/information-services/research-support/research-computing/ecdf/high-performance-computing'// Add parameter to specify extra flags for eddie
-  extra_cluster_options = ""  
-  scratch = true 
+  extra_cluster_options = ""
+  enable_conda = false
+  cache_dir = null
+  max_memory = 2048.GB
+  max_cpus = 64
+  max_time = 240.h
+  scratch = false
   queue_size = 100
- }
+  rl9 = true
+  project = "uoe_baseline"
+}
 
 executor {
   name = "sge"
   queueSize = "${params.queue_size}"
 }
 
 process {
-  clusterOptions = { task.memory ? "-l h_vmem=${task.memory.bytes/task.cpus} ${params.extra_cluster_options}" : null }
+  clusterOptions = { task.memory ? "-l h_vmem=${task.memory.bytes/task.cpus} -R y -l rl9=${params.rl9} -P ${params.project} ${params.extra_cluster_options}" : "-R y -l rl9=${params.rl9} -P ${params.project ?: ''} ${params.extra_cluster_options}" }
   scratch = params.scratch
   penv = { task.cpus > 1 ? "sharedmem" : null }
 
@@ -31,20 +41,11 @@ process {
   beforeScript =
   """
   . /etc/profile.d/modules.sh
-  module load roslin/singularity/3.5.3
+  module load singularity
   export SINGULARITY_TMPDIR="\$TMPDIR"
   """
 }
 
-params {
-  saveReference = true
-  // iGenomes reference base
-  igenomes_base = '/exports/igmm/eddie/BioinformaticsResources/igenomes'
-  max_memory = 384.GB
-  max_cpus = 32
-  max_time = 240.h
-}
-
 env {
   MALLOC_ARENA_MAX=1
 }

diff --git a/conf/eddie_conda.config b/conf/eddie_conda.config
@@ -4,22 +4,32 @@
  * ------------------------------------------------------
  */
  //Profile config names for nf-core/configs
- params {
+params {
+  // iGenomes reference base
+  saveReference = true
+  igenomes_base = '/exports/igmm/eddie/BioinformaticsResources/igenomes'
   config_profile_description = 'University of Edinburgh (eddie) cluster profile using anaconda tweaked by nf-core/configs.'
   config_profile_contact = 'Andrea Talenti (@RenzoTale88)'
   config_profile_url = 'https://www.ed.ac.uk/information-services/research-support/research-computing/ecdf/high-performance-computing'// Add parameter to specify extra flags for eddie
-  extra_cluster_options = ""  
-  scratch = false 
+  extra_cluster_options = ""
+  enable_conda = false
+  cache_dir = null
+  max_memory = 2048.GB
+  max_cpus = 64
+  max_time = 240.h
+  scratch = false
   queue_size = 100
- }
+  rl9 = true
+  project = "uoe_baseline"
+}
 
 executor {
   name = "sge"
   queueSize = "${params.queue_size}"
 }
 
 process {
-  clusterOptions = { task.memory ? "-l h_vmem=${task.memory.bytes/task.cpus} ${params.extra_cluster_options}" : null }
+  clusterOptions = { task.memory ? "-l h_vmem=${task.memory.bytes/task.cpus} -R y -l rl9=${params.rl9} -P ${params.project} ${params.extra_cluster_options}" : "-R y -l rl9=${params.rl9} -P ${params.project ?: ''} ${params.extra_cluster_options}" }
   scratch = params.scratch
   penv = { task.cpus > 1 ? "sharedmem" : null }
 
@@ -31,23 +41,14 @@ process {
   beforeScript =
   """
   . /etc/profile.d/modules.sh
-  module load anaconda/5.3.1
+  module load anaconda/2024.02
   """
 
   withName: mafstats {
     conda = "$projectDir/assets/maf-environment.yml" 
   }
 }
 
-params {
-  saveReference = true
-  // iGenomes reference base
-  igenomes_base = '/exports/igmm/eddie/BioinformaticsResources/igenomes'
-  max_memory = 384.GB
-  max_cpus = 32
-  max_time = 240.h
-}
-
 env {
   MALLOC_ARENA_MAX=1
 }
diff --git a/conf/params.config b/conf/params.config
@@ -28,9 +28,6 @@ params {
     reciprocal_best = false
     minimap2_threads = 2
     gsalign_threads = 2
-    max_memory = (Runtime.runtime.maxMemory() as nextflow.util.MemoryUnit).toGiga().GB
-    max_cpus = Runtime.runtime.availableProcessors() == 1 ? 1 : Runtime.runtime.availableProcessors() - 1
-    max_time = '240.h'
     help = false
     mamba = false
     publish_dir_mode = 'copy'

diff --git a/conf/test.config b/conf/test.config
@@ -24,9 +24,6 @@ params {
 
     // Other parameters    
     outdir = "${PWD}/outputs" 
-    max_memory = "4.GB"
-    max_cpus = Runtime.runtime.availableProcessors() - 1
-    max_time = '240.h'
     help = false
     publish_dir_mode = 'copy'
   }
diff --git a/legacy/README.md b/legacy/README.md