Merge pull request #74 from mpi2/dev

v2.0.0: SLURM pipeline

Author: marinak-ebi

Extract details from StatPackets/ExtractPVals.R

@@ -367,7 +367,7 @@ DRSummaryAcross = function(x) {
 
 
 ########## Main function
-f = function(start, end, file = 'Index_DR101_V1.txt') {
+f = function(start, end, file = 'Index_DR101_V1.txt', mp_chooser_file = mp_chooser_file) {
   if (is.na(end))
     end = start
   ofname = paste0('R', '_', start, '-', end, '_pval.tsv')
@@ -421,12 +421,14 @@ f = function(start, end, file = 'Index_DR101_V1.txt') {
       stringsAsFactors = FALSE
     )
     rN = DRrequiredAgeing:::annotationChooser(statpacket = r0,
-                                              level = .0001)
+                                              level = .0001,
+                                              mp_chooser_file = mp_chooser_file)
     rW = DRrequiredAgeing:::annotationChooser(
       statpacket = r0,
       level = .0001,
       resultKey = 'Windowed result',
-      TermKey = 'WMPTERM'
+      TermKey = 'WMPTERM',
+      mp_chooser_file = mp_chooser_file
     )
 
 
@@ -733,13 +735,13 @@ makejobs = function(path = getwd()) {
     bf = basename(file)
     n = length(parameters)
     jobs = paste0 (
-      'bsub -J IMPC_stats_pipeline_lsf_jobs -M 16000 -e err/err',
+      'sbatch --job-name=impc_stats_pipeline_job --mem=16G --time=2-00 -e err/err',
       bf,
       1:n,
       ' -o out/out',
       bf,
       1:n,
-      ' "Rscript ExtractPVals.R ',
+      ' --wrap="Rscript ExtractPVals.R ',
       parameters,
       ' ',
       file,
@@ -758,7 +760,7 @@ makejobs = function(path = getwd()) {
 #makejobs()
 #qvalue2AllZips()
 #parameter2qvalue(args[1], args[2])
-ignore.my.name = f(start =  as.numeric(args[1]), end = as.numeric(args[2]),file = args[3])
+ignore.my.name = f(start =  as.numeric(args[1]), end = as.numeric(args[2]), file = args[3], mp_chooser_file = args[4])
 
 
 

Extract details from StatPackets/ExtractPValsPhenStat.R

@@ -637,13 +637,13 @@ makejobs = function(path = getwd()) {
     bf = basename(file)
     n = length(parameters)
     jobs = paste0 (
-      'bsub -J IMPC_stats_pipeline_lsf_jobs -M 16000 -e err/err',
+      'sbatch --job-name=impc_stats_pipeline_job --mem=16G --time=2-00 -e err/err',
       bf,
       1:n,
       ' -o out/out',
       bf,
       1:n,
-      ' "Rscript ExtractPVals.R ',
+      ' --wrap="Rscript ExtractPVals.R ',
       parameters,
       ' ',
       file,
@@ -661,7 +661,7 @@ makejobs = function(path = getwd()) {
 #makejobs()
 #qvalue2AllZips()
 #parameter2qvalue(args[1], args[2])
-ignore.my.name = f(start =  as.numeric(args[1]), end = as.numeric(args[2]),file = args[3])
+ignore.my.name = f(start =  as.numeric(args[1]), end = as.numeric(args[2]), file = args[3])
 
 
 

IMPC annotation pipeline/UsefulCommands.txt

@@ -1,23 +1,19 @@
 # Split a text files based on #lines
 split -8500 Index_DR101_V1.txt split_index_
 
-
-# for creating LSF jobs in R
+# for creating jobs in R
 DRrequiredAgeing:::annotationIndexCreator()
 
 ### Logs
 find ./*/*_RawData/ClusterOut/ -name *ClusterOut -type f  |xargs cp --backup=numbered -t ~/NoBckDir/DR10/logs/
 find ./*/*_RawData/ClusterErr/ -name *ClusterErr -type f  |xargs cp --backup=numbered -t ~/NoBckDir/DR10/logs/
 
-
 ### Logs
 find ./*/*_RawData/ClusterOut/ -name *ClusterOut -type f  |xargs rm
 find ./*/*_RawData/ClusterErr/ -name *ClusterErr -type f  |xargs rm
 
-
 ### Merge files
 cat *.tsv >> all.tsv
 
-
-### Compress a directory
-bsub -J IMPC_stats_pipeline_lsf_jobs -q research-rh74  "tar -zcvf ~/NoBckDir/DR9.2/jo
+### Compress a directory: don't forget to specify memory and time
+sbatch --job-name=impc_stats_pipeline_job --mem=1G --time=xx:xx:xx --wrap="tar -zcvf ~/NoBckDir/DR9.2/jo"

IMPC annotation pipeline/loader.R

@@ -51,7 +51,8 @@ mainAgeing = function(file = NULL                                    ,
                       # Only for Batch generator
                       BatchProducer          =  FALSE      ,
                       cpu = 1                              ,
-                      memory = 8000                        ,
+                      memory = "7G"                        ,
+                      time = "08:00:00"                    ,
                       nMax                   = 10000       ,
                       ChunkSize              = 24          ,
                       MinColoniesInChunks    = 32          ,
@@ -105,7 +106,7 @@ mainAgeing = function(file = NULL                                    ,
   EA2LAMApping                   = read.csv(file = file.path(local(), 'EA2LA_parameter_mappings_2019-09-24.csv'))
   MetaDataList                   = read.csv(file = file.path(local(), 'metadataParameters.csv'))
   exceptionList                  = unique(c(exceptionList,MetaDataList$parameter_stable_id))
-  # CategoricalCategoryBlackList   = readFile(file = 'CategoricalCategoryBlackList.list')
+  
   # Main subdirectory/working directory
   message0('Preparing the working directory ...')
   cwd = getwd()
@@ -299,6 +300,7 @@ mainAgeing = function(file = NULL                                    ,
                         center               = center           ,
                         cpu                  = cpu              ,
                         memory               = memory           ,
+                        time                 = time             ,
                         extraBatchParameters =  extraBatchParameters
                       )
                       write(