Changed A97 to array job, split in chunk

code/A97_ElevationExtract.R

 
-ElevationExtract <- function(header, output, ncores){
+ElevationExtract <- function(header, output, ncores, chunk.i){
   print("load packages")
   require(tidyverse)
   
@@ -54,8 +54,13 @@ ElevationExtract <- function(header, output, ncores){
   todo <- sample(todo, replace=F)
   print(paste(length(todo), "tiles to do"))
   
-  print("start main foreach loop")
-  foreach(i =todo) %dopar% {  
+  print("divide in chunks")
+  #divide in chunks #should take around 40'
+  indices <- 1:length(todo)
+  todo.chunks <- split(indices, sort(indices%%99))
+  
+  print(paste("start main foreach loop on chunk n=", chunk.i))
+  foreach(i = todo.cunks[[chunk.i]]) %dopar% {  
 	print(paste("doing", i))
     #create sp and project data
     if(nrow(header.tiles %>% 

code/cli_A97.r

@@ -18,8 +18,15 @@ options <- list (
 		default = default.ncores,
 		help    = paste0("number of CPU cores to use, defaults to ", default.ncores),
 		metavar = "4"
+	), 
+	make_option(
+	  opt_str = c("-i", "--chunk.i"),
+	  dest    = "chunkn",
+	  type    = "integer",
+	  default = 1,
+	  help    = "which chunk out of 99",
+	  metavar = "4"
 	)
-	
 )
 
 parser <- OptionParser(
@@ -35,9 +42,10 @@ cli <- parse_args(parser, positional_arguments = 2)
 # assign a few shortcuts
 # ------------------------------------------------------------------------------
 
-header   	   <- cli$args[1]
+header   	     <- cli$args[1]
 output  	     <- cli$args[2]
 ncores   	     <- cli$options$ncores
+chunkn         <- cli$options$chunk.i
 
 
 # ------------------------------------------------------------------------------
@@ -46,5 +54,5 @@ ncores   	     <- cli$options$ncores
 
 source("A97_ElevationExtract.R")
 
-ElevationExtract(header, output, ncores)
+ElevationExtract(header, output, ncores, chunk.i)
 

code/submit-A97.sh

@@ -3,16 +3,18 @@
 #$ -S /bin/bash
 #$ -N ElevationExtract
 
-#$ -o /work/$USER/$JOB_NAME-$JOB_ID.log
+#$ -o /work/$USER/$JOB_NAME-$JOB_ID-$TASK_ID.log
 #$ -j y
 
-#$ -l h_rt=00:30:00:00
+#$ -l h_rt=00:20:00:00
 #$ -l h_vmem=21G
 
 #$ -pe smp 8-32
 
 #$ -cwd
 
+#$ -t 1-99
+
 
 module load R
 
@@ -21,5 +23,6 @@ output=/data/splot/_data_splot3/output_pred/elevatr/
 Rscript \
     cli_A97.r \
     --cores ${NSLOTS:-1} \
+    --chunk.i $SGE_TASK_ID \
     /data/splot/_data_splot3/header.shp.shp \
     "$output"