diff --git a/code/A97_ElevationExtract.R b/code/A97_ElevationExtract.R index 8bca8f88c4a5ce27f9b9cb3250d3caf8d127a257..611eb55f711d3e2e6e2079f17ef694f6c05187f0 100644 --- a/code/A97_ElevationExtract.R +++ b/code/A97_ElevationExtract.R @@ -1,5 +1,5 @@ -ElevationExtract <- function(header, output, ncores){ +ElevationExtract <- function(header, output, ncores, chunk.i){ print("load packages") require(tidyverse) @@ -54,8 +54,13 @@ ElevationExtract <- function(header, output, ncores){ todo <- sample(todo, replace=F) print(paste(length(todo), "tiles to do")) - print("start main foreach loop") - foreach(i =todo) %dopar% { + print("divide in chunks") + #divide in chunks #should take around 40' + indices <- 1:length(todo) + todo.chunks <- split(indices, sort(indices%%99)) + + print(paste("start main foreach loop on chunk n=", chunk.i)) + foreach(i = todo.cunks[[chunk.i]]) %dopar% { print(paste("doing", i)) #create sp and project data if(nrow(header.tiles %>% diff --git a/code/cli_A97.r b/code/cli_A97.r index b305bfcfc0654cebdbc1ec9e7d05504db5fbf0b9..653390dca27b1c3cfd433353a3aca3f8c08507ad 100644 --- a/code/cli_A97.r +++ b/code/cli_A97.r @@ -18,8 +18,15 @@ options <- list ( default = default.ncores, help = paste0("number of CPU cores to use, defaults to ", default.ncores), metavar = "4" + ), + make_option( + opt_str = c("-i", "--chunk.i"), + dest = "chunkn", + type = "integer", + default = 1, + help = "which chunk out of 99", + metavar = "4" ) - ) parser <- OptionParser( @@ -35,9 +42,10 @@ cli <- parse_args(parser, positional_arguments = 2) # assign a few shortcuts # ------------------------------------------------------------------------------ -header <- cli$args[1] +header <- cli$args[1] output <- cli$args[2] ncores <- cli$options$ncores +chunkn <- cli$options$chunk.i # ------------------------------------------------------------------------------ @@ -46,5 +54,5 @@ ncores <- cli$options$ncores source("A97_ElevationExtract.R") -ElevationExtract(header, output, ncores) +ElevationExtract(header, output, ncores, chunk.i) diff --git a/code/submit-A97.sh b/code/submit-A97.sh index 7c9a778b41d25d7c5f8bc10085fc6fd09554fab9..e0b14feba12130822606a87239f741578f849404 100644 --- a/code/submit-A97.sh +++ b/code/submit-A97.sh @@ -3,16 +3,18 @@ #$ -S /bin/bash #$ -N ElevationExtract -#$ -o /work/$USER/$JOB_NAME-$JOB_ID.log +#$ -o /work/$USER/$JOB_NAME-$JOB_ID-$TASK_ID.log #$ -j y -#$ -l h_rt=00:30:00:00 +#$ -l h_rt=00:20:00:00 #$ -l h_vmem=21G #$ -pe smp 8-32 #$ -cwd +#$ -t 1-99 + module load R @@ -21,5 +23,6 @@ output=/data/splot/_data_splot3/output_pred/elevatr/ Rscript \ cli_A97.r \ --cores ${NSLOTS:-1} \ + --chunk.i $SGE_TASK_ID \ /data/splot/_data_splot3/header.shp.shp \ "$output"