From 377baebf8f8f85d7845fb641a900195ad43b9f2c Mon Sep 17 00:00:00 2001
From: Francesco Sabatini <francesco.sabatini@idiv.de>
Date: Fri, 21 Feb 2020 23:30:09 +0100
Subject: [PATCH] Fixes to A97 to run on cluster
---
code/A97_ElevationExtract.R | 7 +++----
code/A98_PredictorsExtract.R | 2 +-
code/cli_A97.r | 15 +++------------
code/submit-A97.sh | 8 ++++----
4 files changed, 11 insertions(+), 21 deletions(-)
diff --git a/code/A97_ElevationExtract.R b/code/A97_ElevationExtract.R
index 863d8db..6d0fa8c 100644
--- a/code/A97_ElevationExtract.R
+++ b/code/A97_ElevationExtract.R
@@ -54,9 +54,8 @@ ElevationExtract <- function(header, output, ncores){
print(paste(length(todo), "tiles to do"))
print("start main foreach loop")
-#foreach(i = 1:nlevels(header.sp$tilenam)) %do% {
- foreach(i =todo[1:6]) %dopar% {
- #for(i in todo){
+ foreach(i =todo) %dopar% {
+ print(paste("doing", i))
#create sp and project data
if(nrow(header.tiles %>%
filter(tilenam %in% levels(header.tiles$tilenam)[i])) ==0 ) next()
@@ -102,7 +101,7 @@ ElevationExtract <- function(header, output, ncores){
DEM.res=res(raster.tile)[1])
#save output
- save(output.tile, file = paste(output, "_elevation_tile_", i, ".RData", sep=""))
+ save(output.tile, file = paste(output, "elevation_tile_", i, ".RData", sep=""))
print(i)
}
stopCluster(cl)
diff --git a/code/A98_PredictorsExtract.R b/code/A98_PredictorsExtract.R
index 7d7aaf0..5cb6f4b 100644
--- a/code/A98_PredictorsExtract.R
+++ b/code/A98_PredictorsExtract.R
@@ -48,7 +48,7 @@ PredExtr <- function(x.shp, myfunction=NA, output,
print(paste(length(toassign), "plots not matched directely - seek for NN"))
if(length(toassign)>0){
- nearest <- foreach(i=1:length(toassign), .packages=c('raster'), .combine=rbind) %do% {
+ nearest <- foreach(i=1:length(toassign), .packages=c('raster'), .combine=rbind) %dopar% {
print(i)
nearest.tmp <- tryCatch(geosphere::dist2Line(header.shp[i,], mypredictor),
error = function(e){data.frame(distance=NA, lon=NA, lat=NA,ID=NA)}
diff --git a/code/cli_A97.r b/code/cli_A97.r
index 5dbe37b..b305bfc 100644
--- a/code/cli_A97.r
+++ b/code/cli_A97.r
@@ -18,16 +18,8 @@ options <- list (
default = default.ncores,
help = paste0("number of CPU cores to use, defaults to ", default.ncores),
metavar = "4"
- ),
-
- make_option(
- opt_str = c("-t", "--typp"),
- dest = "typp",
- type = "character",
- default = default.typp,
- help = "Which data type? raster or shp",
- metavar = "4"
)
+
)
parser <- OptionParser(
@@ -43,7 +35,7 @@ cli <- parse_args(parser, positional_arguments = 2)
# assign a few shortcuts
# ------------------------------------------------------------------------------
-header <- cli$args[1]
+header <- cli$args[1]
output <- cli$args[2]
ncores <- cli$options$ncores
@@ -54,6 +46,5 @@ ncores <- cli$options$ncores
source("A97_ElevationExtract.R")
-PredExtr(x.shp, myfunction, output, toextract, typp, ncores, chunkn, chunk.i)
-
+ElevationExtract(header, output, ncores)
diff --git a/code/submit-A97.sh b/code/submit-A97.sh
index e0edf5c..a672b92 100644
--- a/code/submit-A97.sh
+++ b/code/submit-A97.sh
@@ -3,13 +3,13 @@
#$ -S /bin/bash
#$ -N ElevationExtract
-#$ -o /work/$USER/$JOB_NAME-$JOB_ID-$TASK_ID.log
+#$ -o /work/$USER/$JOB_NAME-$JOB_ID.log
#$ -j y
#$ -l h_rt=00:30:00:00
-#$ -l h_vmem=60G
+#$ -l h_vmem=1G
-#$ -pe smp 20-32
+#$ -pe smp 4-32
#$ -cwd
@@ -22,4 +22,4 @@ Rscript \
cli_A97.r \
--cores ${NSLOTS:-1} \
/data/splot/_data_splot3/header.shp.shp \
- "$output"
\ No newline at end of file
+ "$output"
--
GitLab