From 8e89cb52767ac4f7c73a5eb71ca20b023cacb0b0 Mon Sep 17 00:00:00 2001
From: Daniel Vedder <daniel.vedder@idiv.de>
Date: Wed, 1 Feb 2023 10:01:36 +0100
Subject: [PATCH] Removed parallelisation again, wasn't working yet
Multiple runs and parameter scanning works, but parallelising was (as
always) a bit trickier than expected... Will try to get this working
soon, but for now I need to focus on more important things.
---
example.sh | 11 +++++++++++
run.sh | 11 -----------
src/core/input.jl | 16 +++++++---------
src/core/output.jl | 6 +++++-
src/core/simulation.jl | 4 ++--
test/io_tests.jl | 13 +------------
test/paramscan.toml | 2 +-
test/runtests.jl | 5 +++++
test/simulation_tests.jl | 1 -
9 files changed, 32 insertions(+), 37 deletions(-)
create mode 100755 example.sh
delete mode 100755 run.sh
diff --git a/example.sh b/example.sh
new file mode 100755
index 0000000..0044006
--- /dev/null
+++ b/example.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+## A short shell script to test the current development version of Persephone.
+
+if [[ -d "example_results" ]]
+then
+ rm -r example_results
+fi
+
+./run.jl -o example_results
+
+src/analysis/analyse_nature.R example_results
diff --git a/run.sh b/run.sh
deleted file mode 100755
index fefbafe..0000000
--- a/run.sh
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/bash
-## A short shell script to test the current development version of Persephone.
-
-if [[ -d "temp_results" ]]
-then
- rm -r temp_results
-fi
-
-./run.jl -o temp_results
-
-src/analysis/analyse_nature.R temp_results
diff --git a/src/core/input.jl b/src/core/input.jl
index 8060d2a..d950738 100644
--- a/src/core/input.jl
+++ b/src/core/input.jl
@@ -84,15 +84,13 @@ function preprocessparameters(settings::Dict{String,Any}, defaultoutdir::String)
if settings["core.startdate"] > settings["core.enddate"]
Base.error("Enddate is earlier than startdate.") #TODO replace with exception
end
- settings["core.loglevel"] == "debug" ? settings["core.loglevel"] = Logging.Debug :
- settings["core.loglevel"] == "warn" ? settings["core.loglevel"] = Logging.Warn :
- settings["core.loglevel"] = Logging.Info
- if !isempty(settings["internal.scanparams"])
- # https://docs.julialang.org/en/v1/manual/distributed-computing/#code-availability
- addprocs(settings["core.processors"])
- #addprocs(exeflags="--project") ?
- @everywhere include("../Persephone.jl") #FIXME
- end
+ #FIXME enable parallelisation
+ # if !isempty(settings["internal.scanparams"]) && (nprocs() < 2)
+ # @warn "To parallelise multiple simulation runs, use `julia -p <n>`."
+ # # https://docs.julialang.org/en/v1/manual/distributed-computing/#code-availability
+ # #addprocs(settings["core.processors"]; exeflags="--project")
+ # #@everywhere include("../src/Persephone.jl")
+ # end
settings
end
diff --git a/src/core/output.jl b/src/core/output.jl
index 470d4b0..8bf8dcc 100644
--- a/src/core/output.jl
+++ b/src/core/output.jl
@@ -36,9 +36,13 @@ Create a logger object that writes output both to screen and to a logfile.
This object is stored as `model.logger` and can then be used with `with_logger()`.
Note: requires `createdatadir()` to be run first.
"""
-function modellogger(loglevel::LogLevel, outdir::String)
+function modellogger(loglevel::String, outdir::String)
!isdir(outdir) && #TODO replace with exception
Base.error("$(outdir) does not exist. Call `createdatadir()` before `modellogger()`.")
+ loglevel == "debug" ? loglevel = Logging.Debug :
+ loglevel == "warn" ? loglevel = Logging.Warn :
+ loglevel == "info" ? loglevel = Logging.Info :
+ Base.error("Invalid loglevel $loglevel, should be debug/info/warn.") #TODO make exception
#XXX If this is a parallel run, should we turn off logging to screen?
logfile = open(joinpath(outdir, LOGFILE), "w+")
TeeLogger(ConsoleLogger(logfile, loglevel),
diff --git a/src/core/simulation.jl b/src/core/simulation.jl
index 492d3e7..e455fce 100644
--- a/src/core/simulation.jl
+++ b/src/core/simulation.jl
@@ -16,7 +16,7 @@ This is the default way to run a Persephone simulation.
function simulate(config::String=PARAMFILE, seed::Union{Int64,Nothing}=nothing)
models = initialise(config, seed)
isa(models, Vector) ?
- pmap(simulate!, models) :
+ map(simulate!, models) : #TODO parallelise
simulate!(models)
end
@@ -48,7 +48,7 @@ function initialise(config::String=PARAMFILE, seed::Union{Int64,Nothing}=nothing
delete!(settings, "internal.scanparams")
isempty(scanparams) ?
initmodel(settings) :
- pmap(initmodel, paramscan(settings, scanparams))
+ map(initmodel, paramscan(settings, scanparams)) #TODO parallelise
end
"""
diff --git a/test/io_tests.jl b/test/io_tests.jl
index be9683e..2180c59 100644
--- a/test/io_tests.jl
+++ b/test/io_tests.jl
@@ -4,7 +4,6 @@
###
@testset "Model configuration" begin
- # Test the configuration file
settings = Ps.getsettings(TESTPARAMETERS)
properties = Dict{Symbol,Any}(:settings=>settings)
space = GridSpace((10,10), periodic=false)
@@ -12,20 +11,10 @@
@test @param(core.configfile) == basename(TESTPARAMETERS)
@test @param(core.startdate) == Date(2022, 2, 1)
- @test @param(core.loglevel) == Logging.Warn
+ @test @param(core.loglevel) == "warn"
@test @param(nature.targetspecies) == ["Wolpertinger", "Wyvern"]
@param(core.enddate) = Date(2022,1,3)
@test @param(core.enddate) == Date(2022,1,3)
- # Test the parsing of commandline parameters
- #XXX ARGS not recognised immediately - redefining a constant is problematic anyway
- # Base.ARGS = ["--configfile", "test.toml", "-s", "314",
- # "--outdir", "random_results", "-l", "info"]
- # settings = Ps.getsettings(TESTPARAMETERS)
- # @test @param(core.configfile) == "test.toml"
- # @test @param(core.seed) == 314
- # @test @param(core.outdir) == "random_results"
- # @test @param(core.loglevel) == Logging.Info
- # Base.ARGS = []
end
@testset "Output functions" begin
diff --git a/test/paramscan.toml b/test/paramscan.toml
index caac133..cd3f513 100644
--- a/test/paramscan.toml
+++ b/test/paramscan.toml
@@ -9,7 +9,7 @@ landcovermap = "landcover_jena.tif" # location of the landcover map
farmfieldsmap = "fields_jena.tif" # location of the field geometry map
outdir = "results_test_paramscan" # location and name of the output folder
overwrite = ["ask",true] # overwrite the output directory? (true/false/"ask")
-loglevel = ["debug", "info"] # verbosity level: "debug", "info", "warn"
+loglevel = ["warn", "info"] # verbosity level: "debug", "info", "warn"
seed = [1,2,3] # seed value for the RNG (0 -> random value)
# dates to start and end the simulation
startdate = 2022-01-01
diff --git a/test/runtests.jl b/test/runtests.jl
index 363ffea..1542cc2 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -37,3 +37,8 @@ const TESTSETTINGS = Ps.getsettings(TESTPARAMETERS)
end
rm(TESTSETTINGS["core.outdir"], force=true, recursive=true)
end
+
+# NOTE: Due to an issue with Julia (https://github.com/JuliaLang/julia/issues/48456),
+# whenever we are using `@test_logs` with a function that takes a model object, we have
+# to wrap that model object in `withtestlogger()`. (For an example, see the "Model
+# simulation" testset in simulation_tests.jl.)
diff --git a/test/simulation_tests.jl b/test/simulation_tests.jl
index 6c19e32..7701033 100644
--- a/test/simulation_tests.jl
+++ b/test/simulation_tests.jl
@@ -27,7 +27,6 @@ end
scan = Ps.paramscan(settings, scanparams)
outdirs = (s["core.outdir"] for s in scan)
@test length(outdirs) == 12
- #FIXME On worker 2: KeyError: key Persephone [039acd1d-2a07-4b33-b082-83a1ff0fd136] not found
@test length(initialise(config)) == 12 #XXX This takes a long time
for dir in testdirs
@test dir in outdirs
--
GitLab