Implemented parameter scanning

This is a major commit with quite deep-reaching changes in the code
base. It enables the user to specify multiple values for a model
parameter, which will then be combined into a full-factorial
simulation experiment. Everything seems to work as intended, although
some refactoring would be good, and we still need to add parallelisation
for issue #42.

And yes, I coded all of this between 23:30 and 03:00 on a Saturday
night  

src/core/input.jl

@@ -6,6 +6,13 @@
 ## Note: much of this code was adapted from the GeMM model by Leidinger et al.
 ## (https://github.com/CCTB-Ecomods/gemm/blob/master/src/input.jl)
 
+"""
+The file that stores all default parameters.
+"""
+const PARAMFILE = joinpath(pkgdir(Persephone), "src/parameters.toml")
+## (DO NOT CHANGE THIS VALUE! Instead, specify simulation-specific configuration files
+## by using the "--configfile" commandline argument, or when invoking simulate().) 
+
 """
     @param(domainparam)
 
@@ -21,7 +28,25 @@ object is available.
 macro param(domainparam)
     domain = String(domainparam.args[1])
     paramname = String(domainparam.args[2].value)
-    :($(esc(:model)).settings[$domain][$paramname])
+    :($(esc(:model)).settings[$(domain*"."*paramname)])
+end
+
+"""
+    flattenTOML(dict)
+
+An internal utility function to convert the two-dimensional dict returned
+by `TOML.parsefile()` into a one-dimensional dict, so that instead of
+writing `settings["domain"]["param"]` one can use `settings["domain.param"]`.
+Can be reversed with `expandTOML()`.
+"""
+function flattenTOML(tomldict)
+    flatdict = Dict{String, Any}()
+    for domain in keys(tomldict)
+        for param in keys(tomldict[domain])
+            flatdict[domain*"."*param] = tomldict[domain][param]
+        end
+    end
+    flatdict
 end
 
 """
@@ -32,39 +57,47 @@ Precedence: commandline parameters - user config file - default values
 """
 function getsettings(configfile::String, seed::Union{Int64,Nothing}=nothing)
     # read in and merge configurations from the commandline, the default config file
-    # and a user-supplied config file
-    defaults::Dict{String, Dict{String, Any}} = TOML.parsefile(configfile)
+    # and a config file supplied by the user (via the --configfile option)
+    defaults::Dict{String, Any} = TOML.parsefile(PARAMFILE) |> flattenTOML
     commandline = parsecommandline()
+    scanparams = Vector{String}()
     if haskey(commandline, "configfile") && isfile(commandline["configfile"])
-        configs = TOML.parsefile(commandline["configfile"])
+        configs = TOML.parsefile(commandline["configfile"]) |> flattenTOML
+    elseif configfile != PARAMFILE && isfile(configfile)
+        configs = TOML.parsefile(configfile) |> flattenTOML
     else
         configs = nothing
     end
     settings = deepcopy(defaults)
-    for domain in keys(defaults)
-        for param in keys(defaults[domain])
-            if param in keys(commandline)
-                settings[domain][param] = commandline[param]
-            elseif !isnothing(configs) && param in keys(configs[domain])
-                settings[domain][param] = configs[domain][param]
+    for param in keys(defaults)
+        if split(param, ".")[2] in keys(commandline)
+            settings[param] = commandline[split(param, ".")[2]]
+        elseif !isnothing(configs) && param in keys(configs)
+            settings[param] = configs[param]
+            # check whether a parameter is given as a list for parameter scanning
+            vectup = Union{Vector,Tuple}
+            if (isa(configs[param], vectup) && !isa(defaults[param], vectup)) ||
+                isa(configs[param], Union{Vector{vectup},Tuple{vectup}})
+                push!(scanparams, param)
             end
         end
     end
     # pre-process certain parameters
     if !isnothing(seed)
-        settings["core"]["seed"] = seed
+        settings["core.seed"] = seed
     end
-    if settings["core"]["seed"] == 0
-        settings["core"]["seed"] = abs(rand(RandomDevice(), Int32))
+    if settings["core.seed"] == 0
+        settings["core.seed"] = abs(rand(RandomDevice(), Int32))
     end
-    defaultoutdir = defaults["core"]["outdir"]
-    if settings["core"]["outdir"] == defaultoutdir
-        outdir = defaultoutdir*"_"*string(Dates.today())*"_s"*string(settings["core"]["seed"])
-        settings["core"]["outdir"] = outdir
+    defaultoutdir = defaults["core.outdir"]
+    if settings["core.outdir"] == defaultoutdir
+        outdir = defaultoutdir*"_"*string(Dates.today())*"_s"*string(settings["core.seed"])
+        settings["core.outdir"] = outdir
     end
-    if settings["core"]["startdate"] > settings["core"]["enddate"]
+    if settings["core.startdate"] > settings["core.enddate"]
         Base.error("Enddate is earlier than startdate.")
     end
+    settings["internal.scanparams"] = scanparams
     settings
 end
 

src/core/output.jl

@@ -57,7 +57,7 @@ function setupdatadir(model::AgentBasedModel)
             println(f, "# WARNING: Your repository contains uncommitted changes. This may")
             println(f, "#          compromise the reproducibility of this simulation run.\n")
         end
-        TOML.print(f, model.settings)
+        TOML.print(f, expandTOML(model.settings))
     end
     # Copy the map files to the output folder
     lcmap = @param(core.landcovermap)
@@ -68,6 +68,23 @@ function setupdatadir(model::AgentBasedModel)
     cp(ffmap, joinpath(@param(core.outdir), basename(ffmap)), force = true)
 end
 
+
+"""
+    expandTOML(dict)
+
+An internal utility function to re-convert the one-dimensional dict created
+by `flattenTOML()` into the two-dimensional dict needed by `TOML.print()`.
+"""
+function expandTOML(settingsdict)
+    fulldict = Dict{String, Dict{String, Any}}()
+    for parameter in keys(settingsdict)
+        domain, param = split(parameter, ".")
+        !(domain in keys(fulldict)) && (fulldict[domain] = Dict{String,Any}())
+        fulldict[domain][param] = settingsdict[parameter]
+    end
+    fulldict
+end
+
 """
     DataOutput
 

src/core/simulation.jl

@@ -3,40 +3,90 @@
 ### This file includes the core functions for initialising and running simulations.
 ###
 
-"""
-The file that stores all default parameters.
-"""
-const PARAMFILE = "src/parameters.toml"
-## (DO NOT CHANGE THIS VALUE! Instead, specify simulation-specific configuration files
-## by using the "--configfile" commandline argument, or when invoking simulate().) 
+#XXX With the parameter scanning, code execution has become rather difficult to follow.
+# Can I refactor this into two clear, separate paths - one for the normal case (single
+# run) and one for parameter scanning?
 
 """
-    initialise(config=PARAMFILE, seed=nothing)
+    initmodels(settings)
 
-Initialise the model: read in parameters, create the output data directory,
-and instantiate the AgentBasedModel object. Optionally allows specifying the
-configuration file and overriding the `seed` parameter.
+Initialise one or more model objects using ready-made settings dicts. This is
+a helper function for `initialise()`.
 """
-function initialise(config::String=PARAMFILE, seed::Union{Int64,Nothing}=nothing)
-    @info "Simulation run started at $(Dates.now())."
-    settings = getsettings(config, seed)
+function initmodels(settingsdicts::Vector{Dict{String, Any}})
+    #TODO parallelise model initialisation
+    @debug "Initialising model object(s)."
+    # generate a set of models from these dicts
+    models = Vector{AgentBasedModel}()
+    for settings in settingsdicts
         events = Vector{FarmEvent}()
         dataoutputs = Vector{DataOutput}()
-    landscape = initlandscape(settings["core"]["landcovermap"], settings["core"]["farmfieldsmap"])
+        landscape = initlandscape(settings["core.landcovermap"], settings["core.farmfieldsmap"])
         space = GridSpace(size(landscape), periodic=false)
         properties = Dict{Symbol,Any}(:settings=>settings,
-                                  :date=>settings["core"]["startdate"],
+                                      :date=>settings["core.startdate"],
                                       :landscape=>landscape,
                                       :dataoutputs=>dataoutputs,
                                       :events=>events)
-    @debug "Setting up model."
         model = AgentBasedModel(Union{Farmer,Animal,FarmPlot}, space, properties=properties,
-                            rng=StableRNG(settings["core"]["seed"]), warn=false)
+                                rng=StableRNG(settings["core.seed"]), warn=false)
         setupdatadir(model)
         initfarms!(model)
         initfields!(model)
         initnature!(model)
-    model
+        push!(models, model)
+    end
+    models
+end
+
+"""
+    paramscan(settings)
+
+Initialise a list of model objects, covering all possible parameter combinations
+given by the settings (i.e. a full-factorial experiment). This is a helper function
+for `initialise()`.
+"""
+function paramscan(settings::Dict{String,Any}, scanparams::Vector{String})
+    # recursively generate a set of settings dicts covering all combinations
+    function generatecombinations(params::Vector{String})
+        (length(params) == 0) && return [settings]
+        param = pop!(params)
+        combinations = Vector{Dict{String,Any}}()
+        for comb in generatecombinations(params)
+            for value in settings[param]
+                newcombination = deepcopy(comb)
+                newcombination[param] = value
+                if comb["core.outdir"] == settings["core.outdir"]
+                    outdir = joinpath(comb["core.outdir"], "$(split(param, ".")[2])_$(value)")
+                else
+                    outdir = "$(comb["core.outdir"])_$(split(param, ".")[2])_$(value)"
+                end
+                newcombination["core.outdir"] = outdir
+                push!(combinations, newcombination)
+            end
+        end
+        combinations
+    end
+    generatecombinations(scanparams)
+end
+
+"""
+    initialise(config=PARAMFILE, seed=nothing)
+
+Initialise the model: read in parameters, create the output data directory,
+and instantiate the AgentBasedModel object(s). Optionally allows specifying the
+configuration file and overriding the `seed` parameter. This returns a single
+model object unless the config file contains multiple values for one or more
+parameters, in which case it creates a full-factorial simulation experiment
+and returns a vector of model objects.
+"""
+function initialise(config::String=PARAMFILE, seed::Union{Int64,Nothing}=nothing)
+    @info "Simulation run started at $(Dates.now())."
+    settings = getsettings(config, seed)
+    scanparams = settings["internal.scanparams"]
+    delete!(settings, "internal.scanparams")
+    models = initmodels(paramscan(settings, scanparams))
+    (length(models) == 1) ? models[1] : models
 end
 
 """
@@ -85,10 +135,19 @@ end
 """
     simulate(config=PARAMFILE, seed=nothing)
 
-Initialise a model object and carry out a complete simulation run, optionally
-specifying a configuration file and a seed for the RNG.
+Initialise one or more model objects and carry out a full simulation experiment,
+optionally specifying a configuration file and a seed for the RNG.
+
+This is the default way to run a Persephone simulation.
 """
 function simulate(config::String=PARAMFILE, seed::Union{Int64,Nothing}=nothing)
-    model = initialise(config, seed)
-    simulate!(model)
+    models = initialise(config, seed)
+    if isa(models, Vector)
+        for m in models
+            @info "Executing run $(m.settings["core.outdir"])"
+            simulate!(m) #TODO parallelise
+        end
+    else
+        simulate!(models)
+    end
 end

test/io_tests.jl

@@ -9,7 +9,7 @@
     space = GridSpace((10,10), periodic=false)
     model = AgentBasedModel(Animal, space, properties=properties, warn=false)
 
-    @test @param(core.configfile) == TESTPARAMETERS
+    @test @param(core.configfile) == basename(TESTPARAMETERS)
     @test @param(core.startdate) == Date(2022, 2, 1)
     @test @param(nature.targetspecies) == ["Wolpertinger", "Wyvern"]
     @param(core.enddate) = Date(2022,1,3)
@@ -24,8 +24,9 @@ end
     model = AgentBasedModel(Animal, space, properties=properties, warn=false)
     # test that the output directory is created with all files
     outdir = @param(core.outdir)
-    logstring = "Setting up output directory results_testsuite_$(Dates.today())_s1."
-    @test_logs (:debug, logstring) min_level=Logging.Debug Ps.setupdatadir(model)
+    @test_logs((:debug, "Setting up output directory results_testsuite."),
+               min_level=Logging.Debug, match_mode=:any,
+               Ps.setupdatadir(model))
     @test isdir(outdir)
     @test isfile(joinpath(outdir, @param(core.landcovermap)))
     @test isfile(joinpath(outdir, @param(core.farmfieldsmap)))

test/landscape_tests.jl

@@ -25,19 +25,19 @@ function smalltestlandscape(agenttype::Type=Animal)
     end
     landscape[6,4] = Pixel(Ps.water, 0, [])
     space = GridSpace(size(landscape), periodic=false)
-    properties = Dict{Symbol,Any}(:date=>TESTSETTINGS["core"]["startdate"],
+    properties = Dict{Symbol,Any}(:date=>TESTSETTINGS["core.startdate"],
                                   :landscape=>landscape,
                                   :events=>Vector{FarmEvent}(),
                                   :dataoutputs=>Vector{DataOutput}(),
                                   :settings=>TESTSETTINGS)
     return AgentBasedModel(agenttype, space, properties=properties,
-                           rng=StableRNG(TESTSETTINGS["core"]["seed"]), warn=false)
+                           rng=StableRNG(TESTSETTINGS["core.seed"]), warn=false)
 end
 
 @testset "Landscape initialisation" begin
     # initialise the landscape part of the model
-    landscape = Ps.initlandscape(TESTSETTINGS["core"]["landcovermap"],
-                                 TESTSETTINGS["core"]["farmfieldsmap"])
+    landscape = Ps.initlandscape(TESTSETTINGS["core.landcovermap"],
+                                 TESTSETTINGS["core.farmfieldsmap"])
     space = GridSpace(size(landscape), periodic=false)
     properties = Dict{Symbol,Any}(:landscape=>landscape, :settings=>TESTSETTINGS)
     model = AgentBasedModel(FarmPlot, space, properties=properties, warn=false)

test/paramscan.toml

+### Persephone - a socio-economic-ecological model of European agricultural landscapes.
+###
+### This configuration file is intended to test the parameter scanning feature.
+###
+
+[core]
+configfile = "test/paramscan.toml" # location of the configuration file
+landcovermap = "landcover_jena.tif" # location of the landcover map
+farmfieldsmap = "fields_jena.tif" # location of the field geometry map
+outdir = "results_test_paramscan" # location and name of the output folder
+overwrite = ["ask",true] # overwrite the output directory? (true/false/"ask")
+loglevel = ["debug", "info"] # verbosity level: "debug", "info", "quiet"
+seed = [1,2,3] # seed value for the RNG (0 -> random value)
+# dates to start and end the simulation
+startdate = 2022-01-01
+enddate = 2022-01-02
+
+[farm]
+farmmodel = "FieldManager" # which version of the farm model to use (not yet implemented)
+
+[nature]
+targetspecies = ["Wolpertinger", "Wyvern"] # list of target species to simulate
+popoutfreq = "daily" # output frequency population-level data, daily/monthly/yearly/end/never
+indoutfreq = "end" # output frequency individual-level data, daily/monthly/yearly/end/never
+	
+[crop]
+cropmodel = "linear" # crop growth model to use, "linear" or "aquacrop" (not yet implemented)
+

test/runtests.jl

@@ -16,7 +16,7 @@ using Test
 
 const Ps = Persephone
 
-const TESTPARAMETERS = "test_parameters.toml"
+const TESTPARAMETERS = joinpath(pkgdir(Persephone), "test/test_parameters.toml")
 const TESTSETTINGS = Ps.getsettings(TESTPARAMETERS)
 
 @testset "Persephone Tests" begin
@@ -34,5 +34,5 @@ const TESTSETTINGS = Ps.getsettings(TESTPARAMETERS)
     @testset "Farm model" begin
         include("farm_tests.jl")
     end
-    rm(TESTSETTINGS["core"]["outdir"], force=true, recursive=true)
+    rm(TESTSETTINGS["core.outdir"], force=true, recursive=true)
 end

test/simulation_tests.jl

@@ -5,7 +5,7 @@
 
 @testset "Model initialisation" begin
     model = initialise(TESTPARAMETERS)
-    @test typeof(model.settings) == Dict{String, Dict{String, Any}}
+    @test typeof(model.settings) == Dict{String, Any}
     @test model.date == Date(2022,2,1)
     @test typeof(model.landscape) == Matrix{Pixel}
     @test typeof(model.dataoutputs) == Vector{DataOutput}
@@ -14,6 +14,26 @@
     @test nagents(model) == 2092+10+28
 end
 
+@testset "Parameter scanning" begin
+    config = "paramscan.toml"
+    testdirs = ["results_test_paramscan/seed_1_loglevel_debug_overwrite_ask",
+                "results_test_paramscan/seed_1_loglevel_debug_overwrite_true",
+                "results_test_paramscan/seed_2_loglevel_info_overwrite_true"]
+    settings = Ps.getsettings(config)
+    scanparams = settings["internal.scanparams"]
+    @test length(scanparams) == 3
+    @test sort(["core.overwrite", "core.loglevel", "core.seed"]) == sort(scanparams)
+    scan = Ps.paramscan(settings, scanparams)
+    outdirs = (s["core.outdir"] for s in scan)
+    @test length(outdirs) == 12
+    @test length(initialise(config)) == 12 #XXX This takes a long time
+    for dir in testdirs
+        @test dir in outdirs
+        @test isdir(dir)
+    end
+    rm("results_test_paramscan", force=true, recursive=true)
+end
+
 @testset "Model simulation" begin
     # The primary reason for this testset is to make sure that a complete
     # simulation will run through without errors. Thus, there are few tests.