Switched to using local loggers

Previously, a global logger was used, which would have given problems
on a parallel run. Also split up the `setupdatadir()` function to
improve code structure.

src/Persephone.jl

@@ -66,8 +66,10 @@ export
 include("core/input.jl")
 include("core/output.jl")
 include("core/landscape.jl")
+
 include("farm/farm.jl")
 include("crop/crops.jl")
+
 include("nature/nature.jl")
 include("nature/populations.jl")
 include("nature/ecologicaldata.jl")
@@ -79,6 +81,7 @@ include("nature/ecologicaldata.jl")
 #include("nature/species/skylark.jl")
 include("nature/species/wolpertinger.jl")
 include("nature/species/wyvern.jl")
-include("core/simulation.jl")
+
+include("core/simulation.jl") #this must be last
 
 end

src/core/output.jl

@@ -5,20 +5,18 @@
 
 const LOGFILE = "simulation.log"
 
-## Note: `setupdatadir()` was adapted from the GeMM model by Leidinger et al.
+## Much of this code was adapted from the GeMM model by Leidinger et al.
 ## (https://github.com/CCTB-Ecomods/gemm/blob/master/src/output.jl)
 
 """
-    setupdatadir()
+    createdatadir(outdir, overwrite)
 
-Creates the output directory and copies relevant files into it.
+Creates the output directory, dealing with possible conflicts.
 """
-function setupdatadir(model::AgentBasedModel)
-    # Check whether the output directory already exists and handle conflicts
-    if isdir(@param(core.outdir))
-        overwrite = @param(core.overwrite)
-        if @param(core.overwrite) == "ask"
-            println("The chosen output directory $(@param(core.outdir)) already exists.")
+function createdatadir(outdir::String, overwrite::Union{Bool,String})
+    if isdir(outdir)
+        if overwrite == "ask"
+            println("The chosen output directory $(outdir) already exists.")
             println("Type 'yes' to overwrite this directory. Otherwise, the simulation will abort.")
             print("Overwrite? ")
             answer = readline()
@@ -27,30 +25,46 @@ function setupdatadir(model::AgentBasedModel)
         if !overwrite
             Base.error("Output directory exists, will not overwrite. Aborting.")
         else
-            @warn "Overwriting existing output directory $(@param(core.outdir))."
+            @warn "Overwriting existing output directory $(outdir)."
         end
     end
-    mkpath(@param(core.outdir))
-    # Setup the logging system and logfile
+    mkpath(outdir)
+end
+
+"""
+    modellogger(logsetting, outdir)
+
+Create a logger object that writes output both to screen and to a logfile.
+This object is stored as `model.logger` and can then be used with `with_logger()`.
+Note: requires `createdatadir()` to be run first.
+"""
+function modellogger(logsetting::String, outdir::String)
+    #XXX If this is a parallel run, should we turn off logging to screen?
     loglevel = Logging.Info
-    if @param(core.loglevel) == "debug"
+    if logsetting == "debug"
         loglevel = Logging.Debug
-    elseif @param(core.loglevel) == "quiet"
+    elseif logsetting == "warn"
         loglevel = Logging.Warn
     end
-    logfile = open(joinpath(@param(core.outdir), LOGFILE), "w+")
-    simulationlogger = TeeLogger(ConsoleLogger(logfile, loglevel),
-                                 ConsoleLogger(stdout, loglevel))
-    global_logger(simulationlogger)
+    logfile = open(joinpath(outdir, LOGFILE), "w+")
+    TeeLogger(ConsoleLogger(logfile, loglevel),
+              ConsoleLogger(stdout, loglevel))
+end
+
+"""
+    saveinputfiles(model)
+
+Copy all input files into the output directory, including the actual parameter
+settings used. This allows replicating a run in future.
+"""
+function saveinputfiles(model::AgentBasedModel)
     @debug "Setting up output directory $(@param(core.outdir))."
-    # Export a copy of the current parameter settings to the output folder.
-    # This can be used to replicate this exact run in future, and also
-    # records the current time and git commit.
     currentcommit = read(`git rev-parse HEAD`, String)[1:8]
     open(joinpath(@param(core.outdir), basename(@param(core.configfile))), "w") do f
         println(f, "#\n# --- Persephone configuration parameters ---")
         println(f, "# This file was generated automatically.")
         println(f, "# Simulation run on $(string(Dates.format(Dates.now(), "d u Y HH:MM:SS"))),")
+        # Record the current git commit and versions of dependencies for reproducibility
         println(f, "# with Persephone $(pkgversion(Persephone)), git commit $(currentcommit),")
         println(f, "# running on Julia $(VERSION) with Agents.jl $(pkgversion(Agents)).\n#\n")
         if !isempty(strip(read(`git status -s`, String)))
@@ -68,7 +82,6 @@ function setupdatadir(model::AgentBasedModel)
     cp(ffmap, joinpath(@param(core.outdir), basename(ffmap)), force = true)
 end
 
-
 """
     expandTOML(dict)
 
@@ -111,8 +124,8 @@ end
 Create and register a new data output. This function must be called by all submodels
 that want to have their output functions called regularly.
 """
-function newdataoutput(model::AgentBasedModel, filename::String, header::String,
-                       outputfunction::Function, frequency::String)
+function newdataoutput!(model::AgentBasedModel, filename::String, header::String,
+                        outputfunction::Function, frequency::String)
     if !(frequency in ("daily", "monthly", "yearly", "end", "never"))
         Base.error("Invalid frequency '$frequency' for $filename.")
     end

src/core/simulation.jl

@@ -8,33 +8,48 @@
 # run) and one for parameter scanning?
 
 """
-    initmodels(settings)
+    initmodel(settings)
 
-Initialise one or more model objects using ready-made settings dicts. This is
-a helper function for `initialise()`.
+Initialise a model object using a ready-made settings dict. This is
+a helper function for `initialise()` and `initmodelsparallel()`.
 """
-function initmodels(settingsdicts::Vector{Dict{String, Any}})
-    #TODO parallelise model initialisation
-    @debug "Initialising model object(s)."
-    # generate a set of models from these dicts
-    models = Vector{AgentBasedModel}()
-    for settings in settingsdicts
+function initmodel(settings::Dict{String, Any})
+    @debug "Initialising model object."
+    createdatadir(settings["core.outdir"], settings["core.overwrite"])
+    logger = modellogger(settings["core.loglevel"], settings["core.outdir"])
+    with_logger(logger) do
         events = Vector{FarmEvent}()
         dataoutputs = Vector{DataOutput}()
         landscape = initlandscape(settings["core.landcovermap"], settings["core.farmfieldsmap"])
         space = GridSpace(size(landscape), periodic=false)
         properties = Dict{Symbol,Any}(:settings=>settings,
+                                      :logger=>logger,
                                       :date=>settings["core.startdate"],
                                       :landscape=>landscape,
                                       :dataoutputs=>dataoutputs,
                                       :events=>events)
         model = AgentBasedModel(Union{Farmer,Animal,FarmPlot}, space, properties=properties,
                                 rng=StableRNG(settings["core.seed"]), warn=false)
-        setupdatadir(model)
+        saveinputfiles(model)
         initfarms!(model)
         initfields!(model)
         initnature!(model)
-        push!(models, model)
+        model
+    end
+end
+
+"""
+    initmodelsparallel(settings)
+
+Initialise multiple model objects using ready-made settings dicts. This is
+a helper function for `initialise()`.
+"""
+function initmodelsparallel(settingsdicts::Vector{Dict{String, Any}})
+    #TODO parallelise model initialisation
+    @debug "Beginning to initialise model objects."
+    models = Vector{AgentBasedModel}()
+    for settings in settingsdicts
+        push!(models, initmodel(settings))
     end
     models
 end
@@ -85,8 +100,7 @@ function initialise(config::String=PARAMFILE, seed::Union{Int64,Nothing}=nothing
     settings = getsettings(config, seed)
     scanparams = settings["internal.scanparams"]
     delete!(settings, "internal.scanparams")
-    models = initmodels(paramscan(settings, scanparams))
-    (length(models) == 1) ? models[1] : models
+    isempty(scanparams) ? initmodel(settings) : initmodelsparallel(settings, scanparameters)
 end
 
 """
@@ -95,18 +109,21 @@ end
 Execute one update of the model.
 """
 function stepsimulation!(model::AgentBasedModel)
-    @info "Simulating day $(model.date)."
-    for a in Schedulers.ByType((Farmer,FarmPlot,Animal), true)(model)
-        try #The animal may have been killed
-            stepagent!(model[a], model)
-        catch exc
-            # check if the KeyError comes from the `model[a]` or the function call
-            isa(exc, KeyError) && isa(exc.key, Int) ? continue : throw(exc)
+    with_logger(model.logger) do
+        @info "Simulating day $(model.date)."
+        for a in Schedulers.ByType((Farmer,FarmPlot,Animal), true)(model)
+            try #The animal may have been killed
+                stepagent!(model[a], model)
+            catch exc
+                # check if the KeyError comes from the `model[a]` or the function call
+                isa(exc, KeyError) && isa(exc.key, Int) ? continue : throw(exc)
+            end
         end
+        updateevents!(model)
+        outputdata(model)
+        model.date += Day(1)
+        model
     end
-    updateevents!(model)
-    outputdata(model)
-    model.date += Day(1)
 end
 
 """
@@ -115,10 +132,12 @@ end
 Wrap up the simulation. Currently doesn't do anything except print some information.
 """
 function finalise!(model::AgentBasedModel)
-    @info "Simulated $(model.date-@param(core.startdate))."
-    @info "Simulation run completed at $(Dates.now()),\nwrote output to $(@param(core.outdir))."
-    #XXX is there anything to do here?
-    model
+    with_logger(model.logger) do
+        @info "Simulated $(model.date-@param(core.startdate))."
+        @info "Simulation run completed at $(Dates.now()),\nwrote output to $(@param(core.outdir))."
+        #XXX is there anything to do here?
+        model
+    end
 end
 
 """

src/nature/ecologicaldata.jl

@@ -12,10 +12,10 @@ const INDFILE = "individuals.csv"
 Create output files for each data group collected by the nature model.
 """
 function initecologicaldata(model::AgentBasedModel)
-    newdataoutput(model, POPFILE, "Date;Species;Abundance",
-                  savepopulationdata, @param(nature.popoutfreq))
-    newdataoutput(model, INDFILE, "Date;ID;X;Y;Species;Sex;Age",
-                  saveindividualdata, @param(nature.indoutfreq))
+    newdataoutput!(model, POPFILE, "Date;Species;Abundance",
+                   savepopulationdata, @param(nature.popoutfreq))
+    newdataoutput!(model, INDFILE, "Date;ID;X;Y;Species;Sex;Age",
+                   saveindividualdata, @param(nature.indoutfreq))
 end
 
 """

src/parameters.toml

@@ -12,7 +12,7 @@ landcovermap = "data/landcover_jena.tif" # location of the landcover map
 farmfieldsmap = "data/fields_jena.tif" # location of the field geometry map
 outdir = "results" # location and name of the output folder
 overwrite = "ask" # overwrite the output directory? (true/false/"ask")
-loglevel = "debug" # verbosity level: "debug", "info", "quiet"
+loglevel = "debug" # verbosity level: "debug", "info", "warn"
 seed = 2 # seed value for the RNG (0 -> random value)
 # dates to start and end the simulation
 startdate = 2022-01-01

test/paramscan.toml

@@ -9,7 +9,7 @@ landcovermap = "landcover_jena.tif" # location of the landcover map
 farmfieldsmap = "fields_jena.tif" # location of the field geometry map
 outdir = "results_test_paramscan" # location and name of the output folder
 overwrite = ["ask",true] # overwrite the output directory? (true/false/"ask")
-loglevel = ["debug", "info"] # verbosity level: "debug", "info", "quiet"
+loglevel = ["debug", "info"] # verbosity level: "debug", "info", "warn"
 seed = [1,2,3] # seed value for the RNG (0 -> random value)
 # dates to start and end the simulation
 startdate = 2022-01-01

test/test_parameters.toml

@@ -9,7 +9,7 @@ landcovermap = "landcover_jena.tif" # location of the landcover map
 farmfieldsmap = "fields_jena.tif" # location of the field geometry map
 outdir = "results_testsuite" # location and name of the output folder
 overwrite = true # overwrite the output directory? (true/false/"ask")
-loglevel = "quiet" # verbosity level: "debug", "info", "quiet"
+loglevel = "warn" # verbosity level: "debug", "info", "warn"
 seed = 1 # seed value for the RNG (0 -> random value)
 # dates to start and end the simulation
 startdate = 2022-02-01