From aa6310677e673c11b9b4df238e77b28b44043ea5 Mon Sep 17 00:00:00 2001
From: Daniel Vedder <daniel.vedder@idiv.de>
Date: Wed, 7 Aug 2024 10:15:12 +0200
Subject: [PATCH] Refactored DataOutput
---
src/analysis/makieplots.jl | 6 ++--
src/core/output.jl | 56 +++++++++++++++++++++++-------------
src/core/simulation.jl | 6 ++--
src/nature/ecologicaldata.jl | 12 ++++----
test/runtests.jl | 3 +-
5 files changed, 49 insertions(+), 34 deletions(-)
diff --git a/src/analysis/makieplots.jl b/src/analysis/makieplots.jl
index 9721aca..cc10094 100644
--- a/src/analysis/makieplots.jl
+++ b/src/analysis/makieplots.jl
@@ -25,7 +25,7 @@ function visualisemap(model::SimulationModel,date=nothing,landcover=nothing)
image!(f[1,1], landcover)
ax.aspect = DataAspect()
# check if there are individuals and plot them
- inds = model.datatables["individuals"]
+ inds = @subset(model.dataoutputs["individuals"].datastore, :X .== -1) #remove migrants
if iszero(size(inds)[1])
@debug "No individual data to map"
return f
@@ -54,7 +54,7 @@ Plot a line graph of population sizes of each species over time.
Returns a Makie figure object.
"""
function populationtrends(model::SimulationModel)
- pops = model.datatables["populations"]
+ pops = model.dataoutputs["populations"].datastore
ncolors = max(2, length(@param(nature.targetspecies)))
update_theme!(palette=(color=cgrad(:seaborn_bright, ncolors),), cycle=[:color])
f = Figure()
@@ -78,7 +78,7 @@ Plot a line graph of total population size and individual demographics of skylar
Returns a Makie figure object.
"""
function skylarkpopulation(model::SimulationModel)
- pops = model.datatables["skylark_abundance"]
+ pops = model.dataoutputs["skylark_abundance"].datastore
f = Figure()
dates = @param(core.startdate):@param(core.enddate)
axlimits = (1, length(dates), 0, maximum(pops[!,:TotalAbundance]))
diff --git a/src/core/output.jl b/src/core/output.jl
index 19a3743..555c4be 100644
--- a/src/core/output.jl
+++ b/src/core/output.jl
@@ -153,31 +153,34 @@ Struct fields:
- frequency: how often to call the output function (daily/monthly/yearly/end/never)
- plotfunction: a function that takes a model object and returns a Makie figure object (optional)
"""
-struct DataOutput
- name::String
- header::Vector{String}
- outputfunction::Function
+mutable struct DataOutput
+ #FIXME update docstring
frequency::String
+ databuffer::Vector{Vector}
+ datastore::DataFrame
+ outputfunction::Union{Function,Nothing}
plotfunction::Union{Function,Nothing} #XXX remove this? (#81)
end
+"Retrieve the data stored in a DataOutput (assumes `core.storedata` is true)."
+data(d::DataOutput) = d.datastore
+
##TODO what about output types that don't fit neatly into the standard CSV table format?
## (e.g. end-of-run stats, map data)
"""
- newdataoutput!(model, name, header, outputfunction, frequency)
+ newdataoutput!(model, name, header, frequency, outputfunction, plotfunction)
Create and register a new data output. This function must be called by all submodels
that want to have their output functions called regularly.
"""
-function newdataoutput!(model::SimulationModel, name::String, header::Vector{String},
- outputfunction::Function, frequency::String,
+function newdataoutput!(model::SimulationModel, name::String,
+ header::Vector{String}, frequency::String,
+ outputfunction::Union{Function,Nothing}=nothing,
plotfunction::Union{Function,Nothing}=nothing) #XXX remove this? (#81)
if !(frequency in ("daily", "monthly", "yearly", "end", "never"))
Base.error("Invalid frequency '$frequency' for $name.") #TODO replace with exception
end
- ndo = DataOutput(name, header, outputfunction, frequency, plotfunction)
- append!(model.dataoutputs, [ndo])
if frequency != "never"
if @param(core.csvoutput)
open(joinpath(@param(core.outdir), name*".csv"), "w") do f
@@ -189,9 +192,10 @@ function newdataoutput!(model::SimulationModel, name::String, header::Vector{Str
for h in header
df[!,h] = Any[] #XXX allow specifying types?
end
- model.datatables[name] = df
end
end
+ ndo = DataOutput(frequency, [[]], df, outputfunction, plotfunction)
+ model.dataoutputs[name] = ndo
end
"""
@@ -202,36 +206,48 @@ configured frequency. If `force` is `true`, activate all outputs regardless
of their configuration.
"""
function outputdata(model::SimulationModel, force=false)
- #XXX enable output every X days, or weekly?
+ #XXX enable output weekly, or on set dates?
#XXX all output functions except for "end" are run on the first update
# -> should they all be run on the last update, too?
startdate = @param(core.startdate)
isnextmonth = d -> (day(d) == day(startdate))
isnextyear = d -> (month(d) == month(startdate) && day(d) == day(startdate))
- for output in model.dataoutputs
+ for o in keys(model.dataoutputs)
+ output = model.dataoutputs[o]
(!force && output.frequency == "never") && continue
# check if this output should be activated today
if force || (output.frequency == "daily") ||
(output.frequency == "monthly" && isnextmonth(model.date)) ||
(output.frequency == "yearly" && isnextyear(model.date)) ||
(output.frequency == "end" && model.date == @param(core.enddate))
- data = output.outputfunction(model)
+ !isnothing(output.outputfunction) && (output.databuffer = output.outputfunction(model))
+ isempty(output.databuffer) && continue
if @param(core.csvoutput)
- open(joinpath(@param(core.outdir), output.name*".csv"), "a") do f
- for row in data
+ open(joinpath(@param(core.outdir), o*".csv"), "a") do f
+ for row in output.databuffer
println(f, join(row, ","))
end
end
end
if @param(core.storedata)
- for row in data
- push!(model.datatables[output.name], row)
+ for row in output.databuffer
+ push!(output.datastore, row)
end
end
+ output.databuffer = [[]]
end
end
end
+"""
+ record(model, outputname, data)
+
+Append an observation vector to the given output.
+"""
+function record(model::SimulationModel, outputname::String, data::Vector)
+ push!(model.dataoutputs[outputname].databuffer, data)
+end
+
"""
visualiseoutput(model)
@@ -241,11 +257,11 @@ saving each figure to file.
function visualiseoutput(model::SimulationModel) #XXX remove this? (#81)
@debug "Visualising output."
CairoMakie.activate!() # make sure we're using Cairo
- for output in model.dataoutputs
+ for o in keys(model.dataoutputs)
+ output = model.dataoutputs[o]
isnothing(output.plotfunction) && continue
figure = output.plotfunction(model)
- save(joinpath(@param(core.outdir), output.name*"."*@param(core.figureformat)),
- figure)
+ save(joinpath(@param(core.outdir), o*"."*@param(core.figureformat)), figure)
end
end
diff --git a/src/core/simulation.jl b/src/core/simulation.jl
index 420b24d..5d150d5 100644
--- a/src/core/simulation.jl
+++ b/src/core/simulation.jl
@@ -16,8 +16,7 @@ mutable struct AgricultureModel{Tcroptype,Tcropstate} <: SimulationModel
settings::Dict{String,Any}
rng::AbstractRNG
logger::AbstractLogger
- dataoutputs::Vector{DataOutput}
- datatables::Dict{String,DataFrame}
+ dataoutputs::Dict{String,DataOutput}
date::Date
landscape::Matrix{Pixel}
weather::Dict{Date,Weather}
@@ -126,8 +125,7 @@ function initmodel(settings::Dict{String, Any})
settings,
StableRNG(settings["core.seed"]),
logger,
- Vector{DataOutput}(),
- Dict{String, DataFrame}(),
+ Dict{String,DataOutput}(),
settings["core.startdate"],
landscape,
weather,
diff --git a/src/nature/ecologicaldata.jl b/src/nature/ecologicaldata.jl
index 52c02e6..c350b99 100644
--- a/src/nature/ecologicaldata.jl
+++ b/src/nature/ecologicaldata.jl
@@ -11,9 +11,9 @@ Create output files for each data group collected by the nature model.
"""
function initecologicaldata(model::SimulationModel)
newdataoutput!(model, "populations", ["Date", "Species", "Abundance"],
- savepopulationdata, @param(nature.popoutfreq), populationtrends)
+ @param(nature.popoutfreq), savepopulationdata, populationtrends)
newdataoutput!(model, "individuals", ["Date","ID","X","Y","Species","Sex","Age"],
- saveindividualdata, @param(nature.indoutfreq), visualisemap)
+ @param(nature.indoutfreq), saveindividualdata, visualisemap)
initskylarkdata(model)
end
@@ -50,12 +50,14 @@ monthly, yearly, or at the end of a simulation, depending on the parameter
`nature.indoutfreq`. WARNING: Produces very big files!
"""
function saveindividualdata(model::SimulationModel)
- #XXX doesn't include migrants!
data = []
for a in model.animals
isnothing(a) && continue
push!(data, [model.date,a.id,a.pos[1],a.pos[2],speciesof(a),a.sex,a.age])
end
+ for m in model.migrants
+ push!(data, [model.date, m[1].id, -1, -1, speciesof(m[1]), m[1].sex, m[1].age])
+ end
data
end
@@ -66,11 +68,11 @@ function initskylarkdata(model::SimulationModel)
newdataoutput!(model, "skylark_abundance",
["Date", "TotalAbundance", "Mating", "Breeding",
"Nonbreeding", "Juvenile", "Migrants"],
- skylarkabundance, "daily", skylarkpopulation)
+ "daily", skylarkabundance, skylarkpopulation)
# newdataoutput!(model, "skylark_territories", ["Date", "ID", "X", "Y"],
# skylarkterritories, "monthly") #TODO add plotting function
newdataoutput!(model, "skylark_nests", ["Date", "ID", "X", "Y", "Landcover", "Crop"],
- skylarknests, "monthly") #TODO add plotting function
+ "monthly", skylarknests) #TODO add plotting function
# newdataoutput!(model, "skylark_mortality", ["Date", "N", "Cause"],
# skylarkmortality, "daily") #TODO add plotting function
end
diff --git a/test/runtests.jl b/test/runtests.jl
index 4aaf78e..4ee512e 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -52,8 +52,7 @@ function inittestmodel(smallmap=true)
TESTSETTINGS,
StableRNG(TESTSETTINGS["core.seed"]),
global_logger(),
- Vector{DataOutput}(),
- Dict{String, DataFrame}(),
+ Dict{String,DataOutput}(),
TESTSETTINGS["core.startdate"],
landscape,
weather,
--
GitLab