Refactored DataOutput

src/analysis/makieplots.jl

@@ -25,7 +25,7 @@ function visualisemap(model::SimulationModel,date=nothing,landcover=nothing)
     image!(f[1,1], landcover)
     ax.aspect = DataAspect()
     # check if there are individuals and plot them
-    inds = model.datatables["individuals"]
+    inds = @subset(model.dataoutputs["individuals"].datastore, :X .== -1) #remove migrants
     if iszero(size(inds)[1])
         @debug "No individual data to map"
         return f
@@ -54,7 +54,7 @@ Plot a line graph of population sizes of each species over time.
 Returns a Makie figure object.
 """
 function populationtrends(model::SimulationModel)
-    pops = model.datatables["populations"]
+    pops = model.dataoutputs["populations"].datastore
     ncolors = max(2, length(@param(nature.targetspecies)))
     update_theme!(palette=(color=cgrad(:seaborn_bright, ncolors),), cycle=[:color])
     f = Figure()
@@ -78,7 +78,7 @@ Plot a line graph of total population size and individual demographics of skylar
 Returns a Makie figure object.
 """
 function skylarkpopulation(model::SimulationModel)
-    pops = model.datatables["skylark_abundance"]
+    pops = model.dataoutputs["skylark_abundance"].datastore
     f = Figure()
     dates = @param(core.startdate):@param(core.enddate)
     axlimits = (1, length(dates), 0, maximum(pops[!,:TotalAbundance]))

src/core/output.jl

@@ -153,31 +153,34 @@ Struct fields:
     - frequency: how often to call the output function (daily/monthly/yearly/end/never)
     - plotfunction: a function that takes a model object and returns a Makie figure object (optional)
 """
-struct DataOutput
-    name::String
-    header::Vector{String}
-    outputfunction::Function
+mutable struct DataOutput
+    #FIXME update docstring
     frequency::String
+    databuffer::Vector{Vector}
+    datastore::DataFrame
+    outputfunction::Union{Function,Nothing}
     plotfunction::Union{Function,Nothing} #XXX remove this? (#81)
 end
 
+"Retrieve the data stored in a DataOutput (assumes `core.storedata` is true)."
+data(d::DataOutput) = d.datastore
+
 ##TODO what about output types that don't fit neatly into the standard CSV table format?
 ## (e.g. end-of-run stats, map data)
 
 """
-    newdataoutput!(model, name, header, outputfunction, frequency)
+    newdataoutput!(model, name, header, frequency, outputfunction, plotfunction)
 
 Create and register a new data output. This function must be called by all submodels
 that want to have their output functions called regularly.
 """
-function newdataoutput!(model::SimulationModel, name::String, header::Vector{String},
-                        outputfunction::Function, frequency::String,
+function newdataoutput!(model::SimulationModel, name::String,
+                        header::Vector{String}, frequency::String,
+                        outputfunction::Union{Function,Nothing}=nothing,
                         plotfunction::Union{Function,Nothing}=nothing) #XXX remove this? (#81)
     if !(frequency in ("daily", "monthly", "yearly", "end", "never"))
         Base.error("Invalid frequency '$frequency' for $name.") #TODO replace with exception
     end
-    ndo = DataOutput(name, header, outputfunction, frequency, plotfunction)
-    append!(model.dataoutputs, [ndo])
     if frequency != "never"
         if @param(core.csvoutput)
             open(joinpath(@param(core.outdir), name*".csv"), "w") do f
@@ -189,9 +192,10 @@ function newdataoutput!(model::SimulationModel, name::String, header::Vector{Str
             for h in header
                 df[!,h] = Any[] #XXX allow specifying types?
             end
-            model.datatables[name] = df
         end
     end
+    ndo = DataOutput(frequency, [[]], df, outputfunction, plotfunction)
+    model.dataoutputs[name] = ndo
 end
 
 """
@@ -202,36 +206,48 @@ configured frequency. If `force` is `true`, activate all outputs regardless
 of their configuration.
 """
 function outputdata(model::SimulationModel, force=false)
-    #XXX enable output every X days, or weekly?
+    #XXX enable output weekly, or on set dates?
     #XXX all output functions except for "end" are run on the first update
     # -> should they all be run on the last update, too?
     startdate = @param(core.startdate)
     isnextmonth = d -> (day(d) == day(startdate))
     isnextyear = d -> (month(d) == month(startdate) && day(d) == day(startdate))
-    for output in model.dataoutputs
+    for o in keys(model.dataoutputs)
+        output = model.dataoutputs[o]
         (!force && output.frequency == "never") && continue
         # check if this output should be activated today
         if force || (output.frequency == "daily") ||
             (output.frequency == "monthly" && isnextmonth(model.date)) ||
             (output.frequency == "yearly" && isnextyear(model.date)) ||
             (output.frequency == "end" && model.date == @param(core.enddate))
-            data = output.outputfunction(model)
+            !isnothing(output.outputfunction) && (output.databuffer = output.outputfunction(model))
+            isempty(output.databuffer) && continue
             if @param(core.csvoutput)
-                open(joinpath(@param(core.outdir), output.name*".csv"), "a") do f
-                    for row in data
+                open(joinpath(@param(core.outdir), o*".csv"), "a") do f
+                    for row in output.databuffer
                         println(f, join(row, ","))
                     end
                 end                
             end
             if @param(core.storedata)
-                for row in data
-                    push!(model.datatables[output.name], row)
+                for row in output.databuffer
+                    push!(output.datastore, row)
                 end
             end
+            output.databuffer = [[]]
         end
     end
 end
 
+"""
+    record(model, outputname, data)
+
+Append an observation vector to the given output.
+"""
+function record(model::SimulationModel, outputname::String, data::Vector)
+    push!(model.dataoutputs[outputname].databuffer, data)
+end
+    
 """
     visualiseoutput(model)
 
@@ -241,11 +257,11 @@ saving each figure to file.
 function visualiseoutput(model::SimulationModel) #XXX remove this? (#81)
     @debug "Visualising output."
     CairoMakie.activate!() # make sure we're using Cairo
-    for output in model.dataoutputs
+    for o in keys(model.dataoutputs)
+        output = model.dataoutputs[o]
         isnothing(output.plotfunction) && continue
         figure = output.plotfunction(model)
-        save(joinpath(@param(core.outdir), output.name*"."*@param(core.figureformat)),
-             figure)
+        save(joinpath(@param(core.outdir), o*"."*@param(core.figureformat)), figure)
     end
 end
 

src/core/simulation.jl

@@ -16,8 +16,7 @@ mutable struct AgricultureModel{Tcroptype,Tcropstate} <: SimulationModel
     settings::Dict{String,Any}
     rng::AbstractRNG
     logger::AbstractLogger
-    dataoutputs::Vector{DataOutput}
-    datatables::Dict{String,DataFrame}
+    dataoutputs::Dict{String,DataOutput}
     date::Date
     landscape::Matrix{Pixel}
     weather::Dict{Date,Weather}
@@ -126,8 +125,7 @@ function initmodel(settings::Dict{String, Any})
             settings,
             StableRNG(settings["core.seed"]),
             logger,
-            Vector{DataOutput}(),
-            Dict{String, DataFrame}(),
+            Dict{String,DataOutput}(),
             settings["core.startdate"],
             landscape,
             weather,

src/nature/ecologicaldata.jl

@@ -11,9 +11,9 @@ Create output files for each data group collected by the nature model.
 """
 function initecologicaldata(model::SimulationModel)
     newdataoutput!(model, "populations", ["Date", "Species", "Abundance"],
-                   savepopulationdata, @param(nature.popoutfreq), populationtrends)
+                   @param(nature.popoutfreq), savepopulationdata, populationtrends)
     newdataoutput!(model, "individuals", ["Date","ID","X","Y","Species","Sex","Age"],
-                   saveindividualdata, @param(nature.indoutfreq), visualisemap)
+                   @param(nature.indoutfreq), saveindividualdata, visualisemap)
     initskylarkdata(model)
 end
 
@@ -50,12 +50,14 @@ monthly, yearly, or at the end of a simulation, depending on the parameter
 `nature.indoutfreq`. WARNING: Produces very big files!
 """
 function saveindividualdata(model::SimulationModel)
-    #XXX doesn't include migrants!
     data = []
     for a in model.animals
         isnothing(a) && continue
         push!(data, [model.date,a.id,a.pos[1],a.pos[2],speciesof(a),a.sex,a.age])
     end
+    for m in model.migrants
+        push!(data, [model.date, m[1].id, -1, -1, speciesof(m[1]), m[1].sex, m[1].age])
+    end
     data
 end
 
@@ -66,11 +68,11 @@ function initskylarkdata(model::SimulationModel)
     newdataoutput!(model, "skylark_abundance",
                    ["Date", "TotalAbundance", "Mating", "Breeding",
                     "Nonbreeding", "Juvenile", "Migrants"],
-                   skylarkabundance, "daily", skylarkpopulation)
+                   "daily", skylarkabundance, skylarkpopulation)
     # newdataoutput!(model, "skylark_territories", ["Date", "ID", "X", "Y"],
     #                skylarkterritories, "monthly") #TODO add plotting function
     newdataoutput!(model, "skylark_nests", ["Date", "ID", "X", "Y", "Landcover", "Crop"],
-                   skylarknests, "monthly") #TODO add plotting function
+                   "monthly", skylarknests) #TODO add plotting function
     # newdataoutput!(model, "skylark_mortality", ["Date", "N", "Cause"],
     #                skylarkmortality, "daily") #TODO add plotting function
 end

test/runtests.jl

@@ -52,8 +52,7 @@ function inittestmodel(smallmap=true)
         TESTSETTINGS,
         StableRNG(TESTSETTINGS["core.seed"]),
         global_logger(),
-        Vector{DataOutput}(),
-        Dict{String, DataFrame}(),
+        Dict{String,DataOutput}(),
         TESTSETTINGS["core.startdate"],
         landscape,
         weather,