output.jl

### Persephone - a socio-economic-ecological model of European agricultural landscapes.
###
### This file includes functions for saving the model output.
###

const LOGFILE = "simulation.log"

## Much of this code was adapted from the GeMM model by Leidinger et al.
## (https://github.com/CCTB-Ecomods/gemm/blob/master/src/output.jl)

"""
    createdatadir(outdir, overwrite)

Creates the output directory, dealing with possible conflicts.
"""
function createdatadir(outdir::String, overwrite::Union{Bool,String})
    if isdir(outdir)
        if overwrite == "ask"
            println("The chosen output directory $(outdir) already exists.")
            println("Type 'yes' to overwrite this directory. Otherwise, the simulation will abort.")
            print("Overwrite? ")
            answer = readline()
            (answer == "yes") && (overwrite = true)
        end
        if !overwrite
            #TODO replace with exception
            Base.error("Output directory exists, will not overwrite. Aborting.") 
        else
            @warn "Overwriting existing output directory $(outdir)."
        end
    end
    mkpath(outdir)
end

"""
    modellogger(logsetting, outdir)

Create a logger object that writes output both to screen and to a logfile.
This object is stored as `model.logger` and can then be used with `with_logger()`.
Note: requires `createdatadir()` to be run first.
"""
function modellogger(logsetting::String, outdir::String)
    #XXX If this is a parallel run, should we turn off logging to screen?
    loglevel = Logging.Info
    if logsetting == "debug"
        loglevel = Logging.Debug
    elseif logsetting == "warn"
        loglevel = Logging.Warn
    end
    logfile = open(joinpath(outdir, LOGFILE), "w+")
    TeeLogger(ConsoleLogger(logfile, loglevel),
              ConsoleLogger(stdout, loglevel))
end

"""
    saveinputfiles(model)

Copy all input files into the output directory, including the actual parameter
settings used. This allows replicating a run in future.
"""
function saveinputfiles(model::AgentBasedModel)
    @debug "Setting up output directory $(@param(core.outdir))."
    currentcommit = read(`git rev-parse HEAD`, String)[1:8]
    open(joinpath(@param(core.outdir), basename(@param(core.configfile))), "w") do f
        println(f, "#\n# --- Persephone configuration parameters ---")
        println(f, "# This file was generated automatically.")
        println(f, "# Simulation run on $(string(Dates.format(Dates.now(), "d u Y HH:MM:SS"))),")
        # Record the current git commit and versions of dependencies for reproducibility
        println(f, "# with Persephone $(pkgversion(Persephone)), git commit $(currentcommit),")
        println(f, "# running on Julia $(VERSION) with Agents.jl $(pkgversion(Agents)).\n#\n")
        if !isempty(strip(read(`git status -s`, String)))
            println(f, "# WARNING: Your repository contains uncommitted changes. This may")
            println(f, "#          compromise the reproducibility of this simulation run.\n")
        end
        TOML.print(f, expandTOML(model.settings))
    end
    # Copy the map files to the output folder
    lcmap = @param(core.landcovermap)
    ffmap = @param(core.farmfieldsmap)
    #TODO replace errors with exceptions
    !(isfile(lcmap)) && Base.error("The map file $(lcmap) doesn't exist.")
    !(isfile(ffmap)) && Base.error("The map file $(ffmap) doesn't exist.")
    cp(lcmap, joinpath(@param(core.outdir), basename(lcmap)), force = true)
    cp(ffmap, joinpath(@param(core.outdir), basename(ffmap)), force = true)
end

"""
    expandTOML(dict)

An internal utility function to re-convert the one-dimensional dict created
by `flattenTOML()` into the two-dimensional dict needed by `TOML.print()`.
"""
function expandTOML(settingsdict)
    fulldict = Dict{String, Dict{String, Any}}()
    for parameter in keys(settingsdict)
        domain, param = split(parameter, ".")
        !(domain in keys(fulldict)) && (fulldict[domain] = Dict{String,Any}())
        fulldict[domain][param] = settingsdict[parameter]
    end
    fulldict
end

"""
    DataOutput

A struct for organising model output. This is designed for text-based data output
that is updated more or less regularly (e.g. population data in csv files).
Submodels can register their own output functions using `newdataoutput()`.

Struct fields:
    - filename: the name of the file to be created in the user-specified output directory
    - header: a string to be written to the start of the file as it is initialised
    - outputfunction: a function that takes a model object and returns a string to write to file
    - frequency: how often to call the output function (daily/monthly/yearly/end/never)
"""
struct DataOutput
    filename::String
    header::String
    outputfunction::Function
    frequency::String
end

"""
    newdataoutput(model, filename, header, outputfunction, frequency)

Create and register a new data output. This function must be called by all submodels
that want to have their output functions called regularly.
"""
function newdataoutput!(model::AgentBasedModel, filename::String, header::String,
                        outputfunction::Function, frequency::String)
    if !(frequency in ("daily", "monthly", "yearly", "end", "never"))
        Base.error("Invalid frequency '$frequency' for $filename.") #TODO replace with exception
    end
    ndo = DataOutput(filename, header, outputfunction, frequency)
    append!(model.dataoutputs, [ndo])
    if frequency != "never"
        open(joinpath(@param(core.outdir), filename), "w") do f
            println(f, header)
        end
    end
end

"""
    outputdata(model)
 
Cycle through all registered data outputs and activate them according to their
configured frequency.
"""
function outputdata(model::AgentBasedModel)
    #XXX all output functions are run on the first update (regardless of frequency)
    # -> should they all be run on the last update, too?
    startdate = @param(core.startdate)
    isnextmonth = d -> (day(d) == day(startdate))
    isnextyear = d -> (month(d) == month(startdate) && day(d) == day(startdate))
    for output in model.dataoutputs
        (output.frequency == "never") && continue
        # check if this output should be activated today
        if (output.frequency == "daily") ||
            (output.frequency == "monthly" && isnextmonth(model.date)) ||
            (output.frequency == "yearly" && isnextyear(model.date)) ||
            (output.frequency == "end" && model.date == @param(core.enddate))
            open(joinpath(@param(core.outdir), output.filename), "a") do f
                outstring = output.outputfunction(model)
                (outstring[end] != '\n') && (outstring *= '\n')
                print(f, outstring)
            end                
        end
    end
end