diff --git a/Project.toml b/Project.toml index a41f96c0bc..ab805d1d1f 100644 --- a/Project.toml +++ b/Project.toml @@ -48,7 +48,7 @@ MathOptInterface = "1" PowerModels = "~0.19" PowerNetworkMatrices = "^0.9" PowerSystems = "^3" -PrettyTables = "^1.3, 2" +PrettyTables = "2" ProgressMeter = "^1.5" SHA = "0.7" Serialization = "1" diff --git a/src/core/dataset.jl b/src/core/dataset.jl index 6340551e1f..f263dfb711 100644 --- a/src/core/dataset.jl +++ b/src/core/dataset.jl @@ -20,9 +20,9 @@ end # Values field is accessed with dot syntax to avoid type instability -mutable struct InMemoryDataset <: AbstractDataset - "Data with dimensions (column names, row indexes)" - values::DenseAxisArray{Float64, 2} +mutable struct InMemoryDataset{N} <: AbstractDataset + "Data with dimensions (N column names, row indexes)" + values::DenseAxisArray{Float64, N} # We use Array here to allow for overwrites when updating the state timestamps::Vector{Dates.DateTime} # Resolution is needed because AbstractDataset might have just one row @@ -33,12 +33,12 @@ mutable struct InMemoryDataset <: AbstractDataset end function InMemoryDataset( - values::DenseAxisArray{Float64, 2}, + values::DenseAxisArray{Float64, N}, timestamps::Vector{Dates.DateTime}, resolution::Dates.Millisecond, end_of_step_index::Int, -) - return InMemoryDataset( +) where {N} + return InMemoryDataset{N}( values, timestamps, resolution, @@ -48,8 +48,8 @@ function InMemoryDataset( ) end -function InMemoryDataset(values::DenseAxisArray{Float64, 2}) - return InMemoryDataset( +function InMemoryDataset(values::DenseAxisArray{Float64, N}) where {N} + return InMemoryDataset{N}( values, Vector{Dates.DateTime}(), Dates.Second(0.0), @@ -59,17 +59,62 @@ function InMemoryDataset(values::DenseAxisArray{Float64, 2}) ) end -get_num_rows(s::InMemoryDataset) = size(s.values)[2] +# Helper method for one dimensional cases +function InMemoryDataset( + fill_val::Float64, + initial_time::Dates.DateTime, + resolution::Dates.Millisecond, + end_of_step_index::Int, + row_count::Int, + column_names::Vector{String}) + return InMemoryDataset( + fill_val, + initial_time, + resolution, + end_of_step_index, + row_count, + (column_names,), + ) +end + +function InMemoryDataset( + fill_val::Float64, + initial_time::Dates.DateTime, + resolution::Dates.Millisecond, + end_of_step_index::Int, + row_count::Int, + column_names::NTuple{N, <:Any}) where {N} + return InMemoryDataset( + fill!( + DenseAxisArray{Float64}(undef, column_names..., 1:row_count), + fill_val, + ), + collect( + range( + initial_time; + step = resolution, + length = row_count, + ), + ), + resolution, + end_of_step_index, + ) +end + +get_num_rows(s::InMemoryDataset{N}) where {N} = size(s.values)[N] function make_system_state( - values::DenseAxisArray{Float64, 2}, timestamp::Dates.DateTime, resolution::Dates.Millisecond, -) - return InMemoryDataset(values, [timestamp], resolution, 0, 1, UNSET_INI_TIME) + columns::NTuple{N, <:Any}, +) where {N} + return InMemoryDataset(NaN, timestamp, resolution, 0, 1, columns) end -function get_dataset_value(s::InMemoryDataset, date::Dates.DateTime) +function get_dataset_value( + s::T, + date::Dates.DateTime, +) where {T <: Union{InMemoryDataset{1}, InMemoryDataset{2}}} s_index = find_timestamp_index(s.timestamps, date) if isnothing(s_index) error("Request time stamp $date not in the state") @@ -77,16 +122,32 @@ function get_dataset_value(s::InMemoryDataset, date::Dates.DateTime) return s.values[:, s_index] end -get_column_names(s::InMemoryDataset) = axes(s.values)[1] -get_column_names(::OptimizationContainerKey, s::InMemoryDataset) = get_column_names(s) +function get_dataset_value(s::InMemoryDataset{3}, date::Dates.DateTime) + s_index = find_timestamp_index(s.timestamps, date) + if isnothing(s_index) + error("Request time stamp $date not in the state") + end + return s.values[:, :, s_index] +end + +function get_column_names(k::OptimizationContainerKey, s::InMemoryDataset) + return get_column_names(k, s.values) +end -function get_last_recorded_value(s::InMemoryDataset) +function get_last_recorded_value(s::InMemoryDataset{2}) if get_last_recorded_row(s) == 0 error("The Dataset hasn't been written yet") end return s.values[:, get_last_recorded_row(s)] end +function get_last_recorded_value(s::InMemoryDataset{3}) + if get_last_recorded_row(s) == 0 + error("The Dataset hasn't been written yet") + end + return s.values[:, :, get_last_recorded_row(s)] +end + function get_end_of_step_timestamp(s::InMemoryDataset) return s.timestamps[s.end_of_step_index] end @@ -110,18 +171,26 @@ function get_value_timestamp(s::InMemoryDataset, date::Dates.DateTime) return s.timestamps[s_index] end -function set_value!(s::InMemoryDataset, vals::DenseAxisArray{Float64, 2}, index::Int) +# These set_value! methods expect a single time_step value because they are used to update +#the state so the incoming vals will have one dimension less than the DataSet. The exception +# is for vals of Dimension 1 which are still stored in DataSets of dimension 2. +function set_value!(s::InMemoryDataset{2}, vals::DenseAxisArray{Float64, 2}, index::Int) s.values[:, index] = vals[:, index] return end -function set_value!(s::InMemoryDataset, vals::DenseAxisArray{Float64, 1}, index::Int) +function set_value!(s::InMemoryDataset{2}, vals::DenseAxisArray{Float64, 1}, index::Int) s.values[:, index] = vals return end +function set_value!(s::InMemoryDataset{3}, vals::DenseAxisArray{Float64, 2}, index::Int) + s.values[:, :, index] = vals + return +end + # HDF5Dataset does not account of overwrites in the data. Values are written sequentially. -mutable struct HDF5Dataset <: AbstractDataset +mutable struct HDF5Dataset{N} <: AbstractDataset values::HDF5.Dataset column_dataset::HDF5.Dataset write_index::Int @@ -129,20 +198,31 @@ mutable struct HDF5Dataset <: AbstractDataset resolution::Dates.Millisecond initial_timestamp::Dates.DateTime update_timestamp::Dates.DateTime - column_names::Vector{String} + column_names::NTuple{N, Vector{String}} - function HDF5Dataset(values, column_dataset, write_index, last_recorded_row, resolution, + function HDF5Dataset{N}(values, + column_dataset, + write_index, + last_recorded_row, + resolution, initial_timestamp, - update_timestamp, column_names, - ) - new(values, column_dataset, write_index, last_recorded_row, resolution, + update_timestamp, + column_names::NTuple{N, Vector{String}}, + ) where {N} + new{N}(values, column_dataset, write_index, last_recorded_row, resolution, initial_timestamp, update_timestamp, column_names) end end -HDF5Dataset(values, column_dataset, resolution, initial_time) = - HDF5Dataset( +function HDF5Dataset{1}( + values::HDF5.Dataset, + column_dataset::HDF5.Dataset, + ::Tuple, + resolution::Dates.Millisecond, + initial_time::Dates.DateTime, +) + HDF5Dataset{1}( values, column_dataset, 1, @@ -150,10 +230,65 @@ HDF5Dataset(values, column_dataset, resolution, initial_time) = resolution, initial_time, UNSET_INI_TIME, - column_dataset[:], + (column_dataset[:],), ) +end -get_column_names(::OptimizationContainerKey, s::HDF5Dataset) = s.column_names +function HDF5Dataset{2}( + values::HDF5.Dataset, + column_dataset::HDF5.Dataset, + dims::NTuple{4, Int}, + resolution::Dates.Period, + initial_time::Dates.DateTime, +) + # The indexing is done in this way because we save all the names in an + # adjacent column entry in the HDF5 Datatset. The indexes for each column + # are known because we know how many elements are in each dimension. + # the names for the first column are store in the 1:first_column_number_of_elements. + col1 = column_dataset[1:dims[2]] + # the names for the second column are store in the first_column_number_of elements + 1:end of the column with the names. + col2 = column_dataset[(dims[2] + 1):end] + HDF5Dataset{2}( + values, + column_dataset, + 1, + 0, + resolution, + initial_time, + UNSET_INI_TIME, + (col1, col2), + ) +end + +function HDF5Dataset{2}( + values::HDF5.Dataset, + column_dataset::HDF5.Dataset, + dims::NTuple{5, Int}, + resolution::Dates.Period, + initial_time::Dates.DateTime, +) + # The indexing is done in this way because we save all the names in an + # adjacent column entry in the HDF5 Datatset. The indexes for each column + # are known because we know how many elements are in each dimension. + # the names for the first column are store in the 1:first_column_number_of_elements. + col1 = column_dataset[1:dims[2]] + # the names for the second column are store in the first_column_number_of elements + 1:end of the column with the names. + col2 = column_dataset[(dims[2] + 1):end] + HDF5Dataset{2}( + values, + column_dataset, + 1, + 0, + resolution, + initial_time, + UNSET_INI_TIME, + (col1, col2), + ) +end + +function get_column_names(::OptimizationContainerKey, s::HDF5Dataset) + return s.column_names +end """ Return the timestamp from most recent data row updated in the dataset. This value may not be the same as the result from `get_update_timestamp` diff --git a/src/core/definitions.jl b/src/core/definitions.jl index 0097b123a2..b51e41e7db 100644 --- a/src/core/definitions.jl +++ b/src/core/definitions.jl @@ -24,6 +24,7 @@ const JuMPVariableMatrix = DenseAxisArray{ JuMP.Containers._AxisLookup{Tuple{Int64, Int64}}, }, } +const JuMPFloatMatrix = DenseAxisArray{Float64, 2} const JuMPFloatArray = DenseAxisArray{Float64} const JuMPVariableArray = DenseAxisArray{JuMP.VariableRef} diff --git a/src/core/optimizer_stats.jl b/src/core/optimizer_stats.jl index 6e2c98806a..19475381c6 100644 --- a/src/core/optimizer_stats.jl +++ b/src/core/optimizer_stats.jl @@ -100,5 +100,5 @@ function to_dict(stats::OptimizerStats) end function get_column_names(::Type{OptimizerStats}) - return collect(string.(fieldnames(OptimizerStats))) + return (collect(string.(fieldnames(OptimizerStats))),) end diff --git a/src/core/results_by_time.jl b/src/core/results_by_time.jl index 827b32d4c4..f35267e0c3 100644 --- a/src/core/results_by_time.jl +++ b/src/core/results_by_time.jl @@ -1,21 +1,26 @@ -mutable struct ResultsByTime{T} +mutable struct ResultsByTime{T, N} key::OptimizationContainerKey data::SortedDict{Dates.DateTime, T} resolution::Dates.Period - column_names::Vector{String} + column_names::NTuple{N, Vector{String}} end -function ResultsByTime(key, data, resolution, column_names) +function ResultsByTime( + key::OptimizationContainerKey, + data::SortedDict{Dates.DateTime, T}, + resolution::Dates.Period, + column_names, +) where {T} _check_column_consistency(data, column_names) ResultsByTime(key, data, resolution, column_names) end function _check_column_consistency( data::SortedDict{Dates.DateTime, DenseAxisArray{Float64, 2}}, - cols::Vector{String}, + cols::Tuple{Vector{String}}, ) for val in values(data) - if axes(val)[1] != cols + if axes(val)[1] != cols[1] error("Mismatch in DenseAxisArray column names: $(axes(val)[1]) $cols") end end @@ -23,15 +28,26 @@ end function _check_column_consistency( data::SortedDict{Dates.DateTime, Matrix{Float64}}, - cols::Vector{String}, + cols::Tuple{Vector{String}}, ) for val in values(data) - if size(val)[2] != length(cols) - error("Mismatch in length of Matrix columns: $(size(val)[2]) $(length(cols))") + if size(val)[2] != length(cols[1]) + error( + "Mismatch in length of Matrix columns: $(size(val)[2]) $(length(cols[1]))", + ) end end end +function _check_column_consistency( + data::SortedDict{Dates.DateTime, DenseAxisArray{Float64, 2}}, + cols::NTuple{N, Vector{String}}, +) where {N} + # TODO: +end + +# TODO: Implement consistency check for other sizes + # This struct behaves like a dict, delegating to its 'data' field. Base.length(res::ResultsByTime) = length(res.data) Base.iterate(res::ResultsByTime) = iterate(res.data) @@ -73,6 +89,19 @@ function make_dataframe( return df end +function make_dataframe( + results::ResultsByTime{DenseAxisArray{Float64, 3}}, + timestamp::Dates.DateTime, +) + df = DataFrames.DataFrame() + array = results.data[timestamp] + for idx in Iterators.product(array.axes[1:2]...) + df[!, "$(idx)"] = array[idx..., :].data + end + # _add_timestamps!(df, results, timestamp, array) + return df +end + function make_dataframe(results::ResultsByTime{Matrix{Float64}}, timestamp::Dates.DateTime) array = results.data[timestamp] df = DataFrames.DataFrame(array, results.column_names) diff --git a/src/operation/decision_model_store.jl b/src/operation/decision_model_store.jl index bf10abcf42..b8c2591164 100644 --- a/src/operation/decision_model_store.jl +++ b/src/operation/decision_model_store.jl @@ -49,7 +49,7 @@ function initialize_storage!( for timestamp in range(initial_time; step = model_interval, length = num_of_executions) data[timestamp] = fill!( - DenseAxisArray{Float64}(undef, column_names, 1:time_steps_count), + DenseAxisArray{Float64}(undef, column_names..., 1:time_steps_count), NaN, ) end @@ -133,5 +133,5 @@ end function get_column_names(store::DecisionModelStore, key::OptimizationContainerKey) container = getfield(store, get_store_container_type(key)) - return axes(first(values(container[key])))[1] + return get_column_names(key, first(values(container[key]))) end diff --git a/src/operation/emulation_model_store.jl b/src/operation/emulation_model_store.jl index fe468b6483..deca8dbca6 100644 --- a/src/operation/emulation_model_store.jl +++ b/src/operation/emulation_model_store.jl @@ -77,7 +77,7 @@ function initialize_storage!( column_names = get_column_names(key, field_container) results_container[key] = InMemoryDataset( fill!( - DenseAxisArray{Float64}(undef, column_names, 1:num_of_executions), + DenseAxisArray{Float64}(undef, column_names..., 1:num_of_executions), NaN, ), ) @@ -139,7 +139,7 @@ end function get_column_names(store::EmulationModelStore, key::OptimizationContainerKey) container = get_data_field(store, get_store_container_type(key)) - return axes(container[key].values)[1] + return get_column_names(key, container[key].values) end function get_dataset_size(store::EmulationModelStore, key::OptimizationContainerKey) diff --git a/src/operation/operation_model_interface.jl b/src/operation/operation_model_interface.jl index e3511a754c..97c9faa313 100644 --- a/src/operation/operation_model_interface.jl +++ b/src/operation/operation_model_interface.jl @@ -87,10 +87,11 @@ function solve_impl!(model::OperationModel) if status != RunStatus.SUCCESSFUL settings = get_settings(model) model_name = get_name(model) + ts = get_current_timestamp(model) if !get_allow_fails(settings) - error("Solving model $(model_name) failed") + error("Solving model $(model_name) failed at $(ts)") else - @error "Solving model $(model_name) failed. Failure Allowed" + @error "Solving model $(model_name) failed at $(ts). Failure Allowed" end end return diff --git a/src/parameters/update_parameters.jl b/src/parameters/update_parameters.jl index ac021131bb..bc81e0259c 100644 --- a/src/parameters/update_parameters.jl +++ b/src/parameters/update_parameters.jl @@ -25,28 +25,6 @@ function _set_param_value!(param::JuMPFloatArray, value::Float64, name::String, return end -function _set_param_value!( - param::SparseAxisArray{Union{Nothing, JuMP.VariableRef}}, - value::Float64, - name::String, - subcomp::String, - t::Int, -) - fix_parameter_value(param[name, subcomp, t], value) - return -end - -function _set_param_value!( - param::SparseAxisArray{Float64}, - value::Float64, - name::String, - subcomp::String, - t::Int, -) - param[name, subcomp, t] = value - return -end - function _update_parameter_values!( parameter_array::AbstractArray{T}, attributes::TimeSeriesAttributes{U}, @@ -171,16 +149,19 @@ function _update_parameter_values!( current_time = get_current_time(model) state_values = get_dataset_values(state, get_attribute_key(attributes)) component_names, time = axes(parameter_array) - resolution = get_resolution(model) - + model_resolution = get_resolution(model) state_data = get_dataset(state, get_attribute_key(attributes)) state_timestamps = state_data.timestamps max_state_index = get_num_rows(state_data) - + if model_resolution < state_data.resolution + t_step = 1 + else + t_step = model_resolution ÷ state_data.resolution + end state_data_index = find_timestamp_index(state_timestamps, current_time) - sim_timestamps = range(current_time; step = resolution, length = time[end]) + sim_timestamps = range(current_time; step = model_resolution, length = time[end]) for t in time - timestamp_ix = min(max_state_index, state_data_index + 1) + timestamp_ix = min(max_state_index, state_data_index + t_step) @debug "parameter horizon is over the step" max_state_index > state_data_index + 1 if state_timestamps[timestamp_ix] <= sim_timestamps[t] state_data_index = timestamp_ix @@ -211,16 +192,19 @@ function _update_parameter_values!( current_time = get_current_time(model) state_values = get_dataset_values(state, get_attribute_key(attributes)) component_names, time = axes(parameter_array) - resolution = get_resolution(model) - + model_resolution = get_resolution(model) state_data = get_dataset(state, get_attribute_key(attributes)) state_timestamps = state_data.timestamps max_state_index = get_num_rows(state_data) - + if model_resolution < state_data.resolution + t_step = 1 + else + t_step = model_resolution ÷ state_data.resolution + end state_data_index = find_timestamp_index(state_timestamps, current_time) - sim_timestamps = range(current_time; step = resolution, length = time[end]) + sim_timestamps = range(current_time; step = model_resolution, length = time[end]) for t in time - timestamp_ix = min(max_state_index, state_data_index + 1) + timestamp_ix = min(max_state_index, state_data_index + t_step) @debug "parameter horizon is over the step" max_state_index > state_data_index + 1 if state_timestamps[timestamp_ix] <= sim_timestamps[t] state_data_index = timestamp_ix @@ -251,17 +235,20 @@ function _update_parameter_values!( current_time = get_current_time(model) state_values = get_dataset_values(state, get_attribute_key(attributes)) component_names, time = axes(parameter_array) - resolution = get_resolution(model) - + model_resolution = get_resolution(model) state_data = get_dataset(state, get_attribute_key(attributes)) state_timestamps = state_data.timestamps max_state_index = get_num_rows(state_data) - + if model_resolution < state_data.resolution + t_step = 1 + else + t_step = model_resolution ÷ state_data.resolution + end state_data_index = find_timestamp_index(state_timestamps, current_time) - sim_timestamps = range(current_time; step = resolution, length = time[end]) + sim_timestamps = range(current_time; step = model_resolution, length = time[end]) for t in time - timestamp_ix = min(max_state_index, state_data_index + 1) + timestamp_ix = min(max_state_index, state_data_index + t_step) @debug "parameter horizon is over the step" max_state_index > state_data_index + 1 if state_timestamps[timestamp_ix] <= sim_timestamps[t] state_data_index = timestamp_ix @@ -485,7 +472,7 @@ end function _fix_parameter_value!( container::OptimizationContainer, - parameter_array::JuMPFloatArray, + parameter_array::DenseAxisArray{Float64, 2}, parameter_attributes::VariableValueAttributes, ) affected_variable_keys = parameter_attributes.affected_keys diff --git a/src/simulation/decision_model_simulation_results.jl b/src/simulation/decision_model_simulation_results.jl index 38e6bb958c..f770ef3b12 100644 --- a/src/simulation/decision_model_simulation_results.jl +++ b/src/simulation/decision_model_simulation_results.jl @@ -114,12 +114,24 @@ function get_forecast_horizon(res::SimulationProblemResults{DecisionModelSimulat end function _get_store_value( - ::Type{T}, res::SimulationProblemResults{DecisionModelSimulationResults}, container_keys::Vector{<:OptimizationContainerKey}, timestamps, ::Nothing, -) where {T <: Union{Matrix{Float64}, DenseAxisArray{Float64, 2}}} +) + simulation_store_path = joinpath(get_execution_path(res), "data_store") + return open_store(HdfSimulationStore, simulation_store_path, "r") do store + _get_store_value(res, container_keys, timestamps, store) + end +end + +function _get_store_value( + T::Type{Matrix{Float64}}, + res::SimulationProblemResults{DecisionModelSimulationResults}, + container_keys::Vector{<:OptimizationContainerKey}, + timestamps::Vector{Dates.DateTime}, + ::Nothing, +) simulation_store_path = joinpath(get_execution_path(res), "data_store") return open_store(HdfSimulationStore, simulation_store_path, "r") do store _get_store_value(T, res, container_keys, timestamps, store) @@ -127,57 +139,111 @@ function _get_store_value( end function _get_store_value( - ::Type{DenseAxisArray{Float64, 2}}, sim_results::SimulationProblemResults{DecisionModelSimulationResults}, container_keys::Vector{<:OptimizationContainerKey}, - timestamps, + timestamps::Vector{Dates.DateTime}, store::SimulationStore, ) - base_power = get_model_base_power(sim_results) - results_by_key = - Dict{OptimizationContainerKey, ResultsByTime{DenseAxisArray{Float64, 2}}}() + results_by_key = Dict{OptimizationContainerKey, ResultsByTime}() model_name = Symbol(get_model_name(sim_results)) + for ckey in container_keys + n_dims = get_number_of_dimensions(store, DecisionModelIndexType, model_name, ckey) + container_type = DenseAxisArray{Float64, n_dims + 1} + results_by_key[ckey] = _get_store_value(container_type, + sim_results, + ckey, + timestamps, store) + end + return results_by_key +end + +function _get_store_value( + ::Type{T}, + sim_results::SimulationProblemResults{DecisionModelSimulationResults}, + key::OptimizationContainerKey, + timestamps::Vector{Dates.DateTime}, + store::SimulationStore, +) where {T <: DenseAxisArray{Float64, 2}} resolution = get_resolution(sim_results) horizon = get_forecast_horizon(sim_results) + base_power = get_model_base_power(sim_results) + model_name = Symbol(get_model_name(sim_results)) + results_by_time = ResultsByTime( + key, + SortedDict{Dates.DateTime, T}(), + resolution, + get_column_names(store, DecisionModelIndexType, model_name, key), + ) + array_size::Union{Nothing, Tuple{Int, Int}} = nothing + for ts in timestamps + array = read_result(DenseAxisArray, store, model_name, key, ts) + if isnothing(array_size) + array_size = size(array) + elseif size(array) != array_size + error( + "Arrays for $(encode_key_as_string(key)) at different timestamps have different sizes", + ) + end + if convert_result_to_natural_units(key) + array.data .*= base_power + end + if array_size[2] != horizon + @warn "$(encode_key_as_string(key)) has a different horizon than the " * + "problem specification. Can't assign timestamps to the resulting DataFrame." + results_by_time.resolution = Dates.Period(Dates.Millisecond(0)) + end + results_by_time[ts] = array + end - for key in container_keys - results_by_time = ResultsByTime( - key, - SortedDict{Dates.DateTime, DenseAxisArray{Float64, 2}}(), - resolution, - get_column_names(store, DecisionModelIndexType, model_name, key), - ) - array_size::Union{Nothing, Tuple{Int, Int}} = nothing - for ts in timestamps - array = read_result(DenseAxisArray, store, model_name, key, ts) - if isnothing(array_size) - array_size = size(array) - elseif size(array) != array_size - error( - "Arrays for $(encode_key_as_string(key)) at different timestamps have different sizes", - ) - end - if convert_result_to_natural_units(key) - array.data .*= base_power - end - if array_size[2] != horizon - @warn "$(encode_key_as_string(key)) has a different horizon than the " * - "problem specification. Can't assign timestamps to the resulting DataFrame." - results_by_time.resolution = Dates.Period(Dates.Millisecond(0)) - end - results_by_time[ts] = array + return results_by_time +end + +function _get_store_value( + ::Type{T}, + sim_results::SimulationProblemResults{DecisionModelSimulationResults}, + key::OptimizationContainerKey, + timestamps::Vector{Dates.DateTime}, + store::SimulationStore, +) where {T <: DenseAxisArray{Float64, 3}} + resolution = get_resolution(sim_results) + horizon = get_forecast_horizon(sim_results) + base_power = get_model_base_power(sim_results) + model_name = Symbol(get_model_name(sim_results)) + results_by_time = ResultsByTime( + key, + SortedDict{Dates.DateTime, T}(), + resolution, + get_column_names(store, DecisionModelIndexType, model_name, key), + ) + array_size::Union{Nothing, Tuple{Int, Int, Int}} = nothing + for ts in timestamps + array = read_result(DenseAxisArray, store, model_name, key, ts) + if isnothing(array_size) + array_size = size(array) + elseif size(array) != array_size + error( + "Arrays for $(encode_key_as_string(key)) at different timestamps have different sizes", + ) end - results_by_key[key] = results_by_time + if convert_result_to_natural_units(key) + array.data .*= base_power + end + if array_size[3] != horizon + @warn "$(encode_key_as_string(key)) has a different horizon than the " * + "problem specification. Can't assign timestamps to the resulting DataFrame." + results_by_time.resolution = Dates.Period(Dates.Millisecond(0)) + end + results_by_time[ts] = array end - return results_by_key + return results_by_time end function _get_store_value( ::Type{Matrix{Float64}}, sim_results::SimulationProblemResults{DecisionModelSimulationResults}, container_keys::Vector{<:OptimizationContainerKey}, - timestamps, + timestamps::Vector{Dates.DateTime}, store::SimulationStore, ) base_power = get_model_base_power(sim_results) @@ -186,7 +252,13 @@ function _get_store_value( resolution = get_resolution(sim_results) for key in container_keys - results_by_time = ResultsByTime{Matrix{Float64}}( + n_dims = get_number_of_dimensions(store, DecisionModelIndexType, model_name, key) + if n_dims != 1 + error( + "The number of dimensions $(n_dims) is not supported for $(encode_key_as_string(key))", + ) + end + results_by_time = ResultsByTime{Matrix{Float64}, 1}( key, SortedDict{Dates.DateTime, Matrix{Float64}}(), resolution, @@ -233,13 +305,30 @@ function _process_timestamps( end function _read_results( - ::Type{T}, + T::Type{Matrix{Float64}}, + res::SimulationProblemResults{DecisionModelSimulationResults}, + result_keys, + timestamps::Vector{Dates.DateTime}, + store::Nothing, +) + isempty(result_keys) && + return Dict{OptimizationContainerKey, ResultsByTime{Matrix{Float64}}}() + + if res.store !== nothing + # In this case we have an InMemorySimulationStore. + store = res.store + end + return _get_store_value(T, res, result_keys, timestamps, store) +end + +function _read_results( res::SimulationProblemResults{DecisionModelSimulationResults}, result_keys, timestamps::Vector{Dates.DateTime}, store::Union{Nothing, <:SimulationStore}, -) where {T <: Union{Matrix{Float64}, DenseAxisArray{Float64, 2}}} - isempty(result_keys) && return Dict{OptimizationContainerKey, ResultsByTime{T}}() +) + isempty(result_keys) && + return Dict{OptimizationContainerKey, ResultsByTime{DenseAxisArray{Float64, 2}}}() if store === nothing && res.store !== nothing # In this case we have an InMemorySimulationStore. @@ -253,7 +342,7 @@ function _read_results( vals = Dict(k => cached_results[k] for k in result_keys) else @debug "reading results from data store" - vals = _get_store_value(T, res, result_keys, timestamps, store) + vals = _get_store_value(res, result_keys, timestamps, store) end return vals end @@ -285,9 +374,7 @@ function read_variable( ) key = _deserialize_key(VariableKey, res, args...) timestamps = _process_timestamps(res, initial_time, count) - return make_dataframes( - _read_results(DenseAxisArray{Float64, 2}, res, [key], timestamps, store)[key], - ) + return make_dataframes(_read_results(res, [key], timestamps, store)[key]) end """ @@ -311,7 +398,7 @@ function read_dual( key = _deserialize_key(ConstraintKey, res, args...) timestamps = _process_timestamps(res, initial_time, count) return make_dataframes( - _read_results(DenseAxisArray{Float64, 2}, res, [key], timestamps, store)[key], + _read_results(res, [key], timestamps, store)[key], ) end @@ -335,7 +422,7 @@ function read_parameter( key = _deserialize_key(ParameterKey, res, args...) timestamps = _process_timestamps(res, initial_time, count) return make_dataframes( - _read_results(DenseAxisArray{Float64, 2}, res, [key], timestamps, store)[key], + _read_results(res, [key], timestamps, store)[key], ) end @@ -359,7 +446,7 @@ function read_aux_variable( key = _deserialize_key(AuxVarKey, res, args...) timestamps = _process_timestamps(res, initial_time, count) return make_dataframes( - _read_results(DenseAxisArray{Float64, 2}, res, [key], timestamps, store)[key], + _read_results(res, [key], timestamps, store)[key], ) end @@ -383,7 +470,7 @@ function read_expression( key = _deserialize_key(ExpressionKey, res, args...) timestamps = _process_timestamps(res, initial_time, count) return make_dataframes( - _read_results(DenseAxisArray{Float64, 2}, res, [key], timestamps, store)[key], + _read_results(res, [key], timestamps, store)[key], ) end @@ -432,8 +519,8 @@ end function _are_results_cached( res::SimulationProblemResults{DecisionModelSimulationResults}, - output_keys, - timestamps, + output_keys::Vector{<:OptimizationContainerKey}, + timestamps::Vector{Dates.DateTime}, cached_keys, ) return isempty(setdiff(timestamps, res.results_timestamps)) && @@ -483,7 +570,6 @@ function load_results!( merge!( get_cached_variables(res), _read_results( - DenseAxisArray{Float64, 2}, res, variable_keys, res.results_timestamps, @@ -493,7 +579,6 @@ function load_results!( merge!( get_cached_duals(res), _read_results( - DenseAxisArray{Float64, 2}, res, dual_keys, res.results_timestamps, @@ -503,7 +588,6 @@ function load_results!( merge!( get_cached_parameters(res), _read_results( - DenseAxisArray{Float64, 2}, res, parameter_keys, res.results_timestamps, @@ -513,7 +597,6 @@ function load_results!( merge!( get_cached_aux_variables(res), _read_results( - DenseAxisArray{Float64, 2}, res, aux_variable_keys, res.results_timestamps, @@ -523,7 +606,6 @@ function load_results!( merge!( get_cached_expressions(res), _read_results( - DenseAxisArray{Float64, 2}, res, expression_keys, res.results_timestamps, diff --git a/src/simulation/hdf_simulation_store.jl b/src/simulation/hdf_simulation_store.jl index 223407bea0..09007a7b07 100644 --- a/src/simulation/hdf_simulation_store.jl +++ b/src/simulation/hdf_simulation_store.jl @@ -261,12 +261,20 @@ function initialize_problem_storage!( # Columns can't be stored in attributes because they might be larger than # the max size of 64 KiB. col = _make_column_name(name) - HDF5.write_dataset(group, col, string.(reqs["columns"])) + if length(reqs["columns"]) == 1 + HDF5.write_dataset(group, col, string.(reqs["columns"][1])) + else + col_vals = vcat(reqs["columns"]...) + HDF5.write_dataset(group, col, string.(col_vals)) + end column_dataset = group[col] datasets = getfield(get_dm_data(store)[problem], type) - datasets[key] = HDF5Dataset( + # First dim is Horizon, Last number of steps + n_dims = max(1, length(reqs["dims"]) - 2) + datasets[key] = HDF5Dataset{n_dims}( dataset, column_dataset, + reqs["dims"], get_resolution(problem_params), initial_time, ) @@ -305,12 +313,20 @@ function initialize_problem_storage!( # Columns can't be stored in attributes because they might be larger than # the max size of 64 KiB. col = _make_column_name(name) - HDF5.write_dataset(group, col, string.(reqs["columns"])) + if length(reqs["columns"]) == 1 + HDF5.write_dataset(group, col, string.(reqs["columns"][1])) + else + col_vals = vcat(reqs["columns"]...) + HDF5.write_dataset(group, col, string.(col_vals)) + end column_dataset = group[col] datasets = getfield(store.em_data, type) - datasets[key] = HDF5Dataset( + # First dim is Horizon, Last number of steps + n_dims = max(1, length(reqs["dims"]) - 2) + datasets[key] = HDF5Dataset{n_dims}( dataset, column_dataset, + reqs["dims"], get_resolution(emulation_params), initial_time, ) @@ -330,6 +346,10 @@ end log_cache_hit_percentages(x::HdfSimulationStore) = log_cache_hit_percentages(x.cache) +function _make_dataframe(data::Matrix{Float64}, columns::Tuple{Vector{String}}) + return DataFrames.DataFrame(data, columns[1]; copycols = false) +end + """ Return DataFrame, DenseAxisArray, or Array for a model result at a timestamp. """ @@ -341,11 +361,26 @@ function read_result( index::Union{DecisionModelIndexType, EmulationModelIndexType}, ) data, columns = _read_data_columns(store, model_name, key, index) + return _make_dataframe(data, columns) +end - if (ndims(data) < 2 || size(data)[1] == 1) && size(data)[2] != size(columns)[1] - data = reshape(data, length(data), 1) - end - return DataFrames.DataFrame(data, columns; copycols = false) +function _make_denseaxisarray( + data::Matrix{Float64}, + columns::Tuple{Vector{String}}, +) + return DenseAxisArray(permutedims(data), columns[1], 1:size(data)[1]) +end + +function _make_denseaxisarray( + data::Array{Float64, 3}, + columns::NTuple{2, <:Any}, +) + return DenseAxisArray( + permutedims(data, (2, 3, 1)), + columns[1], + columns[2], + 1:size(data)[1], + ) end function read_result( @@ -361,7 +396,7 @@ function read_result( else data, columns = _read_result(store, model_name, key, index) end - return DenseAxisArray(permutedims(data), columns, 1:size(data)[1]) + return _make_denseaxisarray(data, columns) end function read_result( @@ -397,8 +432,7 @@ function read_results( data = dataset.values[index:(index + len - 1), :] end columns = get_column_names(key, dataset) - @assert_op size(data)[2] == length(columns) - return DenseAxisArray(permutedims(data), columns, 1:size(data)[1]) + return DenseAxisArray(permutedims(data), columns..., 1:size(data)[1]) end function get_column_names( @@ -422,6 +456,23 @@ function get_column_names( return get_column_names(key, dataset) end +function get_number_of_dimensions( + store::HdfSimulationStore, + i::Type{DecisionModelIndexType}, + model_name::Symbol, + key::OptimizationContainerKey, +) + return length(get_column_names(store, i, model_name, key)) +end + +function get_number_of_dimensions( + store::HdfSimulationStore, + i::Type{EmulationModelIndexType}, + key::OptimizationContainerKey, +) + return length(get_column_names(store, i, model_name, key)) +end + function get_emulation_model_dataset_size( store::HdfSimulationStore, key::OptimizationContainerKey, @@ -500,8 +551,8 @@ function _read_result( num_dims = ndims(dset) if num_dims == 3 data = dset[:, :, row_index] - #elseif num_dims == 4 - # data = dset[:, :, :, row_index] + elseif num_dims == 4 + data = dset[:, :, :, row_index] else error("unsupported dims: $num_dims") end @@ -544,6 +595,46 @@ function write_result!( return end +""" +Write a decision model result for a timestamp to the store. +""" +function write_result!( + store::HdfSimulationStore, + model_name::Symbol, + key::OptimizationContainerKey, + index::DecisionModelIndexType, + ::Dates.DateTime, + data::DenseAxisArray{Float64, 3, <:NTuple{3, Any}}, +) + output_cache = get_output_cache(store.cache, model_name, key) + cur_size = get_size(store.cache) + + add_result!( + output_cache, + index, + permutedims(data.data, (3, 1, 2)), + is_full(store.cache, cur_size), + ) + + if get_dirty_size(output_cache) >= get_min_flush_size(store.cache) + discard = !should_keep_in_cache(output_cache) + + # PERF: A potentially significant performance improvement would be to queue several + # flushes and submit them in parallel. + size_flushed = _flush_data!(output_cache, store, model_name, key, discard) + + @debug "flushed data" LOG_GROUP_SIMULATION_STORE key size_flushed discard cur_size + end + + # Disabled because this is currently a noop. + #if is_full(store.cache) + # _flush_data!(store.cache, store) + #end + + @debug "write_result" get_size(store.cache) encode_key_as_string(key) + return +end + """ Write an emulation model result for an execution index value and the timestamp of the update """ @@ -553,10 +644,10 @@ function write_result!( key::OptimizationContainerKey, index::EmulationModelIndexType, simulation_time::Dates.DateTime, - data::Matrix{Float64}, + data::Array{Float64}, ) dataset = _get_em_dataset(store, key) - _write_dataset!(dataset.values, data, index:index) + _write_dataset!(dataset.values, data, index) set_last_recorded_row!(dataset, index) set_update_timestamp!(dataset, simulation_time) return @@ -568,7 +659,21 @@ function write_result!( key::OptimizationContainerKey, index::EmulationModelIndexType, simulation_time::Dates.DateTime, - data, + data::DenseAxisArray{Float64, 2}, +) + data_array = Array{Float64, 3}(undef, size(data)[1], size(data)[2], 1) + data_array[:, :, 1] = data + write_result!(store, model_name, key, index, simulation_time, data_array) + return +end + +function write_result!( + store::HdfSimulationStore, + model_name::Symbol, + key::OptimizationContainerKey, + index::EmulationModelIndexType, + simulation_time::Dates.DateTime, + data::DenseAxisArray{Float64, 1}, ) write_result!(store, model_name, key, index, simulation_time, to_matrix(data)) return @@ -634,10 +739,11 @@ function _deserialize_attributes!(store::HdfSimulationStore) empty!(get_dm_data(store)) for model in HDF5.read(HDF5.attributes(group)["problem_order"]) problem_group = store.file["simulation/decision_models/$model"] + horizon = HDF5.read(HDF5.attributes(problem_group)["horizon"]) model_name = Symbol(model) store.params.decision_models_params[model_name] = ModelStoreParams( HDF5.read(HDF5.attributes(problem_group)["num_executions"]), - HDF5.read(HDF5.attributes(problem_group)["horizon"]), + horizon, Dates.Millisecond(HDF5.read(HDF5.attributes(problem_group)["interval_ms"])), Dates.Millisecond(HDF5.read(HDF5.attributes(problem_group)["resolution_ms"])), HDF5.read(HDF5.attributes(problem_group)["base_power"]), @@ -652,7 +758,15 @@ function _deserialize_attributes!(store::HdfSimulationStore) column_dataset = group[_make_column_name(name)] resolution = get_resolution(get_decision_model_params(store, model_name)) - item = HDF5Dataset(dataset, column_dataset, resolution, initial_time) + dims = (horizon, size(dataset)[2:end]..., size(dataset)[1]) + n_dims = max(1, ndims(dataset) - 2) + item = HDF5Dataset{n_dims}( + dataset, + column_dataset, + dims, + resolution, + initial_time, + ) container_key = container_key_lookup[name] getfield(get_dm_data(store)[model_name], type)[container_key] = item add_output_cache!( @@ -670,6 +784,7 @@ function _deserialize_attributes!(store::HdfSimulationStore) end em_group = _get_emulation_model_path(store) + horizon = HDF5.read(HDF5.attributes(em_group)["horizon"]) model_name = Symbol(HDF5.read(HDF5.attributes(em_group)["name"])) resolution = Dates.Millisecond(HDF5.read(HDF5.attributes(em_group)["resolution_ms"])) store.params.emulation_model_params[model_name] = ModelStoreParams( @@ -686,7 +801,15 @@ function _deserialize_attributes!(store::HdfSimulationStore) if !endswith(name, "columns") dataset = group[name] column_dataset = group[_make_column_name(name)] - item = HDF5Dataset(dataset, column_dataset, resolution, initial_time) + dims = (horizon, size(dataset)[2:end]..., size(dataset)[1]) + n_dims = max(1, ndims(dataset) - 1) + item = HDF5Dataset{n_dims}( + dataset, + column_dataset, + dims, + resolution, + initial_time, + ) container_key = container_key_lookup[name] getfield(store.em_data, type)[container_key] = item add_output_cache!(store.cache, model_name, container_key, CacheFlushRule()) @@ -856,11 +979,16 @@ function _read_data_columns( if is_cached(store.cache, model_name, key, index) data = read_result(store.cache, model_name, key, index) column_dataset = _get_dm_dataset(store, model_name, key).column_dataset - columns = column_dataset[:] + if ndims(column_dataset) == 1 + columns = (column_dataset[:],) + elseif ndims(column_dataset) == 2 + columns = (column_dataset[:, 1], column_dataset[:, 2]) + else + error("Datasets with $(ndims(column_dataset)) columns not supported") + end else data, columns = _read_result(store, model_name, key, index) end - return data, columns end @@ -887,7 +1015,7 @@ function _read_length(::Type{OptimizerStats}, store::HdfSimulationStore) end function _write_dataset!( - dataset, + dataset::HDF5.Dataset, array::Matrix{Float64}, row_range::UnitRange{Int64}, ::Val{3}, @@ -898,7 +1026,7 @@ function _write_dataset!( end function _write_dataset!( - dataset, + dataset::HDF5.Dataset, array::Matrix{Float64}, row_range::UnitRange{Int64}, ::Val{2}, @@ -908,13 +1036,38 @@ function _write_dataset!( return end -function _write_dataset!(dataset, array::Matrix{Float64}, row_range::UnitRange{Int64}) - _write_dataset!(dataset, array, row_range, Val{ndims(dataset)}()) +function _write_dataset!( + dataset::HDF5.Dataset, + array::Array{Float64, 3}, + row_range::UnitRange{Int64}, + ::Val{3}, +) + dataset[row_range, :, :] = array + @debug "wrote dataset" dataset row_range return end -function _write_dataset!(dataset, array::Array{Float64, 3}, row_range::UnitRange{Int64}) +function _write_dataset!(dataset::HDF5.Dataset, array::Array{Float64}, index::Int) + _write_dataset!(dataset, array, index:index, Val{ndims(dataset)}()) + return +end + +function _write_dataset!( + dataset::HDF5.Dataset, + array::Array{Float64, 3}, + row_range::UnitRange{Int64}, +) dataset[:, :, row_range] = array @debug "wrote dataset" dataset row_range return end + +function _write_dataset!( + dataset::HDF5.Dataset, + array::Array{Float64, 4}, + row_range::UnitRange{Int64}, +) + dataset[:, :, :, row_range] = array + @debug "wrote dataset" dataset row_range + return +end diff --git a/src/simulation/in_memory_simulation_store.jl b/src/simulation/in_memory_simulation_store.jl index cff3ef5b60..37beb6f846 100644 --- a/src/simulation/in_memory_simulation_store.jl +++ b/src/simulation/in_memory_simulation_store.jl @@ -17,6 +17,23 @@ function InMemorySimulationStore() ) end +function get_number_of_dimensions( + store::InMemorySimulationStore, + i::Type{EmulationModelIndexType}, + key::OptimizationContainerKey, +) + return length(get_column_names(store, i, model_name, key)) +end + +function get_number_of_dimensions( + store::InMemorySimulationStore, + i::Type{DecisionModelIndexType}, + model_name::Symbol, + key::OptimizationContainerKey, +) + return length(get_column_names(store, i, model_name, key)) +end + function open_store( func::Function, ::Type{InMemorySimulationStore}, @@ -143,7 +160,7 @@ function initialize_problem_storage!( container = get_data_field(get_em_data(store), type) container[key] = InMemoryDataset( fill!( - DenseAxisArray{Float64}(undef, reqs["columns"], 1:reqs["dims"][1]), + DenseAxisArray{Float64}(undef, reqs["columns"]..., 1:reqs["dims"][1]), NaN, ), ) diff --git a/src/simulation/optimization_output_cache.jl b/src/simulation/optimization_output_cache.jl index da76708ff3..276c529273 100644 --- a/src/simulation/optimization_output_cache.jl +++ b/src/simulation/optimization_output_cache.jl @@ -52,7 +52,10 @@ end """ Add result to the cache. """ -function add_result!(cache::OptimizationOutputCache, timestamp, array, system_cache_is_full) +function add_result!(cache::OptimizationOutputCache, + timestamp::Dates.DateTime, + array::Array{Float64}, + system_cache_is_full::Bool) if cache.size_per_entry == 0 cache.size_per_entry = length(array) * sizeof(first(array)) end @@ -77,7 +80,11 @@ function add_result!(cache::OptimizationOutputCache, timestamp, array, system_ca return cache.size_per_entry end -function _add_result!(cache::OptimizationOutputCache, timestamp, data) +function _add_result!( + cache::OptimizationOutputCache, + timestamp::Dates.DateTime, + data::Array{Float64}, +) cache.data[timestamp] = data push!(cache.dirty_timestamps, timestamp) return diff --git a/src/simulation/realized_meta.jl b/src/simulation/realized_meta.jl index 3b8f9c34ea..379a6e6c52 100644 --- a/src/simulation/realized_meta.jl +++ b/src/simulation/realized_meta.jl @@ -46,6 +46,39 @@ function RealizedMeta( ) end +function _make_dataframe( + columns::Tuple{Vector{String}}, + results_by_time::ResultsByTime{Matrix{Float64}, 1}, + num_rows::Int, + meta::RealizedMeta, + key::OptimizationContainerKey, +) + num_cols = length(columns[1]) + matrix = Matrix{Float64}(undef, num_rows, num_cols) + row_index = 1 + for (step, (_, array)) in enumerate(results_by_time) + first_id = step > 1 ? 1 : meta.start_offset + last_id = + step == meta.len ? meta.interval_len - meta.end_offset : meta.interval_len + if last_id - first_id > size(array, 1) + error( + "Variable $(encode_key_as_string(key)) has $(size(array, 1)) number of steps, that is different than the default problem horizon. \ + Can't calculate the realized variables. Use `read_variables` instead and write your own concatenation", + ) + end + row_end = row_index + last_id - first_id + matrix[row_index:row_end, :] = array[first_id:last_id, :] + row_index += last_id - first_id + 1 + end + df = DataFrames.DataFrame(matrix, collect(columns[1]); copycols = false) + DataFrames.insertcols!( + df, + 1, + :DateTime => meta.realized_timestamps, + ) + return df +end + function get_realization( results::Dict{OptimizationContainerKey, ResultsByTime{Matrix{Float64}}}, meta::RealizedMeta, @@ -57,29 +90,7 @@ function get_realization( Threads.@threads for key in collect(keys(results)) results_by_time = results[key] columns = get_column_names(results_by_time) - num_cols = length(columns) - matrix = Matrix{Float64}(undef, num_rows, num_cols) - row_index = 1 - for (step, (_, array)) in enumerate(results_by_time) - first_id = step > 1 ? 1 : meta.start_offset - last_id = - step == meta.len ? meta.interval_len - meta.end_offset : meta.interval_len - if last_id - first_id > size(array, 1) - error( - "Variable $(encode_key_as_string(key)) has $(size(array, 1)) number of steps, that is different than the default problem horizon. \ - Can't calculate the realized variables. Use `read_variables` instead and write your own concatenation", - ) - end - row_end = row_index + last_id - first_id - matrix[row_index:row_end, :] = array[first_id:last_id, :] - row_index += last_id - first_id + 1 - end - df = DataFrames.DataFrame(matrix, collect(columns); copycols = false) - DataFrames.insertcols!( - df, - 1, - :DateTime => meta.realized_timestamps, - ) + df = _make_dataframe(columns, results_by_time, num_rows, meta, key) lock(lk) do realized_values[key] = df end diff --git a/src/simulation/simulation.jl b/src/simulation/simulation.jl index 0b2985a9ec..aa0a77754f 100644 --- a/src/simulation/simulation.jl +++ b/src/simulation/simulation.jl @@ -384,35 +384,36 @@ function _get_emulation_store_requirements(sim::Simulation) !should_write_resulting_value(key) && continue dims = sim_time ÷ get_data_resolution(state_values) cols = get_column_names(key, state_values) - reqs.duals[key] = Dict("columns" => cols, "dims" => (dims, length(cols))) + reqs.duals[key] = Dict("columns" => cols, "dims" => (dims, length.(cols)...)) end for (key, state_values) in get_parameters_values(system_state) !should_write_resulting_value(key) && continue dims = sim_time ÷ get_data_resolution(state_values) cols = get_column_names(key, state_values) - reqs.parameters[key] = Dict("columns" => cols, "dims" => (dims, length(cols))) + reqs.parameters[key] = Dict("columns" => cols, "dims" => (dims, length.(cols)...)) end for (key, state_values) in get_variables_values(system_state) !should_write_resulting_value(key) && continue dims = sim_time ÷ get_data_resolution(state_values) cols = get_column_names(key, state_values) - reqs.variables[key] = Dict("columns" => cols, "dims" => (dims, length(cols))) + reqs.variables[key] = Dict("columns" => cols, "dims" => (dims, length.(cols)...)) end for (key, state_values) in get_aux_variables_values(system_state) !should_write_resulting_value(key) && continue dims = sim_time ÷ get_data_resolution(state_values) cols = get_column_names(key, state_values) - reqs.aux_variables[key] = Dict("columns" => cols, "dims" => (dims, length(cols))) + reqs.aux_variables[key] = + Dict("columns" => cols, "dims" => (dims, length.(cols)...)) end for (key, state_values) in get_expression_values(system_state) !should_write_resulting_value(key) && continue dims = sim_time ÷ get_data_resolution(state_values) cols = get_column_names(key, state_values) - reqs.expressions[key] = Dict("columns" => cols, "dims" => (dims, length(cols))) + reqs.expressions[key] = Dict("columns" => cols, "dims" => (dims, length.(cols)...)) end return reqs end diff --git a/src/simulation/simulation_state.jl b/src/simulation/simulation_state.jl index daaa02fcfa..b2f4a79d4c 100644 --- a/src/simulation/simulation_state.jl +++ b/src/simulation/simulation_state.jl @@ -90,20 +90,12 @@ function _initialize_model_states!( column_names = get_column_names(key, value) if !haskey(field_states, key) || get_num_rows(field_states[key]) < value_counts field_states[key] = InMemoryDataset( - fill!( - DenseAxisArray{Float64}(undef, column_names, 1:value_counts), - NaN, - ), - collect( - range( - simulation_initial_time; - step = params[key].resolution, - length = value_counts, - ), - ), + NaN, + simulation_initial_time, params[key].resolution, Int(simulation_step / params[key].resolution), - ) + value_counts, + column_names) end end end @@ -125,9 +117,9 @@ function _initialize_system_states!( emulator_states, key, make_system_state( - fill!(DenseAxisArray{Float64}(undef, cols, 1:1), NaN), simulation_initial_time, min_res, + cols, ), ) end @@ -143,6 +135,7 @@ function _initialize_system_states!( decision_states = get_decision_states(sim_state) emulator_states = get_system_states(sim_state) emulation_container = get_optimization_container(emulation_model) + min_res = minimum([v.resolution for v in values(params)]) for field in fieldnames(DatasetContainer) field_containers = getfield(emulation_container, field) @@ -153,29 +146,29 @@ function _initialize_system_states!( emulator_states, key, make_system_state( - fill!(DenseAxisArray{Float64}(undef, column_names, 1:1), NaN), simulation_initial_time, - get_resolution(emulation_model), + min_res, + column_names, ), ) end end for key in get_dataset_keys(decision_states) + dm_cols = get_column_names(key, get_dataset(decision_states, key)) if has_dataset(emulator_states, key) - dm_cols = get_column_names(key, get_dataset(decision_states, key)) em_cols = get_column_names(key, get_dataset(emulator_states, key)) @assert_op dm_cols == em_cols continue end - cols = get_column_names(key, get_dataset(decision_states, key)) + set_dataset!( emulator_states, key, make_system_state( - fill!(DenseAxisArray{Float64}(undef, cols, 1:1), NaN), simulation_initial_time, - get_resolution(emulation_model), + min_res, + dm_cols, ), ) end @@ -207,12 +200,12 @@ end function update_decision_state!( state::SimulationState, key::OptimizationContainerKey, - store_data::DenseAxisArray{Float64}, + store_data::DenseAxisArray{Float64, 2}, simulation_time::Dates.DateTime, model_params::ModelStoreParams, ) state_data = get_decision_state_data(state, key) - column_names = get_column_names(state_data) + column_names = get_column_names(key, state_data)[1] model_resolution = get_resolution(model_params) state_resolution = get_data_resolution(state_data) resolution_ratio = model_resolution ÷ state_resolution @@ -249,7 +242,7 @@ end function update_decision_state!( state::SimulationState, key::AuxVarKey{EnergyOutput, T}, - store_data::DenseAxisArray{Float64}, + store_data::DenseAxisArray{Float64, 2}, simulation_time::Dates.DateTime, model_params::ModelStoreParams, ) where {T <: PSY.Component} @@ -291,7 +284,7 @@ end function update_decision_state!( state::SimulationState, key::AuxVarKey{S, T}, - store_data::DenseAxisArray{Float64}, + store_data::DenseAxisArray{Float64, 2}, simulation_time::Dates.DateTime, model_params::ModelStoreParams, ) where {T <: PSY.Component, S <: Union{TimeDurationOff, TimeDurationOn}} @@ -417,7 +410,8 @@ function update_system_state!( set_update_timestamp!(system_dataset, ts) # Keep coordination between fields. System state is an array of size 1 system_dataset.timestamps[1] = ts - set_dataset_values!(state, key, 1, get_dataset_value(decision_dataset, simulation_time)) + data_set_value = get_dataset_value(decision_dataset, simulation_time) + set_dataset_values!(state, key, 1, data_set_value) # This value shouldn't be other than one and after one execution is no-op. set_last_recorded_row!(system_dataset, 1) return diff --git a/src/utils/dataframes_utils.jl b/src/utils/dataframes_utils.jl index 4f78172b67..eb9607a909 100644 --- a/src/utils/dataframes_utils.jl +++ b/src/utils/dataframes_utils.jl @@ -7,12 +7,22 @@ Creates a DataFrame from a JuMP DenseAxisArray or SparseAxisArray. - `array`: JuMP DenseAxisArray or SparseAxisArray to convert - `key::OptimizationContainerKey`: """ -function to_dataframe(array::DenseAxisArray, key::OptimizationContainerKey) - return DataFrames.DataFrame(to_matrix(array), get_column_names(key, array)) +function to_dataframe( + array::DenseAxisArray{T, 2}, + key::OptimizationContainerKey, +) where {T <: Number} + return DataFrames.DataFrame(to_matrix(array), get_column_names(key, array)[1]) +end + +function to_dataframe( + array::DenseAxisArray{T, 1}, + key::OptimizationContainerKey, +) where {T <: Number} + return DataFrames.DataFrame(to_matrix(array), get_column_names(key, array)[1]) end function to_dataframe(array::SparseAxisArray, key::OptimizationContainerKey) - return DataFrames.DataFrame(to_matrix(array), get_column_names(key, array)) + return DataFrames.DataFrame(to_matrix(array), get_column_names(key, array)[1]) end function to_matrix(df::DataFrames.DataFrame) diff --git a/src/utils/jump_utils.jl b/src/utils/jump_utils.jl index 5f1b2aa2ce..c3360c5da5 100644 --- a/src/utils/jump_utils.jl +++ b/src/utils/jump_utils.jl @@ -55,7 +55,7 @@ function to_matrix(::DenseAxisArray{T, N, K}) where {T, N, K <: NTuple{N, Any}} end function get_column_names(key::OptimizationContainerKey) - return [encode_key_as_string(key)] + return ([encode_key_as_string(key)],) end function get_column_names( @@ -66,10 +66,17 @@ function get_column_names( end function get_column_names( - ::OptimizationContainerKey, + k::OptimizationContainerKey, array::DenseAxisArray{T, 2, K}, ) where {T, K <: NTuple{2, Any}} - return string.(axes(array)[1]) + return (string.(axes(array)[1]),) +end + +function get_column_names( + k::OptimizationContainerKey, + array::DenseAxisArray{T, 3, K}, +) where {T, K <: NTuple{3, Any}} + return (string.(axes(array)[1]), string.(axes(array)[2])) end function _get_column_names(arr::SparseAxisArray{T, N, K}) where {T, N, K <: NTuple{N, Any}} @@ -80,7 +87,7 @@ function get_column_names( ::OptimizationContainerKey, array::SparseAxisArray{T, N, K}, ) where {T, N, K <: NTuple{N, Any}} - return get_column_names(array) + return (get_column_names(array),) end function get_column_names(array::SparseAxisArray{T, N, K}) where {T, N, K <: NTuple{N, Any}} @@ -198,12 +205,12 @@ function _calc_dimensions( if length(ax[2]) != horizon @debug "$(encode_key_as_string(key)) has length $(length(ax[1])). Different than horizon $horizon." end - dims = (length(ax[2]), length(columns), num_rows) + dims = (length(ax[2]), length(columns[1]), num_rows) elseif length(ax) == 3 if length(ax[3]) != horizon @debug "$(encode_key_as_string(key)) has length $(length(ax[1])). Different than horizon $horizon." end - dims = (length(ax[2]), length(ax[3]), length(columns), num_rows) + dims = (length(ax[3]), length(columns[1]), length(columns[2]), num_rows) else error("unsupported data size $(length(ax))") end @@ -218,7 +225,7 @@ function _calc_dimensions( horizon::Int, ) columns = get_column_names(key, array) - dims = (horizon, length(columns), num_rows) + dims = (horizon, length.(columns)..., num_rows) return Dict("columns" => columns, "dims" => dims) end diff --git a/src/utils/printing.jl b/src/utils/printing.jl index 47abf8cce6..8cdb139e29 100644 --- a/src/utils/printing.jl +++ b/src/utils/printing.jl @@ -84,7 +84,7 @@ function _show_method( PrettyTables.pretty_table( io, table; - noheader = true, + show_header = false, backend = Val(backend), title = "Duals", alignment = :l, @@ -176,7 +176,7 @@ function _show_method(io::IO, template::ProblemTemplate, backend::Symbol; kwargs io, table; backend = Val(backend), - noheader = true, + show_header = false, title = "Network Model", alignment = :l, kwargs..., @@ -338,7 +338,7 @@ function _show_method(io::IO, sequence::SimulationSequence, backend::Symbol; kwa io, table; backend = Val(backend), - noheader = true, + show_header = false, title = "Simulation Sequence", alignment = :l, kwargs..., @@ -432,7 +432,7 @@ function _show_method(io::IO, sim::Simulation, backend::Symbol; kwargs...) io, table; backend = Val(backend), - noheader = true, + show_header = false, title = "Simulation", alignment = :l, kwargs..., @@ -479,7 +479,7 @@ function _show_method(io::IO, results::SimulationResults, backend::Symbol; kwarg PrettyTables.pretty_table( io, table; - noheader = true, + show_header = false, backend = Val(backend), title = "Emulator Results", alignment = :l, @@ -534,7 +534,7 @@ function _show_method( PrettyTables.pretty_table( io, val; - noheader = true, + show_header = false, backend = Val(backend), title = "$name Problem $k Results", alignment = :l, diff --git a/test/test_simulation_store.jl b/test/test_simulation_store.jl index 5a29a38d35..bb7a27f4c1 100644 --- a/test/test_simulation_store.jl +++ b/test/test_simulation_store.jl @@ -39,7 +39,7 @@ function _initialize!(store, sim, variables, model_defs, cache_rules) for (key, array) in model_defs[model]["variables"] reqs.variables[key] = Dict( "columns" => model_defs[model]["names"], - "dims" => (horizon, length(model_defs[model]["names"]), num_rows), + "dims" => (horizon, length(model_defs[model]["names"][1]), num_rows), ) keep_in_cache = variables[key]["keep_in_cache"] add_rule!(cache_rules, model, key, keep_in_cache) @@ -146,8 +146,8 @@ function _verify_read_results(path, sim, variables, model_defs, seed) end end -function _verify_data(expected, store, model, name, time, columns) - expected_df = DataFrames.DataFrame(expected, columns) +function _verify_data(expected, store, model, name, time, columns::Tuple{Vector{Symbol}}) + expected_df = DataFrames.DataFrame(expected, columns[1]) df = read_result(DataFrames.DataFrame, store, model, name, time) @test expected_df == df end @@ -172,7 +172,7 @@ end :ED => Dict( "execution_count" => 24, "horizon" => 12, - "names" => [:dev1, :dev2, :dev3, :dev4, :dev5], + "names" => ([:dev1, :dev2, :dev3, :dev4, :dev5],), "variables" => Dict(x => ones(12, 5) for x in keys(variables)), "interval" => Dates.Hour(1), "resolution" => Dates.Hour(1), @@ -182,7 +182,7 @@ end :UC => Dict( "execution_count" => 1, "horizon" => 24, - "names" => [:dev1, :dev2, :dev3], + "names" => ([:dev1, :dev2, :dev3],), "variables" => Dict(x => ones(24, 3) for x in keys(variables)), "interval" => Dates.Hour(1), "resolution" => Dates.Hour(24), diff --git a/test/test_utils.jl b/test/test_utils.jl index 5d49df15cf..7e39118246 100644 --- a/test/test_utils.jl +++ b/test/test_utils.jl @@ -23,10 +23,12 @@ end @test two_df == test_df three = PSI.DenseAxisArray{Float64}(undef, [:a], 1:2, 1:3) - @test_throws ErrorException PSI.to_dataframe(three, mock_key) + fill!(three, 1.0) + @test_throws MethodError PSI.to_dataframe(three, mock_key) four = PSI.DenseAxisArray{Float64}(undef, [:a], 1:2, 1:3, 1:5) - @test_throws ErrorException PSI.to_dataframe(four, mock_key) + fill!(three, 1.0) + @test_throws MethodError PSI.to_dataframe(four, mock_key) sparse_num = JuMP.Containers.@container([i = 1:10, j = (i + 1):10, t = 1:24], 0.0 + i + j + t)