Skip to content

Commit

Permalink
Fix MEMORY LEAKAGE
Browse files Browse the repository at this point in the history
@orso82  @daveweisberg

My conclusion on how the memory leakage is happening relates to the garbage collector being unable to do it's job in a pmap while also being in a try-catch structure... see src/optimization for the fix
  • Loading branch information
TimSlendebroek committed Oct 11, 2023
1 parent 6a54ab1 commit 5d8df73
Showing 1 changed file with 30 additions and 3 deletions.
33 changes: 30 additions & 3 deletions src/optimization.jl
Original file line number Diff line number Diff line change
Expand Up @@ -227,7 +227,13 @@ function optimization_engine(
save(savedir, save_dd ? dd : nothing, ini, act; timer=true, memtrace=true, freeze=true, overwrite_files=false)
end
# evaluate multiple objectives
return collect(map(f -> nan2inf(f(dd)), objectives_functions)), collect(map(g -> nan2inf(g(dd)), constraints_functions)), Float64[]
result = collect(map(f -> nan2inf(f(dd)), objectives_functions)), collect(map(g -> nan2inf(g(dd)), constraints_functions)), Float64[]
# need to force garbage collection on dd (memory usage grows otherwise, this is a bug)
dd = nothing
actor = nothing
GC.gc()
return result

catch e
# save empty dd and error to directory
if !isempty(save_folder)
Expand All @@ -239,10 +245,31 @@ function optimization_engine(
end
end
# rethrow(e) # uncomment for debugging purposes
return Float64[Inf for f in objectives_functions], Float64[Inf for g in constraints_functions], Float64[]
result = Float64[Inf for f in objectives_functions], Float64[Inf for g in constraints_functions], Float64[]
# need to force garbage collection on dd (memory usage grows otherwise, this is a bug)
dd = nothing
actor = nothing
GC.gc()
return result
end
end

function _optimization_engine(
ini::ParametersAllInits,
act::ParametersAllActors,
actor_or_workflow::Union{Type{<:AbstractActor},Function},
x::AbstractVector,
objectives_functions::AbstractVector{<:ObjectiveFunction},
constraints_functions::AbstractVector{<:ConstraintFunction},
save_folder::AbstractString,
generation::Int,
save_dd::Bool=true)

return optimization_engine(ini, act, actor_or_workflow, x, objectives_functions,
constraints_functions, save_folder, generation, save_dd)
end


"""
optimization_engine(
ini::ParametersAllInits,
Expand Down Expand Up @@ -271,7 +298,7 @@ function optimization_engine(
# parallel evaluation of a generation
ProgressMeter.next!(p)
tmp = Distributed.pmap(
x -> optimization_engine(ini, act, actor_or_workflow, x, objectives_functions, constraints_functions, save_folder, p.counter, save_dd),
x -> _optimization_engine(ini, act, actor_or_workflow, x, objectives_functions, constraints_functions, save_folder, p.counter, save_dd),
[X[k, :] for k in 1:size(X)[1]]
)
F = zeros(size(X)[1], length(tmp[1][1]))
Expand Down

2 comments on commit 5d8df73

@orso82
Copy link
Member

@orso82 orso82 commented on 5d8df73 Oct 11, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fantastic job @TimSlendebroek !!!

I see that you are doing both GC.gc() calls, as well as wrapping the optimization_engine function? can you please confirm that the latter is indeed necessary?

@bclyons12
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should review this with @sjkelly

Please sign in to comment.