Skip to content

Commit

Permalink
Improve performance of algorithm to determine project deps
Browse files Browse the repository at this point in the history 
Introduces smarter tracking of deps as they are loaded. These changes help to
avoid traversing the same dep sub-trees multiple times. For larger projects,
including umbrella projects, the same can be traversed millions of times.
  • Loading branch information
@dhedlund
dhedlund committed Sep 16, 2023
1 parent 0173ea5 commit c407d7c
Showing 1 changed file with 141 additions and 74 deletions.
215 changes: 141 additions & 74 deletions lib/dialyxir/project.ex
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@ defmodule Dialyxir.Project do
alias Dialyxir.Formatter.Short
alias Dialyxir.Formatter.Utils

# Maximum depth in the dependency tree to traverse before giving up.
@max_dep_traversal_depth 100

def plts_list(deps, include_project \\ true, exclude_core \\ false) do
elixir_apps = [:elixir]
erlang_apps = [:erts, :kernel, :stdlib, :crypto]
Expand Down Expand Up @@ -308,124 +311,136 @@ defmodule Dialyxir.Project do
defp include_deps do
method = dialyzer_config()[:plt_add_deps]

reduce_umbrella_children([], fn deps ->
deps ++
initial_acc = {
_loaded_apps = [],
_unloaded_apps = [],
_initial_load_statuses = %{}
}

{loaded_apps, _unloaded_apps, _final_load_statuses} =
reduce_umbrella_children(initial_acc, fn acc ->
case method do
false ->
[]
acc

# compatibility
true ->
warning(
"Dialyxir has deprecated plt_add_deps: true in favor of apps_direct, which includes only runtime dependencies."
)

deps_project() ++ deps_app(false)
acc
|> load_project_deps()
|> load_external_deps(recursive: false)

:project ->
warning(
"Dialyxir has deprecated plt_add_deps: :project in favor of apps_direct, which includes only runtime dependencies."
)

deps_project() ++ deps_app(false)
acc
|> load_project_deps()
|> load_external_deps(recursive: false)

:apps_direct ->
deps_app(false)
load_external_deps(acc, recursive: false)

:transitive ->
warning(
"Dialyxir has deprecated plt_add_deps: :transitive in favor of app_tree, which includes only runtime dependencies."
)

deps_transitive() ++ deps_app(true)
acc
|> load_transitive_deps()
|> load_external_deps(recursive: true)

_app_tree ->
deps_app(true)
load_external_deps(acc, recursive: true)
end
end)
end)

loaded_apps
end

defp deps_project do
Mix.Project.config()[:deps]
|> Enum.filter(&env_dep(&1))
|> Enum.map(&elem(&1, 0))
defp load_project_deps({loaded_apps, unloaded_apps, load_statuses}) do
apps =
Mix.Project.config()[:deps]
|> Enum.filter(&env_dep(&1))
|> Enum.map(&elem(&1, 0))

app_load_statuses = Map.new(apps, &{elem(&1, 0), :loaded})

update_load_statuses({loaded_apps, unloaded_apps -- apps, load_statuses}, app_load_statuses)
end

defp deps_transitive do
Mix.Project.deps_paths()
|> Map.keys()
defp load_transitive_deps({loaded_apps, unloaded_apps, load_statuses}) do
apps = Mix.Project.deps_paths() |> Map.values()
app_load_statuses = Map.new(apps, &{elem(&1, 0), :loaded})

update_load_statuses({loaded_apps, unloaded_apps -- apps, load_statuses}, app_load_statuses)
end

@spec deps_app(boolean()) :: [atom]
defp deps_app(recursive) do
defp load_external_deps({loaded_apps, _unloaded_apps, load_statuses}, opts) do
# Non-recursive traversal of 2 tries to load the app immediate deps.
traversal_depth =
case Keyword.fetch!(opts, :recursive) do
true -> @max_dep_traversal_depth
false -> 2
end

app = Mix.Project.config()[:app]
deps_app(app, recursive)
end

if System.version() |> Version.parse!() |> then(&(&1.major >= 1 and &1.minor >= 15)) do
@spec deps_app(atom(), boolean()) :: [atom()]
defp deps_app(app, recursive) do
case do_load_app(app) do
:ok ->
with_each =
if recursive do
&deps_app(&1, true)
else
fn _ -> [] end
end
# Even if already loaded, we'll need to traverse it again to get its deps.
load_statuses_w_app = Map.put(load_statuses, app, {:unloaded, :required})
traverse_deps_for_apps({loaded_apps -- [app], [app], load_statuses_w_app}, traversal_depth)
end

# Identify the optional applications which can't be loaded and thus not available
missing_apps =
Application.spec(app, :optional_applications)
|> List.wrap()
|> Enum.reject(&(do_load_app(&1) == :ok))
defp traverse_deps_for_apps({loaded_apps, [] = unloaded_deps, load_statuses}, _rem_depth),
do: {loaded_apps, unloaded_deps, load_statuses}

# Remove the optional applications which are not available from all the applications
required_apps =
Application.spec(app, :applications)
|> List.wrap()
|> Enum.reject(&(&1 in missing_apps))
defp traverse_deps_for_apps({loaded_apps, unloaded_deps, load_statuses}, 0 = _rem_depth),
do: {loaded_apps, unloaded_deps, load_statuses}

required_apps |> Stream.flat_map(&with_each.(&1)) |> Enum.concat(required_apps)
defp traverse_deps_for_apps({loaded_apps, apps_to_load, load_statuses}, rem_depth) do
initial_acc = {loaded_apps, [], load_statuses}

{:error, err} ->
error("Error loading #{app}, dependency list may be incomplete.\n #{inspect(err)}")
{updated_loaded_apps, updated_unloaded_apps, updated_load_statuses} =
Enum.reduce(apps_to_load, initial_acc, fn app, acc ->
required? = Map.fetch!(load_statuses, app) == {:unloaded, :required}
{app_load_status, app_dep_statuses} = load_app(app, required?)

[]
end
end
else
@spec deps_app(atom(), boolean()) :: [atom()]
defp deps_app(app, recursive) do
with_each =
if recursive do
&deps_app(&1, true)
else
fn _ -> [] end
end
acc
|> update_load_statuses(%{app => app_load_status})
|> update_load_statuses(app_dep_statuses)
end)

case do_load_app(app) do
:ok ->
nil
traverse_deps_for_apps(
{updated_loaded_apps, updated_unloaded_apps, updated_load_statuses},
rem_depth - 1
)
end

{:error, err} ->
error("Error loading #{app}, dependency list may be incomplete.\n #{inspect(err)}")
defp load_app(app, required?) do
case do_load_app(app) do
:ok ->
{dependencies, optional_deps} = app_dep_specs(app)

nil
end
dep_statuses =
Map.new(dependencies, fn dep ->
case dep in optional_deps do
true -> {dep, {:unloaded, :optional}}
false -> {dep, {:unloaded, :required}}
end
end)

case Application.spec(app, :applications) do
[] ->
[]
{:loaded, dep_statuses}

nil ->
[]
{:error, err} ->
if required? do
error("Error loading #{app}, dependency list may be incomplete.\n #{inspect(err)}")
end

this_apps ->
Enum.map(this_apps, with_each)
|> List.flatten()
|> Enum.concat(this_apps)
end
{{:error, err}, %{}}
end
end

Expand All @@ -443,6 +458,58 @@ defmodule Dialyxir.Project do
end
end

if System.version() |> Version.parse!() |> then(&(&1.major >= 1 and &1.minor >= 15)) do
defp app_dep_specs(app) do
# Values returned by :optional_applications are also in :applications.
dependencies = Application.spec(app, :applications) || []
optional_deps = Application.spec(app, :optional_applications) || []

{dependencies, optional_deps}
end
else
defp app_dep_specs(app) do
{Application.spec(app, :applications) || [], []}
end
end

defp update_load_statuses({loaded_apps, unloaded_apps, load_statuses}, new_statuses) do
initial_acc = {loaded_apps, unloaded_apps, load_statuses}

Enum.reduce(new_statuses, initial_acc, fn {app, new_status}, acc ->
{current_loaded_apps, current_unloaded_apps, statuses} = acc
existing_status = Map.get(statuses, app, :unset)

{new_loaded_apps, new_unloaded_apps, updated_load_statuses} =
case {existing_status, new_status} do
{:unset, {:unloaded, _} = new_status} ->
# Haven't seen this app before.
{[], [app], Map.put(statuses, app, new_status)}

{{:unloaded, :optional}, {:unloaded, :required}} ->
# A previous app had this as optional, but another one requires it.
{[], [], Map.put(statuses, app, {:unloaded, :required})}

{{:unloaded, _}, :loaded} ->
# Final state. Dependency successfully loaded.
{[app], [], Map.put(statuses, app, :loaded)}

{{:unloaded, _}, {:error, err}} ->
# Final state. Dependency failed to load.
{[], [], Map.put(statuses, app, {:error, err})}

{_prev_unloaded_or_final, _nwe_unloaded_or_final} ->
# No status change, or one that doesn't matter like final to final.
{[], [], statuses}
end

{
new_loaded_apps ++ current_loaded_apps,
new_unloaded_apps ++ current_unloaded_apps,
updated_load_statuses
}
end)
end

defp env_dep(dep) do
only_envs = dep_only(dep)
only_envs == nil || Mix.env() in List.wrap(only_envs)
Expand All @@ -452,7 +519,7 @@ defmodule Dialyxir.Project do
defp dep_only({_, _, opts}) when is_list(opts), do: opts[:only]
defp dep_only(_), do: nil

@spec reduce_umbrella_children(list(), (list() -> list())) :: list()
@spec reduce_umbrella_children(acc, (acc -> acc)) :: acc when acc: term
defp reduce_umbrella_children(acc, f) do
if Mix.Project.umbrella?() do
children = Mix.Dep.Umbrella.loaded()
Expand Down

0 comments on commit c407d7c

Please sign in to comment.