Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Restructure output format #2179

Merged
merged 1 commit into from
Aug 23, 2023
Merged

Restructure output format #2179

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions lib/livebook/live_markdown/export.ex
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ defmodule Livebook.LiveMarkdown.Export do
defp collect_js_output_data(notebook) do
for section <- notebook.sections,
%{outputs: outputs} <- section.cells,
{_idx, {:js, %{js_view: %{ref: ref, pid: pid}, export: %{}}}} <- outputs do
{_idx, %{type: :js, js_view: %{ref: ref, pid: pid}, export: %{}}} <- outputs do
Task.async(fn ->
{ref, get_js_output_data(pid, ref)}
end)
Expand Down Expand Up @@ -212,7 +212,7 @@ defmodule Livebook.LiveMarkdown.Export do
|> Enum.intersperse("\n\n")
end

defp render_output({:terminal_text, text, %{}}, _ctx) do
defp render_output(%{type: :terminal_text, text: text}, _ctx) do
text = String.replace_suffix(text, "\n", "")
delimiter = MarkdownHelpers.code_block_delimiter(text)
text = strip_ansi(text)
Expand All @@ -222,7 +222,7 @@ defmodule Livebook.LiveMarkdown.Export do
end

defp render_output(
{:js, %{export: %{info_string: info_string, key: key}, js_view: %{ref: ref}}},
%{type: :js, export: %{info_string: info_string, key: key}, js_view: %{ref: ref}},
ctx
)
when is_binary(info_string) do
Expand All @@ -241,7 +241,7 @@ defmodule Livebook.LiveMarkdown.Export do
end
end

defp render_output({:tabs, outputs, _info}, ctx) do
defp render_output(%{type: :tabs, outputs: outputs}, ctx) do
Enum.find_value(outputs, :ignored, fn {_idx, output} ->
case render_output(output, ctx) do
:ignored -> nil
Expand All @@ -250,7 +250,7 @@ defmodule Livebook.LiveMarkdown.Export do
end)
end

defp render_output({:grid, outputs, _info}, ctx) do
defp render_output(%{type: :grid, outputs: outputs}, ctx) do
outputs
|> Enum.map(fn {_idx, output} -> render_output(output, ctx) end)
|> Enum.reject(&(&1 == :ignored))
Expand Down
4 changes: 2 additions & 2 deletions lib/livebook/live_markdown/import.ex
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,7 @@ defmodule Livebook.LiveMarkdown.Import do
[{"pre", _, [{"code", [{"class", "output"}], [output], %{}}], %{}} | ast],
outputs
) do
take_outputs(ast, [{:terminal_text, output, %{chunk: false}} | outputs])
take_outputs(ast, [%{type: :terminal_text, text: output, chunk: false} | outputs])
end

defp take_outputs(
Expand All @@ -206,7 +206,7 @@ defmodule Livebook.LiveMarkdown.Import do
],
outputs
) do
take_outputs(ast, [{:terminal_text, output, %{chunk: false}} | outputs])
take_outputs(ast, [%{type: :terminal_text, text: output, chunk: false} | outputs])
end

# Ignore other exported outputs
Expand Down
121 changes: 50 additions & 71 deletions lib/livebook/notebook.ex
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -641,11 +641,11 @@ defmodule Livebook.Notebook do

defp find_assets_info_in_outputs(outputs, hash) do
Enum.find_value(outputs, fn
{_idx, {:js, %{js_view: %{assets: %{hash: ^hash} = assets_info}}}} ->
{_idx, %{type: :js, js_view: %{assets: %{hash: ^hash} = assets_info}}} ->
assets_info

{_idx, {type, outputs, _}} when type in [:frame, :tabs, :grid] ->
find_assets_info_in_outputs(outputs, hash)
{_idx, output} when output.type in [:frame, :tabs, :grid] ->
find_assets_info_in_outputs(output.outputs, hash)

_ ->
nil
Expand All @@ -669,28 +669,15 @@ defmodule Livebook.Notebook do
Automatically merges terminal outputs and updates frames.
"""
@spec add_cell_output(t(), Cell.id(), Livebook.Runtime.output()) :: t()
def add_cell_output(notebook, cell_id, output)

# Map legacy outputs
def add_cell_output(notebook, cell_id, {:text, text}),
do: add_cell_output(notebook, cell_id, {:terminal_text, text, %{chunk: false}})

def add_cell_output(notebook, cell_id, {:plain_text, text}),
do: add_cell_output(notebook, cell_id, {:plain_text, text, %{chunk: false}})

def add_cell_output(notebook, cell_id, {:markdown, text}),
do: add_cell_output(notebook, cell_id, {:markdown, text, %{chunk: false}})

def add_cell_output(notebook, cell_id, output) do
{notebook, counter} = do_add_cell_output(notebook, cell_id, notebook.output_counter, output)
%{notebook | output_counter: counter}
end

defp do_add_cell_output(notebook, _cell_id, counter, {:frame, _outputs, %{type: type}} = frame)
when type != :default do
defp do_add_cell_output(notebook, _cell_id, counter, %{type: :frame_update} = frame_update) do
update_reduce_cells(notebook, counter, fn
%{outputs: _} = cell, counter ->
{outputs, counter} = update_frames(cell.outputs, counter, frame)
{outputs, counter} = update_frames(cell.outputs, counter, frame_update)
{%{cell | outputs: outputs}, counter}

cell, counter ->
Expand All @@ -709,17 +696,17 @@ defmodule Livebook.Notebook do
{notebook, counter}
end

defp update_frames(outputs, counter, {:frame, new_outputs, %{ref: ref, type: type}} = frame) do
defp update_frames(outputs, counter, %{ref: ref} = frame_update) do
Enum.map_reduce(outputs, counter, fn
{idx, {:frame, outputs, %{ref: ^ref} = info}}, counter ->
{idx, %{type: :frame, outputs: outputs, ref: ^ref} = frame}, counter ->
{update_type, new_outputs} = frame_update.update
{new_outputs, counter} = index_outputs(new_outputs, counter)
output = {idx, {:frame, apply_frame_update(outputs, new_outputs, type), info}}
{output, counter}
outputs = apply_frame_update(outputs, new_outputs, update_type)
{{idx, %{frame | outputs: outputs}}, counter}

{idx, {type, outputs, info}}, counter when type in [:frame, :tabs, :grid] ->
{outputs, counter} = update_frames(outputs, counter, frame)
output = {idx, {type, outputs, info}}
{output, counter}
{idx, output}, counter when output.type in [:frame, :tabs, :grid] ->
{outputs, counter} = update_frames(output.outputs, counter, frame_update)
{{idx, %{output | outputs: outputs}}, counter}

output, counter ->
{output, counter}
Expand All @@ -734,43 +721,40 @@ defmodule Livebook.Notebook do
Enum.reduce(Enum.reverse(new_outputs), outputs, &add_output(&2, &1))
end

defp add_output(outputs, {_idx, :ignored}), do: outputs
defp add_output(outputs, {_idx, %{type: :ignored}}), do: outputs

# Session clients prune rendered chunks, we only keep add the new one
# Session clients prune rendered chunks, we only add the new one
defp add_output(
[{idx, {type, :__pruned__, %{chunk: true} = info}} | tail],
{_idx, {type, text, %{chunk: true}}}
[{idx, %{type: type, chunk: true, text: :__pruned__} = output} | tail],
{_idx, %{type: type, chunk: true, text: text}}
)
when type in [:terminal_text, :plain_text, :markdown] do
[{idx, {type, text, info}} | tail]
[{idx, %{output | text: text}} | tail]
end

# Session server keeps all outputs, so we merge consecutive chunks
defp add_output(
[{idx, {:terminal_text, text, %{chunk: true} = info}} | tail],
{_idx, {:terminal_text, cont, %{chunk: true}}}
[{idx, %{type: :terminal_text, chunk: true, text: text} = output} | tail],
{_idx, %{type: :terminal_text, chunk: true, text: cont}}
) do
[{idx, {:terminal_text, normalize_terminal_text(text <> cont), info}} | tail]
[{idx, %{output | text: normalize_terminal_text(text <> cont)}} | tail]
end

defp add_output(outputs, {idx, {:terminal_text, text, info}}) do
[{idx, {:terminal_text, normalize_terminal_text(text), info}} | outputs]
defp add_output(outputs, {idx, %{type: :terminal_text, text: text} = output}) do
[{idx, %{output | text: normalize_terminal_text(text)}} | outputs]
end

defp add_output(
[{idx, {type, text, %{chunk: true} = info}} | tail],
{_idx, {type, cont, %{chunk: true}}}
[{idx, %{type: type, chunk: true, text: text} = output} | tail],
{_idx, %{type: type, chunk: true, text: cont}}
)
when type in [:plain_text, :markdown] do
[{idx, {type, normalize_terminal_text(text <> cont), info}} | tail]
[{idx, %{output | text: text <> cont}} | tail]
end

defp add_output(outputs, {idx, {type, text, info}}) when type in [:plain_text, :markdown] do
[{idx, {type, normalize_terminal_text(text), info}} | outputs]
end

defp add_output(outputs, {idx, {type, container_outputs, info}}) when type in [:frame, :grid] do
[{idx, {type, merge_chunk_outputs(container_outputs), info}} | outputs]
defp add_output(outputs, {idx, output}) when output.type in [:frame, :grid] do
output = update_in(output.outputs, &merge_chunk_outputs/1)
[{idx, output} | outputs]
end

defp add_output(outputs, output), do: [output | outputs]
Expand Down Expand Up @@ -810,9 +794,9 @@ defmodule Livebook.Notebook do
Enum.map_reduce(outputs, counter, &index_output/2)
end

defp index_output({type, outputs, info}, counter) when type in [:frame, :tabs, :grid] do
{outputs, counter} = index_outputs(outputs, counter)
{{counter, {type, outputs, info}}, counter + 1}
defp index_output(output, counter) when output.type in [:frame, :tabs, :grid] do
{outputs, counter} = index_outputs(output.outputs, counter)
{{counter, %{output | outputs: outputs}}, counter + 1}
end

defp index_output(output, counter) do
Expand All @@ -831,18 +815,15 @@ defmodule Livebook.Notebook do
do: frame_output
end

defp do_find_frame_outputs({_idx, {:frame, _outputs, %{ref: ref}}} = output, ref) do
defp do_find_frame_outputs({_idx, %{type: :frame, ref: ref}} = output, ref) do
[output]
end

defp do_find_frame_outputs({_idx, {type, outputs, _info}}, ref)
when type in [:frame, :tabs, :grid] do
Enum.flat_map(outputs, &do_find_frame_outputs(&1, ref))
defp do_find_frame_outputs({_idx, output}, ref) when output.type in [:frame, :tabs, :grid] do
Enum.flat_map(output.outputs, &do_find_frame_outputs(&1, ref))
end

defp do_find_frame_outputs(_output, _ref) do
[]
end
defp do_find_frame_outputs(_output, _ref), do: []

@doc """
Removes outputs that get rendered only once.
Expand All @@ -864,45 +845,43 @@ defmodule Livebook.Notebook do
defp do_prune_outputs([], _appendable?, acc), do: acc

# Keep trailing outputs that can be merged with subsequent outputs
defp do_prune_outputs([{idx, {type, _, %{chunk: true} = info}}], true = _appendable?, acc)
defp do_prune_outputs([{idx, %{type: type, chunk: true} = output}], true = _appendable?, acc)
when type in [:terminal_text, :plain_text, :markdown] do
[{idx, {type, :__pruned__, info}} | acc]
[{idx, %{output | text: :__pruned__}} | acc]
end

# Keep frame and its relevant contents
defp do_prune_outputs([{idx, {:frame, frame_outputs, info}} | outputs], appendable?, acc) do
do_prune_outputs(
outputs,
appendable?,
[{idx, {:frame, prune_outputs(frame_outputs, true), info}} | acc]
)
defp do_prune_outputs([{idx, %{type: :frame} = output} | outputs], appendable?, acc) do
output = update_in(output.outputs, &prune_outputs(&1, true))
do_prune_outputs(outputs, appendable?, [{idx, output} | acc])
end

# Keep layout output and its relevant contents
defp do_prune_outputs([{idx, {:tabs, tabs_outputs, info}} | outputs], appendable?, acc) do
case prune_outputs(tabs_outputs, false) do
defp do_prune_outputs([{idx, %{type: :tabs} = output} | outputs], appendable?, acc) do
case prune_outputs(output.outputs, false) do
[] ->
do_prune_outputs(outputs, appendable?, acc)

pruned_tabs_outputs ->
info = Map.replace(info, :labels, :__pruned__)
do_prune_outputs(outputs, appendable?, [{idx, {:tabs, pruned_tabs_outputs, info}} | acc])
output = %{output | outputs: pruned_tabs_outputs, labels: :__pruned__}
do_prune_outputs(outputs, appendable?, [{idx, output} | acc])
end
end

defp do_prune_outputs([{idx, {:grid, grid_outputs, info}} | outputs], appendable?, acc) do
case prune_outputs(grid_outputs, false) do
defp do_prune_outputs([{idx, %{type: :grid} = output} | outputs], appendable?, acc) do
case prune_outputs(output.outputs, false) do
[] ->
do_prune_outputs(outputs, appendable?, acc)

pruned_grid_outputs ->
do_prune_outputs(outputs, appendable?, [{idx, {:grid, pruned_grid_outputs, info}} | acc])
output = %{output | outputs: pruned_grid_outputs}
do_prune_outputs(outputs, appendable?, [{idx, output} | acc])
end
end

# Keep outputs that get re-rendered
defp do_prune_outputs([{idx, output} | outputs], appendable?, acc)
when elem(output, 0) in [:input, :control, :error] do
when output.type in [:input, :control, :error] do
do_prune_outputs(outputs, appendable?, [{idx, output} | acc])
end

Expand Down
20 changes: 10 additions & 10 deletions lib/livebook/notebook/cell.ex
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,19 +51,19 @@ defmodule Livebook.Notebook.Cell do
@doc """
Extracts all inputs from the given indexed output.
"""
@spec find_inputs_in_output(indexed_output()) :: list(input_attrs :: map())
@spec find_inputs_in_output(indexed_output()) :: list(Livebook.Runtime.input_output())
def find_inputs_in_output(output)

def find_inputs_in_output({_idx, {:input, attrs}}) do
[attrs]
def find_inputs_in_output({_idx, %{type: :input} = input}) do
[input]
end

def find_inputs_in_output({_idx, {:control, %{type: :form, fields: fields}}}) do
def find_inputs_in_output({_idx, %{type: :control, attrs: %{type: :form, fields: fields}}}) do
Keyword.values(fields)
end

def find_inputs_in_output({_idx, {type, outputs, _}}) when type in [:frame, :tabs, :grid] do
Enum.flat_map(outputs, &find_inputs_in_output/1)
def find_inputs_in_output({_idx, output}) when output.type in [:frame, :tabs, :grid] do
Enum.flat_map(output.outputs, &find_inputs_in_output/1)
end

def find_inputs_in_output(_output), do: []
Expand All @@ -74,13 +74,13 @@ defmodule Livebook.Notebook.Cell do
@spec find_assets_in_output(Livebook.Runtime.output()) :: list(asset_info :: map())
def find_assets_in_output(output)

def find_assets_in_output({:js, %{js_view: %{assets: assets_info}}}), do: [assets_info]
def find_assets_in_output(%{type: :js} = output), do: [output.js_view.assets]

def find_assets_in_output({type, outputs, _}) when type in [:frame, :tabs, :grid] do
Enum.flat_map(outputs, &find_assets_in_output/1)
def find_assets_in_output(output) when output.type in [:frame, :tabs, :grid] do
Enum.flat_map(output.outputs, &find_assets_in_output/1)
end

def find_assets_in_output(_), do: []
def find_assets_in_output(_output), do: []

@setup_cell_id "setup"

Expand Down
Loading