From 260e969ce3d444c864a76e1b6db27a0392c321dd Mon Sep 17 00:00:00 2001 From: Connor Burns Date: Wed, 30 Mar 2022 16:32:17 -0600 Subject: [PATCH 01/11] Notebook metadata logic --- src/notebook/Cell.jl | 6 +++--- src/notebook/Notebook.jl | 32 +++++++++++++++++++++++++++++++- src/webserver/Dynamic.jl | 7 +++++++ 3 files changed, 41 insertions(+), 4 deletions(-) diff --git a/src/notebook/Cell.jl b/src/notebook/Cell.jl index 12de893ace..3542737ba9 100644 --- a/src/notebook/Cell.jl +++ b/src/notebook/Cell.jl @@ -1,7 +1,7 @@ import UUIDs: UUID, uuid1 import .ExpressionExplorer: SymbolsState, UsingsImports -const DEFAULT_METADATA = Dict{String, Any}( +const DEFAULT_CELL_METADATA = Dict{String, Any}( "disabled" => false ) @@ -50,7 +50,7 @@ Base.@kwdef mutable struct Cell depends_on_disabled_cells::Bool=false - metadata::Dict{String,Any}=copy(DEFAULT_METADATA) + metadata::Dict{String,Any}=copy(DEFAULT_CELL_METADATA) end Cell(cell_id, code) = Cell(cell_id=cell_id, code=code) @@ -71,4 +71,4 @@ function Base.convert(::Type{UUID}, string::String) end get_cell_metadata(cell::Cell)::Dict{String,Any} = cell.metadata -get_cell_metadata_no_default(cell::Cell)::Dict{String,Any} = Dict{String,Any}(setdiff(pairs(cell.metadata), pairs(DEFAULT_METADATA))) +get_cell_metadata_no_default(cell::Cell)::Dict{String,Any} = Dict{String,Any}(setdiff(pairs(cell.metadata), pairs(DEFAULT_CELL_METADATA))) diff --git a/src/notebook/Notebook.jl b/src/notebook/Notebook.jl index b6296a7e3e..79cb25dae5 100644 --- a/src/notebook/Notebook.jl +++ b/src/notebook/Notebook.jl @@ -5,6 +5,9 @@ import .PkgCompat: PkgCompat, PkgContext import Pkg import TOML + +const DEFAULT_NOTEBOOK_METADATA = Dict{String, Any}() + mutable struct BondValue value::Any end @@ -55,6 +58,8 @@ Base.@kwdef mutable struct Notebook last_hot_reload_time::typeof(time())=zero(time()) bonds::Dict{Symbol,BondValue}=Dict{Symbol,BondValue}() + + metadata::Dict{String, Any}=copy(DEFAULT_NOTEBOOK_METADATA) end _collect_cells(cells_dict::Dict{UUID,Cell}, cells_order::Vector{UUID}) = @@ -90,6 +95,7 @@ function Base.getproperty(notebook::Notebook, property::Symbol) end const _notebook_header = "### A Pluto.jl notebook ###" +const _notebook_metadata_prefix = "#> " # We use a creative delimiter to avoid accidental use in code # so don't get inspired to suddenly use these in your code! const _cell_id_delimiter = "# ╔═╡ " @@ -116,6 +122,17 @@ Have a look at our [JuliaCon 2020 presentation](https://youtu.be/IAF8DjrQSSk?t=1 function save_notebook(io, notebook::Notebook) println(io, _notebook_header) println(io, "# ", PLUTO_VERSION_STR) + + # Notebook metadata + if length(keys(notebook.metadata)) > 0 + nb_metadata_toml = strip(sprint(TOML.print, notebook.metadata)) + if nb_metadata_toml != "" + for line in split(nb_metadata_toml, "\n") + println(io, _notebook_metadata_prefix, line) + end + end + end + # Anything between the version string and the first UUID delimiter will be ignored by the notebook loader. println(io, "") println(io, "using Markdown") @@ -219,6 +236,18 @@ function load_notebook_nobackup(io, path)::Notebook # @info "Loading a notebook saved with Pluto $(file_VERSION_STR). This is Pluto $(PLUTO_VERSION_STR)." end + nb_metadata_toml_lines = String[] + nb_prefix_length = length(_notebook_metadata_prefix) + while !eof(io) + line = String(readline(io)) + if startswith(line, _notebook_metadata_prefix) + push!(nb_metadata_toml_lines, line[begin+nb_prefix_length:end]) + else + break + end + end + notebook_metadata = Dict{String, Any}(DEFAULT_NOTEBOOK_METADATA..., TOML.parse(join(nb_metadata_toml_lines, "\n"))...) + collected_cells = Dict{UUID,Cell}() # ignore first bits of file @@ -255,7 +284,7 @@ function load_notebook_nobackup(io, path)::Notebook code = code_normalised[1:prevind(code_normalised, end, length(_cell_suffix))] # parse metadata - metadata = Dict{String, Any}(DEFAULT_METADATA..., TOML.parse(join(metadata_toml_lines, "\n"))...) + metadata = Dict{String, Any}(DEFAULT_CELL_METADATA..., TOML.parse(join(metadata_toml_lines, "\n"))...) read_cell = Cell(; cell_id, code, metadata) collected_cells[cell_id] = read_cell @@ -333,6 +362,7 @@ function load_notebook_nobackup(io, path)::Notebook path=path, nbpkg_ctx=nbpkg_ctx, nbpkg_installed_versions_cache=nbpkg_cache(nbpkg_ctx), + metadata=notebook_metadata, ) end diff --git a/src/webserver/Dynamic.jl b/src/webserver/Dynamic.jl index 9eceee0ee1..145a9df724 100644 --- a/src/webserver/Dynamic.jl +++ b/src/webserver/Dynamic.jl @@ -154,6 +154,7 @@ function notebook_to_js(notebook::Notebook) "value" => bondvalue.value, ) for (key, bondvalue) in notebook.bonds), + "metadata" => notebook.metadata, "nbpkg" => let ctx = notebook.nbpkg_ctx Dict{String,Any}( @@ -275,6 +276,12 @@ const effects_of_changed_state = Dict( Firebasey.applypatch!(request.notebook, patch) [BondChanged(name, patch isa Firebasey.AddPatch)] end, + ), + "metadata" => Dict( + Wildcard() => function(property; request::ClientRequest, patch::Firebasey.JSONPatch) + Firebasey.applypatch!(request.notebook, patch) + [FileChanged()] + end ) ) From bc4113107e05cdcd6f386e941da3568fed919ae8 Mon Sep 17 00:00:00 2001 From: Connor Burns Date: Tue, 5 Apr 2022 13:39:27 -0600 Subject: [PATCH 02/11] Tests for notebook metadata --- test/Notebook.jl | 50 ++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 4 deletions(-) diff --git a/test/Notebook.jl b/test/Notebook.jl index a65fb41ae2..b6f756a247 100644 --- a/test/Notebook.jl +++ b/test/Notebook.jl @@ -1,5 +1,5 @@ using Test -import Pluto: Notebook, ServerSession, ClientSession, Cell, load_notebook, load_notebook_nobackup, save_notebook, WorkspaceManager, cutename, numbered_until_new, readwrite, without_pluto_file_extension +import Pluto: Notebook, ServerSession, ClientSession, Cell, load_notebook, load_notebook_nobackup, save_notebook, WorkspaceManager, cutename, numbered_until_new, readwrite, without_pluto_file_extension, update_run! import Random import Pkg import UUIDs: UUID @@ -29,7 +29,7 @@ function basic_notebook() ]) |> init_packages! end -function metadata_notebook() +function cell_metadata_notebook() Notebook([ Cell( code="100*a + b", @@ -45,6 +45,23 @@ function metadata_notebook() ]) |> init_packages! end +function notebook_metadata_notebook() + nb = Notebook([ + Cell(code="n * (n + 1) / 2"), + ]) |> init_packages! + nb.metadata = Dict( + "boolean" => true, + "string" => "String", + "number" => 10000, + "ozymandias" => Dict( + "l1" => "And on the pedestal, these words appear:", + "l2" => "My name is Ozymandias, King of Kings;", + "l3" => "Look on my Works, ye Mighty, and despair!", + ), + ) + nb +end + function shuffled_notebook() Notebook([ Cell("z = y"), @@ -157,13 +174,13 @@ end end end - @testset "Metadata" begin + @testset "Cell Metadata" begin 🍭 = ServerSession() 🍭.options.evaluation.workspace_use_distributed = false fakeclient = ClientSession(:fake, nothing) 🍭.connected_clients[fakeclient.id] = fakeclient - nb = metadata_notebook() + nb = cell_metadata_notebook() update_run!(🍭, nb, nb.cells) cell = first(values(nb.cells_dict)) @test cell.metadata == Dict( @@ -190,6 +207,31 @@ end ) end + @testset "Notebook Metadata" begin + 🍭 = ServerSession() + 🍭.options.evaluation.workspace_use_distributed = false + fakeclient = ClientSession(:fake, nothing) + 🍭.connected_clients[fakeclient.id] = fakeclient + + nb = notebook_metadata_notebook() + update_run!(🍭, nb, nb.cells) + + @test nb.metadata == Dict( + "boolean" => true, + "string" => "String", + "number" => 10000, + "ozymandias" => Dict( + "l1" => "And on the pedestal, these words appear:", + "l2" => "My name is Ozymandias, King of Kings;", + "l3" => "Look on my Works, ye Mighty, and despair!", + ), + ) + + save_notebook(nb) + nb_loaded = load_notebook_nobackup(nb.path) + @test nb.metadata == nb_loaded.metadata + end + @testset "I/O overloaded" begin @testset "$(name)" for (name, nb) in nbs let From 71026cce6eee8310d87e90e62bd74c7ce4489883 Mon Sep 17 00:00:00 2001 From: Connor Burns Date: Sun, 17 Apr 2022 20:13:46 -0600 Subject: [PATCH 03/11] Window access to metadata updates --- frontend/components/Editor.js | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/frontend/components/Editor.js b/frontend/components/Editor.js index a13b3b1aef..3f3e610115 100644 --- a/frontend/components/Editor.js +++ b/frontend/components/Editor.js @@ -197,6 +197,7 @@ const first_true_key = (obj) => { * published_objects: { [objectid: string]: any}, * bonds: { [name: string]: any }, * nbpkg: NotebookPkgData?, + * metadata: object, * }} */ @@ -911,6 +912,22 @@ patch: ${JSON.stringify( false ) } + //@ts-ignore + window.getNotebookMetadata = () => { + return this.state.notebook.metadata + } + //@ts-ignore + window.putNotebookMetadata = (key, value) => { + this.actions.update_notebook((notebook) => { + notebook.metadata[key] = value + }) + } + //@ts-ignore + window.deleteNotebookMetadata = (key) => { + this.actions.update_notebook((notebook) => { + delete notebook.metadata[key] + }) + } this.submit_file_change = async (new_path, reset_cm_value) => { const old_path = this.state.notebook.path if (old_path === new_path) { From d338ad428fb41a5d34783e69b46e859af86b002c Mon Sep 17 00:00:00 2001 From: Connor Burns Date: Wed, 27 Apr 2022 21:50:30 -0600 Subject: [PATCH 04/11] Fixes from merge and keywise metadata retrieval --- frontend/components/Editor.js | 4 ++-- src/notebook/Cell.jl | 2 +- src/notebook/Notebook.jl | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/frontend/components/Editor.js b/frontend/components/Editor.js index 786a389109..0e9dc12507 100644 --- a/frontend/components/Editor.js +++ b/frontend/components/Editor.js @@ -965,8 +965,8 @@ patch: ${JSON.stringify( ) } //@ts-ignore - window.getNotebookMetadata = () => { - return this.state.notebook.metadata + window.getNotebookMetadata = (key) => { + return this.state.notebook.metadata[key] } //@ts-ignore window.putNotebookMetadata = (key, value) => { diff --git a/src/notebook/Cell.jl b/src/notebook/Cell.jl index 7daf56fe3c..bf76176355 100644 --- a/src/notebook/Cell.jl +++ b/src/notebook/Cell.jl @@ -52,7 +52,7 @@ Base.@kwdef mutable struct Cell depends_on_disabled_cells::Bool=false - metadata::Dict{String,Any}=copy(DEFAULT_CELL_METADATA) + metadata::Dict{String,Any}=copy(DEFAULT_METADATA) end Cell(cell_id, code) = Cell(cell_id=cell_id, code=code) diff --git a/src/notebook/Notebook.jl b/src/notebook/Notebook.jl index 79cb25dae5..2abfc58d93 100644 --- a/src/notebook/Notebook.jl +++ b/src/notebook/Notebook.jl @@ -284,7 +284,7 @@ function load_notebook_nobackup(io, path)::Notebook code = code_normalised[1:prevind(code_normalised, end, length(_cell_suffix))] # parse metadata - metadata = Dict{String, Any}(DEFAULT_CELL_METADATA..., TOML.parse(join(metadata_toml_lines, "\n"))...) + metadata = Dict{String, Any}(DEFAULT_METADATA..., TOML.parse(join(metadata_toml_lines, "\n"))...) read_cell = Cell(; cell_id, code, metadata) collected_cells[cell_id] = read_cell From 89d9afbc882dbf45fe5430f4ea9810b40b0a14c9 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Tue, 17 May 2022 17:44:04 +0200 Subject: [PATCH 05/11] Update Notebook.jl --- test/Notebook.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Notebook.jl b/test/Notebook.jl index 5d117e3739..de5998de31 100644 --- a/test/Notebook.jl +++ b/test/Notebook.jl @@ -182,7 +182,7 @@ end 🍭.connected_clients[fakeclient.id] = fakeclient @testset "Disabling & Metadata" begin - nb = metadata_notebook() + nb = notebook_metadata_notebook() update_run!(🍭, nb, nb.cells) cell = first(values(nb.cells_dict)) @test get_cell_metadata_no_default(cell) == Dict( From 3b2b1b3bbdc69d0c40f51ce88ea8bed0fa1460cf Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Tue, 17 May 2022 18:21:30 +0200 Subject: [PATCH 06/11] fixiefix --- test/Notebook.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Notebook.jl b/test/Notebook.jl index de5998de31..c59b007f64 100644 --- a/test/Notebook.jl +++ b/test/Notebook.jl @@ -182,7 +182,7 @@ end 🍭.connected_clients[fakeclient.id] = fakeclient @testset "Disabling & Metadata" begin - nb = notebook_metadata_notebook() + nb = cell_metadata_notebook() update_run!(🍭, nb, nb.cells) cell = first(values(nb.cells_dict)) @test get_cell_metadata_no_default(cell) == Dict( From e82bafee26e736db0f69f4f392555a464ecd8099 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Tue, 17 May 2022 19:11:37 +0200 Subject: [PATCH 07/11] tweaks --- src/notebook/Notebook.jl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/notebook/Notebook.jl b/src/notebook/Notebook.jl index 47868d87bb..0a0438b6ff 100644 --- a/src/notebook/Notebook.jl +++ b/src/notebook/Notebook.jl @@ -126,12 +126,10 @@ function save_notebook(io, notebook::Notebook) println(io, "# ", PLUTO_VERSION_STR) # Notebook metadata - if length(keys(notebook.metadata)) > 0 + if !isempty(notebook.metadata) nb_metadata_toml = strip(sprint(TOML.print, notebook.metadata)) - if nb_metadata_toml != "" - for line in split(nb_metadata_toml, "\n") - println(io, _notebook_metadata_prefix, line) - end + for line in split(nb_metadata_toml, "\n") + println(io, _notebook_metadata_prefix, line) end end @@ -239,7 +237,7 @@ function load_notebook_nobackup(@nospecialize(io::IO), @nospecialize(path::Abstr end nb_metadata_toml_lines = String[] - nb_prefix_length = length(_notebook_metadata_prefix) + nb_prefix_length = ncodeunits(_notebook_metadata_prefix) while !eof(io) line = String(readline(io)) if startswith(line, _notebook_metadata_prefix) @@ -248,7 +246,7 @@ function load_notebook_nobackup(@nospecialize(io::IO), @nospecialize(path::Abstr break end end - notebook_metadata = Dict{String, Any}(DEFAULT_NOTEBOOK_METADATA..., TOML.parse(join(nb_metadata_toml_lines, "\n"))...) + notebook_metadata = fastmerge(DEFAULT_NOTEBOOK_METADATA, TOML.parse(join(nb_metadata_toml_lines, "\n"))) collected_cells = Dict{UUID,Cell}() @@ -452,3 +450,5 @@ function sample_notebook(name::String) nb.path = tempname() * ".jl" nb end + +fastmerge(a, b) = isempty(a) ? b : merge(a, b) From ce840fba12686bf78a641de699aa5b691ddd9b8a Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Tue, 17 May 2022 19:12:00 +0200 Subject: [PATCH 08/11] move API to cell scope --- frontend/components/CellOutput.js | 14 ++++++++++++-- frontend/components/Editor.js | 16 ---------------- 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/frontend/components/CellOutput.js b/frontend/components/CellOutput.js index 24f85851f6..d9c9c05c21 100644 --- a/frontend/components/CellOutput.js +++ b/frontend/components/CellOutput.js @@ -271,7 +271,7 @@ const is_displayable = (result) => result instanceof Element && result.nodeType * @typedef PlutoScript * @type {HTMLScriptElement | { pluto_is_loading_me?: boolean }} */ -const execute_scripttags = async ({ root_node, script_nodes, previous_results_map, invalidation }) => { +const execute_scripttags = async ({ root_node, script_nodes, previous_results_map, invalidation, pluto_actions }) => { let results_map = new Map() // Reattach DOM results from old scripts, you might want to skip reading this @@ -333,6 +333,15 @@ const execute_scripttags = async ({ root_node, script_nodes, previous_results_ma currentScript: currentScript, invalidation: invalidation, getPublishedObject: (id) => cell.getPublishedObject(id), + getNotebookMetadataExperimental: (key) => pluto_actions.get_notebook()?.metadata[key], + setNotebookMetadataExperimental: (key, value) => + pluto_actions.update_notebook((notebook) => { + notebook.metadata[key] = value + }), + deleteNotebookMetadataExperimental: (key) => + pluto_actions.update_notebook((notebook) => { + delete notebook.metadata[key] + }), ...observablehq_for_cells, }, code: node.innerText, @@ -444,8 +453,9 @@ export let RawHTMLContainer = ({ body, className = "", persist_js_state = false, previous_results_map.current = await execute_scripttags({ root_node: container.current, script_nodes: new_scripts, - invalidation: invalidation, + invalidation, previous_results_map: persist_js_state ? previous_results_map.current : new Map(), + pluto_actions, }) if (pluto_actions != null) { diff --git a/frontend/components/Editor.js b/frontend/components/Editor.js index 0e9dc12507..d6491ac689 100644 --- a/frontend/components/Editor.js +++ b/frontend/components/Editor.js @@ -964,22 +964,6 @@ patch: ${JSON.stringify( false ) } - //@ts-ignore - window.getNotebookMetadata = (key) => { - return this.state.notebook.metadata[key] - } - //@ts-ignore - window.putNotebookMetadata = (key, value) => { - this.actions.update_notebook((notebook) => { - notebook.metadata[key] = value - }) - } - //@ts-ignore - window.deleteNotebookMetadata = (key) => { - this.actions.update_notebook((notebook) => { - delete notebook.metadata[key] - }) - } this.submit_file_change = async (new_path, reset_cm_value) => { const old_path = this.state.notebook.path if (old_path === new_path) { From 8139f10864919eea8ef65e66f0fe899395034b4a Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Tue, 17 May 2022 21:19:00 +0200 Subject: [PATCH 09/11] some refactors, more flexible parsing --- frontend/components/Editor.js | 10 ++--- src/notebook/Cell.jl | 9 ++-- src/notebook/Notebook.jl | 78 +++++++++++++++++++++++------------ test/Notebook.jl | 8 ++-- test/helpers.jl | 2 +- 5 files changed, 64 insertions(+), 43 deletions(-) diff --git a/frontend/components/Editor.js b/frontend/components/Editor.js index d6491ac689..503fd47f86 100644 --- a/frontend/components/Editor.js +++ b/frontend/components/Editor.js @@ -39,8 +39,8 @@ import { HijackExternalLinksToOpenInNewTab } from "./HackySideStuff/HijackExtern export const default_path = "..." const DEBUG_DIFFING = false -// Be sure to keep this in sync with DEFAULT_METADATA in Cell.jl -const DEFAULT_METADATA = { +// Be sure to keep this in sync with DEFAULT_CELL_METADATA in Cell.jl +const DEFAULT_CELL_METADATA = { disabled: false, show_logs: true, } @@ -386,7 +386,7 @@ export class Editor extends Component { // Fill the cell with empty code remotely, so it doesn't run unsafe code code: "", metadata: { - ...DEFAULT_METADATA, + ...DEFAULT_CELL_METADATA, }, } } @@ -426,7 +426,7 @@ export class Editor extends Component { code: code, code_folded: false, metadata: { - ...DEFAULT_METADATA, + ...DEFAULT_CELL_METADATA, }, } }) @@ -485,7 +485,7 @@ export class Editor extends Component { cell_id: id, code, code_folded: false, - metadata: { ...DEFAULT_METADATA }, + metadata: { ...DEFAULT_CELL_METADATA }, } notebook.cell_order = [...notebook.cell_order.slice(0, index), id, ...notebook.cell_order.slice(index, Infinity)] }) diff --git a/src/notebook/Cell.jl b/src/notebook/Cell.jl index bf76176355..23b2c3dcac 100644 --- a/src/notebook/Cell.jl +++ b/src/notebook/Cell.jl @@ -1,8 +1,8 @@ import UUIDs: UUID, uuid1 import .ExpressionExplorer: SymbolsState, UsingsImports -# Make sure to keep this in sync with DEFAULT_METADATA in ../frontend/components/Editor.js -const DEFAULT_METADATA = Dict{String, Any}( +# Make sure to keep this in sync with DEFAULT_CELL_METADATA in ../frontend/components/Editor.js +const DEFAULT_CELL_METADATA = Dict{String, Any}( "disabled" => false, "show_logs" => true, ) @@ -52,7 +52,7 @@ Base.@kwdef mutable struct Cell depends_on_disabled_cells::Bool=false - metadata::Dict{String,Any}=copy(DEFAULT_METADATA) + metadata::Dict{String,Any}=copy(DEFAULT_CELL_METADATA) end Cell(cell_id, code) = Cell(cell_id=cell_id, code=code) @@ -72,9 +72,6 @@ function Base.convert(::Type{UUID}, string::String) UUID(string) end -create_metadata(metadata::Dict{String,<:Any}) = merge(DEFAULT_METADATA, metadata) -get_cell_metadata(cell::Cell)::Dict{String,Any} = cell.metadata -get_cell_metadata_no_default(cell::Cell)::Dict{String,Any} = Dict{String,Any}(setdiff(pairs(cell.metadata), pairs(DEFAULT_METADATA))) "Returns whether or not the cell is **explicitely** disabled." is_disabled(c::Cell) = get(c.metadata, "disabled", false) diff --git a/src/notebook/Notebook.jl b/src/notebook/Notebook.jl index 0a0438b6ff..d657bf744f 100644 --- a/src/notebook/Notebook.jl +++ b/src/notebook/Notebook.jl @@ -96,6 +96,10 @@ function Base.getproperty(notebook::Notebook, property::Symbol) end end + +emptynotebook(args...) = Notebook([Cell()], args...) + + const _notebook_header = "### A Pluto.jl notebook ###" const _notebook_metadata_prefix = "#> " # We use a creative delimiter to avoid accidental use in code @@ -112,7 +116,6 @@ const _disabled_suffix = "\n ╠═╡ =#" const _ptoml_cell_id = UUID(1) const _mtoml_cell_id = UUID(2) -emptynotebook(args...) = Notebook([Cell()], args...) """ Save the notebook to `io`, `file` or to `notebook.path`. @@ -126,10 +129,11 @@ function save_notebook(io, notebook::Notebook) println(io, "# ", PLUTO_VERSION_STR) # Notebook metadata - if !isempty(notebook.metadata) - nb_metadata_toml = strip(sprint(TOML.print, notebook.metadata)) - for line in split(nb_metadata_toml, "\n") - println(io, _notebook_metadata_prefix, line) + let nb_metadata_toml = strip(sprint(TOML.print, get_metadata_no_default(notebook))) + if !isempty(nb_metadata_toml) + for line in split(nb_metadata_toml, "\n") + println(io, _notebook_metadata_prefix, line) + end end end @@ -149,13 +153,16 @@ function save_notebook(io, notebook::Notebook) for c in cells_ordered println(io, _cell_id_delimiter, string(c.cell_id)) - metadata_toml = strip(sprint(TOML.print, get_cell_metadata_no_default(c))) - if metadata_toml != "" - for line in split(metadata_toml, "\n") - println(io, _cell_metadata_prefix, line) + + let metadata_toml = strip(sprint(TOML.print, get_metadata_no_default(c))) + if metadata_toml != "" + for line in split(metadata_toml, "\n") + println(io, _cell_metadata_prefix, line) + end end end - cell_running_disabled = c.metadata["disabled"] + + cell_running_disabled = get(c.metadata, "disabled", false)::Bool if cell_running_disabled || c.depends_on_disabled_cells print(io, _disabled_prefix) print(io, replace(c.code, _cell_id_delimiter => "# ")) @@ -225,7 +232,10 @@ save_notebook(notebook::Notebook) = save_notebook(notebook, notebook.path) "Load a notebook without saving it or creating a backup; returns a `Notebook`. REMEMBER TO CHANGE THE NOTEBOOK PATH after loading it to prevent it from autosaving and overwriting the original file." function load_notebook_nobackup(@nospecialize(io::IO), @nospecialize(path::AbstractString))::Notebook - firstline = String(readline(io))::String + + ## HEADER + + firstline = String(readline(io)) if firstline != _notebook_header error("File is not a Pluto.jl notebook") @@ -235,24 +245,28 @@ function load_notebook_nobackup(@nospecialize(io::IO), @nospecialize(path::Abstr if file_VERSION_STR != PLUTO_VERSION_STR # @info "Loading a notebook saved with Pluto $(file_VERSION_STR). This is Pluto $(PLUTO_VERSION_STR)." end - - nb_metadata_toml_lines = String[] + + # Read all remaining file contents before the first cell delimiter. + header_content = readuntil(io, _cell_id_delimiter) + header_lines = split(header_content, "\n") + nb_prefix_length = ncodeunits(_notebook_metadata_prefix) - while !eof(io) - line = String(readline(io)) - if startswith(line, _notebook_metadata_prefix) - push!(nb_metadata_toml_lines, line[begin+nb_prefix_length:end]) - else - break - end + nb_metadata_toml_lines = String[ + line[begin+nb_prefix_length:end] + for line in header_lines if startswith(line, _notebook_metadata_prefix) + ] + + notebook_metadata = try + create_notebook_metadata(TOML.parse(join(nb_metadata_toml_lines, "\n"))) + catch e + @error "Failed to parse embedded TOML content" exception=(e, catch_backtrace()) + DEFAULT_NOTEBOOK_METADATA end - notebook_metadata = fastmerge(DEFAULT_NOTEBOOK_METADATA, TOML.parse(join(nb_metadata_toml_lines, "\n"))) + + ### CELLS + collected_cells = Dict{UUID,Cell}() - - # ignore first bits of file - readuntil(io, _cell_id_delimiter) - while !eof(io) cell_id_str = String(readline(io)) if cell_id_str == "Cell order:" @@ -284,7 +298,12 @@ function load_notebook_nobackup(@nospecialize(io::IO), @nospecialize(path::Abstr code = code_normalised[1:prevind(code_normalised, end, length(_cell_suffix))] # parse metadata - metadata = Dict{String, Any}(DEFAULT_METADATA..., TOML.parse(join(metadata_toml_lines, "\n"))...) + metadata = try + create_cell_metadata(TOML.parse(join(metadata_toml_lines, "\n"))) + catch + @error "Failed to parse embedded TOML content" cell_id exception=(e, catch_backtrace()) + DEFAULT_CELL_METADATA + end read_cell = Cell(; cell_id, code, metadata) collected_cells[cell_id] = read_cell @@ -451,4 +470,9 @@ function sample_notebook(name::String) nb end -fastmerge(a, b) = isempty(a) ? b : merge(a, b) +create_cell_metadata(metadata::Dict{String,<:Any}) = merge(DEFAULT_CELL_METADATA, metadata) +get_metadata(cell::Cell)::Dict{String,Any} = cell.metadata +get_metadata_no_default(cell::Cell)::Dict{String,Any} = Dict{String,Any}(setdiff(pairs(cell.metadata), pairs(DEFAULT_CELL_METADATA))) +create_notebook_metadata(metadata::Dict{String,<:Any}) = merge(DEFAULT_NOTEBOOK_METADATA, metadata) +get_metadata(notebook::Notebook)::Dict{String,Any} = notebook.metadata +get_metadata_no_default(notebook::Notebook)::Dict{String,Any} = Dict{String,Any}(setdiff(pairs(notebook.metadata), pairs(DEFAULT_NOTEBOOK_METADATA))) diff --git a/test/Notebook.jl b/test/Notebook.jl index c59b007f64..ba9985163b 100644 --- a/test/Notebook.jl +++ b/test/Notebook.jl @@ -1,5 +1,5 @@ using Test -import Pluto: Notebook, ServerSession, ClientSession, Cell, load_notebook, load_notebook_nobackup, save_notebook, WorkspaceManager, cutename, numbered_until_new, readwrite, without_pluto_file_extension, update_run!, get_cell_metadata_no_default, is_disabled, create_metadata +import Pluto: Notebook, ServerSession, ClientSession, Cell, load_notebook, load_notebook_nobackup, save_notebook, WorkspaceManager, cutename, numbered_until_new, readwrite, without_pluto_file_extension, update_run!, get_metadata_no_default, is_disabled, create_cell_metadata import Pluto.WorkspaceManager: poll, WorkspaceManager import Random import Pkg @@ -41,7 +41,7 @@ function cell_metadata_notebook() "number" => 10000, ), "disabled" => true, - ) |> create_metadata, + ) |> create_cell_metadata, ), ]) |> init_packages! end @@ -185,7 +185,7 @@ end nb = cell_metadata_notebook() update_run!(🍭, nb, nb.cells) cell = first(values(nb.cells_dict)) - @test get_cell_metadata_no_default(cell) == Dict( + @test get_metadata_no_default(cell) == Dict( "a metadata tag" => Dict( "boolean" => true, "string" => "String", @@ -199,7 +199,7 @@ end @test_notebook_inputs_equal(nb, result) cell = first(nb.cells) @test is_disabled(cell) - @test get_cell_metadata_no_default(cell) == Dict( + @test get_metadata_no_default(cell) == Dict( "a metadata tag" => Dict( "boolean" => true, "string" => "String", diff --git a/test/helpers.jl b/test/helpers.jl index 6720378363..2a81f44010 100644 --- a/test/helpers.jl +++ b/test/helpers.jl @@ -155,7 +155,7 @@ macro test_notebook_inputs_equal(nbA, nbB, check_paths_equality::Bool=true) @test getproperty.(nbA.cells, :cell_id) == getproperty.(nbB.cells, :cell_id) @test getproperty.(nbA.cells, :code_folded) == getproperty.(nbB.cells, :code_folded) @test getproperty.(nbA.cells, :code) == getproperty.(nbB.cells, :code) - @test get_cell_metadata_no_default.(nbA.cells) == get_cell_metadata_no_default.(nbB.cells) + @test get_metadata_no_default.(nbA.cells) == get_metadata_no_default.(nbB.cells) end |> Base.remove_linenums! end From c2eebb573988dfb2a817dc41aaad11ea9cb5ee36 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Tue, 17 May 2022 21:27:58 +0200 Subject: [PATCH 10/11] test tricky file header --- src/notebook/Notebook.jl | 4 ++-- test/Notebook.jl | 48 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/src/notebook/Notebook.jl b/src/notebook/Notebook.jl index d657bf744f..6fabee3ce3 100644 --- a/src/notebook/Notebook.jl +++ b/src/notebook/Notebook.jl @@ -471,8 +471,8 @@ function sample_notebook(name::String) end create_cell_metadata(metadata::Dict{String,<:Any}) = merge(DEFAULT_CELL_METADATA, metadata) -get_metadata(cell::Cell)::Dict{String,Any} = cell.metadata -get_metadata_no_default(cell::Cell)::Dict{String,Any} = Dict{String,Any}(setdiff(pairs(cell.metadata), pairs(DEFAULT_CELL_METADATA))) create_notebook_metadata(metadata::Dict{String,<:Any}) = merge(DEFAULT_NOTEBOOK_METADATA, metadata) +get_metadata(cell::Cell)::Dict{String,Any} = cell.metadata get_metadata(notebook::Notebook)::Dict{String,Any} = notebook.metadata +get_metadata_no_default(cell::Cell)::Dict{String,Any} = Dict{String,Any}(setdiff(pairs(cell.metadata), pairs(DEFAULT_CELL_METADATA))) get_metadata_no_default(notebook::Notebook)::Dict{String,Any} = Dict{String,Any}(setdiff(pairs(notebook.metadata), pairs(DEFAULT_NOTEBOOK_METADATA))) diff --git a/test/Notebook.jl b/test/Notebook.jl index ba9985163b..553092a742 100644 --- a/test/Notebook.jl +++ b/test/Notebook.jl @@ -207,6 +207,8 @@ end ), "disabled" => true, ) + + WorkspaceManager.unmake_workspace((🍭, nb); verbose=false) end end @@ -233,6 +235,52 @@ end save_notebook(nb) nb_loaded = load_notebook_nobackup(nb.path) @test nb.metadata == nb_loaded.metadata + + WorkspaceManager.unmake_workspace((🍭, nb); verbose=false) + end + + @testset "More Metadata" begin + test_file_contents = """ + ### A Pluto.jl notebook ### + # v0.19.4 + + @hello from the future where we might put extra stuff here + + #> [hello] + #> world = [1, 2, 3] + + using Markdown + using SecretThings + + # asdfasdf + + # ╔═╡ a86be878-d616-11ec-05a3-c902726cee5f + # ╠═╡ disabled = true + # ╠═╡ fonsi = 123 + #=╠═╡ + 1 + 1 + ╠═╡ =# + + # ╔═╡ Cell order: + # ╠═a86be878-d616-11ec-05a3-c902726cee5f + + # ok thx byeeeee + + """ + + test_filename = tempname() + write(test_filename, test_file_contents) + nb = load_notebook_nobackup(test_filename) + @test nb.metadata == Dict( + "hello" => Dict( + "world" => [1,2,3], + ) + ) + + @test get_metadata_no_default(only(nb.cells)) == Dict( + "disabled" => true, + "fonsi" => 123, + ) end @testset "I/O overloaded" begin From 8b475b6b9caa07a9b3b25a906f9a7d6771816b6f Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Tue, 17 May 2022 21:53:27 +0200 Subject: [PATCH 11/11] cosmetic tweak --- src/notebook/Notebook.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/notebook/Notebook.jl b/src/notebook/Notebook.jl index 6fabee3ce3..83bfd6aba1 100644 --- a/src/notebook/Notebook.jl +++ b/src/notebook/Notebook.jl @@ -131,6 +131,7 @@ function save_notebook(io, notebook::Notebook) # Notebook metadata let nb_metadata_toml = strip(sprint(TOML.print, get_metadata_no_default(notebook))) if !isempty(nb_metadata_toml) + println(io) for line in split(nb_metadata_toml, "\n") println(io, _notebook_metadata_prefix, line) end