Implement File Storage

lib/livebook/file_system/s3.ex

@@ -8,7 +8,7 @@ defmodule Livebook.FileSystem.S3 do
   @type t :: %__MODULE__{
           id: String.t(),
           bucket_url: String.t(),
-          external_id: String.t(),
+          external_id: String.t() | nil,
           region: String.t(),
           access_key_id: String.t(),
           secret_access_key: String.t()
@@ -43,12 +43,10 @@ defmodule Livebook.FileSystem.S3 do
     bucket_url = String.trim_trailing(bucket_url, "/")
     region = opts[:region] || region_from_uri(bucket_url)
 
-    hash = :crypto.hash(:sha256, bucket_url) |> Base.url_encode64(padding: false)
-
     id =
       if prefix = opts[:prefix],
-        do: "#{prefix}-s3-#{hash}",
-        else: "s3-#{hash}"
+        do: "#{prefix}-#{id(bucket_url)}",
+        else: id(bucket_url)
 
     %__MODULE__{
       id: id,
@@ -63,7 +61,13 @@ defmodule Livebook.FileSystem.S3 do
   defp region_from_uri(uri) do
     # For many services the API host is of the form *.[region].[rootdomain].com
     %{host: host} = URI.parse(uri)
-    host |> String.split(".") |> Enum.reverse() |> Enum.at(2, "auto")
+    splitted_host = host |> String.split(".") |> Enum.reverse()
+
+    case Enum.at(splitted_host, 2, "auto") do
+      "s3" -> "us-east-1"
+      "r2" -> "auto"
+      region -> region
+    end
   end
 
   @doc """
@@ -113,8 +117,32 @@ defmodule Livebook.FileSystem.S3 do
       :access_key_id,
       :secret_access_key
     ])
+    |> put_region_from_uri()
     |> validate_required([:bucket_url, :access_key_id, :secret_access_key])
     |> Livebook.Utils.validate_url(:bucket_url)
+    |> put_id()
+  end
+
+  defp put_region_from_uri(changeset) do
+    case get_field(changeset, :bucket_url) do
+      nil -> changeset
+      bucket_url -> put_change(changeset, :region, region_from_uri(bucket_url))
+    end
+  end
+
+  defp put_id(changeset) do
+    if bucket_url = get_field(changeset, :bucket_url) do
+      put_change(changeset, :id, id(bucket_url))
+    else
+      changeset
+    end
+  end
+
+  defp id(bucket_url) do
+    hash = :crypto.hash(:sha256, bucket_url)
+    encrypted_hash = Base.url_encode64(hash, padding: false)
+
+    "s3-#{encrypted_hash}"
   end
 end
 

lib/livebook/file_systems.ex

@@ -9,6 +9,33 @@ defmodule Livebook.FileSystems do
   @spec type(FileSystem.t()) :: String.t()
   def type(%FileSystem.S3{}), do: "s3"
 
+  @doc """
+  Updates file system with the given changes.
+  """
+  @spec update_file_system(FileSystem.t(), map()) ::
+          {:ok, FileSystem.t()} | {:error, Ecto.Changeset.t()}
+  def update_file_system(file_system, attrs) do
+    file_system
+    |> change_file_system(attrs)
+    |> Ecto.Changeset.apply_action(:update)
+  end
+
+  @doc """
+  Returns an `%Ecto.Changeset{}` for tracking file system changes.
+  """
+  @spec change_file_system(FileSystem.t()) :: Ecto.Changeset.t()
+  def change_file_system(file_system) do
+    change_file_system(file_system, %{})
+  end
+
+  @doc """
+  Returns an `%Ecto.Changeset{}` for tracking file system changes.
+  """
+  @spec change_file_system(FileSystem.t(), map()) :: Ecto.Changeset.t()
+  def change_file_system(%FileSystem.S3{} = file_system, attrs) do
+    FileSystem.S3.change_file_system(file_system, attrs)
+  end
+
   @doc """
   Loads the file system from given type and dumped data.
   """

lib/livebook/hubs.ex

@@ -159,6 +159,12 @@ defmodule Livebook.Hubs do
     * `{:secret_updated, %Secret{}}`
     * `{:secret_deleted, %Secret{}}`
 
+  Topic `hubs:file_systems`:
+
+    * `{:file_system_created, FileSystem.t()}`
+    * `{:file_system_updated, FileSystem.t()}`
+    * `{:file_system_deleted, FileSystem.t()}`
+
   """
   @spec subscribe(atom() | list(atom())) :: :ok | {:error, term()}
   def subscribe(topics) when is_list(topics) do
@@ -296,15 +302,28 @@ defmodule Livebook.Hubs do
     Provider.verify_notebook_stamp(hub, notebook_source, stamp)
   end
 
+  @doc """
+  Gets a list of file systems from all hubs.
+  """
+  @spec get_file_systems() :: list(FileSystem.t())
+  def get_file_systems() do
+    file_systems = Enum.flat_map(get_hubs(), &Provider.get_file_systems/1)
+    local_file_system = Livebook.Config.local_file_system()
+
+    [local_file_system | Enum.sort_by(file_systems, & &1.id)]
+  end
+
   @doc """
   Gets a list of file systems for given hub.
   """
-  @spec get_file_systems(Provider.t()) :: list(FileSystem.t())
-  def get_file_systems(hub) do
+  @spec get_file_systems(Provider.t(), keyword()) :: list(FileSystem.t())
+  def get_file_systems(hub, opts \\ []) do
     hub_file_systems = Provider.get_file_systems(hub)
-    local_file_system = Livebook.Config.local_file_system()
+    sorted_hub_file_systems = Enum.sort_by(hub_file_systems, & &1.id)
 
-    [local_file_system | Enum.sort_by(hub_file_systems, & &1.id)]
+    if opts[:hub_only],
+      do: sorted_hub_file_systems,
+      else: [Livebook.Config.local_file_system() | sorted_hub_file_systems]
   end
 
   @doc """

lib/livebook/users/user.ex

@@ -33,10 +33,10 @@ defmodule Livebook.Users.User do
   @doc """
   Generates a new user.
   """
-  @spec new() :: t()
-  def new() do
+  @spec new(String.t()) :: t()
+  def new(id \\ Utils.random_id()) do
     %__MODULE__{
-      id: Utils.random_id(),
+      id: id,
       name: nil,
       email: nil,
       hex_color: Livebook.EctoTypes.HexColor.random()

lib/livebook_web/live/file_select_component.ex

@@ -45,7 +45,8 @@ defmodule LivebookWeb.FileSelectComponent do
        renaming_file: nil,
        renamed_name: nil,
        error_message: nil,
-       file_systems: Livebook.Settings.file_systems()
+       configure_path: nil,
+       file_systems: []
      )
      |> allow_upload(:folder,
        accept: :any,
@@ -70,7 +71,14 @@ defmodule LivebookWeb.FileSelectComponent do
       |> assign(assigns)
       |> update_file_infos(force_reload? or running_files_changed?)
 
-    {:ok, socket}
+    {file_systems, configure_hub_id} =
+      if hub = socket.assigns[:hub],
+        do: {Livebook.Hubs.get_file_systems(hub), hub.id},
+        else: {Livebook.Hubs.get_file_systems(), Livebook.Hubs.Personal.id()}
+
+    configure_path = ~p"/hub/#{configure_hub_id}/file-systems/new"
+
+    {:ok, assign(socket, file_systems: file_systems, configure_path: configure_path)}
   end
 
   @impl true
@@ -83,6 +91,7 @@ defmodule LivebookWeb.FileSelectComponent do
           <.file_system_menu_button
             file={@file}
             file_systems={@file_systems}
+            configure_path={@configure_path}
             file_system_select_disabled={@file_system_select_disabled}
             myself={@myself}
           />
@@ -281,14 +290,15 @@ defmodule LivebookWeb.FileSelectComponent do
       <%= for file_system <- @file_systems do %>
         <%= if file_system == @file.file_system do %>
           <.menu_item variant={:selected}>
-            <button role="menuitem">
+            <button id={"file-system-#{file_system.id}"} role="menuitem">
               <.file_system_icon file_system={file_system} />
               <span><%= file_system_label(file_system) %></span>
             </button>
           </.menu_item>
         <% else %>
           <.menu_item>
             <button
+              id={"file-system-#{file_system.id}"}
               role="menuitem"
               phx-target={@myself}
               phx-click="set_file_system"
@@ -301,7 +311,7 @@ defmodule LivebookWeb.FileSelectComponent do
         <% end %>
       <% end %>
       <.menu_item>
-        <.link navigate={~p"/settings"} class="border-t border-gray-200" role="menuitem">
+        <.link navigate={@configure_path} class="border-t border-gray-200" role="menuitem">
           <.remix_icon icon="settings-3-line" />
           <span>Configure</span>
         </.link>

lib/livebook_web/live/hooks/user_hook.ex

@@ -33,9 +33,7 @@ defmodule LivebookWeb.UserHook do
   # attributes if the socket is connected. Otherwise uses
   # `user_data` from session.
   defp build_current_user(session, socket) do
-    user = User.new()
     identity_data = Map.new(session["identity_data"], fn {k, v} -> {Atom.to_string(k), v} end)
-
     connect_params = get_connect_params(socket) || %{}
     attrs = connect_params["user_data"] || session["user_data"] || %{}
 
@@ -45,6 +43,8 @@ defmodule LivebookWeb.UserHook do
         attrs -> attrs
       end
 
+    user = User.new(attrs["id"])
+
     case Livebook.Users.update_user(user, attrs) do
       {:ok, user} -> user
       {:error, _changeset} -> user

lib/livebook_web/live/hub/edit/personal_component.ex

@@ -11,17 +11,28 @@ defmodule LivebookWeb.Hub.Edit.PersonalComponent do
     socket = assign(socket, assigns)
     changeset = Personal.change_hub(assigns.hub)
     secrets = Hubs.get_secrets(assigns.hub)
+    file_systems = Hubs.get_file_systems(assigns.hub, hub_only: true)
     secret_name = assigns.params["secret_name"]
+    file_system_id = assigns.params["file_system_id"]
 
     secret_value =
       if assigns.live_action == :edit_secret do
         Enum.find_value(secrets, &(&1.name == secret_name and &1.value)) ||
           raise(NotFoundError, "could not find secret matching #{inspect(secret_name)}")
       end
 
+    file_system =
+      if assigns.live_action == :edit_file_system do
+        Enum.find_value(file_systems, &(&1.id == file_system_id && &1)) ||
+          raise(NotFoundError, "could not find file system matching #{inspect(file_system_id)}")
+      end
+
     {:ok,
      assign(socket,
        secrets: secrets,
+       file_system: file_system,
+       file_system_id: file_system_id,
+       file_systems: file_systems,
        changeset: changeset,
        stamp_changeset: changeset,
        secret_name: secret_name,
@@ -90,7 +101,23 @@ defmodule LivebookWeb.Hub.Edit.PersonalComponent do
               id="hub-secrets-list"
               hub={@hub}
               secrets={@secrets}
-              target={@myself}
+            />
+          </div>
+
+          <div class="flex flex-col space-y-4">
+            <h2 class="text-xl text-gray-800 font-medium pb-2 border-b border-gray-200">
+              File Storages
+            </h2>
+
+            <p class="text-gray-700">
+              File storages are used to store notebooks.
+            </p>
+
+            <.live_component
+              module={LivebookWeb.Hub.FileSystemListComponent}
+              id="hub-file-systems-list"
+              hub_id={@hub.id}
+              file_systems={@file_systems}
             />
           </div>
 
@@ -167,6 +194,23 @@ defmodule LivebookWeb.Hub.Edit.PersonalComponent do
           return_to={~p"/hub/#{@hub.id}"}
         />
       </.modal>
+
+      <.modal
+        :if={@live_action in [:new_file_system, :edit_file_system]}
+        id="file-systems-modal"
+        show
+        width={:medium}
+        patch={~p"/hub/#{@hub.id}"}
+      >
+        <.live_component
+          module={LivebookWeb.Hub.FileSystemFormComponent}
+          id="file-systems"
+          hub={@hub}
+          file_system={@file_system}
+          file_system_id={@file_system_id}
+          return_to={~p"/hub/#{@hub.id}"}
+        />
+      </.modal>
     </div>
     """
   end
@@ -193,25 +237,6 @@ defmodule LivebookWeb.Hub.Edit.PersonalComponent do
     {:noreply, validate(params, :stamp_changeset, socket)}
   end
 
-  def handle_event("delete_hub_secret", attrs, socket) do
-    %{hub: hub} = socket.assigns
-
-    on_confirm = fn socket ->
-      {:ok, secret} = Livebook.Secrets.update_secret(%Livebook.Secrets.Secret{}, attrs)
-      _ = Livebook.Hubs.delete_secret(hub, secret)
-
-      socket
-    end
-
-    {:noreply,
-     confirm(socket, on_confirm,
-       title: "Delete hub secret - #{attrs["name"]}",
-       description: "Are you sure you want to delete this hub secret?",
-       confirm_text: "Delete",
-       confirm_icon: "delete-bin-6-line"
-     )}
-  end
-
   defp save(params, changeset_name, socket) do
     case Personal.update_hub(socket.assigns.hub, params) do
       {:ok, hub} ->