feat: import files to MFS (#810)

* upload files to MFS
* format directory path properly on upload
* move open web UI call out of file upload block
* reword upload to import in UI messages
* move default import directory to preferences screen
* Ensure imported files have the same CID when imported from Web UI or ipfs-companion
  This has been done using adding them to IPFS first (ipfs.add) first and then using the
  ipfs files copy api (ipfs.files.cp) to move the files into the desired directory.
* open resource at gateway if using embedded node
* add date symbols to import directory
* add import options to import screen
* fix: only display openViaWebUI option if opening via web UI is possible
* fix: remove wrapping directory from files API import
* upload => import in MFS import variables
* trim double slashes from import directory
* add file import section on options page
* move preloadAtPublicGateway option to file import section
* preload files when opening imported files in web UI
* test quick-upload formatImportDirectory
* add openViaWebUI option to file import form
* Remove low-level pin when using ipfs.add
* Import to MFS from context menu
  This commit required abstracting the ipfs import functionality into a
  separate file to avoid a circular dependency between
  quick-upload and ipfs-companion, and continue to enable testing
  functionality.

add-on/_locales/en/messages.json

@@ -127,17 +127,13 @@
     "message": "This Page",
     "description": "An item in right-click context menu (contextMenu_parentPage)"
   },
-  "contextMenu_AddToIpfsKeepFilename": {
-    "message": "Add to IPFS (Keep Filename)",
-    "description": "An item in right-click context menu (contextMenu_AddToIpfsKeepFilename)"
+  "contextMenu_importToIpfs": {
+    "message": "Import to IPFS",
+    "description": "An item in right-click context menu (contextMenu_importToIpfs)"
   },
-  "contextMenu_AddToIpfsRawCid": {
-    "message": "Add to IPFS",
-    "description": "An item in right-click context menu (contextMenu_AddToIpfsRawCid)"
-  },
-  "contextMenu_AddToIpfsSelection": {
-    "message": "Add Selected Text to IPFS",
-    "description": "An item in right-click context menu (contextMenu_AddToIpfsSelection)"
+  "contextMenu_importToIpfsSelection": {
+    "message": "Import Selected Text to IPFS",
+    "description": "An item in right-click context menu (contextMenu_importToIpfsSelection)"
   },
   "notify_addonIssueTitle": {
     "message": "IPFS Add-on Issue",
@@ -215,6 +211,10 @@
     "message": "IPFS Node",
     "description": "A section header on the Preferences screen (option_header_nodeType)"
   },
+  "option_header_fileImport": {
+    "message": "File Import",
+    "description": "A section header on the Preferences screen (option_header_fileImport)"
+  },
   "option_ipfsNodeType_title": {
     "message": "IPFS Node Type",
     "description": "An option title on the Preferences screen (option_ipfsNodeType_title)"
@@ -411,12 +411,20 @@
     "message": "Manage permissions",
     "description": "Link text for managing permissions"
   },
+  "option_openViaWebUI_title": {
+    "message": "Open imported files in Web UI",
+    "description": "An option title on the Preferences screen (option_openViaWebUI_title)"
+  },
+  "option_openViaWebUI_description": {
+    "message": "Display files in Web UI rather than opening file or parent directory at gateway.",
+    "description": "An option description on the Preferences screen (option_openViaWebUI_description)"
+  },
   "option_preloadAtPublicGateway_title": {
-    "message": "Preload Uploads",
+    "message": "Preload Imports",
     "description": "An option title on the Preferences screen (option_preloadAtPublicGateway_title)"
   },
   "option_preloadAtPublicGateway_description": {
-    "message": "Enables automatic preload of uploaded assets via asynchronous HTTP HEAD request to a Public Gateway",
+    "message": "Enables automatic preload of imported assets via asynchronous HTTP HEAD request to a Public Gateway",
     "description": "An option description on the Preferences screen (option_preloadAtPublicGateway_description)"
   },
   "option_logNamespaces_title": {
@@ -427,6 +435,14 @@
     "message": "Customize which namespaces are logged to Browser Console. Changing this value will trigger extension restart.",
     "description": "An option description for the log level (option_logNamespaces_description)"
   },
+  "option_importDir_title": {
+    "message": "File Import Directory",
+    "description": "An option title on the Preferences screen (option_importDir_title)"
+  },
+  "option_importDir_description": {
+    "message": "Customize the directory used for imported files.",
+    "description": "An option description on the Preferences screen (option_importDir_description)"
+  },
   "option_resetAllOptions_title": {
     "message": "Reset Everything",
     "description": "An option title and button label on the Preferences screen (option_resetAllOptions_title)"
@@ -472,16 +488,16 @@
     "description": "Status label on the share files page (quickUpload_state_buffering)"
   },
   "quickUpload_options_show": {
-    "message": "upload options",
+    "message": "import options",
     "description": "Button on the share files page (quickUpload_options_show)"
   },
-  "quickUpload_options_wrapWithDirectory": {
-    "message": "Wrap single files in a directory to preserve their filenames.",
-    "description": "Checkbox label on the share files page (quickUpload_options_wrapWithDirectory)"
+  "quickUpload_options_importDir": {
+    "message": "Path to store imported files",
+    "description": "Textbox label on the share files page (quickUpload_options_importDir)"
   },
-  "quickUpload_options_pinUpload": {
-    "message": "Pin files so they are retained when performing garbage collection on your IPFS repo.",
-    "description": "Checkbox label on the share files page (quickUpload_options_pinUpload)"
+  "quickUpload_options_openViaWebUI": {
+    "message": "Open in Web UI",
+    "description": "Checkbox label on the share files page (quickUpload_options_openViaWebUI)"
   },
   "page_proxyAcl_title": {
     "message": "Manage Permissions",

add-on/src/lib/context-menus.js

@@ -51,10 +51,9 @@ const menuParentLink = 'contextMenu_parentLink'
 const menuParentPage = 'contextMenu_parentPage'
 // const menuParentText = 'contextMenu_parentText'
 // Generic Add to IPFS
-const contextMenuAddToIpfsRawCid = 'contextMenu_AddToIpfsRawCid'
-const contextMenuAddToIpfsKeepFilename = 'contextMenu_AddToIpfsKeepFilename'
+const contextMenuImportToIpfs = 'contextMenu_importToIpfs'
 // Add X to IPFS
-const contextMenuAddToIpfsSelection = 'contextMenu_AddToIpfsSelection'
+const contextMenuImportToIpfsSelection = 'contextMenu_importToIpfsSelection'
 // Copy X
 const contextMenuCopyCanonicalAddress = 'panelCopy_currentIpfsAddress'
 const contextMenuCopyRawCid = 'panelCopy_copyRawCid'
@@ -78,15 +77,7 @@ function createContextMenus (getState, runtime, ipfsPathValidator, { onAddFromCo
         contexts: [contextType]
       })
     }
-    const createSeparator = (parentId, id, contextType) => {
-      return browser.contextMenus.create({
-        id: `${parentId}_${id}`,
-        parentId,
-        type: 'separator',
-        contexts: ['all']
-      })
-    }
-    const createAddToIpfsMenuItem = (parentId, id, contextType, ipfsAddOptions) => {
+    const createImportToIpfsMenuItem = (parentId, id, contextType, ipfsAddOptions) => {
       const itemId = `${parentId}_${id}`
       apiMenuItems.add(itemId)
       return browser.contextMenus.create({
@@ -125,19 +116,17 @@ function createContextMenus (getState, runtime, ipfsPathValidator, { onAddFromCo
     }
     const buildSubmenu = (parentId, contextType) => {
       createSubmenu(parentId, contextType)
-      createAddToIpfsMenuItem(parentId, contextMenuAddToIpfsKeepFilename, contextType, { wrapWithDirectory: true })
-      createAddToIpfsMenuItem(parentId, contextMenuAddToIpfsRawCid, contextType, { wrapWithDirectory: false })
-      createSeparator(parentId, 'separator-1', contextType)
+      createImportToIpfsMenuItem(parentId, contextMenuImportToIpfs, contextType, { wrapWithDirectory: true, pin: false })
       createCopierMenuItem(parentId, contextMenuCopyAddressAtPublicGw, contextType, onCopyAddressAtPublicGw)
       createCopierMenuItem(parentId, contextMenuCopyCanonicalAddress, contextType, onCopyCanonicalAddress)
       createCopierMenuItem(parentId, contextMenuCopyRawCid, contextType, onCopyRawCid)
     }
 
     /*
     createSubmenu(menuParentText, 'selection')
-    createAddToIpfsMenuItem(menuParentText, contextMenuAddToIpfsSelection, 'selection')
+    createImportToIpfsMenuItem(menuParentText, contextMenuImportToIpfsSelection, 'selection')
     */
-    createAddToIpfsMenuItem(null, contextMenuAddToIpfsSelection, 'selection')
+    createImportToIpfsMenuItem(null, contextMenuImportToIpfsSelection, 'selection', { pin: false })
     buildSubmenu(menuParentImage, 'image')
     buildSubmenu(menuParentVideo, 'video')
     buildSubmenu(menuParentAudio, 'audio')

add-on/src/lib/ipfs-companion.js

@@ -12,9 +12,10 @@ const { optionDefaults, storeMissingOptions, migrateOptions } = require('./optio
 const { initState, offlinePeerCount } = require('./state')
 const { createIpfsPathValidator } = require('./ipfs-path')
 const createDnslinkResolver = require('./dnslink')
-const { createRequestModifier, redirectOptOutHint } = require('./ipfs-request')
+const { createRequestModifier } = require('./ipfs-request')
 const { initIpfsClient, destroyIpfsClient } = require('./ipfs-client')
 const { createIpfsUrlProtocolHandler } = require('./ipfs-protocol')
+const createIpfsImportHandler = require('./ipfs-import')
 const createNotifier = require('./notifier')
 const createCopier = require('./copier')
 const { createRuntimeChecks } = require('./runtime-checks')
@@ -38,6 +39,7 @@ module.exports = async function init () {
   var apiStatusUpdateInterval
   var ipfsProxy
   var ipfsProxyContentScript
+  var ipfsImportHandler
   const idleInSecs = 5 * 60
   const browserActionPortName = 'browser-action-port'
 
@@ -65,6 +67,7 @@ module.exports = async function init () {
 
     dnslinkResolver = createDnslinkResolver(getState)
     ipfsPathValidator = createIpfsPathValidator(getState, getIpfs, dnslinkResolver)
+    ipfsImportHandler = createIpfsImportHandler(getState, getIpfs, ipfsPathValidator, runtime)
     copier = createCopier(notify, ipfsPathValidator)
     contextMenus = createContextMenus(getState, runtime, ipfsPathValidator, {
       onAddFromContext,
@@ -230,6 +233,8 @@ module.exports = async function init () {
       gwURLString: dropSlash(state.gwURLString),
       pubGwURLString: dropSlash(state.pubGwURLString),
       webuiRootUrl: state.webuiRootUrl,
+      importDir: state.importDir,
+      openViaWebUI: state.openViaWebUI,
       apiURLString: dropSlash(state.apiURLString),
       redirect: state.redirect,
       noRedirectHostnames: state.noRedirectHostnames,
@@ -257,40 +262,16 @@ module.exports = async function init () {
     }
   }
 
-  // GUI
-  // ===================================================================
-
-  function preloadAtPublicGateway (path) {
-    if (!state.preloadAtPublicGateway) return
-    // asynchronous HTTP HEAD request preloads triggers content without downloading it
-    return new Promise((resolve, reject) => {
-      const http = new XMLHttpRequest()
-      // Make sure preload request is excluded from global redirect
-      const preloadUrl = ipfsPathValidator.resolveToPublicUrl(`${path}#${redirectOptOutHint}`, state.pubGwURLString)
-      http.open('HEAD', preloadUrl)
-      http.onreadystatechange = function () {
-        if (this.readyState === this.DONE) {
-          console.info(`[ipfs-companion] preloadAtPublicGateway(${path}):`, this.statusText)
-          if (this.status === 200) {
-            resolve(this.statusText)
-          } else {
-            reject(new Error(this.statusText))
-          }
-        }
-      }
-      http.send()
-    })
-  }
-
   // Context Menu Uploader
   // -------------------------------------------------------------------
 
   async function onAddFromContext (context, contextType, options) {
+    const importDir = ipfsImportHandler.formatImportDirectory(state.importDir)
     let result
     try {
       const dataSrc = await findValueForContext(context, contextType)
       if (contextType === 'selection') {
-        result = await ipfs.add(Buffer.from(dataSrc), options)
+        result = await ipfsImportHandler.importFiles(Buffer.from(dataSrc), options, importDir)
       } else {
         // Enchanced addFromURL
         // --------------------
@@ -319,7 +300,7 @@ module.exports = async function init () {
           path: decodeURIComponent(filename),
           content: buffer
         }
-        result = await ipfs.add(data, options)
+        result = await ipfsImportHandler.importFiles(data, options, importDir)
       }
     } catch (error) {
       console.error('Error in upload to IPFS context menu', error)
@@ -334,37 +315,12 @@ module.exports = async function init () {
       }
       return
     }
-
-    return uploadResultHandler({ result, openRootInNewTab: true })
-  }
-
-  // TODO: feature detect and push to client type specific modules.
-  function getIpfsPathAndNativeAddress (hash) {
-    const path = `/ipfs/${hash}`
-    if (runtime.hasNativeProtocolHandler) {
-      return { path, url: `ipfs://${hash}` }
+    ipfsImportHandler.preloadFilesAtPublicGateway(result)
+    if (state.ipfsNodeType === 'embedded' || !state.openViaWebUI) {
+      return ipfsImportHandler.openFilesAtGateway({ result, openRootInNewTab: true })
     } else {
-      // open at public GW (will be redirected to local elsewhere, if enabled)
-      const url = new URL(path, state.pubGwURLString).toString()
-      return { path, url: url }
-    }
-  }
-
-  async function uploadResultHandler ({ result, openRootInNewTab = false }) {
-    for (const file of result) {
-      if (file && file.hash) {
-        const { path, url } = getIpfsPathAndNativeAddress(file.hash)
-        preloadAtPublicGateway(path)
-        console.info('[ipfs-companion] successfully stored', file)
-        // open the wrapping directory (or the CID if wrapping was disabled)
-        if (openRootInNewTab && (result.length === 1 || file.path === '' || file.path === file.hash)) {
-          await browser.tabs.create({
-            url: url
-          })
-        }
-      }
+      return ipfsImportHandler.openFilesAtWebUI(importDir)
     }
-    return result
   }
 
   // Page-specific Actions
@@ -709,12 +665,16 @@ module.exports = async function init () {
           shouldReloadExtension = true
           state[key] = localStorage.debug = change.newValue
           break
+        case 'importDir':
+          state[key] = change.newValue
+          break
         case 'linkify':
         case 'catchUnhandledProtocols':
         case 'displayNotifications':
         case 'automaticMode':
         case 'detectIpfsPathHeader':
         case 'preloadAtPublicGateway':
+        case 'openViaWebUI':
         case 'noRedirectHostnames':
           state[key] = change.newValue
           break
@@ -783,8 +743,8 @@ module.exports = async function init () {
       return notify
     },
 
-    get uploadResultHandler () {
-      return uploadResultHandler
+    get ipfsImportHandler () {
+      return ipfsImportHandler
     },
 
     destroy () {
@@ -796,6 +756,7 @@ module.exports = async function init () {
       dnslinkResolver = null
       modifyRequest = null
       ipfsPathValidator = null
+      ipfsImportHandler = null
       notify = null
       copier = null
       contextMenus = null