Migrate to rust lib (#36)

* Refactor to use distant-lua library, removing need for external client * Integrate dialog to support downloading prebuilt library, building from source, or copying local library * Fix #27 * Add parser for commands to support custom arg format and quoted values * Refactor fn api to be unified across synchronous and asynchronous usage via a callback * Add DistantInstall command to trigger install of C library * Add tests for ssh vs distant mode * Update readme to reflect 0.15.0 and update dockerfile to 0.15.0
chipsenkbeil · Oct 16, 2021 · 4436ce1 · 4436ce1
1 parent 8f6052a
commit 4436ce1
Show file tree

Hide file tree

Showing 72 changed files with 2,188 additions and 4,271 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -31,5 +31,15 @@ jobs:
         run: make docker-build
       - name: Save Docker cache
         run: make docker-save
-      - name: Run tests
-        run: make docker-test
+      - name: Run unit tests
+        run: make docker-test-unit
+      - name: Run e2e tests (distant)
+        if: always()
+        run: make docker-test-e2e
+        env:
+          DISTANT_MODE: distant
+      - name: Run e2e tests (ssh)
+        if: always()
+        run: make docker-test-e2e
+        env:
+          DISTANT_MODE: ssh
diff --git a/.gitignore b/.gitignore
@@ -1,2 +1,10 @@
 vendor/
 !vendor/.placeholder
+**/*.so
+**/*.dll
+
+
+# Added by cargo
+
+/target
+Cargo.lock
diff --git a/Dockerfile b/Dockerfile
@@ -23,24 +23,26 @@ RUN addgroup -S $user \
     && adduser $user wheel \
     && echo "$user ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/$user \
     && chmod 0440 /etc/sudoers.d/$user \
-    && echo "$user:$user" | chpasswd
+    && echo "$user:" | chpasswd
 USER $user
 WORKDIR /home/$user
 
 ARG cargo_bin_dir=/home/$user/.cargo/bin
 RUN mkdir -p $cargo_bin_dir
 
 # Install and configure rust & rls
+# NOTE: Must install to a path like /usr/bin as 
+#       /usr/local/bin is not on path for ssh
 RUN rustup-init -y \
     && source /home/$user/.cargo/env \
     && rustup component add rls \
-    && sudo ln -s $cargo_bin_dir/rls /usr/local/bin/rls
+    && sudo ln -s $cargo_bin_dir/rls /usr/bin/rls
 
 # Install neovim 0.5 binary (from edge)
 RUN sudo apk add neovim \
     --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community/
 
-# Install and configure sshd
+# Install and configure sshd with key using empty password
 #
 # 1. Support openrc not being properly ready (touch softlevel)
 # 2. Generate host keys
@@ -52,18 +54,29 @@ RUN sudo mkdir -p /var/run/sshd /run/openrc \
     && sudo touch /run/openrc/softlevel \
     && sudo ssh-keygen -A \
     && sudo rc-update add sshd \
-    && ssh-keygen -q -t rsa -N '' -f /home/docker/.ssh/id_rsa \
+    && ssh-keygen -q -m PEM -t rsa -N '' -f /home/docker/.ssh/id_rsa \
     && cp /home/$user/.ssh/id_rsa.pub /home/$user/.ssh/authorized_keys \
     && echo 'StrictHostKeyChecking no' > /home/$user/.ssh/config
 
+# Install libc compatibility to provide /lib64/ld-linux-x86-64.so.2 and libgcc_s.so.1
+# Required due to https://github.com/rust-lang/rust/issues/82521
+RUN sudo apk add libc6-compat \
+    && sudo ln -s /lib/libc.musl-x86_64.so.1 /lib/ld-linux-x86-64.so.2
+
 # Install distant binary and make sure its in a path for everyone
-ARG distant_release=https://github.com/chipsenkbeil/distant/releases/download/v0.14.1
+ARG distant_release=https://github.com/chipsenkbeil/distant/releases/download/v0.15.0
 RUN curl -L $distant_release/distant-linux64-musl > $cargo_bin_dir/distant \
     && chmod +x $cargo_bin_dir/distant \
     && sudo ln -s $cargo_bin_dir/distant /usr/local/bin/distant
 
+# Download the lua module so we can copy it into place
+RUN curl -fLo distant_lua.so --create-dirs $distant_release/distant_lua-linux64-musl.so
+
 # Install our repository within a subdirectory of home
 COPY --chown=$user . app/
 
+# Overwrite the distant_lua.so file to ensure it is the right format
+RUN mkdir -p app/lua/ && mv distant_lua.so app/lua/distant_lua.so
+
 # By default, this will run the ssh server
 CMD ["sudo", "/usr/sbin/sshd", "-D", "-e"]
diff --git a/Makefile b/Makefile
@@ -8,6 +8,9 @@ DISTANT_PORT?=22
 DISTANT_IDENTITY_FILE?=
 DISTANT_BIN?="$(HOME)/.cargo/bin/distant"
 DISTANT_LOG_LEVEL?=info
+DISTANT_USER?=$(shell whoami)
+DISTANT_PASSWORD?=
+DISTANT_MODE?=
 
 DOCKER_IMAGE=distant_nvim_test
 DOCKER_NETWORK=distant_nvim_network
@@ -41,8 +44,12 @@ define docker_exec
 	-e DISTANT_HOST=$(DOCKER_SERVER) \
 	-e DISTANT_PORT=22 \
 	-e DISTANT_BIN=/usr/local/bin/distant \
+	-e DISTANT_USER=$(DISTANT_USER) \
+	-e DISTANT_PASSWORD=$(DISTANT_PASSWORD) \
+	-e DISTANT_MODE=$(DISTANT_MODE) \
+	-e DISTANT_LOG_LEVEL=$(DISTANT_LOG_LEVEL) \
 	$(2) \
-	$(DOCKER_IMAGE) sh -c "cd app && $(1)"; \
+	$(DOCKER_IMAGE) sh -c "cd app && ssh-keyscan -H $(DOCKER_SERVER) >> ~/.ssh/known_hosts && $(1)"; \
 	STATUS=$$?; \
 	docker rm -f $(DOCKER_SERVER) > /dev/null 2>&1; \
 	docker network rm $(DOCKER_NETWORK) > /dev/null 2>&1; \
@@ -60,7 +67,7 @@ define test_exec
 	--headless \
 	--noplugin \
 	-u spec/spec.vim \
-	-c "PlenaryBustedDirectory spec/$(1) { minimal_init = 'spec/spec.vim' $(if 2,$(COMMA)$(2)) }"
+	-c "lua require('plenary.test_harness').test_directory('spec/$(1)', {minimal_init='spec/spec.vim'$(if 2,$(COMMA)$(2))})"
 endef
 
 ###############################################################################
@@ -91,37 +98,49 @@ vendor: vendor/plenary.nvim
 
 # Pulls in the latest version of plenary.nvim, which we use to run our tests
 vendor/plenary.nvim:
-	git clone https://github.com/nvim-lua/plenary.nvim.git vendor/plenary.nvim || \
+	@git clone https://github.com/nvim-lua/plenary.nvim.git vendor/plenary.nvim || \
 		( cd vendor/plenary.nvim && git pull --rebase; )
 
+clean: ## Cleans out vendor directory
+	@rm -rf vendor/*
+
 ###############################################################################
 # DOCKER TEST TARGETS
 ###############################################################################
 
+docker-test: DISTANT_USER=docker
 docker-test: ## Runs all tests using a pair of docker containers that have shared SSH keys
 	$(call docker_exec,make test)
 
+docker-test-arg: DISTANT_USER=docker
 docker-test-arg: ## Runs all tests for a custom path (ARG) inside spec/ using a pair of docker containers that have shared SSH keys
 	$(call docker_exec,make test-arg ARG=$(ARG))
 
+docker-test-unit: DISTANT_USER=docker
 docker-test-unit: ## Runs all unit tests using a pair of docker containers that have shared SSH keys
 	$(call docker_exec,make test-unit)
 
+docker-test-e2e: DISTANT_USER=docker
 docker-test-e2e: ## Runs all e2e tests using a pair of docker containers that have shared SSH keys
 	$(call docker_exec,make test-e2e)
 
+docker-build: DISTANT_USER=docker
 docker-build:
 	@docker build . --file Dockerfile --tag $(DOCKER_IMAGE) --cache-from=$(DOCKER_IMAGE)
 
+docker-save: DISTANT_USER=docker
 docker-save:
 	@docker save --output $(DOCKER_OUT_ARCHIVE) $(DOCKER_IMAGE):latest
 
+docker-load: DISTANT_USER=docker
 docker-load:
 	@docker load --input $(DOCKER_OUT_ARCHIVE)
 
+docker-shell: DISTANT_USER=docker
 docker-shell: ## Runs our client docker container in an interactive shell
 	$(call docker_exec,/bin/sh,-it --init)
 
+docker-shutdown: DISTANT_USER=docker
 docker-shutdown:
 	@docker rm -f $(DOCKER_CLIENT) > /dev/null 2>&1 || true
 	@docker rm -f $(DOCKER_SERVER) > /dev/null 2>&1 || true

diff --git a/README.md b/README.md
@@ -6,7 +6,7 @@ A wrapper around [`distant`](https://github.com/chipsenkbeil/distant) that
 enables users to edit remote files from the comfort of their local environment.
 
 - **Requires neovim 0.5+**
-- **Requires distant 0.14.1+**
+- **Requires distant 0.15.0+**
 
 🚧 **(Alpha stage software) This plugin is in rapid development and may
 break or change frequently!** 🚧
@@ -17,7 +17,27 @@ break or change frequently!** 🚧
 
 ## Installation
 
-Using [packer.nvim](https://github.com/wbthomason/packer.nvim)
+Using [packer.nvim](https://github.com/wbthomason/packer.nvim), the quickest
+way to get up and running is the following:
+
+```lua
+use {
+  'chipsenkbeil/distant.nvim',
+  config = function()
+    require('distant').setup {
+      -- Applies Chip's personal settings to every machine you connect to
+      --
+      -- 1. Ensures that distant servers terminate with no connections
+      -- 2. Provides navigation bindings for remote directories
+      -- 3. Provides keybinding to jump into a remote file's parent directory
+      ['*'] = require('distant.settings').chip_default()
+    }
+  end
+}
+```
+
+Normally, you would want to specify your own settings, both across all hosts
+and custom settings for specific remote machines:
 
 ```lua
 use {
@@ -28,13 +48,13 @@ use {
     require('distant').setup {
       -- Apply these settings to the specific host
       ['example.com'] = {
-        launch = {
-          -- Specify a specific location for the distant binary on the remote machine
-          distant = '/path/to/distant',
+        -- Specify a specific location for the distant binary on the remote machine
+        distant = {
+          bin = '/path/to/distant',
         }
 
+        -- Specify an LSP to run for a specific project
         lsp = {
-          -- Specify an LSP to run for a specific project
           ['My Project'] = {
             cmd = '/path/to/rust-analyzer',
             root_dir = '/path/to/project/root',
@@ -49,12 +69,14 @@ use {
       },
 
       -- Apply these settings to any remote host
+      --
+      -- NOTE: These mirror what is returned with
+      -- require('distant.settings').chip_default()
       ['*'] = {
         -- Apply these launch settings to all hosts
-        launch = {
-          -- Apply additional CLI options to the listening server, such as
-          -- shutting down when there is no connection to it after 30 seconds
-          extra_server_args = '"--shutdown-after 30"',
+        distant = {
+          -- Shutdown server after 60 seconds with no active connection
+          args = {'--shutdown-after', '60'},
         },
 
         -- Specify mappings to apply on remote file buffers
@@ -118,51 +140,73 @@ all of the contents of the specified directory.
 ### Blocking Functions
 
 Synchronous functions are available that perform the given operation in a
-blocking fashion. All blocking functions support being provided a timeout and
-interval to check said timeout, defaulting both to the global settings. For
-more details, check out the doc comments for the individual functions.
-
-| Functions             | Description                                                                   |
-|-----------------------|-------------------------------------------------------------------------------|
-| `fn.copy`             | Copies a remote file or directory to another remote location                  |
-| `fn.dir_list`         | Lists remote files & directories for the given path on the remote machine     |
-| `fn.exists`           | Determines whether or not the path exists on the remote machine               |
-| `fn.metadata`         | Retrieves metadata about a remote file, directory, or symlink                 |
-| `fn.mkdir`            | Creates a new directory remotely                                              |
-| `fn.read_file_text`   | Reads a remote file, returning its content as text                            |
-| `fn.remove`           | Removes a remote file or directory                                            |
-| `fn.rename`           | Renames a remote file or directory                                            |
-| `fn.run`              | Runs a remote program to completion, returning stdout, stderr, and exit code  |
-| `fn.system_info`      | Retrieves information about the remote machine such as its os name and arch   |
-| `fn.write_file_text`  | Writes text to a remote file                                                  |
+blocking fashion. Each function takes a single argument, which is a table
+containing the arguments relevant for the function. Each function returns
+two values: `err` being a userdata error or false, and `result` being the
+results of the function call if it has any (like file text).
+
+```lua
+local fn = require('distant.fn')
+
+local err, text = fn.read_file_text({path = 'path/to/file'})
+if err then
+  vim.api.nvim_err_writeln(tostring(err))
+  return
+end
+
+print('Read file contents', text)
+```
+
+| Functions             | Description                                                                           |
+|-----------------------|---------------------------------------------------------------------------------------|
+| `fn.append_file`      | Appends binary data to a remote file                                                  |
+| `fn.append_file_text` | Appends text to a remote file                                                         |
+| `fn.copy`             | Copies a remote file or directory to another remote location                          |
+| `fn.create_dir`       | Creates a new directory remotely                                                      |
+| `fn.exists`           | Determines whether or not the path exists on the remote machine                       |
+| `fn.metadata`         | Retrieves metadata about a remote file, directory, or symlink                         |
+| `fn.read_dir`         | Lists remote files & directories for the given path on the remote machine             |
+| `fn.read_file`        | Reads a remote file, returning its content as a list of bytes                         |
+| `fn.read_file_text`   | Reads a remote file, returning its content as text                                    |
+| `fn.remove`           | Removes a remote file or directory                                                    |
+| `fn.rename`           | Renames a remote file or directory                                                    |
+| `fn.spawn`            | Starts a process, returning it to support writing stdin and reading stdout and stderr |
+| `fn.spawn_wait`       | Runs a remote program to completion, returning table of stdout, stderr, and exit code |
+| `fn.system_info`      | Retrieves information about the remote machine such as its os name and arch           |
+| `fn.write_file`       | Writes binary data to a remote file                                                   |
+| `fn.write_file_text`  | Writes text to a remote file                                                          |
 
 ### Async Functions
 
-Asynchronous functions are available that use callbacks when functions are
-executed. The singular argument to the callback matches that of the return
-value of the synchronous function. For more details, check out the doc comments
-for the individual functions.
-
-| Functions                     | Description                                               |
-|-------------------------------|-----------------------------------------------------------|
-| `fn.async.copy`               | Async variant of `fn.copy` using callbacks                |
-| `fn.async.dir_list`           | Async variant of `fn.dir_list` using callbacks            |
-| `fn.async.exists`             | Async variant of `fn.exists` using callbacks              |
-| `fn.async.metadata`           | Async variant of `fn.metadata` using callbacks            |
-| `fn.async.mkdir`              | Async variant of `fn.mkdir` using callbacks               |
-| `fn.async.read_file_text`     | Async variant of `fn.read_file_text` using callbacks      |
-| `fn.async.remove`             | Async variant of `fn.remove` using callbacks              |
-| `fn.async.rename`             | Async variant of `fn.rename` using callbacks              |
-| `fn.async.run`                | Async variant of `fn.run` using callbacks                 |
-| `fn.async.system_info`        | Async variant of `fn.system_info` using callbacks         |
-| `fn.async.write_file_text`    | Async variant of `fn.write_file_text` using callbacks     |
+Every blocking function above can also be called in a non-blocking fashion.
+This is done by supplying a callback function as the last argument.
+
+```lua
+local fn = require('distant.fn')
+
+fn.read_file_text({path = 'path/to/file'}, function(err, text)
+  if err then
+    vim.api.nvim_err_writeln(tostring(err))
+    return
+  end
+
+  print('Read file contents', text)
+end)
+```
 
 ## Commands
 
 Alongside functions, this plugin also provides vim commands that can be used to
 initiate different tasks remotely. It also includes specialized commands such
 as `DistantLaunch` that is used to start a remote session.
 
+Commands support positional and key=value pairs. Positional arguments are
+relative to each other and are not influenced by key=value pairs inbetween.
+
+```
+:DistantLaunch example.com distant.use_login_shell=true distant.args="--log-file /path/to/file.log --log-level info"
+```
+
 ### Specialized Commands
 
 These commands are geared towards performing actions that expose some dialogs
@@ -172,6 +216,7 @@ or other user interfaces within neovim.
 |-----------------------|-----------------------------------------------------------|
 | `DistantOpen`         | Opens a file for editing or a directory for navigation    |
 | `DistantLaunch`       | Opens a dialog to launch `distant` on a remote machine    |
+| `DistantInstall`      | Triggers installation process for the C library           |
 | `DistantMetadata`     | Presents information about some path on a remote machine  |
 | `DistantSessionInfo`  | Presents information related to the remote connection     |
 | `DistantSystemInfo`   | Presents information about remote machine itself          |
@@ -187,7 +232,7 @@ function's arguments as input.
 | `DistantMkdir`        | Alias to `lua require('distant').fn.mkdir`        |
 | `DistantRemove`       | Alias to `lua require('distant').fn.remove`       |
 | `DistantRename`       | Alias to `lua require('distant').fn.rename`       |
-| `DistantRun`          | Alias to `lua require('distant').fn.run`          |
+| `DistantRun`          | Alias to `lua require('distant').fn.spawn_wait`   |
 
 ## telescope Integration