feat(context): Support unix context and daemonless (#1062)

* feat(context): Support unix context

Signed-off-by: Ce Gao <cegao@tensorchord.ai>

* chore: Auto-build image

Signed-off-by: Ce Gao <cegao@tensorchord.ai>

* chore: Update release workflow

Signed-off-by: Ce Gao <cegao@tensorchord.ai>

* fix: Update name

Signed-off-by: Ce Gao <cegao@tensorchord.ai>

* chore: Add license for bash

Signed-off-by: Ce Gao <cegao@tensorchord.ai>

Signed-off-by: Ce Gao <cegao@tensorchord.ai>

.github/workflows/release.yml

@@ -116,7 +116,32 @@ jobs:
     - name: Docker Buildx
       run: |
           ./base-images/build.sh
-          DOCKER_IMAGE_TAG=latest ./base-images/build.sh
+  envd_image_push:
+    name: Build & push envd images
+    # only trigger on main repo when tag starts with v
+    if: github.repository == 'tensorchord/envd' && startsWith(github.ref, 'refs/tags/v')
+    runs-on: ubuntu-latest
+    needs: goreleaser
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v2
+    - name: Cache Docker layers
+      uses: actions/cache@v3
+      id: cache
+      with:
+        path: /tmp/.buildx-cache
+        key: ${{ runner.os }}-buildx-${{ github.sha }}
+        restore-keys: |
+          ${{ runner.os }}-buildx-
+    - name: Docker Login
+      uses: docker/login-action@v2
+      with:
+        username: ${{ secrets.DOCKERIO_USERNAME }}
+        password: ${{ secrets.DOCKERIO_TOKEN }}
+    - name: Docker Buildx
+      run: |
+          ./base-images/envd/build.sh
   cache_publish:
     name: Build & Push the remote cache
     # only trigger on main repo when tag starts with v

.goreleaser.yaml

@@ -68,10 +68,27 @@ changelog:
       - '^test:'
       - '^chore:'
 dockers:
+- image_templates:
+  - "tensorchord/envd-from-scratch:v{{ .Version }}-amd64"
+  use: buildx
+  dockerfile: base-images/envd/envd.Dockerfile
+  ids:
+  - envd
+  build_flag_templates:
+  - "--platform=linux/amd64"
+- image_templates:
+  - "tensorchord/envd-from-scratch:v{{ .Version }}-arm64v8"
+  use: buildx
+  goarch: arm64
+  ids:
+  - envd
+  dockerfile: base-images/envd/envd.Dockerfile
+  build_flag_templates:
+  - "--platform=linux/arm64/v8"
 - image_templates:
   - "tensorchord/envd-sshd-from-scratch:v{{ .Version }}-amd64"
   use: buildx
-  dockerfile: .goreleaser/envd-sshd.Dockerfile
+  dockerfile: base-images/envd-sshd/envd-sshd.Dockerfile
   ids:
   - envd-sshd
   build_flag_templates:
@@ -82,14 +99,18 @@ dockers:
   goarch: arm64
   ids:
   - envd-sshd
-  dockerfile: .goreleaser/envd-sshd.Dockerfile
+  dockerfile: base-images/envd-sshd/envd-sshd.Dockerfile
   build_flag_templates:
   - "--platform=linux/arm64/v8"
 docker_manifests:
 - name_template: tensorchord/envd-sshd-from-scratch:v{{ .Version }}
   image_templates:
   - tensorchord/envd-sshd-from-scratch:v{{ .Version }}-amd64
   - tensorchord/envd-sshd-from-scratch:v{{ .Version }}-arm64v8
+- name_template: tensorchord/envd-from-scratch:v{{ .Version }}
+  image_templates:
+  - tensorchord/envd-from-scratch:v{{ .Version }}-amd64
+  - tensorchord/envd-from-scratch:v{{ .Version }}-arm64v8
 # See https://github.com/tensorchord/envd/issues/908
 # brews:
 #   - name: envd

base-images/build.sh

@@ -1,4 +1,18 @@
 #!/usr/bin/env bash
+# Copyright 2022 The envd Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 
 set -euo pipefail
 

base-images/envd/buid.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+# Copyright 2022 The envd Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+set -euo pipefail
+
+ROOT_DIR=`dirname $0`
+
+GIT_TAG_VERSION=$(git describe --tags --abbrev=0 | sed -r 's/[v]+//g') # remove v from version
+ENVD_VERSION="${ENVD_VERSION:-$GIT_TAG_VERSION}"
+DOCKER_HUB_ORG="${DOCKER_HUB_ORG:-tensorchord}"
+TAG_SUFFIX="${TAG_SUFFIX:-}"
+
+cd ${ROOT_DIR}
+
+docker buildx build \
+    --build-arg ENVD_VERSION=${ENVD_VERSION} \
+    -t ${DOCKER_HUB_ORG}/envd:${ENVD_VERSION} \
+    --pull --push --platform linux/x86_64,linux/arm64 \
+    -f envd-daemonless.Dockerfile ../../
+
+cd - > /dev/null

base-images/envd/envd-daemonless.Dockerfile

@@ -0,0 +1,7 @@
+FROM tensorchord/envd:latest as envd
+
+FROM moby/buildkit:v0.10.5-rootless
+COPY --from=envd /usr/bin/envd /usr/bin/envd
+COPY scripts/envd-daemonless.sh /envd-daemonless.sh
+
+CMD [ "/envd-daemonless.sh" ]

base-images/envd/envd.Dockerfile

@@ -0,0 +1,2 @@
+FROM scratch
+COPY envd /usr/bin/envd

base-images/remote-cache/build-and-push-remote-cache.sh

@@ -1,4 +1,18 @@
 #!/usr/bin/env bash
+# Copyright 2022 The envd Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 
 set -euo pipefail
 

pkg/app/context_create.go

@@ -35,7 +35,7 @@ var CommandContextCreate = &cli.Command{
 		},
 		&cli.StringFlag{
 			Name:  "builder",
-			Usage: "Builder to use (docker-container, kube-pod, tcp)",
+			Usage: "Builder to use (docker-container, kube-pod, tcp, unix)",
 			Value: string(types.BuilderTypeDocker),
 		},
 		&cli.StringFlag{

pkg/home/context.go

@@ -95,7 +95,8 @@ func (m *generalManager) ContextCreate(ctx types.Context, use bool) error {
 		}
 	}
 	switch ctx.Builder {
-	case types.BuilderTypeDocker, types.BuilderTypeKubernetes, types.BuilderTypeTCP:
+	case types.BuilderTypeDocker, types.BuilderTypeKubernetes,
+		types.BuilderTypeUNIXDomainSocket, types.BuilderTypeTCP:
 		break
 	default:
 		return errors.New("unknown builder type")

pkg/types/envd.go

@@ -127,9 +127,10 @@ type Context struct {
 type BuilderType string
 
 const (
-	BuilderTypeDocker     BuilderType = "docker-container"
-	BuilderTypeKubernetes BuilderType = "kube-pod"
-	BuilderTypeTCP        BuilderType = "tcp"
+	BuilderTypeDocker           BuilderType = "docker-container"
+	BuilderTypeKubernetes       BuilderType = "kube-pod"
+	BuilderTypeTCP              BuilderType = "tcp"
+	BuilderTypeUNIXDomainSocket BuilderType = "unix"
 )
 
 type RunnerType string

scripts/envd-daemonless.sh

@@ -0,0 +1,77 @@
+#!/bin/sh
+
+# Copyright 2022 The envd Authors
+# Copyright 2022 The buildkit Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# envd-daemonless.sh spawns ephemeral buildkitd for executing envd.
+#
+# Usage: envd-daemonless.sh build ...
+#
+# Flags for buildkitd can be specified as $BUILDKITD_FLAGS .
+#
+# The script is compatible with BusyBox shell.
+set -eu
+
+: ${ENVDCTL=envd}
+: ${BUILDCTL=buildctl}
+: ${BUILDCTL_CONNECT_RETRIES_MAX=10}
+: ${BUILDKITD=buildkitd}
+: ${BUILDKITD_FLAGS=}
+: ${ROOTLESSKIT=rootlesskit}
+
+# $tmp holds the following files:
+# * pid
+# * addr
+# * log
+tmp=$(mktemp -d /tmp/envd-daemonless.XXXXXX)
+trap "kill \$(cat $tmp/pid) || true; wait \$(cat $tmp/pid) || true; rm -rf $tmp" EXIT
+
+startBuildkitd() {
+    addr=
+    helper=
+    if [ $(id -u) = 0 ]; then
+        addr=/run/buildkit/buildkitd.sock
+    else
+        addr=$XDG_RUNTIME_DIR/buildkit/buildkitd.sock
+        helper=$ROOTLESSKIT
+    fi
+    $helper $BUILDKITD $BUILDKITD_FLAGS --addr=unix://$addr >$tmp/log 2>&1 &
+    pid=$!
+    echo $pid >$tmp/pid
+    echo $addr >$tmp/addr
+}
+
+# buildkitd supports NOTIFY_SOCKET but as far as we know, there is no easy way
+# to wait for NOTIFY_SOCKET activation using busybox-builtin commands...
+waitForBuildkitd() {
+    addr=unix://$(cat $tmp/addr)
+    try=0
+    max=$BUILDCTL_CONNECT_RETRIES_MAX
+    until $BUILDCTL --addr=$addr debug workers >/dev/null 2>&1; do
+        if [ $try -gt $max ]; then
+            echo >&2 "could not connect to $addr after $max trials"
+            echo >&2 "========== log =========="
+            cat >&2 $tmp/log
+            exit 1
+        fi
+        sleep $(awk "BEGIN{print (100 + $try * 20) * 0.001}")
+        try=$(expr $try + 1)
+    done
+}
+
+startBuildkitd
+waitForBuildkitd
+$ENVDCTL context create --name daemonless --builder unix --builder-address $(cat $tmp/addr) --use
+$ENVDCTL "$@"