diff --git a/configs/config.toml b/configs/firecracker-containerd/config.toml similarity index 100% rename from configs/config.toml rename to configs/firecracker-containerd/config.toml diff --git a/configs/firecracker-runtime.json b/configs/firecracker-containerd/firecracker-runtime.json similarity index 100% rename from configs/firecracker-runtime.json rename to configs/firecracker-containerd/firecracker-runtime.json diff --git a/configs/istio-minimal-operator.yaml b/configs/istio/istio-minimal-operator.yaml similarity index 100% rename from configs/istio-minimal-operator.yaml rename to configs/istio/istio-minimal-operator.yaml diff --git a/configs/knative_workloads/helloworld.yaml b/configs/knative_workloads/helloworld.yaml new file mode 100644 index 000000000..70b16179a --- /dev/null +++ b/configs/knative_workloads/helloworld.yaml @@ -0,0 +1,18 @@ +apiVersion: serving.knative.dev/v1 +kind: Service +metadata: + name: helloworld # The name of the function instance + namespace: default +spec: + template: + spec: + containers: + - image: crccheck/hello-world:latest # Stub image. See https://github.com/ease-lab/vhive/issues/68 + ports: + - name: h2c # For GRPC support + containerPort: 50051 + env: + - name: GUEST_PORT # Port on which the firecracker-containerd container is accepting requests + value: "50051" + - name: GUEST_IMAGE # Container image to use for firecracker-containerd container + value: "ustiugov/helloworld:var_workload" diff --git a/configs/knative_workloads/helloworldSerial.yaml b/configs/knative_workloads/helloworldSerial.yaml new file mode 100644 index 000000000..9560bd80f --- /dev/null +++ b/configs/knative_workloads/helloworldSerial.yaml @@ -0,0 +1,19 @@ +apiVersion: serving.knative.dev/v1 +kind: Service +metadata: + name: helloworldserial # The name of the function instance + namespace: default +spec: + template: + spec: + containerConcurrency: 1 + containers: + - image: crccheck/hello-world:latest # Stub image. See https://github.com/ease-lab/vhive/issues/68 + ports: + - name: h2c # For GRPC support + containerPort: 50051 + env: + - name: GUEST_PORT # Port on which the firecracker-containerd container is accepting requests + value: "50051" + - name: GUEST_IMAGE # Container image to use for firecracker-containerd container + value: "ustiugov/helloworld:var_workload" diff --git a/configs/knative_workloads/pyaes.yaml b/configs/knative_workloads/pyaes.yaml new file mode 100644 index 000000000..081338445 --- /dev/null +++ b/configs/knative_workloads/pyaes.yaml @@ -0,0 +1,18 @@ +apiVersion: serving.knative.dev/v1 +kind: Service +metadata: + name: pyaes # The name of the function instance + namespace: default +spec: + template: + spec: + containers: + - image: crccheck/hello-world:latest # Stub image. See https://github.com/ease-lab/vhive/issues/68 + ports: + - name: h2c # For GRPC support + containerPort: 50051 + env: + - name: GUEST_PORT # Port on which the firecracker-containerd container is accepting requests + value: "50051" + - name: GUEST_IMAGE # Container image to use for firecracker-containerd container + value: "ustiugov/pyaes:var_workload" diff --git a/configs/knative_workloads/rnn_serving.yaml b/configs/knative_workloads/rnn_serving.yaml new file mode 100644 index 000000000..555be6bd9 --- /dev/null +++ b/configs/knative_workloads/rnn_serving.yaml @@ -0,0 +1,18 @@ +apiVersion: serving.knative.dev/v1 +kind: Service +metadata: + name: rnnserving # The name of the function instance + namespace: default +spec: + template: + spec: + containers: + - image: crccheck/hello-world:latest # Stub image. See https://github.com/ease-lab/vhive/issues/68 + ports: + - name: h2c # For GRPC support + containerPort: 50051 + env: + - name: GUEST_PORT # Port on which the firecracker-containerd container is accepting requests + value: "50051" + - name: GUEST_IMAGE # Container image to use for firecracker-containerd container + value: "ustiugov/rnn_serving:var_workload" diff --git a/knative_yamls/serving-core.yaml b/configs/knative_yamls/serving-core.yaml similarity index 100% rename from knative_yamls/serving-core.yaml rename to configs/knative_yamls/serving-core.yaml diff --git a/knative_yamls/serving-crds.yaml b/configs/knative_yamls/serving-crds.yaml similarity index 100% rename from knative_yamls/serving-crds.yaml rename to configs/knative_yamls/serving-crds.yaml diff --git a/knative_yamls/serving-default-domain.yaml b/configs/knative_yamls/serving-default-domain.yaml similarity index 100% rename from knative_yamls/serving-default-domain.yaml rename to configs/knative_yamls/serving-default-domain.yaml diff --git a/knative_yamls/serving-domainmapping-crds.yaml b/configs/knative_yamls/serving-domainmapping-crds.yaml similarity index 100% rename from knative_yamls/serving-domainmapping-crds.yaml rename to configs/knative_yamls/serving-domainmapping-crds.yaml diff --git a/knative_yamls/serving-domainmapping.yaml b/configs/knative_yamls/serving-domainmapping.yaml similarity index 100% rename from knative_yamls/serving-domainmapping.yaml rename to configs/knative_yamls/serving-domainmapping.yaml diff --git a/knative_yamls/serving-hpa.yaml b/configs/knative_yamls/serving-hpa.yaml similarity index 100% rename from knative_yamls/serving-hpa.yaml rename to configs/knative_yamls/serving-hpa.yaml diff --git a/knative_yamls/serving-nscert.yaml b/configs/knative_yamls/serving-nscert.yaml similarity index 100% rename from knative_yamls/serving-nscert.yaml rename to configs/knative_yamls/serving-nscert.yaml diff --git a/knative_yamls/serving-post-install-jobs.yaml b/configs/knative_yamls/serving-post-install-jobs.yaml similarity index 100% rename from knative_yamls/serving-post-install-jobs.yaml rename to configs/knative_yamls/serving-post-install-jobs.yaml diff --git a/knative_yamls/serving-storage-version-migration.yaml b/configs/knative_yamls/serving-storage-version-migration.yaml similarity index 100% rename from knative_yamls/serving-storage-version-migration.yaml rename to configs/knative_yamls/serving-storage-version-migration.yaml diff --git a/configs/metallb-configmap.yaml b/configs/metallb/metallb-configmap.yaml similarity index 100% rename from configs/metallb-configmap.yaml rename to configs/metallb/metallb-configmap.yaml diff --git a/cri/container_create.go b/cri/container_create.go index c00c7ef05..637d9980f 100644 --- a/cri/container_create.go +++ b/cri/container_create.go @@ -50,17 +50,17 @@ func (s *Service) CreateContainer(ctx context.Context, r *criapi.CreateContainer containerName := config.GetMetadata().GetName() if containerName == userContainerName { - return s.processUserContainer(ctx, r) + return s.createUserContainer(ctx, r) } if containerName == queueProxyName { - return s.processQueueProxy(ctx, r) + return s.createQueueProxy(ctx, r) } - // Containers irrelevant to user's workload + // Containers relevant for control plane return s.stockRuntimeClient.CreateContainer(ctx, r) } -func (s *Service) processUserContainer(ctx context.Context, r *criapi.CreateContainerRequest) (*criapi.CreateContainerResponse, error) { +func (s *Service) createUserContainer(ctx context.Context, r *criapi.CreateContainerRequest) (*criapi.CreateContainerResponse, error) { var ( stockResp *criapi.CreateContainerResponse stockErr error @@ -73,26 +73,26 @@ func (s *Service) processUserContainer(ctx context.Context, r *criapi.CreateCont }() config := r.GetConfig() - guestImage, guestPort, err := getGuestImagePort(config) + guestImage, err := getGuestImage(config) if err != nil { log.WithError(err).Error() return nil, err } - fi, err := s.coordinator.startVM(context.Background(), guestImage) + funcInst, err := s.coordinator.startVM(context.Background(), guestImage) if err != nil { log.WithError(err).Error("failed to start VM") return nil, err } - vmConfig := &VMConfig{guestIP: fi.startVMResponse.GuestIP, guestPort: guestPort} - s.insertPodConfig(r.GetPodSandboxId(), vmConfig) + vmConfig := &VMConfig{guestIP: funcInst.startVMResponse.GuestIP, guestPort: guestPortValue} + s.insertPodVMConfig(r.GetPodSandboxId(), vmConfig) // Wait for placeholder UC to be created <-stockDone containerdID := stockResp.ContainerId - err = s.coordinator.insertActive(containerdID, fi) + err = s.coordinator.insertActive(containerdID, funcInst) if err != nil { log.WithError(err).Error("failed to insert active VM") return nil, err @@ -101,16 +101,15 @@ func (s *Service) processUserContainer(ctx context.Context, r *criapi.CreateCont return stockResp, stockErr } -func (s *Service) processQueueProxy(ctx context.Context, r *criapi.CreateContainerRequest) (*criapi.CreateContainerResponse, error) { - vmConfig, err := s.getPodConfig(r.GetPodSandboxId()) +func (s *Service) createQueueProxy(ctx context.Context, r *criapi.CreateContainerRequest) (*criapi.CreateContainerResponse, error) { + vmConfig, err := s.getPodVMConfig(r.GetPodSandboxId()) if err != nil { log.WithError(err).Error() return nil, err } - s.removePodConfig(r.GetPodSandboxId()) + s.removePodVMConfig(r.GetPodSandboxId()) - // Add guest IP and port guestIPKeyVal := &criapi.KeyValue{Key: guestIPEnv, Value: vmConfig.guestIP} guestPortKeyVal := &criapi.KeyValue{Key: guestPortEnv, Value: vmConfig.guestPort} r.Config.Envs = append(r.Config.Envs, guestIPKeyVal, guestPortKeyVal) @@ -124,34 +123,15 @@ func (s *Service) processQueueProxy(ctx context.Context, r *criapi.CreateContain return resp, nil } -func getGuestImagePort(config *criapi.ContainerConfig) (string, string, error) { - var ( - image, port string - imageFound, portFound bool - ) - +func getGuestImage(config *criapi.ContainerConfig) (string, error) { envs := config.GetEnvs() for _, kv := range envs { if kv.GetKey() == guestImageEnv { - image = kv.GetValue() - imageFound = true - } else if kv.GetKey() == guestPortEnv { - port = kv.GetValue() - portFound = true + return kv.GetValue(), nil } - if imageFound && portFound { - break - } - } - // default value if port not provided - if port == "" { - port = guestPortValue } - if image == "" || port == "" { - return "", "", errors.New("failed to provide non empty guest image and port in user container config") - } + return "", errors.New("failed to provide non empty guest image in user container config") - return image, port, nil } diff --git a/cri/service.go b/cri/service.go index d96db6a11..e2ce219a6 100644 --- a/cri/service.go +++ b/cri/service.go @@ -135,21 +135,21 @@ func getDialOpts() []grpc.DialOption { } } -func (s *Service) insertPodConfig(podID string, vmConfig *VMConfig) { +func (s *Service) insertPodVMConfig(podID string, vmConfig *VMConfig) { s.Lock() defer s.Unlock() s.podVMConfigs[podID] = vmConfig } -func (s *Service) removePodConfig(podID string) { +func (s *Service) removePodVMConfig(podID string) { s.Lock() defer s.Unlock() delete(s.podVMConfigs, podID) } -func (s *Service) getPodConfig(podID string) (*VMConfig, error) { +func (s *Service) getPodVMConfig(podID string) (*VMConfig, error) { s.Lock() defer s.Unlock() diff --git a/knative_workloads/helloworld.yaml b/knative_workloads/helloworld.yaml deleted file mode 100644 index 2256efab0..000000000 --- a/knative_workloads/helloworld.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: serving.knative.dev/v1 # Current version of Knative -kind: Service -metadata: - name: helloworld # The name of the app - namespace: default # The namespace the app will use -spec: - template: - spec: - containers: - - image: crccheck/hello-world:latest # Reference to the image of the app - ports: - - name: h2c - containerPort: 50051 - env: - - name: GUEST_PORT # The environment variable printed out by the sample app - value: "50051" - - name: GUEST_IMAGE - value: "ustiugov/helloworld:var_workload" diff --git a/knative_workloads/helloworldSerial.yaml b/knative_workloads/helloworldSerial.yaml deleted file mode 100644 index 25af5fdb2..000000000 --- a/knative_workloads/helloworldSerial.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: serving.knative.dev/v1 # Current version of Knative -kind: Service -metadata: - name: helloworldserial # The name of the app - namespace: default # The namespace the app will use -spec: - template: - spec: - containerConcurrency: 1 - containers: - - image: plamenppetrov/uc-grpc # Reference to the image of the app - ports: - - name: h2c - containerPort: 8080 - env: - - name: GUESTPORT # The environment variable printed out by the sample app - value: "50051" - - name: GUESTIMAGE - value: "ustiugov/helloworld:var_workload" diff --git a/knative_workloads/pyaes.yaml b/knative_workloads/pyaes.yaml deleted file mode 100644 index 362c92f00..000000000 --- a/knative_workloads/pyaes.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: serving.knative.dev/v1 # Current version of Knative -kind: Service -metadata: - name: pyaes # The name of the app - namespace: default # The namespace the app will use -spec: - template: - spec: - containers: - - image: plamenppetrov/uc-grpc # Reference to the image of the app - ports: - - name: h2c - containerPort: 8080 - env: - - name: GUESTPORT # The environment variable printed out by the sample app - value: "50051" - - name: GUESTIMAGE - value: "ustiugov/pyaes:var_workload" diff --git a/knative_workloads/rnn_serving.yaml b/knative_workloads/rnn_serving.yaml deleted file mode 100644 index b38b52450..000000000 --- a/knative_workloads/rnn_serving.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: serving.knative.dev/v1 # Current version of Knative -kind: Service -metadata: - name: rnnserving # The name of the app - namespace: default # The namespace the app will use -spec: - template: - spec: - containers: - - image: plamenppetrov/uc-grpc # Reference to the image of the app - ports: - - name: h2c - containerPort: 8080 - env: - - name: GUESTPORT # The environment variable printed out by the sample app - value: "50051" - - name: GUESTIMAGE - value: "ustiugov/rnn_serving:var_workload" diff --git a/scripts/cluster/setup_master_node.sh b/scripts/cluster/setup_master_node.sh index 2739bc3dd..789f958e2 100755 --- a/scripts/cluster/setup_master_node.sh +++ b/scripts/cluster/setup_master_node.sh @@ -14,19 +14,19 @@ kubectl apply -f - -n kube-system kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.9.4/manifests/namespace.yaml kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.9.4/manifests/metallb.yaml kubectl create secret generic -n metallb-system memberlist --from-literal=secretkey="$(openssl rand -base64 128)" -kubectl apply -f $ROOT/configs/metallb-configmap.yaml +kubectl apply -f $ROOT/configs/metallb/metallb-configmap.yaml # istio curl -L https://istio.io/downloadIstio | ISTIO_VERSION=1.6.11 TARGET_ARCH=x86_64 sh - -./istio-1.6.11/bin/istioctl manifest apply -f $ROOT/configs/istio-minimal-operator.yaml +./istio-1.6.11/bin/istioctl manifest apply -f $ROOT/configs/istio/istio-minimal-operator.yaml # Install KNative in the cluster -kubectl apply --filename $ROOT/knative_yamls/serving-crds.yaml -kubectl apply --filename $ROOT/knative_yamls/serving-core.yaml +kubectl apply --filename $ROOT/configs/knative_yamls/serving-crds.yaml +kubectl apply --filename $ROOT/configs/knative_yamls/serving-core.yaml # magic DNS -kubectl apply --filename $ROOT/knative_yamls/serving-default-domain.yaml +kubectl apply --filename $ROOT/configs/knative_yamls/serving-default-domain.yaml kubectl apply --filename https://github.com/knative/net-istio/releases/download/v0.19.0/release.yaml kubectl --namespace istio-system get service istio-ingressgateway diff --git a/scripts/setup_containerd.sh b/scripts/setup_containerd.sh index c4920909b..7128caa29 100755 --- a/scripts/setup_containerd.sh +++ b/scripts/setup_containerd.sh @@ -5,7 +5,7 @@ set -e DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" ROOT="$( cd $DIR && cd .. && pwd)" BINS=$ROOT/bin -CONFIGS=$ROOT/configs +CONFIGS=$ROOT/configs/firecracker-containerd sudo mkdir -p /etc/firecracker-containerd sudo mkdir -p /var/lib/firecracker-containerd/runtime