style(image_builder): remove unused import

Substra · Jun 19, 2023 · 1de5b32 · 1de5b32
1 parent d039a9f
commit 1de5b32
Show file tree

Hide file tree

Showing 6 changed files with 58 additions and 57 deletions.
diff --git a/backend/substrapp/compute_tasks/image_builder.py b/backend/substrapp/compute_tasks/image_builder.py
@@ -14,7 +14,6 @@
 from substrapp.compute_tasks.datastore import Datastore
 from substrapp.compute_tasks.volumes import get_docker_cache_pvc_name
 from substrapp.compute_tasks.volumes import get_worker_subtuple_pvc_name
-from substrapp.docker_registry import USER_IMAGE_REPOSITORY
 from substrapp.docker_registry import container_image_exists
 from substrapp.kubernetes_utils import delete_pod
 from substrapp.kubernetes_utils import get_pod_logs
@@ -31,14 +30,14 @@
 REGISTRY = settings.REGISTRY
 REGISTRY_SCHEME = settings.REGISTRY_SCHEME
 NAMESPACE = settings.NAMESPACE
-KANIKO_MIRROR = settings.TASK["KANIKO_MIRROR"]
-KANIKO_IMAGE = settings.TASK["KANIKO_IMAGE"]
-KANIKO_DOCKER_CONFIG_SECRET_NAME = settings.TASK["KANIKO_DOCKER_CONFIG_SECRET_NAME"]
-KANIKO_DOCKER_CONFIG_VOLUME_NAME = "docker-config"
+BUILDER_MIRROR = settings.TASK["KANIKO_MIRROR"]
+BUILDER_IMAGE = settings.TASK["KANIKO_IMAGE"]
+BUILDER_DOCKER_CONFIG_SECRET_NAME = settings.TASK["KANIKO_DOCKER_CONFIG_SECRET_NAME"]
+BUILDER_DOCKER_CONFIG_VOLUME_NAME = "docker-config"
 CELERY_WORKER_CONCURRENCY = settings.CELERY_WORKER_CONCURRENCY
 SUBTUPLE_TMP_DIR = settings.SUBTUPLE_TMP_DIR
 MAX_IMAGE_BUILD_TIME = 3 * 60 * 60  # 3 hours
-KANIKO_CONTAINER_NAME = "kaniko"
+BUILDER_CONTAINER_NAME = "buildkit"
 HOSTNAME = settings.HOSTNAME
 
 
@@ -118,7 +117,7 @@ def _get_entrypoint_from_dockerfile(dockerfile_dir: str) -> list[str]:
 def _delete_kaniko_pod(create_pod: bool, k8s_client: kubernetes.client.CoreV1Api, pod_name: str) -> str:
     logs = ""
     if create_pod:
-        logs = get_pod_logs(k8s_client, pod_name, KANIKO_CONTAINER_NAME, ignore_pod_not_found=True)
+        logs = get_pod_logs(k8s_client, pod_name, BUILDER_CONTAINER_NAME, ignore_pod_not_found=True)
         delete_pod(k8s_client, pod_name)
         logger.info(logs or "", pod_name=pod_name)
     return logs
@@ -192,7 +191,7 @@ def _build_pod(dockerfile_mount_path: str, image_tag: str) -> kubernetes.client.
 
 def _build_pod_name(image_tag: str) -> str:
     dns_1123_compliant_tag = image_tag.split("/")[-1].replace("_", "-")
-    return f"kaniko-{dns_1123_compliant_tag}"
+    return f"{BUILDER_CONTAINER_NAME}-{dns_1123_compliant_tag}"
 
 
 def _build_pod_spec(dockerfile_mount_path: str, image_tag: str) -> kubernetes.client.V1PodSpec:
@@ -217,11 +216,11 @@ def _build_pod_spec(dockerfile_mount_path: str, image_tag: str) -> kubernetes.cl
 
     volumes = [cache, dockerfile]
 
-    if KANIKO_DOCKER_CONFIG_SECRET_NAME:
+    if BUILDER_DOCKER_CONFIG_SECRET_NAME:
         docker_config = kubernetes.client.V1Volume(
-            name=KANIKO_DOCKER_CONFIG_VOLUME_NAME,
+            name=BUILDER_DOCKER_CONFIG_VOLUME_NAME,
             secret=kubernetes.client.V1SecretVolumeSource(
-                secret_name=KANIKO_DOCKER_CONFIG_SECRET_NAME,
+                secret_name=BUILDER_DOCKER_CONFIG_SECRET_NAME,
                 items=[kubernetes.client.V1KeyToPath(key=".dockerconfigjson", path="config.json")],
             ),
         )
@@ -269,15 +268,15 @@ def _build_container(dockerfile_mount_path: str, image_tag: str) -> kubernetes.c
     cache = kubernetes.client.V1VolumeMount(name="cache", mount_path="/cache", read_only=True)
     volume_mounts = [dockerfile, cache]
 
-    if KANIKO_DOCKER_CONFIG_SECRET_NAME:
+    if BUILDER_DOCKER_CONFIG_SECRET_NAME:
         docker_config = kubernetes.client.V1VolumeMount(
-            name=KANIKO_DOCKER_CONFIG_VOLUME_NAME, mount_path="/kaniko/.docker"
+            name=BUILDER_DOCKER_CONFIG_VOLUME_NAME, mount_path="/kaniko/.docker"
         )
         volume_mounts.append(docker_config)
 
     return kubernetes.client.V1Container(
-        name=KANIKO_CONTAINER_NAME,
-        image=KANIKO_IMAGE,
+        name=BUILDER_CONTAINER_NAME,
+        image=BUILDER_IMAGE,
         command=None,
         args=args,
         volume_mounts=volume_mounts,
@@ -288,22 +287,17 @@ def _build_container(dockerfile_mount_path: str, image_tag: str) -> kubernetes.c
 def _build_container_args(dockerfile_mount_path: str, image_tag: str) -> list[str]:
     dockerfile_fullpath = os.path.join(dockerfile_mount_path, "Dockerfile")
     args = [
-        f"--dockerfile={dockerfile_fullpath}",
-        f"--context=dir://{dockerfile_mount_path}",
-        f"--destination={REGISTRY}/{USER_IMAGE_REPOSITORY}:{image_tag}",
-        "--cache=true",
-        "--log-timestamp=true",
-        "--snapshotMode=redo",
-        "--push-retry=3",
-        "--cache-copy-layers",
-        "--log-format=text",
-        f"--verbosity={('debug' if settings.LOG_LEVEL == 'DEBUG' else 'info')}",
+        "/usr/bin/buildctl-deamonless.sh",
+        "build",
+        "--frontend dockerfile.v0",
+        f"--local context=dir://{dockerfile_mount_path}",
+        f"--local dockerfile={dockerfile_fullpath}",
     ]
 
     if REGISTRY_SCHEME == "http":
         args.append("--insecure")
 
-    if KANIKO_MIRROR:
+    if BUILDER_MIRROR:
         args.append(f"--registry-mirror={REGISTRY}")
         if REGISTRY_SCHEME == "http":
             args.append("--insecure-pull")

diff --git a/charts/substra-backend/Chart.lock b/charts/substra-backend/Chart.lock
@@ -12,4 +12,4 @@ dependencies:
   repository: https://charts.bitnami.com/bitnami
   version: 11.6.5
 digest: sha256:b561368b783d0059338aded55c7649355a5f341d8d61e72bfcd63b757371e561
-generated: "2022-08-03T14:41:27.225387+02:00"
+generated: "2023-06-19T16:59:39.174321+02:00"
diff --git a/charts/substra-backend/templates/configmap-settings.yaml b/charts/substra-backend/templates/configmap-settings.yaml
@@ -16,8 +16,8 @@ data:
   COMPUTE_POD_RUN_AS_GROUP: {{ .Values.worker.computePod.securityContext.runAsGroup | quote }}
   COMPUTE_POD_FS_GROUP: {{ .Values.worker.computePod.securityContext.fsGroup | quote }}
 
-  KANIKO_IMAGE: {{ include "common.images.name" .Values.kaniko.image }}
-  KANIKO_MIRROR: {{ .Values.kaniko.mirror | quote }}
+  BUILDER_IMAGE: {{ include "common.images.name" .Values.builder.image }}
+  BUILDER_MIRROR: {{ .Values.builder.mirror | quote }}
 
   OBJECTSTORE_URL: {{ .Release.Name }}-minio:9000
 

diff --git a/charts/substra-backend/templates/deployment-registry-prepopulate.yaml b/charts/substra-backend/templates/deployment-registry-prepopulate.yaml
@@ -24,22 +24,29 @@ spec:
       - name: wait-registry
         image: jwilder/dockerize:0.6.1
         command: ['dockerize', '-wait', 'tcp://{{ $.Release.Name }}-docker-registry:5000']
-      - name: kaniko
-        image: {{ include "common.images.name" $.Values.kaniko.image }}
-        args: ["--context=/docker-context",
-                {{- if .dstImage }}
-                "--destination={{ $.Release.Name }}-docker-registry:5000/{{ .dstImage }}",
-                {{- else }}
-                "--destination={{ $.Release.Name }}-docker-registry:5000/{{ .image }}",
-                {{ end }}
-                "--insecure",
-                "--verbosity=debug"]
+      - name: {{ $.Values.builder.name }}
+        image: {{ include "common.images.name" $.Values.builder.image }}
+        command: ['/usr/bin/buildctl-deamonless.sh']
+        args: [
+          'build',
+          '--frontend dockerfile.v0',
+          '--local context=/docker-context',
+          '--local dockerfile=/docker-context',
+          {{- if .dstImage }}
+          '--output type=image,name={{ $.Release.Name }}-docker-registry:5000/{{ .dstImage }},push=true,registry.insecure=true',
+          {{- else }}
+          '--output type=image,name={{ $.Release.Name }}-docker-registry:5000/{{ .image }},push=true,registry.insecure=true',
+          {{ end }}
+        ]
+        env:
+          - name: BUILDKIT_FLAGS
+            value: --oce-worker-no-process-sandbox
         volumeMounts:
           - name: dockerfile
             mountPath: /docker-context
           {{- if .dockerConfigSecretName }}
           - name: docker-config
-            mountPath: /kaniko/.docker
+            mountPath: '{{ $.Values.builder.name }}/.docker'
           {{- end }}
       containers:
       - image: gcr.io/google-containers/pause:3.2

diff --git a/charts/substra-backend/templates/statefulset-worker.yaml b/charts/substra-backend/templates/statefulset-worker.yaml
@@ -79,22 +79,22 @@ spec:
           - mountPath: /tmp/certs/
             name: ssl-certs
       {{- end }}
-      {{- if .Values.kaniko.cache.warmer.cachedImages }}
-      - name: kaniko-cache-warmer
-        image: {{ include "common.images.name" .Values.kaniko.cache.warmer.image }}
+      {{- if .Values.builder.cache.warmer.cachedImages }}
+      - name: '{{ $.Values.builder.name }}-cache-warmer'
+        image: {{ include "common.images.name" .Values.builder.cache.warmer.image }}
         args:
         - "--cache-dir=/cache"
-        {{- range .Values.kaniko.cache.warmer.cachedImages }}
+        {{- range .Values.builder.cache.warmer.cachedImages }}
         - "--image={{ . }}"
         {{- end }}
         - "--verbosity=debug"
         volumeMounts:
           - name: docker-cache
             mountPath: /cache
             readOnly: False
-          {{- if .Values.kaniko.dockerConfigSecretName }}
+          {{- if .Values.builder.dockerConfigSecretName }}
           - name: docker-config
-            mountPath: /kaniko/.docker
+            mountPath: '{{ $.Values.builder.name }}/.docker'
           {{- end }}
       {{- end}}
       - name: wait-postgresql
@@ -153,8 +153,8 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: spec.nodeName
-            - name: KANIKO_DOCKER_CONFIG_SECRET_NAME
-              value: {{ .Values.kaniko.dockerConfigSecretName | quote  }}
+            - name: BUILDER_DOCKER_CONFIG_SECRET_NAME
+              value: {{ .Values.builder.dockerConfigSecretName | quote  }}
             - name: COMPUTE_POD_MAX_STARTUP_WAIT_SECONDS
               value: {{ .Values.worker.computePod.maxStartupWaitSeconds | quote }}
             - name: OBJECTSTORE_URL
@@ -208,10 +208,10 @@ spec:
           secretName: {{ .Values.orchestrator.tls.mtls.clientCertificate }}
       {{ end }}
       {{ end }}
-      {{- if .Values.kaniko.dockerConfigSecretName }}
+      {{- if .Values.builder.dockerConfigSecretName }}
       - name: docker-config
         secret:
-          secretName: {{ .Values.kaniko.dockerConfigSecretName }}
+          secretName: {{ .Values.builder.dockerConfigSecretName }}
           items:
           - key: .dockerconfigjson
             path: config.json
@@ -254,8 +254,8 @@ spec:
       name: docker-cache
     spec:
       accessModes: [ "ReadWriteOnce" ]
-      {{ include "common.storage.class" .Values.kaniko.cache.persistence }}
+      {{ include "common.storage.class" .Values.builder.cache.persistence }}
       resources:
         requests:
-          storage: {{ .Values.kaniko.cache.persistence.size }}
+          storage: {{ .Values.builder.cache.persistence.size }}
 {{- end }}
diff --git a/charts/substra-backend/values.yaml b/charts/substra-backend/values.yaml
@@ -589,15 +589,16 @@ orchestrator:
 
 ## @section Kaniko settings
 ##
-kaniko:
+builder:
   ## @param kaniko.image.registry Kaniko image registry
   ## @param kaniko.image.repository Kaniko image repository
   ## @param kaniko.image.tag Kaniko image tag
   ##
+  name: buildkit
   image:
-    registry: gcr.io
-    repository: kaniko-project/executor
-    tag: v1.8.1
+    # registry: gcr.io
+    repository: moby/buildkit
+    tag: master-rootless@sha256:cb63454a66f5bba35eba3926962fb5a18912aa3e57629f69c714a0cd303241ce
   ## @param kaniko.mirror If set to `true` pull base images from the local registry.
   ##
   mirror: false
@@ -609,7 +610,6 @@ kaniko:
       ## @param kaniko.cache.warmer.image.registry Kaniko cache warmer registry
       ## @param kaniko.cache.warmer.image.repository Kaniko cache warmer repository
       ## @param kaniko.cache.warmer.image.tag Kaniko cache warmer image tag
-      ##
       image:
         registry: gcr.io
         repository: kaniko-project/warmer