Update CI (#102)

* add basic check * apply autoflake * apply isort * apply black * add docker pipeline * maximize build space * use internal actions * make GA happy * update github action * add lock file * pass ci * fix typo * fix typo * check proxy * check proxy * add proxy * sync docker image everywhere * fix config error * add sync docker image in separate PR --------- Co-authored-by: Yi-01-ai <yi@01.ai>
01-ai · Nov 13, 2023 · 2cbbc97 · 2cbbc97
1 parent fba730f
commit 2cbbc97
Show file tree

Hide file tree

Showing 11 changed files with 2,966 additions and 53 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -15,30 +15,65 @@ concurrency:
 jobs:
   basic-check:
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.10"]
+    outputs:
+      tag: ${{ steps.read_version.outputs.value }}
     steps:
       - uses: actions/checkout@v3
-      # - uses: 01-ai/actions/lint@main
+      - name: Checkout Actions 
+        uses: actions/checkout@v3 
+        with: 
+          repository: 01-ai/actions
+          token: ${{ secrets.PAT_TO_CLONE_ACTIONS }} 
+          path: actions
+          ref: main  
+      - name: Basic check
+        uses: ./actions/lint
+      # https://github.com/actions/checkout/issues/692#issuecomment-1502203573
+      - name: Checkout Actions 
+        uses: actions/checkout@v3 
+        with: 
+          repository: 01-ai/actions
+          token: ${{ secrets.PAT_TO_CLONE_ACTIONS }} 
+          path: actions
+          ref: main  
+      # try to create a tag
+      - uses: SebRollen/toml-action@v1.0.2
+        id: read_version
+        with:
+          file: "pyproject.toml"
+          field: "tool.poetry.version"
+      - uses: rickstaa/action-create-tag@v1
+        id: "tag_create"
+        if: (github.ref_name == github.event.repository.default_branch) && !contains(steps.read_version.outputs.value, '-')
+        with:
+          tag: ${{ steps.read_version.outputs.value }}
 
-  # build-and-push:
-  #   needs: basic-check
-  #   runs-on:
-  #     - arc
-  #     - cpu
-  #   steps:
-  #     - name: Checkout
-  #       uses: actions/checkout@v3
-  #     - name: Set proxy
-  #       run: |
-  #         echo "http_proxy=$http_proxy" >> $GITHUB_ENV
-  #         echo "https_proxy=$https_proxy" >> $GITHUB_ENV
-  #     - uses: 01-ai/actions/build_docker_image@main
-  #       id: build-and-push
-  #       with:
-  #         registry: ${{ vars.VOLCES_REGISTRY }}
-  #         namespace: ${{ vars.VOLCES_NAMESPACE }}
-  #         tags: 'latest'
-  #         username: ${{ secrets.VOLCES_USER }}
-  #         password: ${{ secrets.VOLCES_TOKEN }}
+  build-and-push:
+    runs-on:
+      - public
+    needs: basic-check
+    outputs:
+      primary_tag: ${{ steps.build-and-push.outputs.primary_tag }}
+      matrix: ${{ steps.build-and-push.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Checkout Actions 
+        uses: actions/checkout@v3 
+        with: 
+          repository: 01-ai/actions
+          token: ${{ secrets.PAT_TO_CLONE_ACTIONS }} 
+          path: actions
+          ref: main
+      - name: Set proxy
+        run: |
+          echo "http_proxy=$http_proxy" >> $GITHUB_ENV
+          echo "https_proxy=$https_proxy" >> $GITHUB_ENV
+      - name: Build Docker Image
+        id: build-and-push
+        uses: ./actions/build_docker_image
+        with:
+          registry: ${{ secrets.DEFAULT_REGISTRY }}
+          namespace: ci
+          tags: ${{ needs.basic-check.outputs.tag }}
+          username: ${{ secrets.DEFAULT_REGISTRY_USER }}
+          password: ${{ secrets.DEFAULT_REGISTRY_PASSWORD }}
diff --git a/Dockerfile b/Dockerfile
@@ -1,20 +1,49 @@
 ARG REGISTRY="nvcr.io"
 ARG CUDA_VERSION="11.8.0"
+FROM mambaorg/micromamba:1.5.1 as micromamba
 FROM ${REGISTRY}/nvidia/cuda:${CUDA_VERSION}-devel-ubuntu22.04 as base
 
-ARG MAX_JOBS=8
-
+#####
+# Setup user & common tools
+#####
 RUN apt update \
-  && apt install -y python3-pip python3-packaging git ninja-build \
-  && pip3 install -U pip \
-  && ln -s /usr/bin/python3 /usr/bin/python
+  && apt install -y git ninja-build \
+  && rm -rf /var/lib/apt/lists/*
 
-WORKDIR /yi
+#####
+# Setup micromamba
+#####
+
+USER root
+
+ARG MAMBA_USER=yi
+ARG MAMBA_USER_ID=56789
+ARG MAMBA_USER_GID=56789
+ENV MAMBA_USER=$MAMBA_USER
+ENV MAMBA_ROOT_PREFIX="/opt/conda"
+ENV MAMBA_EXE="/bin/micromamba"
+ENV ENV_NAME=yi
 
-RUN pip3 install torch==2.0.1 \
-  && pip3 install flash-attn==2.3.3  --no-build-isolation
+COPY --from=micromamba "$MAMBA_EXE" "$MAMBA_EXE"
+COPY --from=micromamba /usr/local/bin/_activate_current_env.sh /usr/local/bin/_activate_current_env.sh
+COPY --from=micromamba /usr/local/bin/_dockerfile_shell.sh /usr/local/bin/_dockerfile_shell.sh
+COPY --from=micromamba /usr/local/bin/_entrypoint.sh /usr/local/bin/_entrypoint.sh
+COPY --from=micromamba /usr/local/bin/_dockerfile_initialize_user_accounts.sh /usr/local/bin/_dockerfile_initialize_user_accounts.sh
+COPY --from=micromamba /usr/local/bin/_dockerfile_setup_root_prefix.sh /usr/local/bin/_dockerfile_setup_root_prefix.sh
 
-COPY ./requirements.txt .
-RUN pip3 install -r requirements.txt
+RUN /usr/local/bin/_dockerfile_initialize_user_accounts.sh && \
+  /usr/local/bin/_dockerfile_setup_root_prefix.sh
+
+USER $MAMBA_USER
+SHELL ["/usr/local/bin/_dockerfile_shell.sh"]
+ENTRYPOINT ["/usr/local/bin/_entrypoint.sh"]
+CMD ["/bin/bash"]
+
+# Install dependencies
+
+WORKDIR /yi
+COPY ./conda-lock.yml .
+RUN micromamba create -y -n ${ENV_NAME} -f conda-lock.yml && \
+  micromamba clean --all --yes
 
-COPY . .
+COPY . .