.gitlab-ci.yml

image:
  name: alpine/git:v2.32.0
  entrypoint: ["/bin/sh", "-c"]

variables:
  ONEML_PATH: sertiscorp/mle/edge/oneml
  ONEML_REF: v4.5.0  # a tag or branch name to get oneML artifacts from
  ONEML_COMMIT_HASH: ""  # needed only if ONEML_REF is not a tag
  BOOTCAMP_TAG: v0.11.0  # set manually to the latest or new oneML-bootcamp tag
  RELEASE_BODY: ""  # generated automatically
  PRERELEASE: "true"  # set automatically
  BOOTCAMP_REF: develop  # set automatically

stages:
  - setup
  - release
  - docker-base
  - docker
  - env-check
  - build
  - pages

#-------------- Scripts --------------

# A simple script to authenticate git with SSH
.git_auth: &git_auth
  - eval `ssh-agent -s`
  - echo "${SSH_PRIVATE_KEY}" | tr -d '\r' | ssh-add - > /dev/null
  - mkdir -p ~/.ssh && chmod 700 ~/.ssh
  - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
  - git config --global user.name  "oneML-bot"
  - git config --global user.email "oneml@sertiscorp.com"

# A script to asses if a new docker image is necessary for a new branch or not
.docker_check: &docker_check # DOCKER_FILES
  - >
    if [ -z "${CI_COMMIT_TAG}" ] && [ "$CI_COMMIT_REF_NAME" != "develop" ];
    then
      COUNT=0
      git fetch
      for DOCKER_FILE in ${DOCKER_FILES//,/ };
      do
        COUNT=$((COUNT+$(git diff origin/$CI_COMMIT_REF_NAME origin/develop -- ${DOCKER_FILE} | wc -c)));
      done
      if [ ${COUNT} -eq 0 ]; then
        echo "Using develop docker image..."
        exit 0;
      fi
    fi

# A script to identify the right docker image to use for each job
.env_check: &env_check # IMAGE
  - mkdir -p ~/.docker
  - 'echo ''{"experimental": "enabled"}'' > ~/.docker/config.json'
  - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
  - docker manifest inspect $IMAGE && echo "BUILD_IMAGE=$IMAGE" >> env_setup.env || echo "BUILD_IMAGE=$DEVELOP_IMAGE" >> env_setup.env


#-------------- Base CI --------------

.base_ci: # Base CI
  rules:
    - if: '$CI_COMMIT_BRANCH == "main" || $CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_MESSAGE =~ /Creating release/'
      when: never
    - when: on_success

.base_artifacts: # Base CI for pulling artifacts
  stage: setup
  rules:
    - if: '($CI_COMMIT_TAG || $CI_COMMIT_MESSAGE =~ /Creating release/) && $CI_PIPELINE_SOURCE != "merge_request_event"'
      when: always
    - when: never
  script:
    - echo "Pulling from oneML..."
  artifacts:
    paths:
      - build/

.base_docker: # Base CI for docker stage
  stage: docker-base
  image: docker:19.03.11-git
  variables:
    DOCKER_DRIVER: overlay2
    DOCKER_FILES: docker/Dockerfile
  rules:
    - if: '$CI_COMMIT_BRANCH == "main" || $CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_MESSAGE =~ /Creating release/'
      when: never
    - changes:
      - docker/Dockerfile
      - docker/Dockerfile.cpu
  tags:
    - build

.base_env_check: # Base CI env-check stage
  extends: .base_ci
  stage: env-check
  image: docker:19.03.11
  variables:
    IMAGE: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-cpu
    DEVELOP_IMAGE: $CI_REGISTRY_IMAGE:develop-cpu
  script:
    - *env_check
  artifacts:
    reports:
      dotenv: env_setup.env
  tags:
    - build

.base_build: # Base CI for build stage
  extends: .base_ci
  stage: build
  image: $BUILD_IMAGE
  rules:
    - if: '$CI_COMMIT_BRANCH == "main" || $CI_COMMIT_BRANCH =~ /^release\/+/ || $CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_MESSAGE =~ /Creating release/'
      when: never
    - when: on_success
  tags:
    - build
  needs:
    - cpu-env-check

.base_build_gpu: # Base CI for build stage
  extends: .base_build
  needs:
    - gpu-env-check

.base_release: # Base CI for release stage
  stage: release
  rules:
    - if: '($CI_COMMIT_TAG || $CI_COMMIT_MESSAGE =~ /Creating release/) && $CI_PIPELINE_SOURCE != "merge_request_event"'
      when: on_success
    - when: never
  tags:
    - build

#-------------- Jobs --------------

artifacts-pull-aarch64-linux-android:
  extends: .base_artifacts
  needs:
    - project: $ONEML_PATH
      job: build-bootcamp-aarch64-linux-android
      ref: $ONEML_REF
      artifacts: true

artifacts-pull-arm-linux-android:
  extends: .base_artifacts
  needs:
    - project: $ONEML_PATH
      job: build-bootcamp-arm-linux-android
      ref: $ONEML_REF
      artifacts: true

artifacts-pull-aarch64-linux-gnu:
  extends: .base_artifacts
  needs:
    - project: $ONEML_PATH
      job: build-bootcamp-aarch64-linux-gnu
      ref: $ONEML_REF
      artifacts: true

artifacts-pull-arm-linux-gnueabihf:
  extends: .base_artifacts
  needs:
    - project: $ONEML_PATH
      job: build-bootcamp-arm-linux-gnueabihf
      ref: $ONEML_REF
      artifacts: true

artifacts-pull-msvc-x64:
  extends: .base_artifacts
  needs:
    - project: $ONEML_PATH
      job: build-bootcamp-msvc-x64
      ref: $ONEML_REF
      artifacts: true

artifacts-pull-x86_64:
  extends: .base_artifacts
  needs:
    - project: $ONEML_PATH
      job: build-bootcamp-x86_64
      ref: $ONEML_REF
      artifacts: true

artifacts-pull-x86_64-cuda:
  extends: .base_artifacts
  needs:
    - project: $ONEML_PATH
      job: build-bootcamp-x86_64-cuda-tvm
      ref: $ONEML_REF
      artifacts: true

create-release:
  extends: .base_release
  before_script:
    - apk update && apk add --upgrade --no-cache curl jq
  script:
    - if [[ $ONEML_REF == v* ]]; then ONEML_POSTFIX=${ONEML_REF}; else ONEML_POSTFIX=${ONEML_COMMIT_HASH:0:7}; fi
    - >
      BINARIES="";
      BINARIES="${BINARIES} oneml-bootcamp-cpu-x86_64-$ONEML_POSTFIX.tar.gz";
      BINARIES="${BINARIES} oneml-bootcamp-cuda-x86_64-$ONEML_POSTFIX.tar.gz";
      BINARIES="${BINARIES} oneml-bootcamp-cpu-msvc-x64-$ONEML_POSTFIX.tar.gz";
      BINARIES="${BINARIES} oneml-bootcamp-android-aarch64-linux-android-$ONEML_POSTFIX.tar.gz";
      BINARIES="${BINARIES} oneml-bootcamp-android-arm-linux-android-$ONEML_POSTFIX.tar.gz";
      BINARIES="${BINARIES} oneml-bootcamp-cpu-aarch64-linux-gnu-$ONEML_POSTFIX.tar.gz";
      BINARIES="${BINARIES} oneml-bootcamp-cpu-arm-linux-gnueabihf-$ONEML_POSTFIX.tar.gz";
      export BINARIES=${BINARIES};
      LABELS="";
      LABELS="${LABELS} oneml-bootcamp-x86_64.tar.gz";
      LABELS="${LABELS} oneml-bootcamp-x86_64-cuda.tar.gz";
      LABELS="${LABELS} oneml-bootcamp-msvc-x64.tar.gz";
      LABELS="${LABELS} oneml-bootcamp-aarch64-linux-android.tar.gz";
      LABELS="${LABELS} oneml-bootcamp-arm-linux-android.tar.gz";
      LABELS="${LABELS} oneml-bootcamp-aarch64-linux-gnu.tar.gz";
      LABELS="${LABELS} oneml-bootcamp-arm-linux-gnueabihf.tar.gz";
      export LABELS=${LABELS};
    - >
      if [[ -n "$CI_COMMIT_TAG" ]]; then
        export RELEASE_BODY="$(sed '1,6d' CHANGELOG.md | awk -v n=2 '/## \[/{n--}; n > 0' | sed '$ d' | awk '{printf "%s\\n", $0}')";
        export PRERELEASE=false;
        export BOOTCAMP_REF=main;
        export BOOTCAMP_TAG=$CI_COMMIT_TAG;
      fi
    - sh .github/create_release.sh
  needs:
    - artifacts-pull-aarch64-linux-android
    - artifacts-pull-arm-linux-android
    - artifacts-pull-aarch64-linux-gnu
    - artifacts-pull-arm-linux-gnueabihf
    - artifacts-pull-msvc-x64
    - artifacts-pull-x86_64
    - artifacts-pull-x86_64-cuda

docker-build-cpu-base:
  extends: .base_docker
  script:
    - *git_auth
    - *docker_check
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
    - docker build -t $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-cpu-base -f docker/Dockerfile --build-arg base_image=ubuntu:20.04 .
    - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-cpu-base

docker-build-gpu-base:
  extends: .base_docker
  script:
    - *git_auth
    - *docker_check
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
    - docker build -t $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-gpu-base -f docker/Dockerfile --build-arg base_image=nvidia/cuda:11.4.1-cudnn8-runtime-ubuntu20.04 .
    - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-gpu-base
  rules:
    - if: '$CI_COMMIT_BRANCH == "main" || $CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_MESSAGE =~ /Creating release/'
      when: never
    - changes:
      - docker/Dockerfile
      - docker/Dockerfile.gpu

docker-build-cpu:
  extends: .base_docker
  stage: docker
  variables:
    DOCKER_DRIVER: overlay2
    DOCKER_FILES: docker/Dockerfile.cpu,docker/Dockerfile
    IMAGE: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-cpu-base
    DEVELOP_IMAGE: $CI_REGISTRY_IMAGE:develop-cpu-base
  script:
    - *git_auth
    - *docker_check
    - *env_check
    - set -a && source env_setup.env && set +a
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
    - docker pull $BUILD_IMAGE
    - docker build -t $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-cpu -f docker/Dockerfile.cpu --build-arg base_image=$BUILD_IMAGE .
    - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-cpu
  needs: ["docker-build-cpu-base"]

docker-build-gpu:
  extends: .base_docker
  stage: docker
  variables:
    DOCKER_DRIVER: overlay2
    DOCKER_FILES: docker/Dockerfile.gpu,docker/Dockerfile
    IMAGE: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-gpu-base
    DEVELOP_IMAGE: $CI_REGISTRY_IMAGE:develop-gpu-base
  rules:
    - if: '$CI_COMMIT_BRANCH == "main" || $CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_MESSAGE =~ /Creating release/'
      when: never
    - changes:
      - docker/Dockerfile.gpu
      - docker/Dockerfile
  script:
    - *git_auth
    - *docker_check
    - *env_check
    - set -a && source env_setup.env && set +a
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
    - docker pull $BUILD_IMAGE
    - docker build -t $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-gpu -f docker/Dockerfile.gpu --build-arg base_image=$BUILD_IMAGE .
    - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-gpu
  needs: ["docker-build-gpu-base"]

docker-build-android:
  extends: .base_docker
  stage: docker
  variables:
    DOCKER_DRIVER: overlay2
    DOCKER_FILES: docker/Dockerfile.android,docker/Dockerfile
    IMAGE: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-cpu-base
    DEVELOP_IMAGE: $CI_REGISTRY_IMAGE:develop-cpu-base
  rules:
    - if: '$CI_COMMIT_BRANCH == "main" || $CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_MESSAGE =~ /Creating release/'
      when: never
    - changes:
      - docker/Dockerfile.android
      - docker/Dockerfile
  script:
    - *git_auth
    - *docker_check
    - *env_check
    - set -a && source env_setup.env && set +a
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
    - docker pull $BUILD_IMAGE
    - docker build -t $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-android -f docker/Dockerfile.android --build-arg base_image=$BUILD_IMAGE .
    - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-android
  needs: ["docker-build-cpu-base"]

cpu-env-check:
  extends: .base_env_check

gpu-env-check:
  extends: .base_env_check
  variables:
    IMAGE: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-gpu
    DEVELOP_IMAGE: $CI_REGISTRY_IMAGE:develop-gpu

android-env-check:
  extends: .base_env_check
  variables:
    IMAGE: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-android
    DEVELOP_IMAGE: $CI_REGISTRY_IMAGE:develop-android

build-cpp-x86_64:
  extends: .base_build
  script:
    - ./build.sh -t x86_64 -cc --clean

build-cpp-x86_64-cuda:
  extends: .base_build_gpu
  script:
    - ./build.sh -t x86_64-cuda -cc --clean

build-cpp-aarch64-linux-gnu:
  extends: .base_build
  script:
    - ./build.sh -t aarch64-linux-gnu -cc --clean

build-cpp-arm-linux-gnueabihf:
  extends: .base_build
  script:
    - ./build.sh -t arm-linux-gnueabihf -cc --clean

build-cpp-aarch64-linux-android:
  extends: .base_build
  script:
    - ./build.sh -t aarch64-linux-android -cc --clean
  needs:
    - android-env-check

build-cpp-arm-linux-android:
  extends: .base_build
  script:
    - ./build.sh -t arm-linux-android -cc --clean
  needs:
    - android-env-check

build-python-x86-64:
  extends: .base_build
  script:
    - ./build.sh -t x86_64 -py --clean

build-python-x86-64-cuda:
  extends: .base_build_gpu
  script:
    - ./build.sh -t x86_64-cuda -py --clean

build-java-x86_64:
  extends: .base_build
  script:
    - ./build.sh -t x86_64 -jni --clean

build-java-x86_64-cuda:
  extends: .base_build_gpu
  script:
    - ./build.sh -t x86_64-cuda -jni --clean

build-java-aarch64-linux-android:
  extends: .base_build
  script:
    - ./build.sh -t aarch64-linux-android -jni --clean
  needs:
    - android-env-check

build-java-arm-linux-android:
  extends: .base_build
  script:
    - ./build.sh -t arm-linux-android -jni --clean
  needs:
    - android-env-check

build-golang-x86_64:
  extends: .base_build
  script:
    - ./build.sh -t x86_64 -go --clean

build-golang-x86_64-cuda:
  extends: .base_build_gpu
  script:
    - ./build.sh -t x86_64-cuda -go --clean

build-csharp-x86_64:
  extends: .base_build
  script:
    - ./build.sh -t x86_64 -cs --clean

build-csharp-x86_64-cuda:
  extends: .base_build_gpu
  script:
    - ./build.sh -t x86_64-cuda -cs --clean

update-github-pages:
  # NOTE: We publish the API doc to GitHub Pages by mirroring `gh-pages` branch from GitLab to GitHub.
  extends: .base_release
  stage: pages
  variables:
    TARGET_ARCH: x86_64
  script:
    - *git_auth
    - export BINARY_PATH=${PWD}/assets/binaries/${TARGET_ARCH}
    - export BASE_URL=https://github.com/sertiscorp/oneML-bootcamp/releases/download/${CI_COMMIT_TAG}/oneml-bootcamp-${TARGET_ARCH}.tar.gz
    - apk add --upgrade --no-cache curl perl && rm -rf /var/cache/apk/*
    - curl -L ${BASE_URL} > ${BINARY_PATH}/oneml-bootcamp-${TARGET_ARCH}.tar.gz
    - tar xzf ${BINARY_PATH}/oneml-bootcamp-${TARGET_ARCH}.tar.gz -C ${BINARY_PATH}/ --strip-components=1
    # Update `gh-pages` branch
    # Due to the fact that the `gh-pages` branch has its own root parent
    # different from the main branch, we better check out the `gh-pages` branch
    # by cloning a new repo for less complicated steps.
    - git clone -b gh-pages ${CI_REPOSITORY_URL} oneML-bootcamp-gh
    - cd oneML-bootcamp-gh
    - cp -r ${BINARY_PATH}/share/doc/oneML/sphinx/* .
    - cp -r ${BINARY_PATH}/share/doc/oneML/godoc .
    - COUNT=`git diff | wc -l`
    # Need some preprocessing before use it as a remote url
    - export CI_PUSH_REPO=`echo $CI_REPOSITORY_URL | perl -pe 's#.*@(.+?(\:\d+)?)/#git@\1:#'`
    # Make sure to include .nojekyll for bypassing Jekyll processing on GitHub Pages.
    # Otherwise, our `_static` folder cannot be served.
    - touch .nojekyll
    - >
      if [ ${COUNT} -ne 0 ]; then
        echo "Found changes in API docs";
        git add *;
        git commit -m "Update GitHub Pages";
        git remote set-url --push origin "${CI_PUSH_REPO}"
        git push origin gh-pages;
      else
        echo "Not found any changes API in docs. Skipped";
      fi
  rules:
    - if: '$CI_COMMIT_TAG && $CI_PIPELINE_SOURCE != "merge_request_event"'
      when: on_success
    - when: never