.gitlab-ci.yml

image: aistore/ci:1.17

stages:
  - build
  - test-short
  - test-long

variables:
  MODE: debug # run tests with debug asserts
  NUM_TARGET:
    value: "5"
    description: "Number of targets deployed."
  NUM_PROXY:
    value: "5"
    description: "Number of proxies deployed."
  FS_CNT:
    value: "6"
    description: "Number of disks defined for each target."
  AWS_REGION:
    value: "us-east-2"
    description: "Region of AWS bucket."
  CLOUD_BCKS:
    value: "aws://ais-blossom gs://ais-ci-kube"
    description: "Set of buckets used when running cloud tests."


# Templates

.gather_logs_template: &gather_logs_def
  after_script:
    - make kill # To make sure that nodes flushed the logs.
    - mkdir $CI_PROJECT_DIR/logs
    - find /tmp/ais -type f -name "*log*" -exec cp {} $CI_PROJECT_DIR/logs/ \;
  artifacts:
    when: on_failure
    paths: [ logs/ ]
    expire_in: 1 days

.k8s_long_exit_template: &k8s_long_exit_def
  after_script:
    - make kill
    - mkdir $CI_PROJECT_DIR/logs
    - find /tmp/ais -type f -name "*log*" -exec cp {} $CI_PROJECT_DIR/logs/ \;
  artifacts:
    when: on_failure
    paths: [ logs/ ]
    expire_in: 1 days

.default_only_template: &default_only_def
  only:
    - master
    - merge_requests
    - schedules
    - webs

.test_short_template: &test_short_def
  stage: test-short
  tags:
    - ais
  timeout: 21m
  <<: *default_only_def
  except:
    variables:
      - $CI_MERGE_REQUEST_LABELS =~ /.*skip-ci.*/
  <<: *gather_logs_def

.test_short_optional_template: &test_short_optional_def
  stage: test-short
  tags:
    - ais
  timeout: 20m
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_BRANCH == "master" || $CI_PIPELINE_SOURCE == "web"'
      when: manual
      allow_failure: true
  <<: *gather_logs_def

.test_long_template: &test_long_def
  stage: test-long
  tags:
    - ais
  timeout: 2h30m
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_BRANCH == "master" || $CI_PIPELINE_SOURCE == "web"'
      when: manual
      allow_failure: true
  <<: *gather_logs_def

# Stages

build:linux:
  stage: build
  tags:
    - ais
  timeout: 5m
  <<: *default_only_def
  script:
    - MODE="" make node # Build node without backends in production mode.
    - MODE="debug" make node # Build node without backends in debug mode.
    - AIS_BACKEND_PROVIDERS="ais aws azure gcp hdfs" MODE="" make node # Build with all backends (production mode).
    - AIS_BACKEND_PROVIDERS="ais aws azure gcp hdfs" MODE="debug" make node # Build with all backends (debug mode).
    - MEM_PROFILE="/tmp/mem" CPU_PROFILE="/tmp/cpu" make node # Build with profile.
    - TAGS="nethttp" make node # Build with net/http transport support (fasthttp is used by default).
    - TAGS="s3rproxy" make node # Build with reverse proxy support (redirect is used by default).
    - make authn
    - make aisfs
    - make cli
    - make aisloader

lint:linux:
  stage: build
  tags:
    - ais
  timeout: 5m
  <<: *default_only_def
  script:
    - make lint
    - make fmt-check
    - make spell-check

# Runs cluster with 5 proxies and 5 targets (each with 6 mountpaths).
test:short:
  <<: *test_short_def
  variables:
    BUCKET: "ais://test"
  script:
    - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --deployment all
    - make test-short
    - FLAGS="--duration=10s" make test-aisloader

# Runs cluster with 1 proxy and 1 target (with 6 mountpaths).
test:short:minimal:
  <<: *test_short_def
  variables:
    BUCKET: "ais://test"
  script:
    - deploy/scripts/clean_deploy.sh --target-cnt 1 --proxy-cnt 1 --mountpath-cnt $FS_CNT --deployment all
    - NUM_PROXY=1 NUM_TARGET=1 make test-short

test:short:authn:
  <<: *test_short_optional_def
  variables:
    AUTH_ENABLED: "true"
    AUTHN_SU_NAME: "admin"
    AUTHN_SU_PASS: "admin"
    AUTHN_URL: "http://localhost:52001"
    BUCKET: "ais://test"
    RE: "TestAuth"
  script:
    - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT
    - ais auth login $AUTHN_SU_NAME -p $AUTHN_SU_PASS
    - make test-run
    - ais auth logout

test:short:https:
  <<: *test_short_optional_def
  variables:
    AIS_USE_HTTPS: "true"
    AIS_SKIP_VERIFY_CRT: "true"
    AIS_SERVER_CRT: "$CI_PROJECT_DIR/localhost.crt"
    AIS_SERVER_KEY: "$CI_PROJECT_DIR/localhost.key"
    AIS_ENDPOINT: "https://localhost:8080"
    BUCKET: "ais://ais-ci"
  script:
    - openssl req -x509 -out $AIS_SERVER_CRT -keyout $AIS_SERVER_KEY -newkey rsa:2048 -nodes -sha256 -subj '/CN=localhost' -extensions EXT -config <( printf "[dn]\nCN=localhost\n[req]\ndistinguished_name = dn\n[EXT]\nsubjectAltName=DNS:localhost\nkeyUsage=digitalSignature\nextendedKeyUsage=serverAuth")
    - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --deployment all --https
    - make test-short

test:short:s3rproxy:
  <<: *test_short_optional_def
  variables:
    BUCKET: "ais://ais-ci"
    TAGS: "s3rproxy"
    RE: "S3"
  script:
    - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT
    - make test-short

test:long:
  <<: *test_long_def
  variables:
    NUM_PROXY: 6
    BUCKET: "ais://ais-ci"
  script:
    - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT
    - make test-long

test:long:aws:
  <<: *test_long_def
  variables:
    NUM_PROXY: 6
    BUCKET: "aws://ais-cicd"
  script:
    - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --aws
    - make test-long

test:long:gcp:
  <<: *test_long_def
  variables:
    NUM_PROXY: 6
    GOOGLE_APPLICATION_CREDENTIALS: "/tmp/gcs.json"
    BUCKET: "gs://ais-ci"
  script:
    - echo "${GOOGLE_APPLICATION_CREDENTIALS_JSON}" > "${GOOGLE_APPLICATION_CREDENTIALS}"
    - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --gcp
    - make test-long

test:long:hdfs:
  image: aistore/ci:4.0-hdfs
  <<: *test_long_def
  variables:
    NUM_PROXY: 6
    BUCKET: "hdfs://ais-ci"
  script:
    - bash deploy/test/ci/setup_hdfs.sh
    - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --hdfs
    - ais bucket create "${BUCKET}" --bucket-props="extra.hdfs.ref_directory=/"
    - make test-long

test:long:aisloader:
  stage: test-long
  tags:
    - ais
  timeout: 10m
  only:
    - schedules
  script:
    - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT
    - sleep 10 # make sure that cluster properly starts
    - FLAGS="--duration=5m" make test-aisloader

# Kubernetes stages

.test_k8s_template: &test_k8s_def
  tags:
    - ais-k8s
  variables:
    NUM_TARGET: 1
    NUM_PROXY: 1
    GOOGLE_APPLICATION_CREDENTIALS: "/tmp/gcs.json"
    BUCKET: "gs://ais-ci-kube"
    RE: "TestETL|TestConfig|TestMountpath"
    TESTS_DIR: "ais/tests"
  before_script:
    - kubectl delete pods,services -l nvidia.com/ais-etl-name # TODO: this can be removed once the lifecycle of transformers is implemented.
    # Make sure that metrics collection is enabled.
    - git clone https://github.com/prometheus-operator/kube-prometheus.git
    - kubectl apply -f kube-prometheus/manifests/setup && kubectl apply -f kube-prometheus/manifests && rm -rf kube-prometheus

test:short:k8s:
  stage: test-short
  <<: *test_k8s_def
  timeout: 30m
  only:
    - merge_requests
  except:
    variables:
      - $CI_MERGE_REQUEST_LABELS =~ /.*skip-ci.*/
  script:
    - export NUM_TARGET=3
    - echo "${GOOGLE_APPLICATION_CREDENTIALS_JSON}" > "${GOOGLE_APPLICATION_CREDENTIALS}"
    - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --gcp
    - make test-short
    - status=$? make kill && exit $status

test:long:k8s:
  stage: test-long
  <<: *test_k8s_def
  timeout: 2h
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
    - if: '$CI_MERGE_REQUEST_LABELS =~ /.*k8s-ci.*/'
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_BRANCH == "master" || $CI_PIPELINE_SOURCE == "web"'
      when: manual
      allow_failure: true
  script:
    - export NUM_TARGET=5
    - echo "${GOOGLE_APPLICATION_CREDENTIALS_JSON}" > "${GOOGLE_APPLICATION_CREDENTIALS}"
    - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --gcp
    - make test-run
    - status=$? make kill && exit $status
  <<: *k8s_long_exit_def

test:long:k8s:single-target:
  stage: test-long
  <<: *test_k8s_def
  timeout: 2h
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
    - if: '$CI_MERGE_REQUEST_LABELS =~ /.*k8s-ci.*/'
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_BRANCH == "master" || $CI_PIPELINE_SOURCE == "web"'
      when: manual
      allow_failure: true
  script:
    - echo "${GOOGLE_APPLICATION_CREDENTIALS_JSON}" > "${GOOGLE_APPLICATION_CREDENTIALS}"
    - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --gcp
    - status=0
    - make test-run
    - status=$? make kill && exit $status
  <<: *k8s_long_exit_def

test:long:k8s:aisloader:
  stage: test-long
  <<: *test_k8s_def
  timeout: 15m
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
    - if: '$CI_MERGE_REQUEST_LABELS =~ /.*k8s-ci.*/'
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_BRANCH == "master" || $CI_PIPELINE_SOURCE == "web"'
      when: manual
      allow_failure: true
  script:
    - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT
    - sleep 10 # Give some time for the cluster to stabilize.
    - BUCKET="ais://test" FLAGS="--duration=2m --etl" make test-aisloader
    - status=$? make kill && exit $status

test:long:k8s:all:
  stage: test-long
  tags:
    - ais-k8s
  variables:
    GOOGLE_APPLICATION_CREDENTIALS: "/tmp/gcs.json"
  timeout: 5h
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
    - if: '$CI_MERGE_REQUEST_LABELS =~ /.*k8s-ci.*/'
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_BRANCH == "master" || $CI_PIPELINE_SOURCE == "web"'
      when: manual
      allow_failure: true
  before_script:
    - kubectl delete pods,services -l nvidia.com/ais-etl-name # TODO: this can be removed once the lifecycle of transformers is implemented.
    # Make sure that metrics collection is enabled.
    - git clone https://github.com/prometheus-operator/kube-prometheus.git
    - kubectl apply -f kube-prometheus/manifests/setup && kubectl apply -f kube-prometheus/manifests && rm -rf kube-prometheus
  script:
    - echo "${GOOGLE_APPLICATION_CREDENTIALS_JSON}" > "${GOOGLE_APPLICATION_CREDENTIALS}"
    - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --gcp --aws
    - |
      echo "----- RUNNING K8S TESTS -----"
      BUCKET="aws://ais-blossom" RE="TestETL|TestConfig|TestMountpath" make test-run
      exit_code=$?
      result=$((result + exit_code))
      echo "----- K8S TESTS FINISHED WITH: ${exit_code} -----"
    - |
      for bucket in ${CLOUD_BCKS}; do
        echo "----- RUNNING LONG TESTS WITH: ${bucket} -----"
        BUCKET=${bucket} make test-long && make test-aisloader
        exit_code=$?
        result=$((result + exit_code))
        echo "----- LONG TESTS FINISHED WITH: ${exit_code} -----"
      done
    - make kill && exit $result
  <<: *k8s_long_exit_def