.gitlab-ci.yml

image: python:3.8

variables:
  DOCKER_HOST: tcp://docker:2375
  DOCKER_DRIVER: overlay2

  # Enable buildkit
  DOCKER_BUILDKIT: "1"
  COMPOSE_DOCKER_CLI_BUILD: "1"

  # Get submodules
  GIT_SUBMODULE_STRATEGY: recursive

test:
  stage: test
  services:
    - docker:20.10.5-dind
  before_script:
    - curl -Lo /tmp/docker.tgz https://download.docker.com/linux/static/stable/x86_64/docker-20.10.5.tgz && tar -xf /tmp/docker.tgz -C /usr/local && rm /tmp/docker.tgz && export PATH=/usr/local/docker:$PATH
    - docker info
    - curl -Lo /usr/local/bin/docker-compose "https://github.com/docker/compose/releases/download/1.28.5/docker-compose-$(uname -s)-$(uname -m)"
    - chmod +x /usr/local/bin/docker-compose
    # Important note about this: the Docker server is on a separate host,
    # so exposed ports are at 'docker' not 'localhost', and
    # Docker containers can't reach the local runner!
  script:
    - diff -u lib_core/datamart_core/types.py lib_profiler/datamart_profiler/types.py
    - diff -u lib_core/datamart_core/types.py lib_materialize/datamart_materialize/types.py

    - export PYTHONWARNINGS='default,ignore:`np.float` is a deprecated alias for the builtin `float`:DeprecationWarning:'

    # Set up environment for testing
    - cp tests/ci.env .env
    - . scripts/load_env.sh
    - export DATAMART_VERSION=v0.0
    - "sed -i 's/# CI: //' docker-compose.yml Dockerfile"
    - "sed -i '/# NOTCI$/d' docker-compose.yml Dockerfile"
    - "sed -i 's/127\\.0\\.0\\.1:\\([0-9]\\+\\):\\([0-9]\\+\\)/\\1:\\2/' docker-compose.yml"
    - mkdir cov
    - chown 998 cov

    # Build image, using the GitLab registry as a cache
    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN registry.gitlab.com
    - |
      export AUCTUS_IMAGE=$CI_REGISTRY_IMAGE/ci
      # Pull/update the image
      chmod 644 poetry.lock docker/install_deps.py
      touch -t 200001010000.00 poetry.lock docker/install_deps.py
      docker build -t auctus . \
        --cache-from=$AUCTUS_IMAGE \
        --build-arg BUILDKIT_INLINE_CACHE=1 \
        --build-arg version=$DATAMART_VERSION
      # Push the updated image to the registry (might be no-op)
      docker tag auctus $AUCTUS_IMAGE
      docker push $AUCTUS_IMAGE

    # Pull image not to be built
    - docker-compose pull rabbitmq

    # Bring services up
    - scripts/setup.sh
    - docker-compose up -d elasticsearch rabbitmq redis minio
    - |
      # Wait for Elasticsearch to come up
      slept=0; while [ $(curl -s -o /dev/null -w "%{http_code}" http://docker:8020/) != 200 ]; do
        if [ $slept -gt 120 ]; then
          echo "Elasticsearch didn't come up after ${slept}s"
          exit 1
        fi
        sleep 5; slept=$((slept + 5))
      done
      echo "Elasticsearch came up after ${slept}s"
    - docker-compose up -d cache-cleaner coordinator lazo
    - sleep 10
    - docker-compose up -d profiler apiserver apilb test-discoverer
    - |
      # Wait for profiling to end
      slept=30
      sleep 30
      while [ "$(curl -s http://docker:8012/metrics | sed -n '/^rabbitmq_queue_messages{.*queue="profile".* \([0-9]*\)$/s//\1/p')" != 0 ]; do
        sleep 5
        slept=$((slept + 5))
        if [ $slept -gt 240 ]; then
          echo "Profiling didn't end after ${slept}s"
          docker-compose logs profiler
          exit 1
        fi
      done
      echo "Profiling ended after ${slept}s"
    - docker-compose ps
    - docker-compose logs profiler

    # Run the tests in a container
    - |
      # Run tests
        if ! docker run --env-file tests/ci.env -v $(pwd)/tests:/usr/src/app/tests -v $(pwd)/docs:/usr/src/app/docs -v $(pwd)/cov:/cov -e COVERAGE_FILE=/cov/.coverage.testsuite -e "PYTHONWARNINGS=$PYTHONWARNINGS" --network auctus_default $AUCTUS_IMAGE python -m coverage run --branch tests/__main__.py; then docker-compose logs apiserver; docker-compose logs lazo; exit 1; fi
    - docker-compose logs apiserver
    - docker-compose logs lazo

    # Generate coverage report
    - docker-compose down -t 30
    - ls -lA cov/
    - cp docker/coveragerc .coveragerc
    - docker run -v $(pwd)/tests:/usr/src/app/tests -v $(pwd)/cov:/cov -e COVERAGE_FILE=/cov/coverage $AUCTUS_IMAGE sh -c "coverage combine -a /cov/.coverage* && coverage html --show-contexts -d /cov/htmlcov"
  artifacts:
    paths:
      - cov/htmlcov
    expire_in: 1 week

python-style:
  stage: test
  before_script:
    - curl -sSL https://install.python-poetry.org | python3 - && /root/.local/bin/poetry config virtualenvs.create false
    - /root/.local/bin/poetry install
  script:
    - flake8 --ignore=E731,W504,W503,E501
    - |
      # Check READMEs
      find . -name README.rst | while read i; do
        python -m readme_renderer "$i" >/dev/null
      done

frontend:
  stage: test
  services:
    - docker:20.10.5-dind
  before_script:
    - curl -Lo /tmp/docker.tgz https://download.docker.com/linux/static/stable/x86_64/docker-20.10.5.tgz && tar -xf /tmp/docker.tgz -C /usr/local && rm /tmp/docker.tgz && export PATH=/usr/local/docker:$PATH
    - docker info
    # Important note about this: the Docker server is on a separate host,
    # so exposed ports are at 'docker' not 'localhost', and
    # Docker containers can't reach the local runner!
  script:
    # Build image, using the GitLab registry as a cache
    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN registry.gitlab.com
    - |
      export AUCTUS_NPM_IMAGE=$CI_REGISTRY_IMAGE/ci-frontend
      # Pull/update the npm image
      chmod 644 frontend/package.json frontend/package-lock.json
      touch -t 200001010000.00 frontend/package.json frontend/package-lock.json
      docker build -t $AUCTUS_NPM_IMAGE . \
        --cache-from=$AUCTUS_NPM_IMAGE \
        --build-arg BUILDKIT_INLINE_CACHE=1 \
        -f frontend/Dockerfile \
        --target=build
      # Push the updated image to the registry (might be no-op)
      docker push $AUCTUS_NPM_IMAGE

    # Run the frontend tests
    - docker run --rm $AUCTUS_NPM_IMAGE sh -c "CI=true npm run test"

k8s_configs:
  stage: test
  before_script:
    - curl -Lo /tmp/jsonnet.tar.gz https://github.com/google/jsonnet/releases/download/v0.17.0/jsonnet-bin-v0.17.0-linux.tar.gz && tar -xf /tmp/jsonnet.tar.gz -C /usr/local/bin && rm /tmp/jsonnet.tar.gz
    - curl -Lo /tmp/kubeconform.tar.gz https://github.com/yannh/kubeconform/releases/download/v0.4.7/kubeconform-linux-amd64.tar.gz && tar -xf /tmp/kubeconform.tar.gz -C /usr/local/bin && rm /tmp/kubeconform.tar.gz
  script:
    - mkdir contrib/k8s/yaml contrib/k8s/yaml/discovery
    - (cd contrib/k8s; make)
    - kubeconform --kubernetes-version 1.20.7 --strict --summary contrib/k8s/yaml/*.yml
    - find contrib/k8s -name yaml -prune -o \( -type f -not -name README.md -not -name Makefile -exec jsonnetfmt --test {} + \)

pages-test:
  stage: test
  before_script:
    - curl -sSL https://install.python-poetry.org | python3 - && /root/.local/bin/poetry config virtualenvs.create false
  script:
    - /root/.local/bin/poetry install
    - (cd docs/ && make html)
    - cp -r docs/_build/html pages

    - mkdir pages/rest
    - curl -Lo pages/rest/redoc.js https://cdn.jsdelivr.net/npm/redoc@2.0.0-rc.45/bundles/redoc.standalone.js
    - cp -r docs/schemas pages/schemas
    - cp docs/redoc/index.html pages/rest/index.html
  artifacts:
    paths:
      - pages
  except:
    - master

pages:
  stage: deploy
  before_script:
    - curl -sSL https://install.python-poetry.org | python3 - && /root/.local/bin/poetry config virtualenvs.create false
  script:
    - /root/.local/bin/poetry install
    - (cd docs/ && make html)
    - cp -r docs/_build/html public

    - mkdir public/rest
    - curl -Lo public/rest/redoc.js https://cdn.jsdelivr.net/npm/redoc@2.0.0-rc.45/bundles/redoc.standalone.js
    - cp -r docs/schemas public/schemas
    - cp docs/redoc/index.html public/rest/index.html
  artifacts:
    paths:
      - public
  only:
    - master