Merge pull request #117 from Geosyntec/v0.5.0rc2

V0.5.0rc2

.dockerignore

@@ -2,4 +2,4 @@
 **/.pytest_cache
 **/.mypy_cache
 **/.coverage
-**/.env
+**/.env

.github/workflows/build.yml

@@ -0,0 +1,76 @@
+name: Build and Push Image
+
+on:
+  push:
+    branches:
+      - develop
+      - master
+  workflow_dispatch:
+
+jobs:
+  build-and-push:
+    strategy:
+      fail-fast: false
+      matrix:
+        targets: [nereid, redis, celeryworker]
+    runs-on: ubuntu-latest
+    env:
+      DOCKER_BUILDKIT: 1
+      COMPOSE_DOCKER_CLI_BUILD: 1
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Prepare
+        id: prep
+        run: |
+          DOCKER_IMAGE=${{ secrets.ACR_SERVER }}/nereid/${{ matrix.image }}
+          VERSION=edge
+          if [[ $GITHUB_REF == refs/tags/* ]]; then
+            VERSION=${GITHUB_REF#refs/tags/v}
+          fi
+          if [ "${{ github.event_name }}" = "schedule" ]; then
+            VERSION=nightly
+          fi
+          TAGS="${DOCKER_IMAGE}:${VERSION}"
+          if [[ $VERSION =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
+            TAGS="$TAGS,${DOCKER_IMAGE}:latest"
+          fi
+          echo "tags: $TAGS"
+          echo ::set-output name=tags::${TAGS}
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Inspect builder
+        run: |
+          echo "Name:      ${{ steps.buildx.outputs.name }}"
+          echo "Endpoint:  ${{ steps.buildx.outputs.endpoint }}"
+          echo "Status:    ${{ steps.buildx.outputs.status }}"
+          echo "Flags:     ${{ steps.buildx.outputs.flags }}"
+          echo "Platforms: ${{ steps.buildx.outputs.platforms }}"
+
+      - name: Login to Azure
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v1
+        with:
+          registry: ${{ secrets.ACR_SERVER }}
+          username: ${{ secrets.ACR_USERNAME }}
+          password: ${{ secrets.ACR_PASSWORD }}
+
+      - name: Build and Push
+        id: docker_build
+        uses: docker/build-push-action@v2
+        with:
+          builder: ${{ steps.buildx.outputs.name }}
+          context: ./nereid
+          file: ./nereid/Dockerfile.multi
+          target: ${{ matrix.image }}
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.prep.outputs.tags }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Image digest
+        run: echo ${{ steps.docker_build.outputs.digest }}

.github/workflows/test.yml

@@ -9,8 +9,8 @@ on:
   workflow_dispatch:
   schedule:
     # every other sunday at noon.
-    - cron:  '0 12 1-7,15-21,29-31 * 0'
-    
+    - cron: "0 12 1-7,15-21,29-31 * 0"
+
 jobs:
   python-test:
     runs-on: ubuntu-latest
@@ -20,7 +20,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.8
+          python-version: 3.9
       - name: Install Deps
         run: pip install -r nereid/requirements.txt -r nereid/requirements_tests.txt
       - name: Run Tests
@@ -36,11 +36,11 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.8
+          python-version: 3.9
       - name: Install nereid as library
         run: |
-          pip install . 
-          pip install -r nereid/requirements_tests.txt
+          pip install .
+          pip install pytest
       - name: Run Tests
         run: pytest nereid/nereid/tests/test_src -xv
 
@@ -64,7 +64,7 @@ jobs:
       - name: Run Edge Tests
         continue-on-error: true
         run: |
-          pytest nereid/nereid/tests/test_src -xv
+          pytest nereid/nereid/tests -xv
       - name: Exit
         run: exit 0
 
@@ -77,10 +77,12 @@ jobs:
       - name: build stack
         run: |
           docker --version
-          bash ./scripts/build_deploy.sh
+          bash ./scripts/build_dev.sh
           docker-compose up -d nereid-tests
       - name: run tests
-        run: docker-compose exec -T nereid-tests coverage run -m pytest -xv
+        run: |
+          docker compose exec -T nereid-tests coverage run --branch -m pytest nereid/tests -xs
+          docker compose exec -T nereid-tests coverage run -a --branch -m pytest nereid/tests/test_api -xs --async
       - name: coverage
         run: |
           docker-compose exec -T nereid-tests coverage report -mi

docker-compose.deploy.images.yml

@@ -1,12 +1,12 @@
-version: '3.7'
+version: "3.7"
 services:
   nereid:
-    image: sitkacontainers.azurecr.io/ocstormwatertools/nereid_nereid:${NEREID_IMAGE_TAG:-latest}
+    image: ${ACR_SERVER}/nereid/nereid:${NEREID_IMAGE_TAG:-latest}
   celeryworker:
-    image: sitkacontainers.azurecr.io/ocstormwatertools/nereid_celeryworker:${NEREID_IMAGE_TAG:-latest}
+    image: ${ACR_SERVER}/nereid/celeryworker:${NEREID_IMAGE_TAG:-latest}
   nereid-tests:
-    image: sitkacontainers.azurecr.io/ocstormwatertools/nereid_nereid-tests:${NEREID_IMAGE_TAG:-latest}
+    image: ${ACR_SERVER}/nereid/nereid-tests:${NEREID_IMAGE_TAG:-latest}
   redis:
-    image: sitkacontainers.azurecr.io/ocstormwatertools/nereid_redis:${NEREID_IMAGE_TAG:-latest}
+    image: ${ACR_SERVER}/nereid/redis:${NEREID_IMAGE_TAG:-latest}
   flower:
-    image: sitkacontainers.azurecr.io/ocstormwatertools/nereid_flower:${NEREID_IMAGE_TAG:-latest}
+    image: ${ACR_SERVER}/nereid/flower:${NEREID_IMAGE_TAG:-latest}

docker-compose.shared.env.yml

@@ -1,6 +1,8 @@
 version: '3.7'
 services:
   nereid:
+    environment:
+      - NEREID_ASYNC_MODE=replace
     env_file:
       - env.env
   nereid-tests:

docker-compose.shared.ports.yml

@@ -1,8 +1,8 @@
 version: '3.7'
 services:
   redis:
-    ports:
-      - 6379:6379
+    expose:
+      - 6379
   nereid:
     ports:
       - '80:80'

env.env

@@ -1,4 +1,2 @@
-__CELERY_BROKER_URL=redis://guest@redis:6379/0
-__CELERY_RESULT_BACKEND=redis://guest@redis:6379/0
 CELERY_BROKER_URL=redis://redis:6379/0
-CELERY_RESULT_BACKEND=redis://redis:6379/0
+CELERY_RESULT_BACKEND=redis://redis:6379/0

make.bat

@@ -8,10 +8,14 @@ if /i %1 == up goto :up
 if /i %1 == down goto :down
 if /i %1 == typecheck goto :typecheck
 if /i %1 == coverage goto :coverage
+if /i %1 == coverage-sync goto :coverage-sync
+if /i %1 == coverage-async goto :coverage-async
+if /i %1 == coverage-all goto :coverage-all
 if /i %1 == cover-src goto :cover-src
 if /i %1 == dev-server goto :dev-server
 if /i %1 == restart goto :restart
 if /i %1 == lint goto :lint
+if /i %1 == login goto :login
 
 :help
 echo Commands:
@@ -31,10 +35,15 @@ goto :eof
 
 set COMPOSE_DOCKER_CLI_BUILD=1
 
+:login
+bash scripts/az-login.sh
+goto :eof
+
 :test
 call make clean
 call make restart
-docker compose exec nereid-tests pytest %2 %3 %4 %5 %6
+for /f "tokens=1,* delims= " %%a in ("%*") do set ALL_BUT_FIRST=%%b
+docker compose exec nereid-tests pytest %ALL_BUT_FIRST%
 goto :eof
 
 :typecheck
@@ -59,7 +68,29 @@ goto :eof
 :coverage
 call make clean
 call make restart
-docker compose exec nereid-tests coverage run -m pytest -x
+docker compose exec nereid-tests coverage run --branch -m pytest nereid/tests -xs
+docker compose exec nereid-tests coverage report -m
+goto :eof
+
+:coverage-sync
+call make clean
+call make restart
+docker compose exec nereid-tests coverage run --branch -m pytest nereid/tests -xs
+docker compose exec nereid-tests coverage report -m --omit=*test*,*bg_*,*celery_*
+goto :eof
+
+:coverage-async
+call make clean
+call make restart
+docker compose exec nereid-tests coverage run --branch -m pytest nereid/tests -xs --async
+docker compose exec nereid-tests coverage report -m --omit=*test*,*_sync*
+goto :eof
+
+:coverage-all
+call make clean
+call make restart
+docker compose exec nereid-tests coverage run --branch -m pytest nereid/tests -xs
+docker compose exec nereid-tests coverage run -a --branch -m pytest nereid/tests/test_api -xs --async
 docker compose exec nereid-tests coverage report -m
 goto :eof
 
@@ -85,3 +116,8 @@ goto :eof
 :lint
 bash scripts/lint.sh
 goto :eof
+
+rem docker build --target nereid_install -t nereid_install -f nereid/Dockerfile.multi .
+rem docker run -it -v %cd%:/nereid -p 8088:80 --expose 80 nereid_install bash
+rem pip install -e .[app]
+rem uvicorn nereid.main:app --port 80 --host 0.0.0.0 --reload

nereid/.coveragerc

@@ -6,4 +6,4 @@ branch = True
 [report]
 include = nereid/*.py
 ignore_errors = True
-omit = nereid/data/*
+omit = nereid/data/*,*main.py