azure-pipelines.yml

# .azure-pipelines/azure-pipelines.yml
#
# Azure Pipelines configuration file
# project: https://dev.azure.com/jtmmoon/goto_http_redirect_server/_build
# source: https://github.com/jtmoon79/goto_http_redirect_server/blob/master/.azure-pipelines/
#
# YAML Schema
#   https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=azure-devops&tabs=schema
#   (http://archive.fo/TFpib)
# Task Catalog
#   https://docs.microsoft.com/en-us/azure/devops/pipelines/tasks/index?view=azure-devops
#   (http://archive.ph/2q9aS)
# vmImage list
#   https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops
#   (http://archive.ph/Ed9H0)
#   https://github.com/MicrosoftDocs/azure-docs/blob/main/articles/virtual-machines/linux/cli-ps-findimage.md
---
trigger:  # these are branches
  - master
  - azure-pipelines
variables:
  GOTO_PATH_PROGRAM_PY: $(Build.SourcesDirectory)/goto_http_redirect_server/goto_http_redirect_server.py
  GOTO_PROGRAM: goto_http_redirect_server
  GOTO_PACKAGE_NAME: goto_http_redirect_server
  GOTO_PATH_TEST: $(Build.SourcesDirectory)/goto_http_redirect_server/test
  GOTO_PATH_DIST: $(Build.SourcesDirectory)/dist
  GOTO_PATH_SERVICE: $(Build.SourcesDirectory)/service
  GOTO_PATH_TOOLS: $(Build.SourcesDirectory)/tools
  GOTO_PATH_TOOLS_CI: $(Build.SourcesDirectory)/tools/ci
  GOTO_PATH_FLAKE8: $(GOTO_PATH_TOOLS)/flake8.sh
  GOTO_PATH_MYPY: $(GOTO_PATH_TOOLS)/mypy.sh
  GOTO_PATH_YAMLLINT: $(GOTO_PATH_TOOLS)/yamllint.sh
  GOTO_PATH_REALPATH: $(GOTO_PATH_TOOLS)/realpath.sh
  GOTO_PATH_ADD_PIP_SITE: $(GOTO_PATH_TOOLS_CI)/PATH-add-pip-site.sh
  GOTO_PYTEST_REPORT: pytest-report.xml
  GOTO_PYTEST_COVERAGE_REPORT: pytest-cov-coverage.xml
  GOTO_VM_IMAGE: 'ubuntu-22.04'  # default working VM
  GOTO_VM_IMAGE_WIN: 'windows-2019'  # default working VM for Windows
  GOTO_PY_VERSION: '3.7'  # default working Python
  GOTO_WHEEL_ARTIFACT_DEFAULT: artifact_wheel_$(GOTO_VM_IMAGE)  # default wheel package - one of the built wheel artifacts must have this name
stages:
  - stage: stage_env_dump
    displayName: 'stage: «env dump»'
    jobs:
      - job: job_env_dump
        displayName: 'job: «env dump»'
        pool:
          vmImage: $(GOTO_VM_IMAGE)
        steps:
          - task: Bash@3
            displayName: 'task: «Bash@3» vmImage "$(GOTO_VM_IMAGE)"'
            inputs:
              targetType: inline
              script: |
                set -eux
                shopt
                pwd
                whoami
                hostname
                cat /etc/os-release || true
                env | sort
                ls -la .
                uname -a
                docker info || true
                python --version
                python -m pip --version
                python -m pip list --disable-pip-version-check --no-index -vvv
                chmod -v +x -- "${GOTO_PATH_REALPATH}"  # test
  - stage: stage_flake8_yamllint_mypy_pytest
    displayName: 'stage: «flake8 yamllint mypy pytest»'
    jobs:
      - job: job_flake8
        displayName: 'job: «flake8»'
        pool:
          vmImage: $(GOTO_VM_IMAGE)
        steps:
          - task: UsePythonVersion@0
            displayName: 'task: «UsePythonVersion@0» Python $(GOTO_PY_VERSION)'
            inputs:
              versionSpec: $(GOTO_PY_VERSION)
          - task: Bash@3
            displayName: 'task: «Bash@3»'
            inputs:
              targetType: inline
              script: |
                set -eux
                source "${GOTO_PATH_ADD_PIP_SITE}"
                python -m pip install --user -e '.[development-flake8]'
                python -m pip list --disable-pip-version-check --no-index -vvv
                chmod -v +x -- "${GOTO_PATH_FLAKE8}"
                "${GOTO_PATH_FLAKE8}"
      - job: job_yamllint
        displayName: "job: «yamllint»"
        pool:
          vmImage: $(GOTO_VM_IMAGE)
        steps:
          - task: UsePythonVersion@0
            displayName: 'task: «UsePythonVersion@0» Python $(GOTO_PY_VERSION)'
            inputs:
              versionSpec: $(GOTO_PY_VERSION)
          - task: Bash@3
            displayName: 'task: «Bash@3»'
            inputs:
              targetType: inline
              script: |
                set -eux
                source "${GOTO_PATH_ADD_PIP_SITE}"
                python -m pip install --user -e '.[development-yamllint]'
                python -m pip list --disable-pip-version-check --no-index -vvv
                chmod -v +x -- "${GOTO_PATH_YAMLLINT}"
                "${GOTO_PATH_YAMLLINT}"
      - job: job_mypy
        displayName: 'job: «mypy»'
        pool:
          vmImage: $(GOTO_VM_IMAGE)
        steps:
          - task: UsePythonVersion@0
            displayName: 'task: «UsePythonVersion@0» Python $(GOTO_PY_VERSION)'
            inputs:
              versionSpec: $(GOTO_PY_VERSION)
          - task: Bash@3
            displayName: 'task: «Bash@3»'
            inputs:
              targetType: inline
              script: |
                set -eux
                python -m pip install --user -e '.[development-mypy]'
                python -m pip list --disable-pip-version-check --no-index -vvv
                chmod -v +x -- "${GOTO_PATH_MYPY}"
                python -c "import signal, pprint; pprint.pprint(dir(signal));"
                "${GOTO_PATH_MYPY}"
      - job: job_pytest
        displayName: 'job: «pytest»'
        strategy:
          # see https://github.com/actions/python-versions/blob/main/versions-manifest.json
          matrix:
            Python37:
              python.release: $(GOTO_PY_VERSION)
              vmImageName: 'ubuntu-20.04'
            Python38:
              python.release: '3.8'
            Python39:
              python.release: '3.9'
            Python310:
              python.release: '3.10'
            Python311:
              python.release: '3.11'
        pool:
          vmImage: $(GOTO_VM_IMAGE)
        steps:
          - task: UsePythonVersion@0
            displayName: 'task: «UsePythonVersion@0» Python $(python.release)'
            inputs:
              versionSpec: $(python.release)
          - task: Bash@3
            displayName: 'task: «Bash@3» Python $(python.release)'
            inputs:
              targetType: inline
              script: |
                set -eux
                python --version
                python -m pip install --user -e '.[development-pytest]'
                python -m pip list --disable-pip-version-check --no-index -vvv
                chmod -v +x -- ./tools/pytest.sh
                ./tools/pytest.sh
                cat "${GOTO_PYTEST_REPORT}" | python -c "import sys;import xml.dom.minidom;s=sys.stdin.read();print(xml.dom.minidom.parseString(s).toprettyxml())"
                # XXX: is this `cp` necessary?
                cp -av -- "${GOTO_PYTEST_REPORT}" "${BUILD_ARTIFACTSTAGINGDIRECTORY}/"
                cp -av -- "${GOTO_PYTEST_COVERAGE_REPORT}" "${BUILD_ARTIFACTSTAGINGDIRECTORY}/"
          - task: PublishBuildArtifacts@1
            displayName: 'task: «PublishBuildArtifacts@1» $(GOTO_PYTEST_REPORT) "$(Build.ArtifactStagingDirectory)"'
            condition: in(variables['python.release'], variables['GOTO_PY_VERSION'])
            inputs:
              pathtoPublish: '$(Build.ArtifactStagingDirectory)'
              artifactName: artifact_pytest_report
          - task: PublishTestResults@2
            displayName: 'task: «PublishTestResults@2» $(GOTO_PYTEST_REPORT)'
            condition: in(variables['python.release'], variables['GOTO_PY_VERSION'])
            inputs:
              testResultsFiles: '**/$(GOTO_PYTEST_REPORT)'
              testRunTitle: 'pytest for build $(Build.BuildId) commit $(Build.SourceVersion)'
          - task: PublishCodeCoverageResults@1
            displayName: 'task: «PublishCodeCoverageResults@2» $(GOTO_PYTEST_COVERAGE_REPORT)'
            condition: in(variables['python.release'], variables['GOTO_PY_VERSION'])
            inputs:
              codeCoverageTool: cobertura
              summaryFileLocation: '**/$(GOTO_PYTEST_COVERAGE_REPORT)'
              failIfCoverageEmpty: true
  - stage: stage_build
    displayName: 'stage: «build»'
    jobs:
      - job: job_build
        displayName: 'job: «build»'
        strategy:
          matrix:
            Ubuntu1804:
              vmImageName: 'ubuntu-18.04'
            Ubuntu2004:
              vmImageName: 'ubuntu-20.04'
            Ubuntu2204:
              vmImageName: 'ubuntu-22.04'
            Win:
              vmImageName: 'windows-2019'
            MacOS:
              vmImageName: 'macOS-11'
        pool:
          vmImage: $(vmImageName)
        steps:
          - task: UsePythonVersion@0
            displayName: 'task: «UsePythonVersion@0» Python $(GOTO_PY_VERSION)'
            inputs:
              versionSpec: $(GOTO_PY_VERSION)
          - task: Bash@3
            displayName: 'task: «Bash@3» vmImage "$(vmImageName)"'
            condition: not(in(variables['vmImageName'], variables['GOTO_VM_IMAGE_WIN']))
            inputs:
              targetType: inline
              script: |
                # script settings
                set -eux
                # install build packages
                source "${GOTO_PATH_ADD_PIP_SITE}"
                python -m pip install --quiet --user -e '.[build]'
                python -m pip list --disable-pip-version-check --no-index -vvv
                # do the build
                python setup.py -v bdist_wheel
                # check with twine
                ls -l "${GOTO_PATH_DIST}"
                python -m twine check "${GOTO_PATH_DIST}"/*.whl
          - task: PowerShell@2
            displayName: 'task: «PowerShell@2» vmImage "$(vmImageName)"'
            condition: in(variables['vmImageName'], variables['GOTO_VM_IMAGE_WIN'])
            inputs:
              targetType: inline
              script: |
                # script settings
                $PSVersionTable
                $DebugPreference = "Continue"
                $ErrorActionPreference = "Stop"
                Set-PSDebug -Trace 1
                # install build packages
                & python -m pip install --quiet --user -e ".[build]"
                & python -m pip list --disable-pip-version-check --no-index -vvv
                # do the build
                & python setup.py -v bdist_wheel
                # check with twine
                & python -m twine check $env:GOTO_PATH_DIST\*.whl
          - task: CopyFiles@2
            displayName: 'task: «CopyFiles@2» wheel Package to Build Artifact Staging Directory "$(Build.ArtifactStagingDirectory)"'
            inputs:
              sourceFolder: '$(GOTO_PATH_DIST)'
              contents: '*.whl'
              TargetFolder: '$(Build.ArtifactStagingDirectory)'
              preserveTimestamp: true
          - task: PublishBuildArtifacts@1
            displayName: 'task: «PublishBuildArtifacts@1» artifact_wheel_$(vmImageName)'
            inputs:
              pathtoPublish: '$(Build.ArtifactStagingDirectory)'
              artifactName: 'artifact_wheel_$(vmImageName)'
  - stage: stage_install_user_self_test
    displayName: 'stage: «install --user, run self-test»'
    jobs:
      - job: job_run_test
        displayName: 'job: «install --user, run self-test»'
        strategy:
          matrix:
            Ubuntu1804_Python36:
              python.version: '3.6'
              vmImageName: 'ubuntu-18.04'
            Ubuntu1804_Python37:
              python.version: '3.7'
              vmImageName: 'ubuntu-18.04'
            Ubuntu1804_Python38:
              python.version: '3.8'
              vmImageName: 'ubuntu-18.04'
            Ubuntu2004_Python39:
              python.version: '3.9'
              vmImageName: 'ubuntu-20.04'
            Ubuntu2004_Python310:
              python.version: '3.10'
              vmImageName: 'ubuntu-20.04'
            Ubuntu2204_Python311:
              python.version: '3.11'
              vmImageName: 'ubuntu-22.04'
            MacOS:
              python.version: $(GOTO_PY_VERSION)
              vmImageName: 'macOS-11'
            Win:
              python.version: $(GOTO_PY_VERSION)
              vmImageName: 'windows-2019'
        pool:
          vmImage: $(vmImageName)
        steps:
          - task: UsePythonVersion@0
            displayName: 'task: «UsePythonVersion@0» Python $(python.version)'
            inputs:
              versionSpec: '$(python.version)'
          - task: DownloadPipelineArtifact@2
            inputs:
              artifact: $(GOTO_WHEEL_ARTIFACT_DEFAULT)
              path: $(Build.BinariesDirectory)
          - task: Bash@3
            displayName: 'task: «Bash@3» vmImage "$(vmImageName)" Python $(python.version)'
            condition: not(in(variables['vmImageName'], variables['GOTO_VM_IMAGE_WIN']))
            inputs:
              targetType: inline
              script: |
                # script settings
                set -eux

                chmod -v +x "${GOTO_PATH_REALPATH}"

                # update packages
                #python -m pip install --quiet --upgrade pip setuptools
                python -m pip list --disable-pip-version-check --no-index -vvv

                # update PATH with potential pip install locations
                source "${GOTO_PATH_ADD_PIP_SITE}"

                # install wheel package
                cv_whl=$("${GOTO_PATH_REALPATH}" "$(find "${BUILD_BINARIESDIRECTORY}" -name '*.whl' -print -quit)")
                ls -l "${cv_whl}"
                cd /tmp
                python -m pip install --user --verbose "${cv_whl}"
                python -m pip list --disable-pip-version-check --no-index -vvv

                # check can run
                "${GOTO_PROGRAM}" --version

                # server test
                cd -
                SERVER_TEST=$("${GOTO_PATH_REALPATH}" "./tools/ci/server-test.sh")
                chmod -v +x "${SERVER_TEST}"
                "${SERVER_TEST}"

                # uninstall
                python -m pip uninstall --yes --verbose "${GOTO_PACKAGE_NAME}"
          - task: PowerShell@2
            displayName: 'task: «PowerShell@2» vmImage "$(vmImageName)" Python $(python.version)'
            condition: in(variables['vmImageName'], variables['GOTO_VM_IMAGE_WIN'])
            inputs:
              targetType: inline
              script: |
                # script settings
                $PSVersionTable
                $DebugPreference = "Continue"
                $ErrorActionPreference = "Stop"
                Set-PSDebug -Trace 1

                # update packages
                #& python -m pip install --quiet --upgrade pip setuptools
                & python -m pip list --disable-pip-version-check --no-index -vvv

                # update PATH with potential pip install locations
                $usersite = & python -B -c 'import site; print(site.USER_SITE);'
                $userbase = & python -B -c 'import site; print(site.USER_BASE);'
                $env:PATH += [IO.Path]::PathSeparator + $usersite + [IO.Path]::PathSeparator + $userbase

                # install wheel package
                $cv_whl = Get-ChildItem -Path "$env:BUILD_BINARIESDIRECTORY" -Filter '*whl' -File | Select-Object -First 1
                Push-Location '..'
                & python -m pip install --user --verbose $cv_whl.FullName

                # run
                & "$env:GOTO_PROGRAM" --version
                # server test
                Pop-Location
                & bash "./tools/ci/server-test.sh"  # powershell to run bash! LOL!
                # uninstall
                python -m pip uninstall --yes --verbose "$env:GOTO_PACKAGE_NAME"
  - stage: stage_install_systemd
    displayName: 'stage: «install systemd, start service»'
    jobs:
      - job: job_install_systemd
        displayName: 'job: «install systemd, start service»'
        strategy:
          matrix:
            Ubuntu1804:
              vmImageName: 'ubuntu-18.04'
            Ubuntu2004:
              vmImageName: 'ubuntu-20.04'
            Ubuntu2204:
              vmImageName: 'ubuntu-22.04'
        pool:
          vmImage: $(vmImageName)
        steps:
          - task: UsePythonVersion@0
            displayName: 'task: «UsePythonVersion@0» Python $(GOTO_PY_VERSION)'
            inputs:
              versionSpec: $(GOTO_PY_VERSION)
          - task: DownloadPipelineArtifact@2
            inputs:
              artifact: $(GOTO_WHEEL_ARTIFACT_DEFAULT)
              path: $(Build.BinariesDirectory)
          - task: Bash@3
            displayName: "task: «Bash@3» vmImage $(vmImageName) Python $(GOTO_PY_VERSION)"
            inputs:
              targetType: inline
              script: |
                # script settings
                set -eux

                # verify and check system settings
                systemctl --version
                systemctl show-environment
                curl --version
                USERTEST=nobody
                id -u "${USERTEST}" &>/dev/null

                # install authbind
                chmod -vR +x -- "${GOTO_PATH_TOOLS_CI}"
                PORT1=80
                sudo -- "${GOTO_PATH_TOOLS_CI}/authbind-install.sh" "${USERTEST}" ${PORT1}

                chmod -v +x -- "${GOTO_PATH_REALPATH}"

                # update PATH with potential pip install locations
                source "${GOTO_PATH_ADD_PIP_SITE}"

                function sudo_python () {
                    # hack to force sudo shell to include the $PATH (which holds installed Python paths)
                    # XXX: does not handle spaces in arguments, a clever `eval` might improve this
                    sudo -HE -- bash -c "export PATH=${PATH}; python ${*}"
                }
                sudo_python -m pip list --disable-pip-version-check --no-index -vvv

                # install wheel package
                cv_whl=$("${GOTO_PATH_REALPATH}" "$(find "${BUILD_BINARIESDIRECTORY}" -name '*.whl' -print -quit)")
                ls -l "${cv_whl}"
                cd /tmp
                sudo_python -m pip install --verbose "${cv_whl}"
                sudo_python -m pip list --disable-pip-version-check --no-index -vvv
                cd -

                # configure service files
                GOTO_CONFIG=/etc/goto_http_redirect_server.conf
                GOTO_FILE_SCRIPT=/usr/local/bin/goto_http_redirect_server
                GOTO_FILE_REDIRECTS=/usr/local/share/goto_http_redirect_server.csv
                GOTO_FILE_LOG=/var/log/goto_http_redirect_server.log
                GOTO_SERVICE=goto_http_redirect_server.service
                GOTO_LISTEN_IP=0.0.0.0

                # TODO: service-install.sh, use `python setup.py systemd_install`

                # install service files
                chmod -vR +x -- "${GOTO_PATH_SERVICE}"
                sudo -- "${GOTO_PATH_SERVICE}/service-install.sh" --enable
                sudo -- systemctl list-dependencies "${GOTO_SERVICE}"

                "${GOTO_FILE_SCRIPT}" --version  # check can run from new location

                # adjust settings
                echo '/a	http://aaa.com	bob	2000-01-01 00:00:00' | sudo -- tee "${GOTO_FILE_REDIRECTS}"

                # start and check systemd service
                sudo -- systemctl start "${GOTO_SERVICE}"
                sleep 3
                tail "${GOTO_FILE_LOG}" || true
                sudo -- journalctl -u "${GOTO_SERVICE}"
                sudo -- systemctl status "${GOTO_SERVICE}"

                # user query service
                URL="http://${GOTO_LISTEN_IP}:${PORT1}"
                curl -v --fail --output /dev/null "${URL}/a"

                #
                # test all other systemd wrapper configuration options
                #
                GOTO_LISTEN_IP=127.0.0.2
                PORT2=8083
                sudo -- chmod -v 0666 -- "${GOTO_FILE_LOG}"  # allow writing by $USERTEST
                echo "
                GOTO_AUTHBIND_ENABLE=true
                GOTO_SUDOAS_ENABLE=true
                GOTO_SUDOAS_USER=${USERTEST}
                GOTO_NICE_ENABLE=true
                GOTO_NICE_LEVEL=-10
                declare -ag GOTO_ARGV=(
                    '--redirects' '${GOTO_FILE_REDIRECTS}'
                    '--ip' '${GOTO_LISTEN_IP}'
                    '--port' '${PORT2}'
                    '--status-path' '/status'
                    '--reload-path' '/reload'
                    '--debug'
                )
                " | sudo -- tee "${GOTO_CONFIG}"
                sudo -- cat "${GOTO_CONFIG}"  # note contents in debug log

                # use a different port because sometimes the TCP port is held
                # open long after the process has exited.
                URL="http://${GOTO_LISTEN_IP}:${PORT2}"
                sudo -- "${GOTO_PATH_TOOLS_CI}/authbind-install.sh" "${USERTEST}" ${PORT2}

                sudo -- systemctl restart "${GOTO_SERVICE}"
                sleep 20
                tail "${GOTO_FILE_LOG}" || true
                sudo -- journalctl -u "${GOTO_SERVICE}"
                sudo -- systemctl status "${GOTO_SERVICE}"

                # user query various URI paths
                curl -v --fail --output /dev/null "${URL}/a"
                curl -v --fail --output /dev/null "${URL}/reload"
                curl -v --fail --output /dev/null "${URL}/status"

                sudo -- systemctl status "${GOTO_SERVICE}"

                # TODO: use `python setup.py systemd_uninstall`
...