.gitlab-ci.yml

# SPDX-License-Identifier: FSFAP
# Copyright (C) 2019 John Hsu
# Copyright (C) 2019-2024 Colin B. Macdonald
# Copyright (C) 2021 Peter Lee
# Copyright (C) 2021 Morgan Arnold
# Copyright (C) 2022-2023 Edith Coates
# Copyright (C) 2023 Natalie Balashov
# Copyright (C) 2023 Julian Lapenna
# Copyright (C) 2023-2024 Andrew Rechnitzer
# Copyright (C) 2024 Aidan Murphy
#
# Copying and distribution of this file, with or without modification,
# are permitted in any medium without royalty provided the copyright
# notice and this notice are preserved.  This file is offered as-is,
# without any warranty.

# This the CI configuration for Plom

# Issue #1654: This stops multiple pipelines on merge-requests from forks
# But it seems to prevent forks from running their own CI jobs pre-MR
# include:
#   - template: 'Workflows/MergeRequest-Pipelines.gitlab-ci.yml'

# workflow:
#   rules:
#     - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
#     - if: '$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS'
#       when: never
#     - if: '$CI_COMMIT_BRANCH'

image: docker:27.3.1

services:
  - docker:27.3.1-dind

# $CI_REGISTRY_IMAGE is:
# registry.gitlab.com/plom/plom
variables:
  DOCKER_DRIVER: overlay2
  IM: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG
  IM_LATEST: $CI_REGISTRY_IMAGE:latest
  PLOM_NO_SSL_VERIFY: 1
  MACBIN_PY_MAJOR_MINOR_VER: "3.11"
  MACBIN_PY_VER: 3.11.9
  QT_QPA_PLATFORM: offscreen

stages:
  - static_analysis
  - build
  - alt_build
  - test
  - packaging
  - prep
  - release


# Attention maintainers: if you start to see errors in the CI runs like:
#   E: Failed to fetch http://security.ubuntu.com... 404 Not Found [IP: 91...]
# Run with DOCKER_USE_CACHE set to 0, under Pipeline -> Run pipeline
container-image:
  stage: build
  script:
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
    - echo $DOCKER_USE_CACHE
    - >
      if [ "$DOCKER_USE_CACHE" == "1" ]; then
        docker pull $IM_LATEST || true
      fi
    - >
      if [ "$DOCKER_USE_CACHE" == "1" ]; then
        docker build --cache-from $IM_LATEST --tag $IM . -f Containerfile.bulk
      else
        docker build --tag $IM . -f Containerfile.bulk
      fi
    - docker push $IM


# Get fast results by running tests in a copy-pasta of the container-image job
# TODO: why is this failing?  check if client-building works by commenting out
# quick-pytests:
#   stage: build
#   needs: []
#   script:
#     - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
#     - docker pull $IM_LATEST || true
#     - docker build --cache-from $IM_LATEST --tag $IM .
#     - docker run $IM pytest-3 -l --pyargs plom


codespell:
  image: python:3
  stage: static_analysis
  before_script:
    - python3 -m pip install codespell~=2.3.0
  script:
    - echo "Need an exception?  See the .codespell-ignore* files"
    - codespell
  allow_failure: true
  artifacts:
    reports:
      dotenv: release_info.env


# sometimes we write everything twice: this job ensures dupe file/info stay same
wet_no_drift:
  stage: static_analysis
  image: alpine
  before_script:
    - apk add bash
  script:
    - 'echo "Issue #2402: ensure two identical copies of the latex template"'
    - diff -s plom/latexTemplate.tex testTemplates/latexTemplate.tex
    - diff -s plom/latexTemplatev2.tex testTemplates/latexTemplatev2.tex
    - echo "Version info duplicated in two places, ensure same"
    - export VER0=`sed -nr 's/^__version__ = \"(.+)\"/\1/p' plom/__init__.py`
    - export VER1=`sed -nr 's/^\s+version. (.+)/\1/p' AppImageBuilder.yml`
    - echo $VER0
    - echo $VER1
    - bash -c "[[ x$VER0 == x$VER1 ]]"


copyright_year:
  image: python:3
  stage: static_analysis
  before_script:
    - git --version
    - echo "ensure we have the main branch for comparing"
    - git fetch origin main
    - git checkout main
    - git checkout $CI_COMMIT_BRANCH --
    - git branch -avv
  script:
    - echo "Considering the following commits to this branch (since main):"
    - git shortlog origin/main..${CI_COMMIT_BRANCH} --
    - echo "Those commits touch the following files:"
    # git show --pretty="" --name-only origin/main..${CI_COMMIT_BRANCH} -- | uniq -u
    # TODO: tried ${CI_MERGE_REQUEST_TARGET_BRANCH_NAME} instead of main?
    - git diff --name-only origin/main..${CI_COMMIT_BRANCH} --
    - ./contrib/is_copyright_uptodate.py `git diff --name-only origin/main..$CI_COMMIT_BRANCH --`
    - echo "Note that there are some files that do not have/need copyright headers"
  allow_failure: true


djlint:
  image: python:3
  stage: static_analysis
  before_script:
    - pip install djlint~=1.35.3
  script:
    - djlint plom_server/templates/*/*.html plom_server/templates/*/*/*.html --profile django


djlint-check:
  image: python:3
  stage: static_analysis
  before_script:
    - pip install djlint~=1.35.3
  script:
    - djlint plom_server/templates/*/*.html plom_server/templates/*/*/*.html --check


# If on main branch, tag earlier image as "latest" (in Gitlab Container Registry)
# TODO: Issue #3573: did adding `tags` here make this run during the big release on tags pipeline?
gitlab-container-registry-tag-latest:
  stage: packaging
  needs: ["container-image"]
  before_script:
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
  script:
    - docker pull $IM
    - docker tag $IM $IM_LATEST
    - docker push $IM_LATEST
  only:
    - main
    - tags


# push to the image to the docker.io container registry, using credentials from GitLab's secrets
push-to-docker-io:
  stage: prep
  rules:
   - if: $CI_COMMIT_TAG
  before_script:
    - docker login -u $REGISTRY_USER_DOCKER_IO -p $REGISTRY_PASSWORD_DOCKER_IO docker.io
  script:
    - echo $CI_COMMIT_TAG
    - docker pull $IM
    # strip the leading v in vx.y.z
    - export VER=${CI_COMMIT_TAG:1}
    # strip the .z in x.y.z
    - export VER_MINOR=`echo $VER | sed -e "s/^\(\d*\.\d*\).\d*/\1/"`
    - docker tag $IM plomgrading/server:$VER
    - docker tag plomgrading/server:$VER plomgrading/server:$VER_MINOR
    - docker tag plomgrading/server:$VER plomgrading/server:latest
    - docker push plomgrading/server:$VER
    - docker push plomgrading/server:$VER_MINOR
    - docker push plomgrading/server:latest


# to ensure the pypi job works, make sure we can package
# TODO: we make these again during pypi push: better to use these artifacts?
make_package:
  image: python:3.12
  stage: build
  script:
    - pip install --upgrade build
    - python3 -m build
    - ls dist
    - md5sum dist/*
    - mv dist dist_TODO
  artifacts:
    paths:
      - dist_TODO/plom-*.tar.gz
      - dist_TODO/plom-*.whl
    expire_in: 16 days


# If we have a tag, then push to PyPI using TWINE_* env vars
# Debug: `upload -r testpypi`, comment `only`, change and unprotect token
# TODO: maybe we can bring the artifacts we built earlier in `make_package`?
pypi:
  image: python:3.12
  stage: packaging
  rules:
   - if: $CI_COMMIT_TAG
  cache: {}
  script:
    - pip install --upgrade twine wheel setuptools packaging build
    - python3 -m build
    - ls dist
    - pushd dist
    - md5sum *
    - sha256sum *
    - popd
    - python3 -m twine check dist/*
    - python3 -m twine upload --verbose dist/*
  artifacts:
    paths:
      - dist/plom-*.tar.gz
    expire_in: 16 days


release_job:
  stage: release
  image: registry.gitlab.com/gitlab-org/release-cli:latest
  rules:
    - if: $CI_COMMIT_TAG
  script:
    - echo 'running release_job'
    - echo "Draft of release notes follows (newlines eaten)"
    - cat release_desc.md
    - echo "Just debugging:"
    - echo $LINKNAME0
    - echo $URL0
    - echo $FILENAME0
    - echo $LINKNAME1
    - echo $URL1
    - echo $FILENAME1
    - echo $LINKNAME2
    - echo $URL2
    - echo $FILENAME2
    - echo $LINKNAME3
    - echo $URL3
    - echo $FILENAME4
  release:
    name: 'Release $CI_COMMIT_TAG'
    description: './release_desc.md'
    tag_name: '$CI_COMMIT_TAG'
    ref: '$CI_COMMIT_TAG'
    milestones:
      # ${CI_COMMIT_TAG:1} might strip the leading v
      - $CI_COMMIT_TAG
    assets:
      links:
        - name: $LINKNAME0
          filepath: "/$FILENAME0"
          url: $URL0
          link_type: "package"
        - name: $LINKNAME1
          filepath: "/$FILENAME1"
          url: $URL1
          link_type: "package"
        - name: $LINKNAME2
          filepath: "/$FILENAME2"
          url: $URL2
          link_type: "package"
        - name: $LINKNAME3
          filepath: "/$FILENAME3"
          url: $URL3
          link_type: "package"
        - name: $OTHER_LINKNAME1
          url: $OTHER_URL1
          link_type: "other"
        - name: $OTHER_LINKNAME2
          url: $OTHER_URL2
          link_type: "other"
        - name: $OTHER_LINKNAME3
          url: $OTHER_URL3
          link_type: "other"


# block release unless tag matches in-source version
tag_matches_ver:
  stage: static_analysis
  image: python:3
  rules:
   - if: $CI_COMMIT_TAG
  script:
    - export VER=`sed -nr 's/^__version__ = \"(.+)\"/\1/p' plom/__init__.py`
    - echo "Extracted version string '$VER'"
    - echo "Now comparing to CI_COMMIT_TAG '$CI_COMMIT_TAG'"
    - echo $VER
    - echo $CI_COMMIT_TAG
    # note reversed logic and extra "v"
    - python3 -c "exit(not 'v$VER' == '$CI_COMMIT_TAG')"


# Should keep version same or close to that in .pre-commit-config.yaml
black:
  stage: static_analysis
  image: python:3.12
  before_script:
    - pip3 install "black~=24.10.0"
  script:
    - black --check --diff .
  allow_failure: true


unittests:
  stage: test
  needs: ["container-image"]
  image: $IM
  script:
    - pytest-3 -l --pyargs plom


# TODO: may consider merging with unittests job above
# Issue #2291: fix coverage calculations for plom_server/
coverage:
  stage: test
  needs: ["container-image"]
  image: $IM
  script:
    - pip install pytest pytest-cov coverage[toml]~=7.6.4
    - coverage run -m pytest --ignore plom_server
    - coverage report --precision=2
    - coverage xml
  coverage: '/^TOTAL\s+.*\s(\d+\.\d+)%$/'
  artifacts:
    reports:
      coverage_report:
        coverage_format: cobertura
        path: coverage.xml

# Notes:
# masked out some stuff that deps on tensorflow
doctests:
  stage: test
  needs: ["container-image"]
  image: $IM
  before_script:
    - pip3 install --upgrade pytest
  script:
    # TODO how to run only doctests?  here we just ignore-glib on "test_*"
    - pytest --version
    - pytest -l --doctest-modules --doctest-continue-on-failure --ignore-glob="*digitHunter.py" --ignore-glob="*/test_*.py" plom


# needs full dependencies, cannot easily run with other early static analysis
pylint:
  stage: test
  needs: ["container-image"]
  image: $IM
  before_script:
    # These are pinned at 3.x.*: should be manually bumped sometimes
    - python3 -m pip install astroid~=3.3.5  # codespell:ignore astroid
    - python3 -m pip install pylint~=3.3.1
  script:
    - pylint plom
    - pylint plom_server
    - pylint contrib
  allow_failure: true


# needs full dependencies, cannot easily run with other early static analysis
# If exceptions are needed, see "tool.mypy" sections in pyproject.toml
# Note: currently allowed to fail because of partial adoption of type hints and safety in this project.
#       Will make not allowed to fail in the future, once types are more widely adopted in the project.
mypy-type-checking:
  stage: test
  needs: ["container-image"]
  image: $IM
  before_script:
    - python3 -m pip install mypy~=1.13.0
  script:
    - mypy --version
    - mypy plom --python-ver 3.8 --install-types --non-interactive || true
    - mypy plom --python-ver 3.8
    - mypy plom_server --python-ver 3.8 --install-types --non-interactive || true
    - mypy plom_server --python-ver 3.8
  allow_failure: true


pyright:
  stage: test
  needs: ["container-image"]
  image: $IM
  before_script:
    - python3 -m pip install pyright~=1.1.387
  script:
    # for now, just display errors but don't fail (Issue #3237)
    - pyright || true
  allow_failure: true


flake8:
  image: python:3
  stage: static_analysis
  before_script:
    - python3 -m pip install flake8~=7.1.1
  script:
    # first show all errors and warnings but don't fail
    - flake8 --exit-zero
    - flake8


# TODO: does this fully replace flake8?
# TODO: `ruff format` seems not completely compatible with black
ruff:
  image: python:3
  stage: static_analysis
  before_script:
    - python3 -m pip install ruff~=0.7.1
  script:
    - ruff check .
    # ruff format .


# Note: $EXCEPTIONS matches an ignore list in pyproject.toml
count-no-docstring:
  image: python:3
  stage: static_analysis
  before_script:
    - python3 -m pip install ruff~=0.7.1
  script:
    # run once just for log out but don't fail
    - EXCEPTIONS="D101,D102,D103,D104,D106"
    - ruff check --select $EXCEPTIONS --exit-zero
    - HOWMANY=`ruff check --select $EXCEPTIONS --exit-zero | grep "Found .* errors" | grep -o "[0-9]*"`
    - echo $EXCEPTIONS
    - ruff check --select $EXCEPTIONS --statistics --exit-zero
    - echo "Total $HOWMANY things without docstrings, please don't increase it"
    # Dear hackers: please try to decrease this number
    - bash -c "[[ $HOWMANY -le 1533 ]]"
  allow_failure: true


no-question-numbers:
  image: python:3.12
  stage: static_analysis
  script:
    - grep plom_server -rie "question.num"
    - echo "Question number is ambiguous; generally we prefer index or the question label"
    - HOW_MANY=`grep plom_server -rie "question.num" | wc -l`
    # For now, just ensure the total does not increase
    - echo $HOW_MANY
    # Future hackers: consider trying to decrease this number
    - bash -c "[[ $HOW_MANY -le 31 ]]"
  allow_failure: true


# TODO: why do we need to run makemigrations/migrate before tests?
webplom_test:
  stage: test
  needs: ["container-image"]
  image: $IM
  script:
    - export PYTHONPATH=$PWD
    - python3 -m pip install coverage~=7.6.4
    - pushd plom_server
    - export PLOM_DATABASE_BACKEND=sqlite
    - python3 manage.py makemigrations
    - python3 manage.py migrate
    - coverage run --source="." manage.py test
    # -i ignores an error caused by openCV's config-3.py file
    - coverage report -i --precision=2
    - coverage xml -i -o django_coverage.xml
    - popd
  coverage: '/^TOTAL\s+.*\s(\d+\.\d+)%$/'
  artifacts:
    reports:
      coverage_report:
        coverage_format: cobertura
        path: plom_server/django_coverage.xml


webplom_demo:
  services:
    - postgres
  variables:
    POSTGRES_DB: "plom_db"
    POSTGRES_USER: "postgres"
    POSTGRES_PASSWORD: "postgres"
    POSTGRES_HOST_AUTH_METHOD: trust
    PLOM_DATABASE_HOSTNAME: "postgres"
  stage: test
  needs: ["container-image"]
  image: $IM
  script:
    - export PYTHONPATH=$PWD
    - echo $PYTHONPATH
    - pushd plom_server
    - ./Launcher/launch_scripts/launch_plom_demo_server.py --length quick --stop-after reports
    - popd


# a legacy-only job
demoserver:
  stage: test
  needs: ["container-image"]
  image: $IM
  before_script:
    - apt-get update
    - apt-get --no-install-recommends --yes install iproute2 psmisc file curl
  script:
    - mkdir demo
    - pushd demo
    - plom-server init
    - plom-server users --demo
    - plom-create newspec --demo
    # Start server
    - ip addr
    - plom-server launch &
    - sleep 2
    - sleep 2
    - echo "Server should be in background"
    - jobs -l
    - echo "We should be able to connect to it"
    - curl -k https://localhost:41984/Version
    - plom-create uploadspec demoSpec.toml -w 1234
    - plom-create class --demo -w 1234
    - plom-create make -w 1234
    - plom-solutions extract solutionSpec.toml -w 1234
    - plom-solutions extract --upload -w 1234
    - python3 -m plom.create.exam_scribbler -w 1234
    - plom-scan process -w 4567 fake_scribbled_exams1.pdf
    - plom-scan upload -w 4567 fake_scribbled_exams1.pdf
    - plom-scan status -w 4567
    - plom-scan process -w 4567 fake_scribbled_exams2.pdf
    - plom-scan upload -w 4567 fake_scribbled_exams2.pdf
    - plom-scan status -w 4567
    - plom-scan process -w 4567 fake_scribbled_exams3.pdf
    - plom-scan upload -w 4567 fake_scribbled_exams3.pdf
    - plom-scan status -w 4567
    - echo "Now take down the server"
    # https://gitlab.com/gitlab-org/gitlab-runner/issues/2880
    - jobs -l
    - kill %1
    - sleep 2
    - echo "Should be no jobs and this should succeed"
    - jobs -l
    - popd


# a legacy-only job
hwdemoserver:
  stage: test
  needs: ["container-image"]
  image: $IM
  before_script:
    - apt-get update
    - apt-get --no-install-recommends --yes install iproute2 psmisc file curl
  script:
    - mkdir demo
    - pushd demo
    - plom-server init
    - plom-server users --demo
    - plom-create newspec --demo
    # Start server
    - ip addr
    - plom-server launch &
    - sleep 2
    - sleep 2
    - echo "Server should be in background"
    - jobs -l
    - echo "We should be able to connect to it"
    - curl -k https://localhost:41984/Version
    - export PLOM_USER=user0
    - export PLOM_PASSWORD=0123
    - export PLOM_MANAGER_PASSWORD=1234
    - export PLOM_SCAN_PASSWORD=4567
    - plom-create uploadspec demoSpec.toml
    - plom-create class --demo
    - plom-create make
    - plom-solutions extract solutionSpec.toml
    - plom-solutions extract --upload
    - plom-hwscan status
    - python3 -m plom.create.homework_scribbler
    # TODO: the actual hwdemo script does more with qstr, semiloose
    - plom-hwscan allbyq -y
    - plom-hwscan missing -y
    - plom-hwscan submitted
    - plom-hwscan submitted -d
    - python3 -m plom.client.randoMarker
    - plom-finish status
    - plom-finish csv
    - A=`cat marks.csv  | wc -l`     # How many lines?
    - echo $A
    - A="$((A-1))"
    - bash -c "[[ $A -le 10 ]]"      # 10 named papers
    - bash -c "[[ $A == 6 ]]"        # hw_scribber makes 4 "semiloose"
    - file marks.csv
    - file marks.csv | grep text
    - plom-finish reassemble
    - B=`ls reassembled/ | wc -l`    # How many files?
    - echo $B
    - bash -c "[[ $B == $A ]]"
    - plom-finish solutions --mark
    - B=`ls solutions/ | wc -l`    # How many files?
    - echo $B
    - bash -c "[[ $B == $A ]]"
    - echo "Now take down the server"
    # https://gitlab.com/gitlab-org/gitlab-runner/issues/2880
    - jobs -l
    - kill %1
    - sleep 2
    - echo "Should be no jobs and this should succeed"
    - jobs -l
    - popd


# a legacy-only job
fullworkflow:
  stage: test
  needs: ["container-image"]
  image: $IM
  before_script:
    - apt-get update
    - apt-get --no-install-recommends --yes install iproute2 psmisc file curl
  script:
    - pwd
    - ls /
    - cd /exam
    - ls /usr/local/lib/python3.10/dist-packages/plom/
    - ls /src
    - python3 --version
    # Build tests
    - mkdir play
    - pushd play
    - plom-server init
    - plom-server users --demo
    - plom-create newspec
    - python3 -m plom.create.demotools solutions # to get soln pdf copied
    # TODO: override the private seed
    # sed specAndDatabase/verifiedSpec.toml -i -e "s/privateSeed = .*/privateSeed = \"8253996856355501\"/"
    # diff -u specAndDatabase/verifiedSpec.toml ../tmp/resources/verifiedSpec.toml
    # Start server
    - ip addr
    - plom-server launch &
    - sleep 2
    - sleep 2
    - echo "Server should be in background"
    - jobs -l
    - echo "We should be able to connect to it"
    - curl -k https://localhost:41984/Version
    - export PLOM_USER=user0
    - export PLOM_PASSWORD=0123
    - export PLOM_MANAGER_PASSWORD=1234
    - export PLOM_SCAN_PASSWORD=4567
    - plom-create uploadspec
    - plom-create class --demo
    - plom-create make
    - plom-solutions extract solutionSpec.toml
    - plom-solutions extract --upload
    - A=`ls papersToPrint/ | wc -l`  # How many files?
    - bash -c "[[ $A == 20 ]]"       # should be 20
    - python3 -m plom.create.exam_scribbler
    - plom-create rubric --demo
    # Scan and upload
    # supposed to fail:
    - if (plom-scan process); then false; else true; fi
    - plom-scan process --demo fake_scribbled_exams1.pdf
    - plom-scan upload fake_scribbled_exams1.pdf
    - plom-scan upload -u fake_scribbled_exams1.pdf
    # TODO: I removed some -c lines here...
    - plom-scan status
    - plom-scan process --demo fake_scribbled_exams2.pdf
    - plom-scan upload -u fake_scribbled_exams2.pdf
    - plom-scan status
    - plom-scan process --demo fake_scribbled_exams3.pdf
    - plom-scan upload -u fake_scribbled_exams3.pdf
    - plom-scan status
    ## not supposed to be done yet:
    - if (plom-finish status); then false; else true; fi
    - python3 -m plom.client.randoIDer -s localhost -u user0 -w 0123
    - python3 -m plom.client.randoMarker -s localhost -u user0 -w 0123
    - plom-finish status
    - plom-finish csv
    - A=`cat marks.csv  | wc -l`      # How many lines?
    - echo $A
    - A="$((A-1))"
    - bash -c "[[ $A == 19 ]]"        # b/c 1 page from 1 test is deleted
    - file marks.csv
    - file marks.csv | grep text
    - plom-finish reassemble
    - A=`ls reassembled/ | wc -l`    # How many files?
    - bash -c "[[ $A == 19 ]]" # since 1 test incomplete
    - plom-finish solutions --mark
    - A=`ls solutions/ | wc -l`    # How many files?
    - bash -c "[[ $A == 19 ]]"
    - A=`du -sm reassembled/ | cut -f1`  # Don't regress on issue #627
    - bash -c "[[ $A -lt 40 ]]"          # not more than 10 MB
    - plom-finish webpage --solutions
    - A=`ls codedReturn/ | wc -l`    # How many files (inc soln)?
    - bash -c "[[ $A == 39 ]]"       # 2*(20-1) pdf + 1 html
    - echo "Now take down the server"
    # https://gitlab.com/gitlab-org/gitlab-runner/issues/2880
    - jobs -l
    - kill %1
    - sleep 2
    - echo "Should be no jobs and this should succeed"
    - jobs -l
    - popd


appstream-valid:
  stage: static_analysis
  image: alpine
  before_script:
    - apk add appstream
    - apk add appstream-glib
  script:
    - appstream-util --version || true
    - appstream-util validate org.plomgrading.PlomClient.metainfo.xml
    - appstreamcli --version
    # TODO: consider adding --strict on 0.16.4 which supports <developer> tag
    - appstreamcli validate --pedantic --explain org.plomgrading.PlomClient.metainfo.xml


# TODO: perhaps all deps should be in the docker image?
docs_sphinx:
  stage: test
  needs: ["container-image"]
  image: $IM
  before_script:
    - apt-get update
    - apt-get --no-install-recommends --yes install tex-gyre
    - pip install -r doc/requirements.txt
  script:
    - pushd doc
    - ls
    - make autodocs
    - make html
    - make singlehtml
    - make linkcheck
    - make latexpdf
    - popd
  artifacts:
    paths:
      - doc/build/latex/plom.pdf
      - doc/build/html/
    expire_in: 16 days


# get latest pip deps, doesn't use docker, closer to user install
# allowed to fail (some pip stuff might be new) but we want to know
# TODO: `dnf install python3-opencv`, and `sed` out the setup.py dep
fedora_legacy:
  stage: alt_build
  image: fedora:41
  when: manual
  allow_failure: true
  before_script:
    - dnf install -y ImageMagick gcc gcc-c++ cmake
          latexmk tex-dvipng texlive-scheme-basic
          tex-preview tex-charter tex-exam tex-preprint
          python3-passlib python3-pyqt6
          python3-jsmin python3-defusedxml python3-yaml
          python3-urllib3 python3-more-itertools
          python3-seaborn python3-pandas python3-requests-toolbelt
          python3-pip python3-wheel python3-setuptools
          python3-tomlkit python3-pillow python3-tqdm
          python3-appdirs python3-arrow
          python3-aiohttp python3-peewee python3-cryptography
          python3-zxing-cpp
          python3-pytest
          python3-PyMuPDF python3-scikit-learn python3-PyMySQL
          file python3-file-magic
          iproute
    - pip --version
  script:
    # temporary, see https://gitlab.com/plom/plom/-/merge_requests/1634
    - pip install -U pytest
    - pip install .
    # First, run the unit tests
    - which pytest-3
    - which pytest
    - pytest plom
    # Build tests
    - mkdir play
    - pushd play
    - plom-demo . --num-papers 3 --prepare-only
    - ip addr
    - plom-server launch . &
    - sleep 2
    - sleep 2
    - echo "Server should be in background"
    - jobs -l
    - echo "We should be able to connect to it"
    - curl -k https://localhost:41984/Version
    - A=`ls papersToPrint/ | wc -l`  # How many files?
    - bash -c "[[ $A == 3 ]]"        # should be 3
    - export PLOM_MANAGER_PASSWORD=1234
    # not supposed to be done yet:
    - if (plom-finish status); then false; else true; fi
    - python3 -m plom.client.randoIDer -s localhost -u user0 -w 0123
    - python3 -m plom.client.randoMarker -s localhost -u user0 -w 0123
    - plom-finish status
    - plom-finish csv
    - A=`cat marks.csv  | wc -l`      # How many lines?
    - echo $A
    - A="$((A-1))"
    - bash -c "[[ $A == 2 ]]"         # b/c 1 page from 1 test is deleted
    - file marks.csv
    - file marks.csv | grep text
    - plom-finish reassemble
    - A=`ls reassembled/ | wc -l`    # How many files?
    - bash -c "[[ $A == 2 ]]" # since 1 test incomplete
    - A=`du -sm reassembled/ | cut -f1`  # Don't regress on issue #627
    - bash -c "[[ $A -lt 10 ]]"          # not more than 10 MB
    - echo "Now take down the server"
    # https://gitlab.com/gitlab-org/gitlab-runner/issues/2880
    - jobs -l
    - kill %1
    - sleep 2
    - echo "Should be no jobs and this should succeed"
    - jobs -l
    - popd


# get latest pip deps, doesn't use docker, closer to user install
# allowed to fail (some pip stuff might be new) but we want to know
# TODO: `dnf install python3-opencv`, and `sed` out the setup.py dep
fedora_webplom:
  services:
    - postgres
  variables:
    POSTGRES_DB: "plom_db"
    POSTGRES_USER: "postgres"
    POSTGRES_PASSWORD: "postgres"
    POSTGRES_HOST_AUTH_METHOD: trust
    PLOM_DATABASE_HOSTNAME: "postgres"
  stage: alt_build
  needs: []
  image: fedora:41
  when: manual
  allow_failure: true
  before_script:
    - dnf install -y ImageMagick gcc gcc-c++ cmake
          latexmk tex-dvipng texlive-scheme-basic
          tex-preview tex-charter tex-exam tex-preprint
          python3-passlib python3-pyqt6
          python3-jsmin python3-defusedxml python3-yaml
          python3-urllib3 python3-more-itertools
          python3-seaborn python3-pandas python3-requests-toolbelt
          python3-pip python3-wheel python3-setuptools
          python3-tomlkit python3-pillow python3-tqdm
          python3-appdirs python3-arrow
          python3-aiohttp python3-peewee python3-cryptography
          python3-zxing-cpp
          python3-pytest
          python3-PyMuPDF python3-scikit-learn python3-PyMySQL
          file python3-file-magic
          iproute
          python3-django
    - pip --version
  script:
    # temporary, see https://gitlab.com/plom/plom/-/merge_requests/1634
    - pip install -U pytest
    - pip install .
    - export PYTHONPATH=$PWD
    - echo $PYTHONPATH
    - pushd plom_server
    - ./Launcher/launch_scripts/launch_plom_demo_server.py --stop-after reports
    - export PLOM_DATABASE_BACKEND=sqlite
    - python3 manage.py test
    - popd


# get latest pip deps, doesn't use docker, closer to user install
# allowed to fail (some pip stuff might be new) but we want to know
newOS_newdeps_legacy:
  stage: alt_build
  image: ubuntu:22.04
  allow_failure: true
  before_script:
    - apt-get update
    - DEBIAN_FRONTEND=noninteractive apt-get install -y tzdata curl
    - apt-get --no-install-recommends --yes install
      cmake make g++ imagemagick
      dvipng latexmk texlive-latex-extra texlive-fonts-recommended
      libgl1-mesa-glx libsm6 libxrender1 libegl1 libxkbcommon0 libdbus-1-3
      libpango-1.0-0 libpangocairo-1.0-0
      python3-pytest python3-dev
      python3-pip
      iproute2 psmisc file python3-magic
    # apt-get --no-install-recommends --yes install libimage-exiftool-perl
    - python3 -m pip install --upgrade pip setuptools packaging wheel
    - pip --version
  script:
    - pip install .
    # First, run the unit tests
    - pytest-3 -l --pyargs plom
    # Build tests
    - mkdir play
    - pushd play
    - plom-demo . --num-papers 3 --prepare-only
    - ip addr
    - plom-server launch . &
    - sleep 2
    - sleep 2
    - echo "Server should be in background"
    - jobs -l
    - echo "We should be able to connect to it"
    - curl -k https://localhost:41984/Version
    - A=`ls papersToPrint/ | wc -l`  # How many files?
    - bash -c "[[ $A == 3 ]]"        # should be 3
    - export PLOM_MANAGER_PASSWORD=1234
    # not supposed to be done yet:
    - if (plom-finish status); then false; else true; fi
    - python3 -m plom.client.randoIDer -s localhost -u user0 -w 0123
    - python3 -m plom.client.randoMarker -s localhost -u user0 -w 0123
    - plom-finish status
    - plom-finish csv
    - A=`cat marks.csv  | wc -l`      # How many lines?
    - echo $A
    - A="$((A-1))"
    - bash -c "[[ $A == 2 ]]"         # b/c 1 page from 1 test is deleted
    - file marks.csv
    - file marks.csv | grep text
    - plom-finish reassemble
    - A=`ls reassembled/ | wc -l`    # How many files?
    - bash -c "[[ $A == 2 ]]"  # since 1 test incomplete
    - A=`du -sm reassembled/ | cut -f1`  # Don't regress on issue #627
    - bash -c "[[ $A -lt 10 ]]"          # not more than 10 MB
    - plom-finish webpage
    - A=`ls codedReturn/ | wc -l`    # How many files?
    - bash -c "[[ $A == 3 ]]"        # 3-1 pdf + 1 html
    - echo "Now take down the server"
    # https://gitlab.com/gitlab-org/gitlab-runner/issues/2880
    - jobs -l
    - kill %1
    - sleep 2
    - echo "Should be no jobs and this should succeed"
    - jobs -l
    - popd


# get latest pip deps, doesn't use docker, closer to user install
# allowed to fail (some pip stuff might be new) but we want to know
newOS_newdeps_webplom:
  services:
    - postgres
  variables:
    POSTGRES_DB: "plom_db"
    POSTGRES_USER: "postgres"
    POSTGRES_PASSWORD: "postgres"
    POSTGRES_HOST_AUTH_METHOD: trust
    PLOM_DATABASE_HOSTNAME: "postgres"
  stage: alt_build
  image: ubuntu:22.04
  allow_failure: true
  before_script:
    - apt-get update
    - DEBIAN_FRONTEND=noninteractive apt-get install -y tzdata curl
    - apt-get --no-install-recommends --yes install
      cmake make g++ imagemagick
      dvipng latexmk texlive-latex-extra texlive-fonts-recommended
      libgl1-mesa-glx libsm6 libxrender1 libegl1 libxkbcommon0 libdbus-1-3
      libpango-1.0-0 libpangocairo-1.0-0
      python3-pytest python3-dev
      python3-pip
      iproute2 psmisc file python3-magic
    # apt-get --no-install-recommends --yes install libimage-exiftool-perl
    - python3 -m pip install --upgrade pip setuptools packaging wheel
    - pip --version
  script:
    - pip install .
    - export PYTHONPATH=$PWD
    - echo $PYTHONPATH
    - pushd plom_server
    - ./Launcher/launch_scripts/launch_plom_demo_server.py --length quick --stop-after randomarking
    - export PLOM_DATABASE_BACKEND=sqlite
    - python3 manage.py test
    - popd


# Ensure minimum listed dependency versions actually work on older system
# 1. oldest reasonably supported popular OS
# 2. take python deps from package manager
# 3. force the minimum version from setup.py
# Goal here is to catch changes that need newer features of a dependency.
oldOS_mindeps:
  stage: alt_build
  image: ubuntu:20.04
  before_script:
    - apt-get update
    - DEBIAN_FRONTEND=noninteractive apt-get install -y tzdata curl
    - apt-get --no-install-recommends --yes install
      cmake make g++ imagemagick
      dvipng latexmk texlive-latex-extra texlive-fonts-recommended
      libgl1-mesa-glx libsm6 libxrender1 libegl1 libxkbcommon0 libdbus-1-3
      libpango-1.0-0 libpangocairo-1.0-0
      python3 python3-dev
      python3-pip python3-setuptools python3-wheel
      python3-passlib
      python3-requests-toolbelt python3-pil python3-tqdm
      python3-defusedxml python3-jsmin python3-packaging
      iproute2 psmisc file python3-magic
    # TODO: is it possible to run with OOTB pip etc?
    - python3 -m pip install --upgrade pip setuptools packaging wheel
    - python3 -m pip --version
  script:
    - python3 -m pip install -r requirements.txt.tempminima
    - python3 -m pip install .
    # First, run the unit tests
    - pytest -l --pyargs plom
    # Build tests
    - mkdir play
    - pushd play
    - plom-demo . --num-papers 3 --prepare-only
    - ip addr
    - plom-server launch . &
    - sleep 2
    - sleep 2
    - echo "Server should be in background"
    - jobs -l
    - echo "We should be able to connect to it"
    - curl -k https://localhost:41984/Version
    - A=`ls papersToPrint/ | wc -l`  # How many files?
    - bash -c "[[ $A == 3 ]]"        # should be 3
    - export PLOM_MANAGER_PASSWORD=1234
    # not supposed to be done yet:
    - if (plom-finish status); then false; else true; fi
    - python3 -m plom.client.randoIDer -s localhost -u user0 -w 0123
    - python3 -m plom.client.randoMarker -s localhost -u user0 -w 0123
    - plom-finish status
    - plom-finish csv
    - A=`cat marks.csv  | wc -l`      # How many lines?
    - echo $A
    - A="$((A-1))"
    - bash -c "[[ $A == 2 ]]"         # b/c 1 page from 1 test is deleted
    - file marks.csv
    - file marks.csv | grep text
    - plom-finish reassemble
    - A=`ls reassembled/ | wc -l`    # How many files?
    - bash -c "[[ $A == 2 ]]"  # since 1 test incomplete
    - A=`du -sm reassembled/ | cut -f1`  # Don't regress on issue #627
    - bash -c "[[ $A -lt 10 ]]"          # not more than 10 MB
    - echo "Now take down the server"
    # https://gitlab.com/gitlab-org/gitlab-runner/issues/2880
    - jobs -l
    - kill %1
    - sleep 2
    - echo "Should be no jobs and this should succeed"
    - jobs -l
    - popd


# Minimum Python, minimum deps
minpy_mindeps:
  stage: alt_build
  image: python:3.8
  before_script:
    - apt-get update
    - apt-get --no-install-recommends --yes install
      cmake make imagemagick dvipng g++
      texlive-latex-extra latexmk texlive-fonts-recommended
      libgl1-mesa-glx libsm6 libxrender1 libegl1 libxkbcommon0 libdbus-1-3
    - pip install pytest
  script:
    - pip install -r requirements.txt.tempminima
    - pip install .
    - pytest -l --pyargs plom
    - plom-demo . --num-papers 3 --prepare-only


# Minimum Python, minimum deps
minpy_mindeps_webplom:
  services:
    - postgres
  variables:
    POSTGRES_DB: "plom_db"
    POSTGRES_USER: "postgres"
    POSTGRES_PASSWORD: "postgres"
    POSTGRES_HOST_AUTH_METHOD: trust
    PLOM_DATABASE_HOSTNAME: "postgres"
  stage: alt_build
  image: python:3.8
  before_script:
    - apt-get update
    - apt-get --no-install-recommends --yes install
      cmake make imagemagick dvipng g++
      texlive-latex-extra latexmk texlive-fonts-recommended
      libgl1-mesa-glx libsm6 libxrender1 libegl1 libxkbcommon0 libdbus-1-3
    - pip install pytest
  script:
    - pip install -r requirements.txt.tempminima
    - pip install .
    - export PYTHONPATH=$PWD
    - echo $PYTHONPATH
    - pushd plom_server
    - ./Launcher/launch_scripts/launch_plom_demo_server.py --length quick --stop-after randomarking
    - export PLOM_DATABASE_BACKEND=sqlite
    - python3 manage.py test
    - popd


# upload binaries are generic gitlab packages
binary_upload:
  stage: prep
  image: curlimages/curl:latest
  rules:
   - if: $CI_COMMIT_TAG
  script:
    - ls
    - md5sum PlomClient*
    - |
      tee release_desc.md <<EOF
      ## Installation instructions

      Please see [plomgrading.org](https://plomgrading.org).
      If you're here looking for Clients, see "Plom Client" packages above.


      #### Changes in this release

      See [the Changelog](https://gitlab.com/plom/plom/-/blob/$CI_COMMIT_TAG/CHANGELOG.md).


      EOF
    # strip the leading v in v0.x.y: don't see how to do this without dotenv
    - export VER=${CI_COMMIT_TAG:1}
    - echo $VER
    - export LINKNAME0="Plom Client for GNU/Linux (AppImage)"
    - export LINKNAME1="Plom Client for Windows (Windows ≥ 10, x86-64)"
    - export LINKNAME2="Plom Client for older Mac (x86_64, macOS ≥ 13)"
    - export LINKNAME3="Plom Client for newer Mac (arm64, macOS ≥ 13)"
    - export FILENAME0="PlomClient-$VER-x86_64.AppImage"
    - export FILENAME1="PlomClient-$VER.exe"
    - export FILENAME2="PlomClient-$VER-macos13-x86_64.zip"
    - export FILENAME3="PlomClient-$VER-macos13-arm64.zip"
    - echo "#### artifact md5 hashes" >> "release_desc.md"
    - echo "" >> "release_desc.md"
    - md5sum dist/plom*.tar.gz | sed -e "s/^/    /" | sed -e "s/dist\///" >> "release_desc.md"
    - md5sum $FILENAME0 | sed -e "s/^/    /" >> "release_desc.md"
    - md5sum $FILENAME1 | sed -e "s/^/    /" >> "release_desc.md"
    - md5sum $FILENAME2 | sed -e "s/^/    /" >> "release_desc.md"
    - md5sum $FILENAME3 | sed -e "s/^/    /" >> "release_desc.md"
    - echo "" >> "release_desc.md"
    - echo "" >> "release_desc.md"
    - echo "#### artifact sha256 hashes" >> "release_desc.md"
    - echo "" >> "release_desc.md"
    - sha256sum dist/plom*.tar.gz | sed -e "s/^/    /" | sed -e "s/dist\///" >> "release_desc.md"
    - sha256sum $FILENAME0 | sed -e "s/^/    /" >> "release_desc.md"
    - sha256sum $FILENAME1 | sed -e "s/^/    /" >> "release_desc.md"
    - sha256sum $FILENAME2 | sed -e "s/^/    /" >> "release_desc.md"
    - sha256sum $FILENAME3 | sed -e "s/^/    /" >> "release_desc.md"
    - export URL0="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/PlomClient-linux/$VER/$FILENAME0"
    - export URL1="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/PlomClient-windows/$VER/$FILENAME1"
    - export URL2="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/PlomClient-macos/$VER/$FILENAME2"
    - export URL3="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/PlomClient-macos/$VER/$FILENAME3"
    - export OTHER_URL1="https://pypi.org/project/plom"
    - export OTHER_URL2="https://flathub.org/apps/details/org.plomgrading.PlomClient"
    - export OTHER_URL3="https://hub.docker.com/r/plomgrading/server"
    - export OTHER_LINKNAME1="Find Plom on PyPI"
    - export OTHER_LINKNAME2="Install Plom Client from Flathub (GNU/Linux)"
    - export OTHER_LINKNAME3="Plom Server on DockerHub"
    - echo "OTHER_URL1=$OTHER_URL1" >> release_info.env
    - echo "OTHER_URL2=$OTHER_URL2" >> release_info.env
    - echo "OTHER_URL3=$OTHER_URL3" >> release_info.env
    - echo "OTHER_LINKNAME1=$OTHER_LINKNAME1" >> release_info.env
    - echo "OTHER_LINKNAME2=$OTHER_LINKNAME2" >> release_info.env
    - echo "OTHER_LINKNAME3=$OTHER_LINKNAME3" >> release_info.env
    - echo "LINKNAME0=$LINKNAME0" >> release_info.env
    - echo "LINKNAME1=$LINKNAME1" >> release_info.env
    - echo "LINKNAME2=$LINKNAME2" >> release_info.env
    - echo "LINKNAME3=$LINKNAME3" >> release_info.env
    - echo "URL0=$URL0" >> release_info.env
    - echo "URL1=$URL1" >> release_info.env
    - echo "URL2=$URL2" >> release_info.env
    - echo "URL3=$URL3" >> release_info.env
    - echo "FILENAME0=$FILENAME0" >> release_info.env
    - echo "FILENAME1=$FILENAME1" >> release_info.env
    - echo "FILENAME2=$FILENAME2" >> release_info.env
    - echo "FILENAME3=$FILENAME3" >> release_info.env
    # actually do the uploads
    - |
      curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file $FILENAME0 "$URL0"
    - |
      curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file $FILENAME1 "$URL1"
    - |
      curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file $FILENAME2 "$URL2"
    - |
      curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file $FILENAME3 "$URL3"
    - cat release_desc.md
  artifacts:
    paths:
      - release_desc.md
    reports:
      dotenv: release_info.env
    expire_in: 16 days


# Build Client Binaries
# These are run once tests pass: to change re-add `needs: []`
linux_client_appimage:
  stage: packaging
  script:
    - docker build --tag appimagebuilder_tmp -f AppImageBuilder.Containerfile .
    - docker create -ti --name dummy appimagebuilder_tmp bash
    - export VER=`sed -nr 's/^__version__ = \"(.+)\"/\1/p' plom/__init__.py`
    - export NAME=PlomClient-$VER-x86_64.AppImage
    - docker cp dummy:/app/$NAME .
    - docker rm -f dummy
    - md5sum $NAME > md5sum
    - ls $NAME
    - cat md5sum
  artifacts:
    paths:
    - PlomClient*.AppImage
    - md5sum
    expire_in: 16 days


# TODO: broken by the upgrade to PyQt6, currently set to manual
linuxbin_ubuntu2004:
  stage: packaging
  when: manual
  allow_failure: true
  image: ubuntu:20.04
  before_script:
    - apt-get update
    - DEBIAN_FRONTEND=noninteractive apt-get install -y tzdata curl
    - apt-get --no-install-recommends --yes install iproute2 psmisc file binutils
      python3 python3-dev python3-wheel python3-setuptools python3-pip
    - apt-get install -y libglib2.0 libgl1-mesa-glx libsm6 libxrender1 libegl1 libxkbcommon0 libdbus-1-3 libxcb-cursor0
    - pip install --upgrade pip
    - pip install pyinstaller
  script:
    - pip install -r requirements.txt.client
    - pyinstaller client.linux.spec
    - export VER=$(python3 -c "from plom import __version__; print(__version__)")
    - echo "Extracted version string '$VER'"
    - export NAME=PlomClient-$VER-linux-ubuntu2004.bin
    - mv dist/PlomClient*.bin $NAME
    - chmod 755 $NAME
    - md5sum $NAME > md5sum
    - ls $NAME
    - cat md5sum
  artifacts:
    paths:
    - PlomClient*.bin
    - md5sum
    expire_in: 16 days


linuxbin_ubuntu2204:
  stage: packaging
  image: ubuntu:22.04
  before_script:
    - apt-get update
    - DEBIAN_FRONTEND=noninteractive apt-get install -y tzdata curl
    - apt-get --no-install-recommends --yes install iproute2 psmisc file binutils
      python3 python3-dev python3-wheel python3-setuptools python3-pip
    - apt-get install -y libglib2.0 libgl1-mesa-glx libsm6 libxrender1 libegl1 libxkbcommon0 libdbus-1-3 libxcb-cursor0
    - pip install --upgrade pip
    - pip install pyinstaller
    - pip install . --no-deps  # pyinstaller spec needs version
  script:
    - pip install -r requirements.txt.client
    - pyinstaller client.linux.spec
    - export VER=$(python3 -c "from plom import __version__; print(__version__)")
    - echo "Extracted version string '$VER'"
    - export NAME=PlomClient-$VER-linux-ubuntu2204.bin
    - mv dist/PlomClient*.bin $NAME
    - chmod 755 $NAME
    - md5sum $NAME > md5sum
    - ls $NAME
    - cat md5sum
  artifacts:
    paths:
    - PlomClient*.bin
    - md5sum
    expire_in: 16 days


# Notes on macOS client builds
# - June 2023, we now have only M1 (arm64) runners:
#     - goal remains universal2 binary: Issue #1655
#     - for now, we use Rosetta2 to cross compile onto x86_64
# - Aug 2021, based on https://github.com/danhper/asdf-python/issues/38 we used:
#     - PYTHON_CONFIGURE_OPTS="--enable-framework" asdf install python $MACBIN_PY_VER
#   Not sure what this was for but it broke in 2023-Jan (Issue #2478).  I tried:
#   "--enable-framework=/Users/gitlab/.asdf/installs/Library/Frameworks" asdf ...
#   Also "--enable-shared", but it seems to work without any of these things?
# - macOS <= 10.14: older asdf, use `asdf local` not `asdf shell`
# - to get hardcoded path: verbose `pip install -v -v pyinstaller`
macos13_universal_client:
  image: macos-13-xcode-14
  tags:
  - saas-macos-medium-m1
  rules:
  - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
    when: never
  - if: $CI_PROJECT_PATH == "plom/plom"
    when: manual
  allow_failure: true
  stage: alt_build
  needs: []
  variables:
    HOMEBREW_NO_AUTO_UPDATE: 1
    HOMEBREW_NO_INSTALL_CLEANUP: 1
    HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1
  script:
  - pwd
  - sw_vers
  - uname -a
  - uname -m
  - echo $SHELL
  - which python
  - python --version
  - pip --version
  - uname -m
  - arch -x86_64 uname -m
  - brew --version
  - brew install ninja
  - brew install cmake
  - brew install numpy
  # quite likely the built-in python would work...  but below we install latest
  - arch -x86_64 /usr/bin/python3 --version
  - arch -arm64 /usr/bin/python3 --version
  - /usr/bin/python3 -m pip --version
  # install specific python
  - curl https://www.python.org/ftp/python/$MACBIN_PY_VER/python-$MACBIN_PY_VER-macos11.pkg --output python.pkg
  - sudo installer -pkg python.pkg -target /
  - python3 --version
  - /usr/bin/python3 --version
  - /usr/local/bin/python3 --version
  - arch -x86_64 /usr/local/bin/python3 --version
  - /usr/local/bin/python3 -m pip install --upgrade pip
  - which pip
  - which pip3
  - pip --version
  # hacks for charset_normalizer, following https://github.com/machawk1/wail/pull/562/
  - arch -x86_64 /usr/local/bin/python3 -m pip install "charset_normalizer<3.0"
  # /usr/local/bin/python3 -m pip install "charset_normalizer<3.0"
  # arch -x86_64 /usr/local/bin/python3 -m pip install Pillow --no-binaries
  # numpy fails complaining about blas, try from brew instead?
  # /usr/local/bin/python3 -m pip install numpy --no-binary ":all:"
  - /usr/local/bin/python3 -m pip install Pillow --no-binary ":all:"
  - /usr/local/bin/python3 -m pip install -r requirements.txt.client
  - arch -x86_64 /usr/local/bin/python3 -m pip install -r requirements.txt.client
  - /usr/local/bin/python3 -m pip install pyinstaller~=6.10.0
  - /usr/local/bin/python3 -m pip install . --no-deps  # pyinstaller spec needs version
  - /Library/Frameworks/Python.framework/Versions/${MACBIN_PY_MAJOR_MINOR_VER}/bin/pyinstaller client.macos-universal2.spec
  - ls dist/
  - export VER=`sed -nr 's/^__version__ = \"(.+)\"/\1/p' plom/__init__.py`
  - echo "Extracted version string '$VER'"
  - export ZIPNAME=PlomClient-$VER-macos13.zip
  - pushd dist
  - ls
  - ls -sklR PlomClient-$VER-universal2.app
  - zip -r $ZIPNAME PlomClient-$VER.app
  - popd
  - mv dist/$ZIPNAME $ZIPNAME
  - md5 $ZIPNAME > md5sum
  - ls $ZIPNAME
  - cat md5sum
  artifacts:
    paths:
    - PlomClient*.zip
    - md5sum
    expire_in: 16 days


macos13_arm64_client:
  image: macos-13-xcode-14
  tags:
  - saas-macos-medium-m1
  rules:
  - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
    when: never
  - if: $CI_PROJECT_PATH == "plom/plom"
  stage: packaging
  script:
  - pwd
  - sw_vers
  - uname -a
  - uname -m
  - echo $SHELL
  - which python
  - python --version
  - pip --version
  - uname -m
  - /usr/bin/python3 -m pip --version
  - curl https://www.python.org/ftp/python/$MACBIN_PY_VER/python-$MACBIN_PY_VER-macos11.pkg --output python.pkg
  - sudo installer -pkg python.pkg -target /
  - python3 --version
  - /usr/bin/python3 --version
  - /usr/local/bin/python3 --version
  - /usr/local/bin/python3 -m pip install --upgrade pip
  - /usr/local/bin/python3 -m pip install -r requirements.txt.client
  - /usr/local/bin/python3 -m pip install pyinstaller~=6.10.0
  - /usr/local/bin/python3 -m pip install . --no-deps  # pyinstaller spec needs version
  - /Library/Frameworks/Python.framework/Versions/${MACBIN_PY_MAJOR_MINOR_VER}/bin/pyinstaller client.macos.spec
  - ls dist/
  - export VER=`sed -nr 's/^__version__ = \"(.+)\"/\1/p' plom/__init__.py`
  - echo "Extracted version string '$VER'"
  - export ZIPNAME=PlomClient-$VER-macos13-arm64.zip
  - pushd dist
  - ls
  - ls -sklR PlomClient-$VER-arm64.app
  - zip -r $ZIPNAME PlomClient-$VER-arm64.app
  - popd
  - mv dist/$ZIPNAME $ZIPNAME
  - md5 $ZIPNAME > md5sum
  - ls $ZIPNAME
  - cat md5sum
  artifacts:
    paths:
    - PlomClient*.zip
    - md5sum
    expire_in: 16 days


macos13_x86_64_client:
  image: macos-13-xcode-14
  tags:
  - saas-macos-medium-m1
  rules:
  - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
    when: never
  - if: $CI_PROJECT_PATH == "plom/plom"
  stage: packaging
  script:
  - pwd
  - sw_vers
  - uname -a
  - uname -m
  - echo $SHELL
  - which python
  - python --version
  - pip --version
  - uname -m
  - arch -x86_64 uname -m
  # quite likely the built-in python would work...  but below we install latest
  - arch -x86_64 /usr/bin/python3 --version
  # install specific python
  - curl https://www.python.org/ftp/python/$MACBIN_PY_VER/python-$MACBIN_PY_VER-macos11.pkg --output python.pkg
  - sudo installer -pkg python.pkg -target /
  - python3 --version
  - /usr/bin/python3 --version
  - /usr/local/bin/python3 --version
  - arch -x86_64 /usr/local/bin/python3 --version
  - arch -x86_64 /usr/local/bin/python3 -m pip install --upgrade pip
  # hacks for charset_normalizer, following https://github.com/machawk1/wail/pull/562/
  - arch -x86_64 /usr/local/bin/python3 -m pip install "charset_normalizer<3.0"
  - arch -x86_64 /usr/local/bin/python3 -m pip install -r requirements.txt.client
  - arch -x86_64 /usr/local/bin/python3 -m pip install pyinstaller~=6.10.0
  - arch -x86_64 /usr/local/bin/python3 -m pip install . --no-deps  # pyinstaller spec needs version
  - arch -x86_64 /Library/Frameworks/Python.framework/Versions/${MACBIN_PY_MAJOR_MINOR_VER}/bin/pyinstaller client.macos-x86_64.spec
  - ls dist/
  - export VER=`sed -nr 's/^__version__ = \"(.+)\"/\1/p' plom/__init__.py`
  - echo "Extracted version string '$VER'"
  - export ZIPNAME=PlomClient-$VER-macos13-x86_64.zip
  - pushd dist
  - ls
  - ls -sklR PlomClient-$VER-x86_64.app
  - zip -r $ZIPNAME PlomClient-$VER-x86_64.app
  - popd
  - mv dist/$ZIPNAME $ZIPNAME
  - md5 $ZIPNAME > md5sum
  - ls $ZIPNAME
  - cat md5sum
  artifacts:
    paths:
    - PlomClient*.zip
    - md5sum
    expire_in: 16 days


# this was slow and failed, went back to "brew"
#   asdf plugin-add imagemagick
#   asdf install imagemagick 7.0.11-5
# Runs Python 3.10 as of 2023-10
# As of 2023-10, we only try to run webplom tests, see TODO about legacy
macos_server_webplom:
  variables:
    HOMEBREW_NO_AUTO_UPDATE: 1
    HOMEBREW_NO_INSTALL_CLEANUP: 1
    HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1
  image: macos-13-xcode-14
  tags:
  - saas-macos-medium-m1
  rules:
  - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
    when: never
  - if: $CI_PROJECT_PATH == "plom/plom"
    when: manual
  allow_failure: true
  stage: alt_build
  needs: []
  script:
  - sw_vers
  - echo $SHELL
  - which python
  - python --version
  - pip --version
  - python -m pip install --upgrade pip
  - pip --version
  - echo $USER
  - brew --version
  # brew update
  # brew --version
  - brew install cmake pango
  - brew install gobject-introspection
  - brew install cffi
  - brew install imagemagick
  - time brew install basictex
  - echo $PATH
  # pdflatex et al not in path until we do this:
  - eval "$(/usr/libexec/path_helper)"
  - echo $PATH
  # alternative slower tex install:
  # time brew install mactex-no-gui
  - ls /Library/TeX/texbin/
  - which pdflatex
  - which tlmgr
  # need sudo, "You don't have permission to change the installation in any way"
  - time sudo tlmgr update --self
  - sudo tlmgr install latexmk
  - sudo tlmgr install dvipng
  - sudo tlmgr install preview exam preprint
  - which latexmk
  - pip install pytest
  - pip install .
  - python -m plom.create --version
  - python -m plom.server --version
  # TODO this fails b/c it runs python3, which is not the right Python
  # python -m pytest -l --pyargs plom
  # Instead test webplom
  - export PYTHONPATH=$PWD
  - echo $PYTHONPATH
  - pushd plom_server
  # no postgres yet
  # ./Launcher/launch_scripts/launch_plom_demo_server.py --length quick --stop-after randomarking
  - export PLOM_DATABASE_BACKEND=sqlite
  - python manage.py test
  - popd


# winmintest:
#   tags:
#   - saas-windows-medium-amd64
#   stage: test
#   needs: []
#   script:
#   - choco install -y python311
#   - refreshenv
#   - c:\Python311\Scripts\pip install -r requirements.txt.client
#   - refreshenv
#   - echo $Env:Path
#   - c:\Python311\python -m plom.client -h


# Build Windows client binaries
# This section written in PowerShell
winpyinst:
  tags:
  - saas-windows-medium-amd64
  stage: packaging
  before_script:
  - choco install -y python311
  - Import-Module $env:ChocolateyInstall\helpers\chocolateyProfile.psm1
  - refreshenv
  - c:\python311\python -m pip --version
  - c:\python311\python -m pip install --user --upgrade pip
  - c:\python311\python -m pip --version
  - c:\python311\python -m pip install --user --upgrade setuptools packaging wheel
  - refreshenv
  script:
  - c:\python311\python -m pip install -r requirements.txt.client
  - refreshenv
  # Delay pyinstaller for a month or so, sometimes output flagged as virus (Issue #1353)
  - c:\python311\python -m pip install pyinstaller~=6.10.0
  - refreshenv
  - c:\python311\python -m pip install . --no-deps  # pyinstaller spec needs version
  - refreshenv
  - c:\python311\python -m plom.client -h
  - c:\python311\scripts\pyinstaller client.windows.spec
  - $VER = &"c:\python311\python" -c "from plom import __version__; print(__version__)"
  - echo "Extracted version string '$VER'"
  - $NAME = "PlomClient-$VER.exe"
  - move dist\PlomClient*.exe $NAME
  - CertUtil -hashfile $NAME MD5 > md5sum.txt
  - ls $NAME
  - type md5sum.txt
  artifacts:
    paths:
    - PlomClient*.exe
    - md5sum.txt
    expire_in: 16 days