diff --git a/.github/workflows/makefile.yml b/.github/workflows/makefile.yml new file mode 100644 index 0000000000..35d7d59922 --- /dev/null +++ b/.github/workflows/makefile.yml @@ -0,0 +1,280 @@ +name: Makefile CI + +on: + push: + pull_request: +env: + # BASIC_PACKAGES is used fo the basic tools (make) + BASIC_PACKAGES: git build-essential ccache libgc-dev graphviz libunwind-dev libreadline-dev pkg-config + # MORE_PACKAGES is used for more tools and libs (make more) + MORE_PACKAGES: libgmp-dev libcurl4-openssl-dev libevent-dev openjdk-17-jdk-headless libgles-dev libegl-dev libsdl1.2-dev libsdl2-dev libsdl2-mixer-dev libsdl2-gfx-dev libsdl2-image-dev libsdl2-net-dev libsdl2-ttf-dev libgtk-3-dev inkscape postgresql libxdg-basedir-dev gettext libsqlite3-dev libpq-dev libmongoc-dev openmpi-bin libopenmpi-dev + JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64/ + JNI_LIB_PATH: /usr/lib/jvm/java-17-openjdk-amd64/lib/server/ + LD_LIBRARY_PATH: /usr/lib/jvm/java-17-openjdk-amd64/lib/server/ + +jobs: + build: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - name: Install dependencies + run: sudo apt-get update && sudo apt-get install -y $BASIC_PACKAGES + - run: make + - uses: actions/upload-artifact@v4 + with: + path: bin + name: bin + + sanity-check: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - run: misc/jenkins/ensuregit.sh + - run: misc/jenkins/checkwhitespaces.sh + if: success() || failure() + - run: misc/jenkins/checklicense.sh + if: success() || failure() + - run: misc/jenkins/checksignedoffby.sh + if: success() || failure() + + tests-some: + runs-on: ubuntu-24.04 + needs: [build] + container: "ubuntu:24.04" + services: + postgres: + image: postgres + env: + POSTGRES_HOST_AUTH_METHOD: trust + mongo: + image: mongo + neo4j: + image: 'neo4j:2.3' + env: + NEO4J_AUTH: none + strategy: + fail-fast: false + matrix: + engine: [nitcg, nitcs, nitcsg, nitce, niti] + steps: + - name: Install dependencies + run: apt-get update && apt-get install -y $BASIC_PACKAGES $MORE_PACKAGES + - uses: actions/checkout@v4 + - run: misc/jenkins/ensuregit.sh + - uses: actions/download-artifact@v4 + - run: chmod +x bin/* + - run: make src/version.nit + - run: cd tests && ./search_tests_git.sh origin/master HEAD > list + - run: cd tests && ./tests.sh --engine ${{ matrix.engine }} `cat list` + - uses: actions/upload-artifact@v4 + if: success() || failure() + with: + path: 'tests/*.xml' + name: tests-some-${{ matrix.engine }} + - uses: actions/upload-artifact@v4 + if: success() || failure() + with: + path: 'tests/out' + name: tests-some-${{ matrix.engine }}-res + - uses: EnricoMi/publish-unit-test-result-action@v2 + if: success() || failure() + with: + files: "tests/*.xml" + comment_mode: off + check_name: tests-some (${{ matrix.engine }}) Results + + tests-full: + runs-on: ubuntu-24.04 + needs: [tests-some] + container: "ubuntu:24.04" + services: + postgres: + image: postgres + env: + POSTGRES_HOST_AUTH_METHOD: trust + mongo: + image: mongo + neo4j: + image: 'neo4j:2.3' + env: + NEO4J_AUTH: none + strategy: + fail-fast: false + matrix: + engine: [nitcg, nitcs, nitcsg, nitce, niti] + steps: + - name: Install dependencies + run: apt-get update && apt-get install -y $BASIC_PACKAGES $MORE_PACKAGES + - uses: actions/checkout@v4 + - run: misc/jenkins/ensuregit.sh + - uses: actions/download-artifact@v4 + - run: chmod +x bin/* + - run: make src/version.nit + - name: Run tests + run: cd tests && ./testfull.sh --engine ${{ matrix.engine }} + - uses: actions/upload-artifact@v4 + if: success() || failure() + with: + path: 'tests/*.xml' + name: tests-full-${{ matrix.engine }} + - uses: actions/upload-artifact@v4 + if: success() || failure() + with: + path: 'tests/out' + name: tests-full-${{ matrix.engine }}-res + - uses: EnricoMi/publish-unit-test-result-action@v2 + if: success() || failure() + with: + files: "tests/*.xml" + comment_mode: off + check_name: tests-full (${{ matrix.engine }}) Results + + nitunit-some: + runs-on: ubuntu-24.04 + needs: [build] + container: "ubuntu:24.04" + services: + postgres: + image: postgres + env: + POSTGRES_HOST_AUTH_METHOD: trust + mongo: + image: mongo + neo4j: + image: 'neo4j:2.3' + env: + NEO4J_AUTH: none + steps: + - name: Install dependencies + run: apt-get update && apt-get install -y $BASIC_PACKAGES $MORE_PACKAGES + - uses: actions/checkout@v4 + - run: misc/jenkins/ensuregit.sh + - uses: actions/download-artifact@v4 + - run: chmod +x bin/* + - run: make nitunit-some + - uses: actions/upload-artifact@v4 + if: success() || failure() + with: + path: '*.xml' + name: nitunit-some + - uses: EnricoMi/publish-unit-test-result-action@v2 + if: success() || failure() + with: + files: "*.xml" + comment_mode: off + check_name: 'nitunit-some Results' + + nitunit-full: + runs-on: ubuntu-24.04 + needs: [nitunit-some] + container: "ubuntu:24.04" + services: + postgres: + image: postgres + env: + POSTGRES_HOST_AUTH_METHOD: trust + mongo: + image: mongo + neo4j: + image: 'neo4j:2.3' + env: + NEO4J_AUTH: none + steps: + - name: Install dependencies + run: apt-get update && apt-get install -y $BASIC_PACKAGES $MORE_PACKAGES + - uses: actions/checkout@v4 + - run: misc/jenkins/ensuregit.sh + - uses: actions/download-artifact@v4 + - run: chmod +x bin/* + - run: make nitunit-full + - uses: actions/upload-artifact@v4 + if: success() || failure() + with: + path: '*.xml' + name: nitunit-full + - uses: EnricoMi/publish-unit-test-result-action@v2 + if: success() || failure() + with: + files: "*.xml" + comment_mode: off + check_name: 'nitunit-full Results' + + test-contrib: + runs-on: ubuntu-24.04 + needs: [build] + steps: + - uses: actions/checkout@v4 + - name: Install dependencies + run: sudo apt-get update && sudo apt-get install -y $BASIC_PACKAGES $MORE_PACKAGES + - uses: actions/download-artifact@v4 + - run: chmod +x bin/* + - run: 'PATH=$PWD/bin:$PATH make test-contrib' + - uses: actions/upload-artifact@v4 + if: success() || failure() + with: + path: '*.xml' + name: test-contrib + - uses: EnricoMi/publish-unit-test-result-action@v2 + if: success() || failure() + with: + files: "*.xml" + comment_mode: off + check_name: 'test-contrib Results' + + bootstrap-full: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - name: Install dependencies + run: sudo apt-get update && sudo apt-get install -y $BASIC_PACKAGES + - run: make bootstrap-full + + bench-fast: + runs-on: ubuntu-24.04 + needs: [build] + steps: + - uses: actions/checkout@v4 + - name: Install dependencies + run: sudo apt-get update && sudo apt-get install -y $BASIC_PACKAGES gnuplot + - uses: actions/download-artifact@v4 + - run: chmod +x bin/* + - run: 'PATH=$PWD/bin:$PATH make bench-fast' + - uses: actions/upload-artifact@v4 + if: success() || failure() + with: + path: |- + benchmarks/*.dat + benchmarks/*.gnu + benchmarks/*.png + benchmarks/*.html + name: bench-fast + + valgrind: + runs-on: ubuntu-24.04 + needs: [build] + steps: + - uses: actions/checkout@v4 + - name: Install dependencies + run: sudo apt-get update && sudo apt-get install -y $BASIC_PACKAGES valgrind + - uses: actions/download-artifact@v4 + - run: chmod +x bin/* + - run: make valgrind + - uses: actions/upload-artifact@v4 + if: success() || failure() + with: + path: 'valgrind.out' + name: valgrind + + info: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - run: pwd + - run: id + - run: ls -al + - run: git branch -av || true + - run: lscpu || true + - run: git log --oneline --graph --decorate | head -n 50 + - run: cat /etc/issue + - run: cat /etc/os-release + - run: env diff --git a/Makefile b/Makefile index 4f4f71b96d..48c3fe7cab 100644 --- a/Makefile +++ b/Makefile @@ -49,7 +49,16 @@ bin/nitdoc: cd src; make ../bin/nitdoc bin/nitls: - cd src; make ../bin/nitls + cd src; make + +bin/nitunit: + cd src; make + +bin/nitpick: + cd src; make + +src/version.nit: + cd src && ./git-gen-version.sh doc/stdlib/index.html: bin/nitdoc bin/nitls @echo '***************************************************************' @@ -87,4 +96,77 @@ clean: (echo "CLEANING $$directory" && cd "$$directory" && $(MAKE) clean); \ done @echo "" - @echo "Nit was succesfully cleaned." \ No newline at end of file + @echo "Nit was succesfully cleaned." + +# BASIC CI-CD JOBS ######################################################## + +sanity-check: + misc/jenkins/checkwhitespaces.sh + misc/jenkins/checksignedoffby.sh + misc/jenkins/checkbinaryfiles.sh + misc/jenkins/checklicense.sh + +nitunit-some: bin/nitls bin/nitunit src/version.nit + misc/jenkins/nitunit-some.sh + +nitpick-full: bin/nitls bin/nitpick src/version.nit + bin/nitls lib src examples contrib + bin/nitls -Pp lib src examples | grep -v -f tests/gitlab_ci.skip > list.txt || true # filter what is skipped by tests.sh + xargs bin/nitpick < list.txt + + +# LIB, CONTRIB AND OOT ############################################## + +nitunit-full: bin/nitls bin/nitunit src/version.nit + bin/nitls -Pp lib src examples | grep -v -f tests/gitlab_ci.skip > list.txt || true # filter what is skipped by tests.sh + ls -1 doc/manual/*.md >> list.txt + xargs bin/nitunit -v < list.txt + +nitunit-src: bin/nitls bin/nitunit src/version.nit + bin/nitls -Pp src examples | grep -v -f tests/gitlab_ci.skip > list.txt || true # filter what is skipped by tests.sh + xargs bin/nitunit -v < list.txt + +test-contrib: src/version.nit + misc/jenkins/check_contrib.sh all check + grep 'error message' *.xml > status.txt || true + test ! -s status.txt # no lines, no errors + +# MISC ############################################################## + +bootstrap-full: + cd src && ./full_bootstrap && ./ncall.sh + +bench-fast: + cd benchmarks && ./bench_engines.sh --fast --fast --html options + +nitunit-manual: bin/nitunit + bin/nitunit doc/manual/*.md + +# MORE TOOLS ######################################################## + +build-more-tools: more + $(MAKE) -C contrib/nitcc + +valgrind: src/version.nit + mkdir -p valgrind.out + bin/nitc src/nitc.nit # To warm-up the cache + src/valgrind.sh --callgrind-out-file=valgrind.out/nitc.nitc.out bin/nitc src/nitc.nit -vv + callgrind_annotate valgrind.out/nitc.nitc.out > valgrind.out/nitc.nitc.txt + src/valgrind.sh --callgrind-out-file=valgrind.out/niti.niti.out bin/nit -- src/nit.nit tests/base_simple3.nit -vv + callgrind_annotate valgrind.out/niti.niti.out > valgrind.out/niti.niti.txt + +build-doc: + nitdoc -d nitdoc.out --keep-going lib src + +build-manual: + # apt-get update && apt-get install --yes --no-install-recommends pandoc texlive texlive-latex-extra lmodern + $(make) -C doc/manual + +nitmetrics: + mkdir -p nitmetrics.out + nitmetrics --all --log --log-dir nitmetrics.out --dir nitmetrics.out --keep-going lib src | tee nitmetrics.out/metrics.txt + +build-catalog: + misc/jenkins/check_contrib.sh pre-build + cd contrib && ./oot.sh update && ./oot.sh pre-build + nitcatalog -d catalog.out lib/ examples/ contrib/ contrib/oot/ diff --git a/misc/jenkins/ensuregit.sh b/misc/jenkins/ensuregit.sh new file mode 100755 index 0000000000..9fa7fb6d1f --- /dev/null +++ b/misc/jenkins/ensuregit.sh @@ -0,0 +1,35 @@ +#!/bin/sh +# This file is part of NIT ( http://www.nitlanguage.org ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script ensure that the root direcroty of nit is a git repository with a fully fetched origin. +# It helps CI/CD in environments (like github action) that trim and hack the initial fetch. + +if [ ! -f src/nitc.nit ]; then + echo "Error: run the script in the root directory of nit" + exit 1 +fi +if [ ! -d .git ]; then + echo "No git directory" + exit 1 +fi + +set -x +git config --global --add safe.directory "$PWD" # Because github messes with the homedir +git fetch --unshallow # Because github does --depth=1 +git fetch origin # Ensure origin/master is here +# Outputs some info... +git remote -v +git branch -av +git log --oneline --graph --decorate origin/master..HEAD diff --git a/misc/jenkins/nitunit-some.sh b/misc/jenkins/nitunit-some.sh new file mode 100755 index 0000000000..54560a0850 --- /dev/null +++ b/misc/jenkins/nitunit-some.sh @@ -0,0 +1,25 @@ +#!/bin/sh +# This file is part of NIT ( http://www.nitlanguage.org ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script run nitunit of nit packages modified between revisions + +from=${1:-origin/master} +to=${2:-HEAD} + +git diff --name-only "$from..$to" -- "*.nit" "*.res" "README.*" | grep -v "^tests/\|contrib/" > list0.txt || true +test -s list0.txt || exit 0 +xargs bin/nitls -pP < list0.txt > list.txt +test -s list.txt || exit 0 +xargs bin/nitunit -v < list.txt