From 3e4b76e76116737fbf2709595142c98529177dc2 Mon Sep 17 00:00:00 2001 From: Govind Kamat Date: Sat, 7 Sep 2024 15:59:45 -0700 Subject: [PATCH] More updates to the OSB build. Signed-off-by: Govind Kamat --- .ci/build.sh | 24 +++++++-------- .github/workflows/integ-test.yml | 4 +-- ...elease-drafter.yml => publish-release.yml} | 19 ++++++------ .github/workflows/unit-test.yml | 4 +-- DEVELOPER_GUIDE.md | 19 ++++++++++-- Makefile | 30 ++++++++++++------- docker/Dockerfile | 10 ++++--- 7 files changed, 62 insertions(+), 48 deletions(-) rename .github/workflows/{release-drafter.yml => publish-release.yml} (79%) diff --git a/.ci/build.sh b/.ci/build.sh index e3c4d5e22..dea1f918b 100644 --- a/.ci/build.sh +++ b/.ci/build.sh @@ -1,22 +1,20 @@ #!/usr/bin/env bash -pyenv_init() { - PATH=$HOME/.pyenv/shims:$PATH:$HOME/.pyenv/bin -} - function setup { - export THESPLOG_FILE="${THESPLOG_FILE:-${BENCHMARK_HOME}/.benchmark/logs/actor-system-internal.log}" - # this value is in bytes, the default is 50kB. We increase it to 200kiB. - export THESPLOG_FILE_MAXSIZE=${THESPLOG_FILE_MAXSIZE:-204800} - # adjust the default log level from WARNING - export THESPLOG_THRESHOLD="INFO" + export BENCHMARK_HOME=$GITHUB_WORKSPACE + + export THESPLOG_FILE=$BENCHMARK_HOME/.benchmark/logs/actor-system-internal.log + export THESPLOG_FILE_MAXSIZE=204800 # default is 50 KiB + export THESPLOG_THRESHOLD=INFO # default log level is WARNING - pyenv_init export TERM=dumb export LC_ALL=en_US.UTF-8 + + # Init pyenv. + PATH=$HOME/.pyenv/shims:$PATH:$HOME/.pyenv/bin } -function build { +function build_and_unit_test { setup set -e @@ -25,11 +23,9 @@ function build { make test } -function build_it { +function run_it { setup - export BENCHMARK_HOME="$GITHUB_WORKSPACE" - docker pull ubuntu/squid:latest # make it38, it39, etc. so they run as concurrent GHA jobs diff --git a/.github/workflows/integ-test.yml b/.github/workflows/integ-test.yml index a8c3cf444..9195f0eaa 100644 --- a/.github/workflows/integ-test.yml +++ b/.github/workflows/integ-test.yml @@ -42,6 +42,4 @@ jobs: - run: echo "JAVA17_HOME=$JAVA_HOME" >> $GITHUB_ENV - name: Run the CI build script - run: bash .ci/build.sh build_it ${{ matrix.python-version }} - env: - BENCHMARK_HOME: env.GITHUB_WORKSPACE + run: bash .ci/build.sh run_it ${{ matrix.python-version }} diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/publish-release.yml similarity index 79% rename from .github/workflows/release-drafter.yml rename to .github/workflows/publish-release.yml index 54e12cf82..ab07aadd5 100644 --- a/.github/workflows/release-drafter.yml +++ b/.github/workflows/publish-release.yml @@ -1,4 +1,4 @@ -name: Release drafter +name: Publish Release to GitHub on: push: @@ -6,7 +6,7 @@ on: - "*" jobs: - draft-a-release: + publish-release: runs-on: ubuntu-latest steps: - name: Checkout Repository @@ -22,21 +22,20 @@ jobs: issue-title: 'Release opensearch-benchmark' issue-body: "Please approve or deny the release of opensearch-benchmark. **Tag**: ${{ github.ref_name }} **Commit**: ${{ github.sha }}" exclude-workflow-initiator-as-approver: true + - name: Set up Python 3 uses: actions/setup-python@v3 with: python-version: '3.x' - - name: Install build tools - run: | - python -m pip install --upgrade build + - name: Build project for distribution run: | - python -m build - tar -zvcf artifacts.tar.gz dist - - name: Release + make build + tar zcvf artifacts.tar.gz dist + + - name: Publish release uses: softprops/action-gh-release@v1 with: draft: true generate_release_notes: true - files: | - artifacts.tar.gz \ No newline at end of file + files: artifacts.tar.gz diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 44393e995..5ae43cbf7 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -17,6 +17,4 @@ jobs: run: git clone https://github.com/pyenv/pyenv.git ~/.pyenv - name: Run the CI build script - run: bash .ci/build.sh build - env: - BENCHMARK_HOME: env.GITHUB_WORKSPACE + run: bash .ci/build.sh build_and_unit_test diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index b895bb6fe..79eb63a74 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -23,16 +23,29 @@ This document will walk you through on what's needed to start contributing code ### Prerequisites - - **Pyenv**: Install `pyenv` and follow the instructions in the output of `pyenv init` to set up your shell and restart it before proceeding. - For more details please refer to the [PyEnv installation instructions](https://github.com/pyenv/pyenv#installation). + - **pyenv**: Install `pyenv` and follow the instructions in the output of `pyenv init` to set up your shell and restart it before proceeding. + For more details please refer to the [pyenv installation instructions](https://github.com/pyenv/pyenv#installation). - **Optional Step:** For Debian-based systems, install the following modules to continue with the next steps: + `pyenv` requires that the C compiler and development libraries be installed, so that the specified Python versions can be build from source. The installation instructions vary from platform to platform. + + For Debian-based systems, install the following modules to continue with the next steps: ``` sudo apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev \ libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \ xz-utils tk-dev libffi-dev liblzma-dev git ``` + For Amazon Linux 2023, run the following command: + ``` + sudo yum -y install gcc openssl-devel bzip2-devel libffi-devel ncurses-devel sqlite-devel readline-devel zlib-devel xz-devel + ``` + + On the Mac platform, XCode needs to be installed as well as some additional required libraries: + ``` + xcode-select --install + brew install pyenv jq zlib xz + ``` + - **JDK**: Although OSB is a Python application, it optionally builds and provisions OpenSearch clusters. JDK version 17 is used to build the current version of OpenSearch. Please refer to the [build setup requirements](https://github.com/opensearch-project/OpenSearch/blob/ca564fd04f5059cf9e3ce8aba442575afb3d99f1/DEVELOPER_GUIDE.md#install-prerequisites). Note that the `javadoc` executable should be available in the JDK installation. An earlier version of the JDK can be used, but not all the integration tests will pass. diff --git a/Makefile b/Makefile index 55f4a0248..b3a96d6c3 100644 --- a/Makefile +++ b/Makefile @@ -16,11 +16,14 @@ # under the License. SHELL = /bin/bash +PYTHON = python3 PIP = pip3 VERSIONS = $(shell jq -r '.python_versions | .[]' .ci/variables.json | sed '$$d') VERSION38 = $(shell jq -r '.python_versions | .[]' .ci/variables.json | sed '$$d' | grep 3\.8) PYENV_ERROR = "\033[0;31mIMPORTANT\033[0m: Please install pyenv and run \033[0;31meval \"\$$(pyenv init -)\"\033[0m.\n" +all: install-devel + pyinst: @which pyenv > /dev/null 2>&1 || { printf $(PYENV_ERROR); exit 1; } @for i in $(VERSIONS); do pyenv install --skip-existing $$i; done @@ -31,6 +34,10 @@ pyinst38: pyenv install --skip-existing $(VERSION38) pyenv local $(VERSION38) +check-pip: + # Install pyenv if Python environment is externally managed. + @if ! $(PIP) > /dev/null 2>&1 || ! $(PIP) install pip > /dev/null 2>&1; then make pyinst38; fi + check-java: @if ! test "$(JAVA_HOME)" || ! java --version > /dev/null 2>&1 || ! javadoc --help > /dev/null 2>&1; then \ echo "Java installation issues for running integration tests" >&2; \ @@ -40,23 +47,24 @@ check-java: echo "NOTE: Java version 17 required to have all integration tests pass" >&2; \ fi -install-deps: pyinst38 - # @if test `uname` = Darwin -o `python3 --version | sed 's/.* 3.\([0-9]*\).*/3\1/'` -lt 38; then make pyinst38; fi +install-deps: check-pip $(PIP) install --upgrade pip setuptools wheel install-user: install-deps PIP_ONLY_BINARY=h5py $(PIP) install -e . -install-devel: install-deps - $(PIP) install -e .[develop] +# pylint does not work with Python versions >3.8: +# Value 'Optional' is unsubscriptable (unsubscriptable-object) +install-devel: pyinst38 install-deps + PIP_ONLY_BINARY=h5py $(PIP) install -e .[develop] -wheel: - $(PIP) install --upgrade pip setuptools wheel - PIP_ONLY_BINARY=h5py $(PIP) wheel . +build: install-deps + $(PIP) install --upgrade build + $(PYTHON) -m build -install: wheel - PIP_ONLY_BINARY=h5py $(PIP) install opensearch_benchmark-*.whl - rm -r *.whl *.egg-info +install: build + PIP_ONLY_BINARY=h5py $(PIP) install dist/opensearch_benchmark-*.whl + rm -rf dist clean: rm -rf .benchmarks .eggs .tox .benchmark_it .cache build dist *.egg-info logs junit-py*.xml *.whl NOTICE.txt @@ -75,7 +83,7 @@ tox-env-clean: lint: @find osbenchmark benchmarks scripts tests it -name "*.py" -exec pylint -j0 -rn --load-plugins pylint_quotes --rcfile=$(CURDIR)/.pylintrc \{\} + -test: +test: install-devel pytest tests/ it: pyinst check-java python-caches-clean tox-env-clean diff --git a/docker/Dockerfile b/docker/Dockerfile index 104a10972..fd5b8cbd6 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -14,7 +14,10 @@ RUN apt-get -y update && \ COPY . opensearch-benchmark -RUN cd opensearch-benchmark; make wheel +# There is no binary package currently available for yappi on ARM. +RUN cd opensearch-benchmark; \ + make build; \ + if test "`uname -m`" = aarch64; then pip wheel yappi && cp yappi-*.whl dist; fi # @@ -30,15 +33,14 @@ RUN groupadd --gid 1000 opensearch-benchmark && \ RUN mkdir -p /opensearch-benchmark/.benchmark && \ chown -R 1000:0 /opensearch-benchmark/.benchmark -COPY --from=build-stage /opensearch-benchmark/opensearch-benchmark/yappi-*.whl /opensearch-benchmark/opensearch-benchmark/opensearch_benchmark-*.whl ./ +COPY --from=build-stage /opensearch-benchmark/opensearch-benchmark/dist/*.whl ./ -# There is no binary package currently available for yappi on ARM. RUN set -ex; \ apt-get -y update; \ apt-get install -y git pbzip2; \ apt-get -y upgrade; \ rm -rf /var/lib/apt/lists/*; \ - PIP_ONLY_BINARY=h5py pip install yappi-*.whl opensearch_benchmark-*.whl; \ + PIP_ONLY_BINARY=h5py pip install *.whl; \ rm *.whl USER 1000