Merge branch 'v1.0' of https://github.com/CrayLabs/SmartSim into tf_w…

…orker_1.0
CrayLabs · Oct 31, 2024 · 6a08b82 · 6a08b82
2 parents f23c267 + 904b5cc
commit 6a08b82
Show file tree

Hide file tree

Showing 404 changed files with 29,163 additions and 15,822 deletions.
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -32,108 +32,61 @@ on:
   release:
     types: [published]
 
-
-env:
-  HOMEBREW_NO_ANALYTICS: "ON" # Make Homebrew installation a little quicker
-  HOMEBREW_NO_AUTO_UPDATE: "ON"
-  HOMEBREW_NO_BOTTLE_SOURCE_FALLBACK: "ON"
-  HOMEBREW_NO_GITHUB_API: "ON"
-  HOMEBREW_NO_INSTALL_CLEANUP: "ON"
-  CIBW_SKIP: "pp* *i686*" # skip building for PyPy
-  CIBW_ARCHS_MACOS: x86_64
-  CIBW_ARCHS_LINUX: x86_64  # ppc64le # uncomment to enable powerPC build
-  CIBW_ENVIRONMENT_MACOS: PATH="$(brew --prefix)/opt/make/libexec/gnubin:$PATH"
-  MACOSX_DEPLOYMENT_TARGET: "10.09"
-
-
 jobs:
-  build_wheels:
-    name: Build wheels on ${{ matrix.os }}
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-22.04, macos-12]
-
+  build_dists:
+    name: Build Distributions
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
+        with:
+          python-version: '3.9'
 
-      - name: Install cibuildwheel
-        run: python -m pip install cibuildwheel>=2.12.3
+      - name: Install build
+        run: python -m pip install 'build>=1.2.2,<2'
 
       - name: Install build-essentials
-        if: contains(matrix.os, 'ubuntu')
         run: |
           sudo add-apt-repository ppa:ubuntu-toolchain-r/test
           sudo apt-get update
-          sudo apt-get install -y build-essential
-          sudo apt-get install -y wget
+          sudo apt-get install -y build-essential wget
 
-      - name: Install GNU make for MacOS
-        if: contains(matrix.os, 'macos')
-        run: brew install make || true
+      - name: Build Distributions
+        run: python -m build .
 
-      - name: list target wheels
-        run: |
-          python -m cibuildwheel . --print-build-identifiers
-
-      - name: Build wheels
-        run: python -m cibuildwheel --output-dir wheelhouse
-        env:
-          CIBW_ENVIRONMENT_MACOS: PATH="$(brew --prefix)/opt/make/libexec/gnubin:$PATH"
-          MACOSX_DEPLOYMENT_TARGET: "10.09"
-
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v3
         with:
-          path: ./wheelhouse/*.whl
-
-
-  build_sdist:
-    name: Build source distribution
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-
-      - uses: actions/setup-python@v5
-        name: Install Python
-        with:
-          python-version: '3.9'
-
-      - name: Build sdist
-        run: |
-          python -m pip install cmake>=3.13
-          python setup.py sdist
-
-      - uses: actions/upload-artifact@v2
-        with:
-          path: dist/*.tar.gz
+          name: distributables
+          path: ./dist/*
 
 
   upload_pypi:
-    needs: [build_wheels, build_sdist]
-    runs-on: ubuntu-latest
+    needs: [build_dists]
+    runs-on: ubuntu-22.04
     steps:
-      - uses: actions/download-artifact@v2
+      - uses: actions/download-artifact@v3
         with:
-          name: artifact
+          name: distributables
           path: dist
 
       - uses: pypa/gh-action-pypi-publish@release/v1
         with:
           user: __token__
           password: ${{ secrets.PYPI }}
-          #repository_url: https://test.pypi.org/legacy/
-
+          # repository-url: https://test.pypi.org/legacy/
 
   createPullRequest:
-    runs-on: ubuntu-latest
+    needs: [upload_pypi]
+    runs-on: ubuntu-22.04
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
 
       - name: Create pull request
         run: |
-          gh pr create -B develop -H master --title 'Merge master into develop' --body 'This PR brings develop up to date with master for release.'
+          gh pr create -B develop \
+                       -H master \
+                       --title 'Merge master into develop' \
+                       --body 'This PR brings develop up to date with master for release.'
         env:
             GH_TOKEN: ${{ github.token }}
diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml
@@ -49,7 +49,7 @@ env:
 
 jobs:
   run_tests:
-    name: Run tests ${{ matrix.subset }} with ${{ matrix.os }}, Python ${{ matrix.py_v}}, RedisAI ${{ matrix.rai }}
+    name: Run tests ${{ matrix.subset }} with ${{ matrix.os }}, Python ${{ matrix.py_v}}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -63,15 +63,20 @@ jobs:
           - os: macos-14
             py_v: "3.9"
 
-    env:
-      SMARTSIM_REDISAI: ${{ matrix.rai }}
-
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.py_v }}
 
+      - name: Check Test Files are Marked
+        run: |
+          diff <(find tests -path tests/_legacy -prune -o -type f -name 'test_*.py' -print \
+                 | xargs grep -l 'pytestmark' \
+                 | sort) \
+               <(find tests -path tests/_legacy -prune -o -type f -name 'test_*.py' -print \
+                 | sort)
+
       - name: Install build-essentials for Ubuntu
         if: contains( matrix.os, 'ubuntu' )
         run: |
@@ -109,36 +114,34 @@ jobs:
       - name: Install SmartSim (with ML backends)
         run: |
           python -m pip install git+https://github.com/CrayLabs/SmartRedis.git@develop#egg=smartredis
-          python -m pip install .[dev,mypy,ml]
+          python -m pip install .[dev,mypy]
 
-      - name: Install ML Runtimes with Smart (with pt, tf, and onnx support)
-        if: (contains( matrix.os, 'ubuntu' ) || contains( matrix.os, 'macos-12')) && ( matrix.subset != 'dragon' )
-        run: smart build --device cpu --onnx -v
+      - name: Install ML Runtimes
+        if: matrix.subset != 'dragon'
+        run: smart build --device cpu -v
 
-      - name: Install ML Runtimes with Smart (with pt, tf, dragon, and onnx support)
-        if: (contains( matrix.os, 'ubuntu' ) || contains( matrix.os, 'macos-12')) && ( matrix.subset == 'dragon' )
+
+      - name: Install ML Runtimes (with dragon)
+        if: matrix.subset == 'dragon'
         env:
           SMARTSIM_DRAGON_TOKEN: ${{ secrets.DRAGON_TOKEN }}
         run: |
           if [ -n "${SMARTSIM_DRAGON_TOKEN}" ]; then
-            smart build --device cpu --onnx -v --dragon-repo dragonhpc/dragon-nightly --dragon-version 0.10
+            smart build --device cpu -v --dragon-repo dragonhpc/dragon-nightly --dragon-version 0.10
           else
-            smart build --device cpu --onnx -v --dragon
+            smart build --device cpu -v --dragon
           fi
           SP=$(python -c 'import site; print(site.getsitepackages()[0])')/smartsim/_core/config/dragon/.env
           LLP=$(cat $SP | grep LD_LIBRARY_PATH | awk '{split($0, array, "="); print array[2]}')
           echo "LD_LIBRARY_PATH=$LLP:$LD_LIBRARY_PATH" >> $GITHUB_ENV
 
-      - name: Install ML Runtimes with Smart (no ONNX,TF on Apple Silicon)
-        if: contains( matrix.os, 'macos-14' )
-        run: smart build --device cpu --no_tf -v
-
       - name: Run mypy
         run: |
           make check-mypy
 
-      - name: Run Pylint
-        run: make check-lint
+      # TODO: Re-enable static analysis once API is firmed up
+      # - name: Run Pylint
+      #   run: make check-lint
 
       # Run isort/black style check
       - name: Run isort

diff --git a/.gitignore b/.gitignore
@@ -12,6 +12,7 @@ tests/test_output
 # Dependencies
 smartsim/_core/.third-party
 smartsim/_core/.dragon
+smartsim/_core/build
 
 # Docs
 _build

diff --git a/.pylintrc b/.pylintrc
@@ -167,7 +167,7 @@ max-module-lines=1000
 
 # Allow the body of a class to be on the same line as the declaration if body
 # contains single statement.
-single-line-class-stmt=no
+single-line-class-stmt=yes
 
 # Allow the body of an if to be on the same line as the test if there is no
 # else.

diff --git a/.wci.yml b/.wci.yml
@@ -22,8 +22,8 @@
     language: Python
 
     release:
-      version: 0.7.0
-      date: 2024-05-14
+      version: 0.8.0
+      date: 2024-09-25
 
     documentation:
       general: https://www.craylabs.org/docs/overview.html

diff --git a/Makefile b/Makefile
@@ -150,11 +150,11 @@ tutorials-dev:
 	@docker compose build tutorials-dev
 	@docker run -p 8888:8888 smartsim-tutorials:dev-latest
 
-# help: tutorials-prod                 - Build and start a docker container to run the tutorials (v0.7.0)
+# help: tutorials-prod                 - Build and start a docker container to run the tutorials (v0.8.0)
 .PHONY: tutorials-prod
 tutorials-prod:
 	@docker compose build tutorials-prod
-	@docker run -p 8888:8888 smartsim-tutorials:v0.7.0
+	@docker run -p 8888:8888 smartsim-tutorials:v0.8.0
 
 
 # help:
@@ -164,22 +164,22 @@ tutorials-prod:
 # help: test                           - Run all tests
 .PHONY: test
 test:
-	@python -m pytest --ignore=tests/full_wlm/ --ignore=tests/dragon
+	@python -m pytest --ignore=tests/full_wlm/ --ignore=tests/dragon_wlm
 
 # help: test-verbose                   - Run all tests verbosely
 .PHONY: test-verbose
 test-verbose:
-	@python -m pytest -vv --ignore=tests/full_wlm/ --ignore=tests/dragon
+	@python -m pytest -vv --ignore=tests/full_wlm/ --ignore=tests/dragon_wlm
 
 # help: test-debug                     - Run all tests with debug output
 .PHONY: test-debug
 test-debug:
-	@SMARTSIM_LOG_LEVEL=developer python -m pytest -s -o log_cli=true -vv --ignore=tests/full_wlm/ --ignore=tests/dragon
+	@SMARTSIM_LOG_LEVEL=developer python -m pytest -s -o log_cli=true -vv --ignore=tests/full_wlm/ --ignore=tests/dragon_wlm
 
 # help: test-cov                       - Run all tests with coverage
 .PHONY: test-cov
 test-cov:
-	@python -m pytest -vv --cov=./smartsim --cov-config=${COV_FILE} --ignore=tests/full_wlm/ --ignore=tests/dragon
+	@python -m pytest -vv --cov=./smartsim --cov-config=${COV_FILE} --ignore=tests/full_wlm/ --ignore=tests/dragon_wlm
 
 
 # help: test-full                      - Run all WLM tests with Python coverage (full test suite)
@@ -196,4 +196,4 @@ test-wlm:
 # help: test-dragon                   - Run dragon-specific tests
 .PHONY: test-dragon
 test-dragon:
-	@dragon pytest tests/dragon
+	@dragon pytest tests/dragon_wlm
diff --git a/README.md b/README.md
@@ -643,11 +643,11 @@ from C, C++, Fortran and Python with the SmartRedis Clients:
     <tr>
       <td rowspan="3">1.2.7</td>
       <td>PyTorch</td>
-      <td>2.0.1</td>
+      <td>2.1.0</td>
     </tr>
     <tr>
       <td>TensorFlow\Keras</td>
-      <td>2.13.1</td>
+      <td>2.15.0</td>
     </tr>
     <tr>
       <td>ONNX</td>