Add logic for handling large files in MultipartWriter uploads to s3 (… #744
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test | |
| on: [push, pull_request] | |
| jobs: | |
| linters: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Setup up Python 3.11 | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: "3.11" | |
| - name: Update pip | |
| run: python -m pip install -U pip | |
| - name: Install dependencies | |
| run: python -m pip install flake8 | |
| - name: Run flake8 linter (source) | |
| run: flake8 --show-source smart_open | |
| unit_tests: | |
| needs: [linters] | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| include: | |
| - {python: '3.8', os: ubuntu-20.04} | |
| - {python: '3.9', os: ubuntu-20.04} | |
| - {python: '3.10', os: ubuntu-20.04} | |
| - {python: '3.11', os: ubuntu-20.04} | |
| - {python: '3.8', os: windows-2019} | |
| - {python: '3.9', os: windows-2019} | |
| - {python: '3.10', os: windows-2019} | |
| - {python: '3.11', os: windows-2019} | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Update pip | |
| run: python -m pip install -U pip | |
| # | |
| # https://askubuntu.com/questions/1428181/module-lib-has-no-attribute-x509-v-flag-cb-issuer-check | |
| # | |
| - name: Upgrade PyOpenSSL | |
| run: python -m pip install pyOpenSSL --upgrade | |
| - name: Install smart_open and its dependencies | |
| run: pip install -e .[test] | |
| - name: Run unit tests | |
| run: pytest smart_open -v -rfxECs --durations=20 | |
| doctest: | |
| needs: [linters,unit_tests] | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| include: | |
| - {python: '3.8', os: ubuntu-20.04} | |
| - {python: '3.9', os: ubuntu-20.04} | |
| - {python: '3.10', os: ubuntu-20.04} | |
| - {python: '3.11', os: ubuntu-20.04} | |
| # | |
| # Some of the doctests don't pass on Windows because of Windows-specific | |
| # character encoding issues. | |
| # | |
| # - {python: '3.7', os: windows-2019} | |
| # - {python: '3.8', os: windows-2019} | |
| # - {python: '3.9', os: windows-2019} | |
| # - {python: '3.10', os: windows-2019} | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Update pip | |
| run: python -m pip install -U pip | |
| - name: Upgrade PyOpenSSL | |
| run: python -m pip install pyOpenSSL --upgrade | |
| - name: Install smart_open and its dependencies | |
| run: pip install -e .[test] | |
| - name: Run doctests | |
| run: python ci_helpers/doctest.py | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| integration: | |
| needs: [linters,unit_tests] | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| include: | |
| - {python: '3.8', os: ubuntu-20.04} | |
| - {python: '3.9', os: ubuntu-20.04} | |
| - {python: '3.10', os: ubuntu-20.04} | |
| - {python: '3.11', os: ubuntu-20.04} | |
| # Not sure why we exclude these, perhaps for historical reasons? | |
| # | |
| # - {python: '3.7', os: windows-2019} | |
| # - {python: '3.8', os: windows-2019} | |
| # - {python: '3.9', os: windows-2019} | |
| # - {python: '3.10', os: windows-2019} | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Update pip | |
| run: python -m pip install -U pip | |
| - name: Upgrade PyOpenSSL | |
| run: python -m pip install pyOpenSSL --upgrade | |
| - run: python -m pip install numpy | |
| - name: Install smart_open and its dependencies | |
| run: pip install -e .[test] | |
| - run: bash ci_helpers/helpers.sh enable_moto_server | |
| if: ${{ matrix.moto_server }} | |
| - run: | | |
| sudo apt-get install vsftpd | |
| sudo bash ci_helpers/helpers.sh create_ftp_ftps_servers | |
| - name: Run integration tests | |
| run: python ci_helpers/run_integration_tests.py | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| - run: bash ci_helpers/helpers.sh disable_moto_server | |
| if: ${{ matrix.moto_server }} | |
| - run: sudo bash ci_helpers/helpers.sh delete_ftp_ftps_servers | |
| benchmarks: | |
| needs: [linters,unit_tests] | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| include: | |
| - {python: '3.8', os: ubuntu-20.04} | |
| - {python: '3.9', os: ubuntu-20.04} | |
| - {python: '3.10', os: ubuntu-20.04} | |
| - {python: '3.11', os: ubuntu-20.04} | |
| # - {python: '3.7', os: windows-2019} | |
| # - {python: '3.8', os: windows-2019} | |
| # - {python: '3.9', os: windows-2019} | |
| # - {python: '3.10', os: windows-2019} | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Update pip | |
| run: python -m pip install -U pip | |
| - name: Upgrade PyOpenSSL | |
| run: python -m pip install pyOpenSSL --upgrade | |
| - name: Install smart_open and its dependencies | |
| run: pip install -e .[test] | |
| - run: pip install awscli pytest_benchmark | |
| - name: Run benchmarks | |
| run: python ci_helpers/run_benchmarks.py | |
| env: | |
| SO_BUCKET: smart-open | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| # | |
| # The test_coverage environment in tox.ini generates coverage data and | |
| # saves it to disk. This step uploads that data. We do it | |
| # separately from the tox env because the upload can fail for various | |
| # reasons (e.g. https://github.com/lemurheavy/coveralls-public/issues/1392) | |
| # and we don't want it to break the build. | |
| # | |
| # Looks like there's a github action for this | |
| # (https://github.com/coverallsapp/github-action/issues/30) but it does | |
| # not work with pytest output. | |
| # | |
| # - name: Upload code coverage to coveralls.io | |
| # if: ${{ matrix.coveralls }} | |
| # continue-on-error: true | |
| # env: | |
| # GITHUB_TOKEN: ${{ github.token }} | |
| # run: | | |
| # pip install coveralls | |
| # coveralls |