From e8b9205a4280164c3528d0863dd28d7d2609f7ff Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Sun, 27 Dec 2020 15:40:48 -0600 Subject: [PATCH] CI: Move sql builds from Travis to Github actions (#38344) --- .github/workflows/database.yml | 180 ++++++++++++++++++ .travis.yml | 17 +- ...travis-37-cov.yaml => actions-37-cov.yaml} | 2 +- ...-37-locale.yaml => actions-37-locale.yaml} | 7 +- ci/deps/azure-38-slow.yaml | 1 - ci/deps/azure-windows-38.yaml | 2 +- doc/source/getting_started/install.rst | 8 +- doc/source/whatsnew/v1.2.1.rst | 7 + pandas/tests/io/test_sql.py | 12 +- 9 files changed, 204 insertions(+), 32 deletions(-) create mode 100644 .github/workflows/database.yml rename ci/deps/{travis-37-cov.yaml => actions-37-cov.yaml} (97%) rename ci/deps/{travis-37-locale.yaml => actions-37-locale.yaml} (89%) diff --git a/.github/workflows/database.yml b/.github/workflows/database.yml new file mode 100644 index 00000000000000..5fe7fc17a98cb6 --- /dev/null +++ b/.github/workflows/database.yml @@ -0,0 +1,180 @@ +name: Database + +on: + push: + branches: [master] + pull_request: + branches: + - master + - 1.2.x + +env: + PYTEST_WORKERS: "auto" + PANDAS_CI: 1 + PATTERN: ((not slow and not network and not clipboard) or (single and db)) + +jobs: + Linux_py37_locale: + runs-on: ubuntu-latest + defaults: + run: + shell: bash -l {0} + + env: + ENV_FILE: ci/deps/actions-37-locale.yaml + LOCALE_OVERRIDE: zh_CN.UTF-8 + + services: + mysql: + image: mysql + env: + MYSQL_ALLOW_EMPTY_PASSWORD: yes + MYSQL_DATABASE: pandas + options: >- + --health-cmd "mysqladmin ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 3306:3306 + + postgres: + image: postgres + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: pandas + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + + steps: + - name: Checkout + uses: actions/checkout@v1 + + - name: Cache conda + uses: actions/cache@v1 + env: + CACHE_NUMBER: 0 + with: + path: ~/conda_pkgs_dir + key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ + hashFiles('${{ env.ENV_FILE }}') }} + + - uses: conda-incubator/setup-miniconda@v2 + with: + activate-environment: pandas-dev + channel-priority: strict + environment-file: ${{ env.ENV_FILE }} + use-only-tar-bz2: true + + - name: Environment Detail + run: | + conda info + conda list + + - name: Build Pandas + run: | + python setup.py build_ext -j 2 + python -m pip install -e . --no-build-isolation --no-use-pep517 + + - name: Test + run: ci/run_tests.sh + if: always() + + - name: Publish test results + uses: actions/upload-artifact@master + with: + name: Test results + path: test-data.xml + if: failure() + + - name: Print skipped tests + run: python ci/print_skipped.py + + Linux_py37_cov: + runs-on: ubuntu-latest + defaults: + run: + shell: bash -l {0} + + env: + ENV_FILE: ci/deps/actions-37-cov.yaml + PANDAS_TESTING_MODE: deprecate + COVERAGE: true + + services: + mysql: + image: mysql + env: + MYSQL_ALLOW_EMPTY_PASSWORD: yes + MYSQL_DATABASE: pandas + options: >- + --health-cmd "mysqladmin ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 3306:3306 + + postgres: + image: postgres + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: pandas + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + + steps: + - name: Checkout + uses: actions/checkout@v1 + + - name: Cache conda + uses: actions/cache@v1 + env: + CACHE_NUMBER: 0 + with: + path: ~/conda_pkgs_dir + key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ + hashFiles('${{ env.ENV_FILE }}') }} + + - uses: conda-incubator/setup-miniconda@v2 + with: + activate-environment: pandas-dev + channel-priority: strict + environment-file: ${{ env.ENV_FILE }} + use-only-tar-bz2: true + + - name: Environment Detail + run: | + conda info + conda list + + - name: Build Pandas + run: | + python setup.py build_ext -j 2 + python -m pip install -e . --no-build-isolation --no-use-pep517 + + - name: Test + run: ci/run_tests.sh + if: always() + + - name: Publish test results + uses: actions/upload-artifact@master + with: + name: Test results + path: test-data.xml + if: failure() + + - name: Print skipped tests + run: python ci/print_skipped.py diff --git a/.travis.yml b/.travis.yml index 31edc4872e9070..8ede978074a9cf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,13 +16,13 @@ services: # travis cache --delete inside the project directory from the travis command line client # The cache directories will be deleted if anything in ci/ changes in a commit cache: + apt: true ccache: true directories: - $HOME/.cache # cython cache env: global: - - PYTEST_WORKERS="auto" # create a github personal access token # cd pandas-dev/pandas # travis encrypt 'PANDAS_GH_TOKEN=personal_access_token' -r pandas-dev/pandas @@ -35,25 +35,10 @@ matrix: fast_finish: true include: - - env: - - JOB="3.7, locale" ENV_FILE="ci/deps/travis-37-locale.yaml" PATTERN="((not slow and not network and not clipboard) or (single and db))" LOCALE_OVERRIDE="zh_CN.UTF-8" SQL="1" - services: - - mysql - - postgresql - - arch: arm64 env: - JOB="3.7, arm64" PYTEST_WORKERS=1 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard and not arm_slow)" - - env: - # Enabling Deprecations when running tests - # PANDAS_TESTING_MODE="deprecate" causes DeprecationWarning messages to be displayed in the logs - # See pandas/_testing.py for more details. - - JOB="3.7, coverage" ENV_FILE="ci/deps/travis-37-cov.yaml" PATTERN="((not slow and not network and not clipboard) or (single and db))" PANDAS_TESTING_MODE="deprecate" COVERAGE=true SQL="1" - services: - - mysql - - postgresql - allow_failures: # Moved to allowed_failures 2020-09-29 due to timeouts https://github.com/pandas-dev/pandas/issues/36719 - arch: arm64 diff --git a/ci/deps/travis-37-cov.yaml b/ci/deps/actions-37-cov.yaml similarity index 97% rename from ci/deps/travis-37-cov.yaml rename to ci/deps/actions-37-cov.yaml index b68ff0672888a7..5381caaa242cf3 100644 --- a/ci/deps/travis-37-cov.yaml +++ b/ci/deps/actions-37-cov.yaml @@ -15,7 +15,7 @@ dependencies: - beautifulsoup4 - botocore>=1.11 - dask - - fastparquet>=0.3.2 + - fastparquet>=0.4.0 - fsspec>=0.7.4 - gcsfs>=0.6.0 - geopandas diff --git a/ci/deps/travis-37-locale.yaml b/ci/deps/actions-37-locale.yaml similarity index 89% rename from ci/deps/travis-37-locale.yaml rename to ci/deps/actions-37-locale.yaml index 60a92c4dfd3c6b..4f9918ca2f0c01 100644 --- a/ci/deps/travis-37-locale.yaml +++ b/ci/deps/actions-37-locale.yaml @@ -1,6 +1,5 @@ name: pandas-dev channels: - - defaults - conda-forge dependencies: - python=3.7.* @@ -18,9 +17,9 @@ dependencies: # optional - beautifulsoup4 - - blosc=1.15.0 + - blosc=1.17.0 - python-blosc - - fastparquet=0.3.2 + - fastparquet=0.4.0 - html5lib - ipython - jinja2 @@ -43,5 +42,5 @@ dependencies: # sql - psycopg2=2.7 - - pymysql=0.7.11 + - pymysql=0.8.1 - sqlalchemy=1.3.0 diff --git a/ci/deps/azure-38-slow.yaml b/ci/deps/azure-38-slow.yaml index 9651837f261141..fd40f40294b7f9 100644 --- a/ci/deps/azure-38-slow.yaml +++ b/ci/deps/azure-38-slow.yaml @@ -1,6 +1,5 @@ name: pandas-dev channels: - - defaults - conda-forge dependencies: - python=3.8.* diff --git a/ci/deps/azure-windows-38.yaml b/ci/deps/azure-windows-38.yaml index 08693e02aa8d3f..661d8813d32d21 100644 --- a/ci/deps/azure-windows-38.yaml +++ b/ci/deps/azure-windows-38.yaml @@ -15,7 +15,7 @@ dependencies: # pandas dependencies - blosc - bottleneck - - fastparquet>=0.3.2 + - fastparquet>=0.4.0 - flask - fsspec>=0.8.0 - matplotlib=3.1.3 diff --git a/doc/source/getting_started/install.rst b/doc/source/getting_started/install.rst index 2c829bc664c6c9..49039f05b889a0 100644 --- a/doc/source/getting_started/install.rst +++ b/doc/source/getting_started/install.rst @@ -263,12 +263,12 @@ Jinja2 2.10 Conditional formatting with DataFra PyQt4 Clipboard I/O PyQt5 Clipboard I/O PyTables 3.5.1 HDF5-based reading / writing -SQLAlchemy 1.2.8 SQL support for databases other than sqlite +SQLAlchemy 1.3.0 SQL support for databases other than sqlite SciPy 1.12.0 Miscellaneous statistical functions xlsxwriter 1.0.2 Excel writing -blosc 1.15.0 Compression for HDF5 +blosc 1.17.0 Compression for HDF5 fsspec 0.7.4 Handling files aside from local and HTTP -fastparquet 0.3.2 Parquet reading / writing +fastparquet 0.4.0 Parquet reading / writing gcsfs 0.6.0 Google Cloud Storage access html5lib 1.0.1 HTML parser for read_html (see :ref:`note `) lxml 4.3.0 HTML parser for read_html (see :ref:`note `) @@ -278,7 +278,7 @@ openpyxl 2.6.0 Reading / writing for xlsx files pandas-gbq 0.12.0 Google Big Query access psycopg2 2.7 PostgreSQL engine for sqlalchemy pyarrow 0.15.0 Parquet, ORC, and feather reading / writing -pymysql 0.7.11 MySQL engine for sqlalchemy +pymysql 0.8.1 MySQL engine for sqlalchemy pyreadstat SPSS files (.sav) reading pyxlsb 1.0.6 Reading for xlsb files qtpy Clipboard I/O diff --git a/doc/source/whatsnew/v1.2.1.rst b/doc/source/whatsnew/v1.2.1.rst index 3f3eb9e4b605c7..c630fc26a93a26 100644 --- a/doc/source/whatsnew/v1.2.1.rst +++ b/doc/source/whatsnew/v1.2.1.rst @@ -23,6 +23,13 @@ Fixed regressions Bug fixes ~~~~~~~~~ + +I/O +^^^ + +- Bumped minimum fastparquet version to 0.4.0 to avoid ``AttributeError`` from numba (:issue:`38344`) +- Bumped minimum pymysql version to 0.8.1 to avoid test failures (:issue:`38344`) + - - diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index df0815fc52bbae..cb5539b9b5ddf1 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -1334,7 +1334,7 @@ def test_sql_open_close(self): @pytest.mark.skipif(SQLALCHEMY_INSTALLED, reason="SQLAlchemy is installed") def test_con_string_import_error(self): - conn = "mysql://root@localhost/pandas_nosetest" + conn = "mysql://root@localhost/pandas" msg = "Using URI string without sqlalchemy installed" with pytest.raises(ImportError, match=msg): sql.read_sql("SELECT * FROM iris", conn) @@ -2071,11 +2071,12 @@ class _TestMySQLAlchemy: """ flavor = "mysql" + port = 3306 @classmethod def connect(cls): return sqlalchemy.create_engine( - f"mysql+{cls.driver}://root@localhost/pandas_nosetest", + f"mysql+{cls.driver}://root@localhost:{cls.port}/pandas", connect_args=cls.connect_args, ) @@ -2140,11 +2141,12 @@ class _TestPostgreSQLAlchemy: """ flavor = "postgresql" + port = 5432 @classmethod def connect(cls): return sqlalchemy.create_engine( - f"postgresql+{cls.driver}://postgres@localhost/pandas_nosetest" + f"postgresql+{cls.driver}://postgres:postgres@localhost:{cls.port}/pandas" ) @classmethod @@ -2760,7 +2762,7 @@ class TestXMySQL(MySQLMixIn): @pytest.fixture(autouse=True, scope="class") def setup_class(cls): pymysql = pytest.importorskip("pymysql") - pymysql.connect(host="localhost", user="root", passwd="", db="pandas_nosetest") + pymysql.connect(host="localhost", user="root", passwd="", db="pandas") try: pymysql.connect(read_default_group="pandas") except pymysql.ProgrammingError as err: @@ -2780,7 +2782,7 @@ def setup_class(cls): @pytest.fixture(autouse=True) def setup_method(self, request, datapath): pymysql = pytest.importorskip("pymysql") - pymysql.connect(host="localhost", user="root", passwd="", db="pandas_nosetest") + pymysql.connect(host="localhost", user="root", passwd="", db="pandas") try: pymysql.connect(read_default_group="pandas") except pymysql.ProgrammingError as err: