Skip to content

Commit

Permalink
Separate verify RC CI jobs from packages (#99)
Browse files Browse the repository at this point in the history
Closes GH-98
  • Loading branch information
kou authored Sep 11, 2023
1 parent 9a0c36f commit c2e780e
Show file tree
Hide file tree
Showing 6 changed files with 175 additions and 149 deletions.
104 changes: 0 additions & 104 deletions .github/workflows/package.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -187,107 +187,3 @@ jobs:
local \
package/${PACKAGE}/${TASK_NAMESPACE}/repositories
popd
verify-source:
name: Verify source on ${{ matrix.os }}
needs:
- source
runs-on: ${{ matrix.os }}-latest
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
os:
- macos
- ubuntu
postgresql-version:
- "15"
env:
BUNDLE_GEMFILE: ${{ github.workspace }}/apache-arrow-flight-sql-postgresql/Gemfile
steps:
- uses: actions/download-artifact@v3
with:
name: source
- name: Extract source archive
run: |
tar_gz=$(echo apache-arrow-flight-sql-postgresql-*.tar.gz)
tar xf ${tar_gz}
ln -s ${tar_gz%.tar.gz} apache-arrow-flight-sql-postgresql
- name: Install dependencies on Ubuntu
if: matrix.os == 'ubuntu'
run: |
sudo apt update
sudo apt install -y -V \
ca-certificates \
gpg \
lsb-release \
wget
os=$(lsb_release --id --short | tr 'A-Z' 'a-z')
code_name=$(lsb_release --codename --short)
apt_source_deb=apache-arrow-apt-source-latest-${code_name}.deb
wget https://apache.jfrog.io/artifactory/arrow/${os}/${apt_source_deb}
sudo apt install -y -V ./${apt_source_deb}
wget -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | \
sudo gpg --no-default-keyring --keyring /usr/share/keyrings/pgdg.gpg --import -
(echo "Types: deb"; \
echo "URIs: http://apt.postgresql.org/pub/repos/apt"; \
echo "Suites: ${code_name}-pgdg"; \
echo "Components: main"; \
echo "Signed-By: /usr/share/keyrings/pgdg.gpg") | \
sudo tee /etc/apt/sources.list.d/pgdg.sources
sudo apt update
sudo apt -y -V purge '^postgresql'
sudo apt install -y -V \
libarrow-flight-sql-glib-dev \
libkrb5-dev \
meson \
ninja-build \
postgresql-${{ matrix.postgresql-version }} \
postgresql-server-dev-${{ matrix.postgresql-version }}
- name: Install dependencies on macOS
if: matrix.os == 'macos'
run: |
pushd apache-arrow-flight-sql-postgresql
sed \
-i '' \
-e 's/postgresql@[0-9]*/postgresql@${{ matrix.postgresql-version }}/g' \
Brewfile
brew update
brew bundle
echo "$(brew --prefix postgresql@${{ matrix.postgresql-version }})/bin" >> $GITHUB_PATH
popd
- uses: ruby/setup-ruby@v1
with:
ruby-version: ruby
bundler-cache: true
cache-version: 1
- name: Run
run: |
pushd apache-arrow-flight-sql-postgresql
bundle exec dev/release/verify-rc.sh
popd
env:
TEST_DEFAULT: "0"
TEST_SOURCE: "1"

verify-binaries:
name: Verify binaries
if: >-
github.ref_type == 'tag'
needs:
- linux
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@v4
- name: Run
run: |
version=${GITHUB_REF_NAME%-rc*}
rc=${GITHUB_REF_NAME#*-rc}
dev/release/verify-rc.sh ${version} ${rc}
env:
TEST_BINARIES: "1"
TEST_DEFAULT: "0"
56 changes: 11 additions & 45 deletions .github/workflows/test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -43,50 +43,9 @@ jobs:
- "15"
steps:
- uses: actions/checkout@v4
- name: Install dependencies on Ubuntu
if: matrix.os == 'ubuntu'
- name: Prepare
run: |
sudo apt update
sudo apt install -y -V \
ca-certificates \
gpg \
lsb-release \
wget
os=$(lsb_release --id --short | tr 'A-Z' 'a-z')
code_name=$(lsb_release --codename --short)
apt_source_deb=apache-arrow-apt-source-latest-${code_name}.deb
wget https://apache.jfrog.io/artifactory/arrow/${os}/${apt_source_deb}
sudo apt install -y -V ./${apt_source_deb}
wget -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | \
sudo gpg --no-default-keyring --keyring /usr/share/keyrings/pgdg.gpg --import -
(echo "Types: deb"; \
echo "URIs: http://apt.postgresql.org/pub/repos/apt"; \
echo "Suites: ${code_name}-pgdg"; \
echo "Components: main"; \
echo "Signed-By: /usr/share/keyrings/pgdg.gpg") | \
sudo tee /etc/apt/sources.list.d/pgdg.sources
sudo apt update
sudo apt -y -V purge '^postgresql'
sudo apt install -y -V \
libarrow-flight-sql-glib-dev \
libkrb5-dev \
meson \
ninja-build \
postgresql-${{ matrix.postgresql-version }} \
postgresql-server-dev-${{ matrix.postgresql-version }}
- name: Install dependencies on macOS
if: matrix.os == 'macos'
run: |
sed \
-i '' \
-e 's/postgresql@[0-9]*/postgresql@${{ matrix.postgresql-version }}/g' \
Brewfile
brew update
brew bundle
echo "$(brew --prefix postgresql@${{ matrix.postgresql-version }})/bin" >> $GITHUB_PATH
ci/scripts/prepare-${{ matrix.os }}.sh ${{ matrix.postgresql-version }}
- name: Install Apache Arrow Flight SQL adapter
run: |
setup_args=(
Expand All @@ -109,6 +68,13 @@ jobs:
ruby-version: ruby
bundler-cache: true
cache-version: 1
- name: Test
- name: Run
run: |
PATH=$(pg_config --bindir):$PATH bundle exec test/run.rb
ARROW_FLIGHT_SQL_POSTGRESQL_MESON_SETUP_ARGS="-Dbenchmark=true"
if [ ${{ matrix.os }} = "macos" ]; then
ARROW_FLIGHT_SQL_POSTGRESQL_MESON_SETUP_ARGS+=" --pkg-config-path=$(brew --prefix libpq)/lib/pkgconfig"
fi
bundle exec dev/release/verify-rc.sh
env:
TEST_DEFAULT: "0"
TEST_SOURCE: "1"
79 changes: 79 additions & 0 deletions .github/workflows/verify-rc.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

name: Verify RC

on:
push:
branches:
- "!**"
tags:
- "*-rc*"

permissions:
contents: read

jobs:
source:
name: Source on ${{ matrix.os }}
runs-on: ${{ matrix.os }}-latest
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
os:
- macos
- ubuntu
postgresql-version:
- "15"
env:
BUNDLE_GEMFILE: ${{ github.workspace }}/apache-arrow-flight-sql-postgresql/Gemfile
steps:
- uses: actions/checkout@v4
- name: Prepare
run: |
ci/scripts/prepare-${{ matrix.os }}.sh ${{ matrix.postgresql-version }}
- uses: ruby/setup-ruby@v1
with:
ruby-version: ruby
bundler-cache: true
cache-version: 1
- name: Run
run: |
version=${GITHUB_REF_NAME%-rc*}
rc=${GITHUB_REF_NAME#*-rc}
bundle exec dev/release/verify-rc.sh ${version} ${rc}
env:
TEST_DEFAULT: "0"
TEST_SOURCE: "1"

binaries:
name: Binaries
needs:
- linux
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@v4
- name: Run
run: |
version=${GITHUB_REF_NAME%-rc*}
rc=${GITHUB_REF_NAME#*-rc}
dev/release/verify-rc.sh ${version} ${rc}
env:
TEST_BINARIES: "1"
TEST_DEFAULT: "0"
30 changes: 30 additions & 0 deletions ci/scripts/prepare-macos.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

set -eux

postgresql_version=$1

sed \
-i "" \
-e "s/postgresql@[0-9]*/postgresql@${postgresql_version}/g" \
Brewfile
brew update
brew bundle
echo "$(brew --prefix postgresql@${postgresql_version})/bin" >> "${GITHUB_PATH}"
54 changes: 54 additions & 0 deletions ci/scripts/prepare-ubuntu.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

set -eux

postgresql_version=$1

sudo apt update
sudo apt install -y -V \
ca-certificates \
gpg \
lsb-release \
wget
os=$(lsb_release --id --short | tr 'A-Z' 'a-z')
code_name=$(lsb_release --codename --short)

apt_source_deb=apache-arrow-apt-source-latest-${code_name}.deb
wget https://apache.jfrog.io/artifactory/arrow/${os}/${apt_source_deb}
sudo apt install -y -V ./${apt_source_deb}

wget -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | \
sudo gpg --no-default-keyring --keyring /usr/share/keyrings/pgdg.gpg --import -
(echo "Types: deb"; \
echo "URIs: http://apt.postgresql.org/pub/repos/apt"; \
echo "Suites: ${code_name}-pgdg"; \
echo "Components: main"; \
echo "Signed-By: /usr/share/keyrings/pgdg.gpg") | \
sudo tee /etc/apt/sources.list.d/pgdg.sources

sudo apt update
sudo apt -y -V purge '^postgresql'
sudo apt install -y -V \
libarrow-flight-sql-glib-dev \
libkrb5-dev \
meson \
ninja-build \
postgresql-${postgresql_version} \
postgresql-server-dev-${postgresql_version}
1 change: 1 addition & 0 deletions dev/release/verify-rc.sh
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,7 @@ test_source_distribution() {
meson setup \
--prefix="${PWD}/install" \
-Dpostgresql_dir="$(pg_config --bindir)/.." \
${ARROW_FLIGHT_SQL_POSTGRESQL_MESON_SETUP_ARGS:-} \
build
meson compile -C build
if [ ${TEST_SOURCE_MANUAL} -gt 0 ]; then
Expand Down

0 comments on commit c2e780e

Please sign in to comment.