ARROW-15483: [Release] Revamp the verification scripts

ci/conda_env_cpp.txt

@@ -21,7 +21,8 @@ boost-cpp>=1.68.0
 brotli
 bzip2
 c-ares
-cmake
+# Required due to the AWS SDK C++ pin
+cmake<3.22
 gflags
 glog
 gmock>=1.10.0

ci/conda_env_gandiva.txt

@@ -15,5 +15,5 @@
 # specific language governing permissions and limitations
 # under the License.
 
-clang=11
-llvmdev=11
+clang>=11
+llvmdev>=11

ci/docker/almalinux-8-verify-rc.dockerfile

@@ -0,0 +1,57 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+ARG arch=amd64
+FROM ${arch}/almalinux:8
+
+# A script to install dependencies required for release
+# verification Red Hat Enterprise Linux 8 clones in particular
+# on AlmaLinux 8 and Rocky Linux 8
+
+RUN dnf -y install 'dnf-command(config-manager)' && \
+    dnf config-manager --set-enabled powertools && \
+    dnf -y update && \
+    dnf -y module disable nodejs && \
+    dnf -y module enable nodejs:16 && \
+    dnf -y module disable ruby && \
+    dnf -y module enable ruby:2.7 && \
+    dnf -y groupinstall "Development Tools" && \
+    dnf -y install \
+        cmake \
+        git \
+        gobject-introspection-devel \
+        java-1.8.0-openjdk-devel \
+        libcurl-devel \
+        llvm-devel \
+        llvm-toolset \
+        maven \
+        ncurses-devel \
+        ninja-build \
+        nodejs \
+        openssl-devel \
+        python38-devel \
+        python38-pip \
+        ruby-devel \
+        sqlite-devel \
+        wget \
+        which && \
+    dnf -y clean all
+
+RUN python3 -m pip install -U pip && \
+    alternatives --set python /usr/bin/python3
+
+RUN npm install -g yarn

ci/docker/ubuntu-18.04-verify-rc.dockerfile

@@ -0,0 +1,60 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+ARG arch=amd64
+FROM ${arch}/ubuntu:18.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+ARG llvm=12
+RUN apt-get update -y -q && \
+    apt-get install -y -q --no-install-recommends \
+       apt-transport-https \
+       ca-certificates \
+       gnupg \
+       wget && \
+    wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
+    echo "deb https://apt.llvm.org/bionic/ llvm-toolchain-bionic-${llvm} main" > \
+       /etc/apt/sources.list.d/llvm.list && \
+    apt-get update -y -q && \
+    apt-get install -y -q --no-install-recommends \
+        build-essential \
+        clang \
+        cmake \
+        curl \
+        git \
+        libcurl4-openssl-dev \
+        libgirepository1.0-dev \
+        libglib2.0-dev \
+        libsqlite3-dev \
+        libssl-dev \
+        llvm-${llvm}-dev \
+        maven \
+        ninja-build \
+        openjdk-11-jdk \
+        pkg-config \
+        python3-pip \
+        python3.8-dev \
+        python3.8-venv \
+        ruby-dev \
+        wget \
+        tzdata && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists*
+
+RUN python3.8 -m pip install -U pip && \
+    update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1

ci/docker/ubuntu-20.04-verify-rc.dockerfile

@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+ARG arch=amd64
+FROM ${arch}/ubuntu:20.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update -y -q && \
+    apt-get install -y -q --no-install-recommends \
+        build-essential \
+        clang \
+        cmake \
+        curl \
+        git \
+        libcurl4-openssl-dev \
+        libgirepository1.0-dev \
+        libglib2.0-dev \
+        libsqlite3-dev \
+        libssl-dev \
+        llvm-dev \
+        maven \
+        ninja-build \
+        openjdk-11-jdk \
+        pkg-config \
+        python3-dev \
+        python3-pip \
+        python3-venv \
+        ruby-dev \
+        wget && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists*

cpp/cmake_modules/FindLLVMAlt.cmake

@@ -19,20 +19,41 @@
 #
 #  find_package(LLVMAlt)
 
-set(LLVM_HINTS ${LLVM_ROOT} ${LLVM_DIR} /usr/lib /usr/share)
-if(LLVM_BREW_PREFIX)
-  list(APPEND LLVM_HINTS ${LLVM_BREW_PREFIX})
+if(DEFINED LLVM_ROOT)
+  # if llvm source is set to conda then prefer conda llvm over system llvm even
+  # if the system one is newer
+  foreach(ARROW_LLVM_VERSION ${ARROW_LLVM_VERSIONS})
+    find_package(LLVM
+                 ${ARROW_LLVM_VERSION}
+                 CONFIG
+                 NO_DEFAULT_PATH
+                 HINTS
+                 ${LLVM_ROOT})
+    if(LLVM_FOUND)
+      break()
+    endif()
+  endforeach()
 endif()
-foreach(ARROW_LLVM_VERSION ${ARROW_LLVM_VERSIONS})
-  find_package(LLVM
-               ${ARROW_LLVM_VERSION}
-               CONFIG
-               HINTS
-               ${LLVM_HINTS})
-  if(LLVM_FOUND)
-    break()
+
+if(NOT LLVM_FOUND)
+  set(LLVM_HINTS ${LLVM_ROOT} ${LLVM_DIR} /usr/lib /usr/share)
+  if(LLVM_BREW_PREFIX)
+    list(APPEND LLVM_HINTS ${LLVM_BREW_PREFIX})
   endif()
-endforeach()
+
+  foreach(HINT ${LLVM_HINTS})
+    foreach(ARROW_LLVM_VERSION ${ARROW_LLVM_VERSIONS})
+      find_package(LLVM
+                   ${ARROW_LLVM_VERSION}
+                   CONFIG
+                   HINTS
+                   ${HINT})
+      if(LLVM_FOUND)
+        break()
+      endif()
+    endforeach()
+  endforeach()
+endif()
 
 if(LLVM_FOUND)
   # Find the libraries that correspond to the LLVM components

dev/release/VERIFY.md

@@ -94,8 +94,10 @@ download of a dependency from the internet. It is possible to run
 specific verification tests by setting environment variables, for example
 
 ```console
-% TEST_DEFAULT=0 TEST_GO=1 dev/release/verify-release-candidate.sh source 6.0.0 3
-% TEST_YUM=1 dev/release/verify-release-candidate.sh binaries 6.0.0 3
+% TEST_DEFAULT=0 TEST_SOURCE=1 dev/release/verify-release-candidate.sh 6.0.0 3
+% TEST_DEFAULT=0 TEST_BINARIES=1 dev/release/verify-release-candidate.sh 6.0.0 3
+% TEST_DEFAULT=0 TEST_GO=1 dev/release/verify-release-candidate.sh 6.0.0 3
+% TEST_DEFAULT=0 TEST_YUM=1 dev/release/verify-release-candidate.sh 6.0.0 3
 ```
 
 It is also possible to use

dev/release/setup-rhel-rebuilds.sh

@@ -21,8 +21,7 @@
 # verification Red Hat Enterprise Linux 8 clones in particular
 # on AlmaLinux 8 and Rocky Linux 8
 
-dnf -y install \
-  'dnf-command(config-manager)'
+dnf -y install 'dnf-command(config-manager)'
 dnf config-manager --set-enabled powertools
 dnf -y update
 dnf -y module disable nodejs
@@ -36,20 +35,15 @@ dnf -y install \
   gobject-introspection-devel \
   java-1.8.0-openjdk-devel \
   libcurl-devel \
-  libcurl-devel \
   llvm-devel \
   llvm-toolset \
   maven \
   ncurses-devel \
-  ncurses-devel \
-  ninja-build \
   ninja-build \
   nodejs \
   openssl-devel \
-  python3-devel \
-  python3-devel \
-  python3-pip \
-  python3-pip \
+  python38-devel \
+  python38-pip \
   ruby-devel \
   sqlite-devel \
   wget \

dev/release/setup-ubuntu.sh

@@ -25,6 +25,7 @@ apt-get -y install \
   cmake \
   curl \
   git \
+  libcurl4-openssl-dev \
   libgirepository1.0-dev \
   libglib2.0-dev \
   libsqlite3-dev \
@@ -35,5 +36,6 @@ apt-get -y install \
   openjdk-11-jdk \
   pkg-config \
   python3-pip \
+  python3-venv \
   ruby-dev \
   wget

dev/release/verify-release-candidate.bat

@@ -20,30 +20,49 @@
 
 @echo on
 
+setlocal ENABLEDELAYEDEXPANSION
+
 if not exist "C:\tmp\" mkdir C:\tmp
 if exist "C:\tmp\arrow-verify-release" rd C:\tmp\arrow-verify-release /s /q
 if not exist "C:\tmp\arrow-verify-release" mkdir C:\tmp\arrow-verify-release
 
 set _VERIFICATION_DIR=C:\tmp\arrow-verify-release
 set _VERIFICATION_DIR_UNIX=C:/tmp/arrow-verify-release
 set _VERIFICATION_CONDA_ENV=%_VERIFICATION_DIR%\conda-env
-set _DIST_URL=https://dist.apache.org/repos/dist/dev/arrow
-set _TARBALL=apache-arrow-%1.tar.gz
-set ARROW_SOURCE=%_VERIFICATION_DIR%\apache-arrow-%1
 set INSTALL_DIR=%_VERIFICATION_DIR%\install
 
-@rem Requires GNU Wget for Windows
-wget --no-check-certificate -O %_TARBALL% %_DIST_URL%/apache-arrow-%1-rc%2/%_TARBALL% || exit /B 1
-
-tar xf %_TARBALL% -C %_VERIFICATION_DIR_UNIX%
-
+set VERSION=%1
+set RC_NUMBER=%2
+set TARBALL_NAME=apache-arrow-%VERSION%.tar.gz
+set TARBALL_URL=https://dist.apache.org/repos/dist/dev/arrow/apache-arrow-%VERSION%-rc%RC_NUMBER%/%TARBALL_NAME%
+
+if "%VERSION%"=="" (
+    set ARROW_SOURCE=%~dp0..\..\
+) else (
+    set ARROW_SOURCE=%_VERIFICATION_DIR%\apache-arrow-%1
+    if "%RC_NUMBER%"=="" (
+        @rem verify a specific git revision
+        git clone https://github.com/apache/arrow.git !ARROW_SOURCE!
+        git -C !ARROW_SOURCE! checkout %VERSION%
+    ) else (
+        @rem verify a release candidate tarball
+        @rem Requires GNU Wget for Windows
+        wget --no-check-certificate -O %TARBALL_NAME% %TARBALL_URL% || exit /B 1
+        tar xf %TARBALL_NAME% -C %_VERIFICATION_DIR_UNIX%
+    )
+    git clone https://github.com/apache/arrow-testing.git !ARROW_SOURCE!\testing
+    git clone https://github.com/apache/parquet-testing.git !ARROW_SOURCE!\cpp\submodules\parquet-testing
+)
+
+set ARROW_TEST_DATA=!ARROW_SOURCE!\testing\data
+set PARQUET_TEST_DATA=!ARROW_SOURCE!\cpp\submodules\parquet-testing\data
 set PYTHON=3.8
 
 @rem Using call with conda.bat seems necessary to avoid terminating the batch
 @rem script execution
 call conda create --no-shortcuts -c conda-forge -f -q -y -p %_VERIFICATION_CONDA_ENV% ^
-    --file=ci\conda_env_cpp.txt ^
-    --file=ci\conda_env_python.txt ^
+    --file=!ARROW_SOURCE!\ci\conda_env_cpp.txt ^
+    --file=!ARROW_SOURCE!\ci\conda_env_python.txt ^
     git ^
     python=%PYTHON% ^
     || exit /B 1
@@ -57,15 +76,15 @@ call conda remove -y gtest gmock || exit /B 1
 set GENERATOR=Visual Studio 15 2017 Win64
 set CONFIGURATION=release
 
-pushd %ARROW_SOURCE%
+pushd !ARROW_SOURCE!
 
 set ARROW_HOME=%INSTALL_DIR%
 set PARQUET_HOME=%INSTALL_DIR%
 set PATH=%INSTALL_DIR%\bin;%PATH%
 
 @rem Build and test Arrow C++ libraries
-mkdir %ARROW_SOURCE%\cpp\build
-pushd %ARROW_SOURCE%\cpp\build
+mkdir !ARROW_SOURCE!\cpp\build
+pushd !ARROW_SOURCE!\cpp\build
 
 @rem This is the path for Visual Studio Community 2017
 call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\Common7\Tools\VsDevCmd.bat" -arch=amd64
@@ -96,13 +115,6 @@ cmake -G "%GENERATOR%" ^
 
 cmake --build . --target INSTALL --config Release || exit /B 1
 
-@rem Get testing datasets for Parquet unit tests
-git clone https://github.com/apache/parquet-testing.git %_VERIFICATION_DIR%\parquet-testing
-set PARQUET_TEST_DATA=%_VERIFICATION_DIR%\parquet-testing\data
-
-git clone https://github.com/apache/arrow-testing.git %_VERIFICATION_DIR%\arrow-testing
-set ARROW_TEST_DATA=%_VERIFICATION_DIR%\arrow-testing\data
-
 @rem Needed so python-test.exe works
 set PYTHONPATH_ORIGINAL=%PYTHONPATH%
 set PYTHONPATH=%CONDA_PREFIX%\Lib;%CONDA_PREFIX%\Lib\site-packages;%CONDA_PREFIX%\DLLs;%CONDA_PREFIX%;%PYTHONPATH%
@@ -111,7 +123,7 @@ set PYTHONPATH=%PYTHONPATH_ORIGINAL%
 popd
 
 @rem Build and import pyarrow
-pushd %ARROW_SOURCE%\python
+pushd !ARROW_SOURCE!\python
 
 pip install -r requirements-test.txt || exit /B 1