From 864803c682db7d7a68f83c7d892f32bbed49f12b Mon Sep 17 00:00:00 2001 From: Raymond Xu <2701446+xushiyan@users.noreply.github.com> Date: Wed, 4 May 2022 07:15:54 -0700 Subject: [PATCH] [HUDI-4005] Update release scripts to help validation (#5479) --- scripts/release/deploy_staging_jars.sh | 2 +- scripts/release/validate_staged_bundles.sh | 66 ++++++++++++++++++++++ scripts/release/validate_staged_release.sh | 13 ++++- 3 files changed, 77 insertions(+), 4 deletions(-) create mode 100755 scripts/release/validate_staged_bundles.sh diff --git a/scripts/release/deploy_staging_jars.sh b/scripts/release/deploy_staging_jars.sh index 6928e3600cd1a..fa71faae80d6d 100755 --- a/scripts/release/deploy_staging_jars.sh +++ b/scripts/release/deploy_staging_jars.sh @@ -43,8 +43,8 @@ declare -a ALL_VERSION_OPTS=( "-Dscala-2.11 -Dspark2.4 -Dflink1.13" "-Dscala-2.11 -Dspark2.4 -Dflink1.14" "-Dscala-2.12 -Dspark2.4 -Dflink1.13" -"-Dscala-2.12 -Dspark3.1 -Dflink1.14" "-Dscala-2.12 -Dspark3.2 -Dflink1.14" +"-Dscala-2.12 -Dspark3.1 -Dflink1.14" # run this last to make sure utilities bundle has spark 3.1 ) printf -v joined "'%s'\n" "${ALL_VERSION_OPTS[@]}" diff --git a/scripts/release/validate_staged_bundles.sh b/scripts/release/validate_staged_bundles.sh new file mode 100755 index 0000000000000..b210f39fbfaeb --- /dev/null +++ b/scripts/release/validate_staged_bundles.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# fail immediately +set -o errexit +set -o nounset + +REPO=$1 +VERSION=$2 + +STAGING_REPO="https://repository.apache.org/content/repositories/${REPO}/org/apache/hudi" + +declare -a BUNDLE_URLS=( +"${STAGING_REPO}/hudi-datahub-sync-bundle/${VERSION}/hudi-datahub-sync-bundle-${VERSION}.jar" +"${STAGING_REPO}/hudi-flink1.13-bundle_2.11/${VERSION}/hudi-flink1.13-bundle_2.11-${VERSION}.jar" +"${STAGING_REPO}/hudi-flink1.13-bundle_2.12/${VERSION}/hudi-flink1.13-bundle_2.12-${VERSION}.jar" +"${STAGING_REPO}/hudi-flink1.14-bundle_2.11/${VERSION}/hudi-flink1.14-bundle_2.11-${VERSION}.jar" +"${STAGING_REPO}/hudi-flink1.14-bundle_2.12/${VERSION}/hudi-flink1.14-bundle_2.12-${VERSION}.jar" +"${STAGING_REPO}/hudi-gcp-bundle/${VERSION}/hudi-gcp-bundle-${VERSION}.jar" +"${STAGING_REPO}/hudi-hadoop-mr-bundle/${VERSION}/hudi-hadoop-mr-bundle-${VERSION}.jar" +"${STAGING_REPO}/hudi-hive-sync-bundle/${VERSION}/hudi-hive-sync-bundle-${VERSION}.jar" +"${STAGING_REPO}/hudi-integ-test-bundle/${VERSION}/hudi-integ-test-bundle-${VERSION}.jar" +"${STAGING_REPO}/hudi-kafka-connect-bundle/${VERSION}/hudi-kafka-connect-bundle-${VERSION}.jar" +"${STAGING_REPO}/hudi-presto-bundle/${VERSION}/hudi-presto-bundle-${VERSION}.jar" +"${STAGING_REPO}/hudi-spark-bundle_2.11/${VERSION}/hudi-spark-bundle_2.11-${VERSION}.jar" +"${STAGING_REPO}/hudi-spark-bundle_2.12/${VERSION}/hudi-spark-bundle_2.12-${VERSION}.jar" +"${STAGING_REPO}/hudi-spark2.4-bundle_2.11/${VERSION}/hudi-spark2.4-bundle_2.11-${VERSION}.jar" +"${STAGING_REPO}/hudi-spark2.4-bundle_2.12/${VERSION}/hudi-spark2.4-bundle_2.12-${VERSION}.jar" +"${STAGING_REPO}/hudi-spark3-bundle_2.12/${VERSION}/hudi-spark3-bundle_2.12-${VERSION}.jar" +"${STAGING_REPO}/hudi-spark3.1-bundle_2.12/${VERSION}/hudi-spark3.1-bundle_2.12-${VERSION}.jar" +"${STAGING_REPO}/hudi-spark3.2-bundle_2.12/${VERSION}/hudi-spark3.2-bundle_2.12-${VERSION}.jar" +"${STAGING_REPO}/hudi-timeline-server-bundle/${VERSION}/hudi-timeline-server-bundle-${VERSION}.jar" +"${STAGING_REPO}/hudi-trino-bundle/${VERSION}/hudi-trino-bundle-${VERSION}.jar" +"${STAGING_REPO}/hudi-utilities-bundle_2.11/${VERSION}/hudi-utilities-bundle_2.11-${VERSION}.jar" +"${STAGING_REPO}/hudi-utilities-bundle_2.12/${VERSION}/hudi-utilities-bundle_2.12-${VERSION}.jar" +"${STAGING_REPO}/hudi-utilities-slim-bundle_2.11/${VERSION}/hudi-utilities-slim-bundle_2.11-${VERSION}.jar" +"${STAGING_REPO}/hudi-utilities-slim-bundle_2.12/${VERSION}/hudi-utilities-slim-bundle_2.12-${VERSION}.jar" +) + +NOW=$(date +%s) +TMP_DIR_FOR_BUNDLES=/tmp/${NOW} +mkdir "$TMP_DIR_FOR_BUNDLES" +for url in "${BUNDLE_URLS[@]}"; do + echo "downloading $url" + wget "$url" -P "$TMP_DIR_FOR_BUNDLES" +done + +ls -l "$TMP_DIR_FOR_BUNDLES" diff --git a/scripts/release/validate_staged_release.sh b/scripts/release/validate_staged_release.sh index 0e027442570a5..41295edfee614 100755 --- a/scripts/release/validate_staged_release.sh +++ b/scripts/release/validate_staged_release.sh @@ -78,9 +78,8 @@ pushd $WORK_DIR # Checkout dist repo LOCAL_SVN_DIR=local_svn_dir -ROOT_SVN_URL=https://dist.apache.org/repos/dist/ +ROOT_SVN_URL=https://dist.apache.org/repos/dist REPO_TYPE=${RELEASE_TYPE} -#RELEASE_REPO=release HUDI_REPO=hudi if [ $RC_NUM == -1 ]; then @@ -99,7 +98,15 @@ echo "Downloading from svn co ${ROOT_SVN_URL}/${REPO_TYPE}/${HUDI_REPO}" (bash -c "svn co ${ROOT_SVN_URL}/${REPO_TYPE}/${HUDI_REPO} $REDIRECT") || (echo -e "\t\t Unable to checkout ${ROOT_SVN_URL}/${REPO_TYPE}/${HUDI_REPO} to $REDIRECT. Please run with --verbose to get details\n" && exit -1) echo "Validating hudi-${ARTIFACT_SUFFIX} with release type \"${REPO_TYPE}\"" -cd ${HUDI_REPO}/hudi-${ARTIFACT_SUFFIX} +if [ $RELEASE_TYPE == "release" ]; then + ARTIFACT_PREFIX= +elif [ $RELEASE_TYPE == "dev" ]; then + ARTIFACT_PREFIX='hudi-' +else + echo "Unexpected RELEASE_TYPE: $RELEASE_TYPE" + exit 1; +fi +cd ${HUDI_REPO}/${ARTIFACT_PREFIX}${ARTIFACT_SUFFIX} $SHASUM hudi-${ARTIFACT_SUFFIX}.src.tgz > got.sha512 echo "Checking Checksum of Source Release"