Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added CD pipeline scripts #1224

Merged
merged 3 commits into from
Jul 18, 2023
Merged

Added CD pipeline scripts #1224

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4613b27
Added scripts
ashmeenkaur Jul 14, 2023
dde7d81
review comments
ashmeenkaur Jul 17, 2023
0d2fe98
move cd_scripts to tools/
ashmeenkaur Jul 18, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
121 changes: 121 additions & 0 deletions tools/cd_scripts/e2e_test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
#! /bin/bash
# Copyright 2023 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http:#www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Print commands and their arguments as they are executed.
set -x
# Exit immediately if a command exits with a non-zero status.
set -e

#details.txt file contains the release version and commit hash of the current release.
gsutil cp gs://gcsfuse-release-packages/version-detail/details.txt .
# Writing VM instance name to details.txt (Format: release-test-<os-name>)
curl http://metadata.google.internal/computeMetadata/v1/instance/name -H "Metadata-Flavor: Google" >> details.txt

# Based on the os type(from vm instance name) in detail.txt, run the following commands to add starterscriptuser
if grep -q ubuntu details.txt || grep -q debian details.txt;
then
# For ubuntu and debian os
sudo adduser --ingroup google-sudoers --disabled-password --home=/home/starterscriptuser --gecos "" starterscriptuser
else
# For rhel and centos
sudo adduser -g google-sudoers --home-dir=/home/starterscriptuser starterscriptuser
fi

# Run the following as starterscriptuser
sudo -u starterscriptuser bash -c '
# Exit immediately if a command exits with a non-zero status.
set -e
# Print commands and their arguments as they are executed.
set -x

#Copy details.txt to starterscriptuser home directory and create logs.txt
cd ~/
cp /details.txt .
touch logs.txt

echo User: $USER &>> ~/logs.txt
echo Current Working Directory: $(pwd) &>> ~/logs.txt

# Based on the os type in detail.txt, run the following commands for setup
if grep -q ubuntu details.txt || grep -q debian details.txt;
then
# For Debian and Ubuntu os
sudo apt update

#Install fuse
sudo apt install -y fuse

# download and install gcsfuse deb package
gsutil cp gs://gcsfuse-release-packages/v$(sed -n 1p details.txt)/gcsfuse_$(sed -n 1p details.txt)_amd64.deb .
sudo dpkg -i gcsfuse_$(sed -n 1p details.txt)_amd64.deb |& tee -a ~/logs.txt

# install wget
sudo apt install -y wget

#install git
sudo apt install -y git

#install build-essentials
sudo apt install -y build-essential
else
# For rhel and centos
sudo yum makecache
sudo yum check-update

#Install fuse
sudo yum -y install fuse

#download and install gcsfuse rpm package
gsutil cp gs://gcsfuse-release-packages/v$(sed -n 1p details.txt)/gcsfuse-$(sed -n 1p details.txt)-1.x86_64.rpm .
sudo yum -y localinstall gcsfuse-$(sed -n 1p details.txt)-1.x86_64.rpm

#install wget
sudo yum -y install wget

#install git
sudo yum -y install git

#install Development tools
sudo yum -y install gcc gcc-c++ make
fi

# install go
wget -O go_tar.tar.gz https://go.dev/dl/go1.20.4.linux-amd64.tar.gz
sudo tar -C /usr/local -xzf go_tar.tar.gz
export PATH=${PATH}:/usr/local/go/bin

#Write gcsfuse and go version to log file
gcsfuse --version |& tee -a ~/logs.txt
go version |& tee -a ~/logs.txt

# Clone and checkout gcsfuse repo
export PATH=${PATH}:/usr/local/go/bin
git clone https://github.com/googlecloudplatform/gcsfuse |& tee -a ~/logs.txt
cd gcsfuse
git checkout $(sed -n 2p ~/details.txt) |& tee -a ~/logs.txt

#run tests with testbucket flag
GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/... -p 1 --integrationTest -v --testbucket=$(sed -n 3p ~/details.txt) --testInstalledPackage --timeout=60m &>> ~/logs.txt

if grep -q FAIL ~/logs.txt;
then
echo "Test failures detected" &>> ~/logs.txt
else
touch success.txt
gsutil cp success.txt gs://gcsfuse-release-packages/v$(sed -n 1p ~/details.txt)/$(sed -n 3p ~/details.txt)/
fi

gsutil cp ~/logs.txt gs://gcsfuse-release-packages/v$(sed -n 1p ~/details.txt)/$(sed -n 3p ~/details.txt)/
'
107 changes: 107 additions & 0 deletions tools/cd_scripts/install_test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
#! /bin/bash
# Copyright 2023 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http:#www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Print commands and their arguments as they are executed.
set -x

#details.txt file contains the release version and commit hash of the current release.
gsutil cp gs://gcsfuse-release-packages/version-detail/details.txt .
# Writing VM instance name to details.txt (Format: release-test-<os-name>)
curl http://metadata.google.internal/computeMetadata/v1/instance/name -H "Metadata-Flavor: Google" >> details.txt
touch ~/logs.txt

# Based on the os type(from vm instance name) in detail.txt, run the following commands to install apt-transport-artifact-registry
if grep -q ubuntu details.txt || grep -q debian details.txt;
then
# For ubuntu and debian os
curl https://us-central1-apt.pkg.dev/doc/repo-signing-key.gpg | sudo apt-key add - && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
echo 'deb http://packages.cloud.google.com/apt apt-transport-artifact-registry-stable main' | sudo tee -a /etc/apt/sources.list.d/artifact-registry.list
sudo apt update
sudo apt install apt-transport-artifact-registry
echo 'deb ar+https://us-apt.pkg.dev/projects/gcs-fuse-prod $(lsb_release -cs) main' | sudo tee -a /etc/apt/sources.list.d/artifact-registry.list
sudo apt update

# Install released gcsfuse version
sudo apt install -y gcsfuse=$(sed -n 1p details.txt) -t gcsfuse-$(lsb_release -cs) |& tee -a ~/logs.txt
else
# For rhel and centos
sudo yum makecache
sudo yum -y install yum-plugin-artifact-registry
sudo tee -a /etc/yum.repos.d/artifact-registry.repo << EOF
[gcsfuse-el7-x86-64]
name=gcsfuse-el7-x86-64
baseurl=https://asia-yum.pkg.dev/projects/gcs-fuse-prod/gcsfuse-el7-x86-64
enabled=1
repo_gpgcheck=0
gpgcheck=1
gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg
https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
EOF
sudo yum makecache
sudo yum -y --enablerepo=gcsfuse-el7-x86-64 install gcsfuse-$(sed -n 1p details.txt)-1 |& tee -a ~/logs.txt
fi

# Verify gcsfuse version (successful installation)
gcsfuse --version |& tee version.txt
installed_version=$(echo $(sed -n 1p version.txt) | cut -d' ' -f3)
if grep -q $installed_version details.txt; then
echo "GCSFuse latest version installed correctly." &>> ~/logs.txt
else
echo "Failure detected in latest gcsfuse version installation." &>> ~/logs.txt
fi

# Uninstall gcsfuse latest version and install old version
if grep -q ubuntu details.txt || grep -q debian details.txt;
then
sudo apt remove -y gcsfuse
sudo apt install -y gcsfuse=0.42.5 -t gcsfuse-$(lsb_release -cs) |& tee -a ~/logs.txt
else
sudo yum -y remove gcsfuse
sudo yum -y install gcsfuse-0.42.5-1 |& tee -a ~/logs.txt
fi

# verify old version installation
gcsfuse --version |& tee version.txt
installed_version=$(echo $(sed -n 1p version.txt) | cut -d' ' -f3)
if [ $installed_version == "0.42.5" ]; then
echo "GCSFuse old version (0.42.5) installed successfully" &>> ~/logs.txt
else
echo "Failure detected in GCSFuse old version installation." &>> ~/logs.txt
fi

# Upgrade gcsfuse to latest version
if grep -q ubuntu details.txt || grep -q debian details.txt;
then
sudo apt install --only-upgrade gcsfuse |& tee -a ~/logs.txt
else
sudo yum -y upgrade gcsfuse |& tee -a ~/logs.txt
fi

gcsfuse --version |& tee version.txt
installed_version=$(echo $(sed -n 1p version.txt) | cut -d' ' -f3)
if grep -q $installed_version details.txt; then
echo "GCSFuse successfully upgraded to latest version $installed_version." &>> ~/logs.txt
else
echo "Failure detected in upgrading to latest gcsfuse version." &>> ~/logs.txt
fi

if grep -q Failure ~/logs.txt; then
echo "Test failed" &>> ~/logs.txt ;
else
touch success.txt
gsutil cp success.txt gs://gcsfuse-release-packages/v$(sed -n 1p details.txt)/installation-test/$(sed -n 3p details.txt)/ ;
fi

gsutil cp ~/logs.txt gs://gcsfuse-release-packages/v$(sed -n 1p details.txt)/installation-test/$(sed -n 3p details.txt)/