Skip to content
This repository has been archived by the owner on Apr 17, 2023. It is now read-only.

Commit

Permalink
packaging: changed how we build dependencies
Browse files Browse the repository at this point in the history 
This commit is a follow-up of bd383fb,
and it adapts all the packaging scripts on the idea that the Gemfile and
the Gemfile.lock files cannot be touched. After this commit, the portus
RPM builds again and it's cleaner than ever.

First of all, the `make_spec.sh` has been heavily modified, while
polishing some rough edges (e.g. proper debug functions). It will
require a specific bundler version, because now it will be taken from
the gem requirements instead of the `spec.in` file. The biggest change
is to only use `bundler` for anything related to gems. To list the gems
to be added as dependencies we are using now `bundle show`, which has
the knowledge of gem groups as defined in the previous commit. This way,
without modifying the Gemfile.lock file, we can properly filter the
dependencies. As a side-effect, this simplifies quite a lot the script.

As for the `portus.spec.in`, I have updated it so it uses gem groups,
and it will compile assets after fetching all gems (since it's the proper
order). Finally, I also added a couple of lines to remove all the
unneeded files/directories coming from vendored gems.

Signed-off-by: Miquel Sabaté Solà <msabate@suse.com>
  • Loading branch information
@mssola
mssola committed Jan 3, 2018
1 parent 0527128 commit 0970b99
Show file tree
Hide file tree
Showing 4 changed files with 114 additions and 98 deletions.
9 changes: 0 additions & 9 deletions packaging/suse/README.install
View file

This file was deleted.

113 changes: 71 additions & 42 deletions packaging/suse/make_spec.sh
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,36 @@
#!/bin/bash

##
# Requirements

BUNDLER_VERSION="1.15.4"

##
# Helper functions

log() { (>&2 echo ">>> [make_spec] $@") ; }
debug() { log "DEBUG: $@" ; }
error() { log "ERROR: $@" ; exit 1 ; }

# Depending on the given gem, it will append its native build requirements.
additional_native_build_requirements() {
# NOTE: all echo'ed strings must start with a "\n" character.
if [ $1 == "nokogiri" ];then
echo "\nBuildRequires: libxml2-devel libxslt-devel"
elif [ $1 == "mysql2" ];then
echo "\n%if 0%{?suse_version} <= 1320\nBuildRequires: libmysqlclient-devel < 10.1\nRequires: libmysqlclient18 < 10.1\n%else\nBuildRequires: libmysqlclient-devel\nRequires: libmysqlclient18\n%endif\nRecommends: mariadb"
elif [ $1 == "ethon" ];then
echo "\nBuildRequires: libcurl-devel\nRequires: libcurl4"
elif [ $1 == "ffi" ];then
echo "\nBuildRequires: libffi-devel"
elif [ $1 == "pg" ];then
echo "\nBuildRequires: postgresql-devel\nRequires: postgresql-devel"
fi
}

##
# Initialization

if [ -z "$1" ]; then
cat <<EOF
usage:
Expand All @@ -9,11 +41,11 @@ fi

packagename=$1

bundle version 2>/dev/null
if [ $? != 0 ];then
echo "bundler is not installed. Please install it."
exit -1
bundler_version=$(bundle version 2>/dev/null | awk '{ print $3 }')
if [ "$bundler_version" != "$BUNDLER_VERSION" ];then
error "Bundler $BUNDLER_VERSION required!"
fi

cd $(dirname $0)

if [ $TRAVIS_BRANCH ];then
Expand All @@ -31,68 +63,66 @@ version="$version+git$commit"
date=$(date --rfc-2822)
year=$(date +%Y)

# clean
[ ! -d build ] || rm -rf build
##
# Creating build environment

additional_native_build_requirements() {
if [ $1 == "nokogiri" ];then
echo "BuildRequires: libxml2-devel libxslt-devel\n"
elif [ $1 == "mysql2" ];then
echo "%if 0%{?suse_version} <= 1320\nBuildRequires: libmysqlclient-devel < 10.1\nRequires: libmysqlclient18 < 10.1\n%else\nBuildRequires: libmysqlclient-devel\nRequires: libmysqlclient18\n%endif\nRecommends: mariadb\n"
elif [ $1 == "ethon" ];then
echo "BuildRequires: libcurl-devel\nRequires: libcurl4\n"
elif [ $1 == "ffi" ];then
echo "BuildRequires: libffi-devel\n"
fi
}
# Clean
[ ! -d build ] || rm -rf build

mkdir -p build/$packagename-$branch
cp -v ../../Gemfile* build/$packagename-$branch
cp -v patches/*.patch build/$packagename-$branch
if ls patches/*.patch >/dev/null 2>&1 ;then
cp -v patches/*.patch build/$packagename-$branch
fi

##
# Generating spec file

pushd build/$packagename-$branch/
echo "DEBUG: Gemfile"
cat Gemfile
echo "DEBUG: Gemfile.lock"
cat Gemfile.lock
echo "apply patches if needed"
debug "Apply patches if needed"
if ls *.patch >/dev/null 2>&1 ;then
patchsources="\n# Dynamically defined patches."
for p in *.patch;do
number=$(echo "$p" | cut -d"_" -f1)
patchsources="$patchsources\nPatch$number: $p\n"
patchexecs="$patchexecs\n%patch$number -p1\n"
# skip applying rpm patches
[[ $p =~ .rpm\.patch$ ]] && continue
echo "applying patch $p"
debug "Applying patch $p"
echo "DEBUG"
cat $p
patch -p1 < $p || exit -1
done
fi
echo "generate the Gemfile.lock for packaging"

# Generate the Gemfile.lock file while ignoring some unnecessary groups.
debug "Generate the Gemfile.lock for packaging"
export BUNDLE_GEMFILE=$PWD/Gemfile
cp Gemfile.lock Gemfile.lock.orig
bundle config build.nokogiri --use-system-libraries
export PORTUS_PUMA_DEPLOYMENT=yes
PACKAGING=yes bundle install --retry=3 --no-deployment
grep "git-review" Gemfile.lock
if [ $? == 0 ];then
echo "DEBUG: ohoh something went wrong and you have devel packages"
diff Gemfile.lock Gemfile.lock.orig
exit -1
fi
echo "get requirements from Gemfile.lock"
bundle install --retry=3 --deployment --without assets test development

debug "Diff of old and new Gemfile.lock file"
diff Gemfile.lock Gemfile.lock.orig

debug "Getting requirements from Gemfile.lock"
IFS=$'\n' # do not split on spaces
build_requires=""
for gem in $(cat Gemfile.lock | grep " " | grep " " -v | sort | uniq);do
gem_name=$(echo $gem | cut -d" " -f5)
gem_version=$(echo $gem | cut -d "(" -f2 | cut -d ")" -f1)
build_requires="# Dependencies extracted from the defined Gemfile."

# Bundle's show command will list you the installed gems (the real ones, not
# the ones installed on the Gemfile.lock). We use tail to skip the first line,
# which is irrelevant. Then, with awk, we model the output to be "$gem
# $version", but this $version is inside of parenthesis, so we remove them
# with tr. This way, fetching the name and version is as easy as awk'ing again.
for gem in $(bundle show | tail -n +2 | awk '{ print $2 " " $3 }' | tr -d '()');do
gem_name=$(echo $gem | awk '{ print $1 }')
gem_version=$(echo $gem | awk '{ print $2 }')
build_requires="$build_requires\nBuildRequires: %{rubygem $gem_name} = $gem_version"
build_requires="$build_requires\n$(additional_native_build_requirements $gem_name)"
build_requires="$build_requires$(additional_native_build_requirements $gem_name)"
done
popd

echo "create ${packagename}.spec based on ${packagename}.spec.in"
debug "Creating ${packagename}.spec based on ${packagename}.spec.in"
cp ${packagename}.spec.in ${packagename}.spec
sed -e "s/__BRANCH__/$branch/g" -i ${packagename}.spec
sed -e "s/__RUBYGEMS_BUILD_REQUIRES__/$build_requires/g" -i ${packagename}.spec
Expand All @@ -107,6 +137,5 @@ if [ -f ${packagename}.spec ];then
echo "Done!"
exit 0
else
echo "A problem occured creating the spec file."
exit -1
error "A problem occured creating the spec file."
fi
11 changes: 8 additions & 3 deletions packaging/suse/package_and_push_to_obs.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,16 @@ if [ $TRAVIS_PULL_REQUEST == "false" ] && [ $OBS_BRANCH ] && [ $TRAVIS_COMMIT ]
rm -r *.orig
git checkout -- .

# Generate the spec file and submit it.
packaging/suse/make_spec.sh portus
curl -X PUT -T packaging/suse/portus.spec -u $OSC_CREDENTIALS https://api.opensuse.org/source/$OBS_REPO/portus/portus.spec?comment=update_portus.spec\_to_commit_$TRAVIS_COMMIT\_from_branch_$OBS_BRANCH
for p in packaging/suse/patches/*.patch;do
curl -X PUT -T $p -u $OSC_CREDENTIALS https://api.opensuse.org/source/$OBS_REPO/portus/$(basename $p)?comment=update_patches\_to_commit_$TRAVIS_COMMIT\_from_branch_$OBS_BRANCH
done

# Submit patches if they exist.
if ls packaging/suse/patches/*.patch >/dev/null 2>&1 ;then
for p in packaging/suse/patches/*.patch;do
curl -X PUT -T $p -u $OSC_CREDENTIALS https://api.opensuse.org/source/$OBS_REPO/portus/$(basename $p)?comment=update_patches\_to_commit_$TRAVIS_COMMIT\_from_branch_$OBS_BRANCH
done
fi

# Get the yarn.lock file from OBS and compare it. If they differ, then push a
# new node_modules.tar.gz file.
Expand Down
79 changes: 35 additions & 44 deletions packaging/suse/portus.spec.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,31 +15,23 @@
# Please submit bugfixes or comments via http://bugs.opensuse.org/
#

Name: portus

# When you release a new version, set Version and branch accordingly.
# For example:
# Version: 1.0.0
# %define branch 1.0.0
%define branch __BRANCH__
%define portusdir /srv/Portus

Name: portus
Version: __VERSION__
%define branch __BRANCH__
Release: 0.0.1
License: Apache-2.0
Summary: Authorization service and fronted for Docker registry (v2)
Url: https://github.com/SUSE/Portus
Source0: %{branch}.tar.gz
Group: System/Management

Source0: %{branch}.tar.gz
# Generated with `yarn install` which produces a reproduceable `node_modules`
# directory thanks to the yarn.lock file defined in the Portus repo.
Source1: node_modules.tar.gz

# Dynamically defined patches.
__PATCHSOURCES__

Group: System/Management
%define portusdir /srv/Portus

Requires: ruby >= 2.4
Requires: timezone
Requires: net-tools
Expand All @@ -59,22 +51,23 @@ Obsoletes: Portus = 20151120162040
BuildRequires: nodejs6
BuildRequires: yarn

# Base ruby engine.
%define rb_build_versions ruby24
%define rb_ver 2.4.0
BuildRequires: %{rubydevel}
BuildRequires: %{rubygem gem2rpm}
BuildRequires: %{rubygem bundler} >= 1.15.4

__RUBYGEMS_BUILD_REQUIRES__



BuildRoot: %{_tmppath}/%{name}-%{version}-build

%description
Portus targets version 2 of the Docker registry API. It aims to act both as an authoritzation server and as a user interface for the next generation of the Docker registry.
Portus targets version 2 of the Docker registry API. It aims to act both as an
authoritzation server and as a user interface for the next generation of the
Docker registry.

This package has been built with commit __COMMIT__ from branch __BRANCH__ on date __DATE__
This package has been built with commit __COMMIT__ from branch __BRANCH__ on
date __DATE__

%prep
%setup -q -n Portus-%{branch}
Expand All @@ -89,8 +82,14 @@ tar xzvf node_modules.tar.gz
install -d vendor/cache
cp %{_libdir}/ruby/gems/%{rb_ver}/cache/*.gem vendor/cache

# Deploy gems
bundle config build.nokogiri --use-system-libraries
bundle install --retry=3 --local --deployment --without assets test development

# Install bundler
gem.ruby2.4 install --no-rdoc --no-ri --install-dir vendor/bundle/ruby/%{rb_ver}/ vendor/cache/bundler-*.gem

# Compile assets
export NOKOGIRI_USE_SYSTEM_LIBRARIES=1
SKIP_MIGRATION="yes" \
PORTUS_SECRET_KEY_BASE="ap" PORTUS_KEY_PATH="ap" PORTUS_PASSWORD="ap" \
RAILS_ENV=production NODE_ENV=production \
Expand All @@ -100,15 +99,6 @@ SKIP_MIGRATION="yes" \
APPLICATION_CSS=$(find . -name application-*.css 2>/dev/null)
cp $APPLICATION_CSS public/landing.css

# Run bundle list to redo the Gemfile.lock
bundle list

# Deploy gems
bundle install --retry=3 --local --deployment --without assets test development

# Install bundler
gem install --no-rdoc --no-ri --install-dir vendor/bundle/ruby/%{rb_ver}/ vendor/cache/bundler-*.gem

# Remove unneeded directories/files
rm -rf \
vendor/cache \
Expand All @@ -118,6 +108,23 @@ rm -rf \
examples \
*.orig

# Removing irrelevant files for production.
declare -a ary=(
".gitignore" ".travis.yml" ".pelusa.yml" ".keep" ".rspec" ".codeclimate.yml"
".yardopts" ".ruby-gemset" ".rubocop.yml" ".document" ".eslintrc"
".eslintignore" ".env" ".dockerignore" ".editorconfig" ".erdconfig"
".ruby-version" "*.pem" ".rubocop_todo.yml" ".concourse.yml"
)
for i in "${ary[@]}"; do
find . -name "$i" -type f -delete
done

# Remove directories and empty files.
find . -name ".bundle" -type d -exec rm -rv {} +
find . -name ".github" -type d -exec rm -rv {} +
find . -name ".empty_directory" -type d -delete
find . -size 0 -delete

%install
install -d %{buildroot}/%{portusdir}

Expand All @@ -137,22 +144,6 @@ install -p -m 644 packaging/suse/portusctl/man/man1/*.1 %{buildroot}%{_mandir}/m

%fdupes %{buildroot}/%{portusdir}

%pre

%post
if [ -d /srv/Portus/tmp ];then
chown -R wwwrun:www /srv/Portus/tmp
fi

if [ \! -e "/srv/Portus/config/config-local.yml" ]; then
# First installation of Portus
cat %{portusdir}/packaging/suse/README.install
fi

%preun

%postun

%files
%defattr(-,root,root)
%{portusdir}
Expand Down

0 comments on commit 0970b99

Please sign in to comment.