Skip to content

Commit

Permalink
S7IPA-169 Add support for extra/future docker images (#2)
Browse files Browse the repository at this point in the history
  • Loading branch information
rcmorano authored Jul 7, 2020
1 parent 99c6f48 commit 1779c5a
Show file tree
Hide file tree
Showing 3 changed files with 49 additions and 144 deletions.
101 changes: 41 additions & 60 deletions .github/workflows/docker-builds.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
name: docker builds
on: [push, pull_request]

env:
DOCKER_IMAGE_BASENAME: libpostal
BASE_IMAGES: "ruby:2.4-jessie ruby:2.4.9-stretch ubuntu:16.04 ubuntu:18.04"


jobs:
build:
name: Build OS packages inside a docker container and create release
Expand All @@ -9,76 +14,52 @@ jobs:
- uses: actions/checkout@v2
- name: Make git short-commit available for the different steps
id: metadata
env:
UBUNTU_VERSIONS: 16.04 18.04
run: |
echo "::set-output name=commit::$(echo ${GITHUB_SHA} | head -c7)"
echo "::set-output name=version::$(sed 's|^v||' versions/parser)"
- name: Build
env:
DOCKER_IMAGE_NAME: stuart/libpostal
UBUNTU_VERSIONS: "16.04 18.04"
run: |
pwd
ls -hal
git status -sb
for ubuver in ${UBUNTU_VERSIONS}
mkdir release-files
for base_image in ${BASE_IMAGES}
do
FS_SANITIZED_BASE_IMAGE=$(echo ${base_image} | sed 's|:|_|g')
OUTPUT_DIR=output-${FS_SANITIZED_BASE_IMAGE}
docker build \
--build-arg UBUNTU_VERSION=${ubuver} \
-t ${DOCKER_IMAGE_NAME}:${ubuver} .
CID=$(docker create ${DOCKER_IMAGE_NAME}:${ubuver})
docker cp ${CID}:/output ./output-ubuntu-${ubuver}
--build-arg BASE_IMAGE=${base_image} \
-t ${DOCKER_IMAGE_BASENAME}/${base_image} .
CID=$(docker create ${DOCKER_IMAGE_BASENAME}/${base_image})
docker cp ${CID}:/output ${OUTPUT_DIR}
DEB_FILE=$(ls ${OUTPUT_DIR}/*deb)
DATA_FILE=$(ls ${OUTPUT_DIR}/*gz)
mv ${DEB_FILE} release-files/${FS_SANITIZED_BASE_IMAGE}-$(basename ${DEB_FILE})
mv ${DATA_FILE} release-files/${FS_SANITIZED_BASE_IMAGE}-$(basename ${DATA_FILE})
done
- name: Create Release
id: create_release
uses: actions/create-release@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token
with:
tag_name: ${{ steps.metadata.outputs.version }}-${{ steps.metadata.outputs.commit }}
release_name: Release ${{ steps.metadata.outputs.version }}-${{ steps.metadata.outputs.commit }}
body: |
Automatically generated by GH action on this project
draft: false
prerelease: false
- name: Upload .deb package release asset (ubuntu-16.04)
id: upload-deb-release-asset-ubuntu-16-04
uses: actions/upload-release-asset@v1
- name: Release
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
asset_path: ./output-ubuntu-16.04/libpostal_${{ steps.metadata.outputs.version }}+git${{ steps.metadata.outputs.commit }}_amd64.deb
asset_name: ubuntu-16.04-libpostal_${{ steps.metadata.outputs.version }}+git${{ steps.metadata.outputs.commit }}_amd64.deb #FIXME: find a way to dynamically set upstream version
asset_content_type: application/vnd.debian.binary-package
- name: Upload data release asset (ubuntu-16.04)
id: upload-data-release-asset-ubuntu-16-04
uses: actions/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
asset_path: ./output-ubuntu-16.04/data.tgz
asset_name: ubuntu-16.04-data-${{ steps.metadata.outputs.version }}-${{ steps.metadata.outputs.commit }}.tar.gz #FIXME: find a way to dynamically set upstream version
asset_content_type: application/tar+gzip
- name: Upload .deb package release asset (ubuntu-18.04)
id: upload-deb-release-asset-ubuntu-18-04
uses: actions/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
asset_path: ./output-ubuntu-18.04/libpostal_${{ steps.metadata.outputs.version }}+git${{ steps.metadata.outputs.commit }}_amd64.deb
asset_name: ubuntu-18.04-libpostal_${{ steps.metadata.outputs.version }}+git${{ steps.metadata.outputs.commit }}_amd64.deb #FIXME: find a way to dynamically set upstream version
asset_content_type: application/vnd.debian.binary-package
- name: Upload data release asset (ubuntu-18.04)
id: upload-data-release-asset-ubuntu-18-04
uses: actions/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
asset_path: ./output-ubuntu-18.04/data.tgz
asset_name: ubuntu-18.04-data-${{ steps.metadata.outputs.version }}-${{ steps.metadata.outputs.commit }}.tar.gz #FIXME: find a way to dynamically set upstream version
asset_content_type: application/tar+gzip
RELEASE_TAG: "${{ steps.metadata.outputs.version }}-${{ steps.metadata.outputs.commit }}"
RELEASE_NAME: "Release ${{ steps.metadata.outputs.version }}-${{ steps.metadata.outputs.commit }}"
RELEASE_NOTES: "Automatic release from GH action on this project."
run: |
sudo apt install -y npm
sudo npm install -g publish-release
set -x
GITHUB_OWNER=$(echo ${GITHUB_REPOSITORY} | awk -F/ '{print $1}')
GITHUB_REPO=$(echo ${GITHUB_REPOSITORY} | awk -F/ '{print $2}')
git tag ${RELEASE_TAG}
echo '{}' > package.json
publish-release \
--reuseRelease \
--tag ${RELEASE_TAG} \
--owner ${GITHUB_OWNER} \
--repo ${GITHUB_REPO} \
--token ${GITHUB_TOKEN} \
--name "${RELEASE_NAME}" \
--notes "${RELEASE_NOTES}" \
--assets $(ls release-files/* | xargs echo | sed 's| |,|g')
88 changes: 6 additions & 82 deletions .travis.yml
Original file line number Diff line number Diff line change
@@ -1,83 +1,7 @@
language: c
branches:
only:
- master
env:
global:
- secure: "bHrAu46oecEj3gjamT+XWXtf2J0ZJCFa8tUdgM4evscaJiiwv1TtsGXyhIj/ai7DlRIPVJUtBUy6uoGGjr6GT43zTrzSxYAOMdVXZYsnTDcdL1/0dbwcIK6/u0EI377s1buGIxG1fHveWKXuXwJWDAw4KS+5HU88a42+zMbhKe4="
- secure: "SkvNYucKVns9qDjOEW2WIhDlOMKBOwhzVcwY++HWTRtn04ErrqR4k01Mmho0jGBQD9JrPLhDgnX1BNy5s+Kmq/bxn9OZm7K1z24qBKb0mBBiNEnf2jvT0AvF5xxM+cJf4KKNL+CC0MwNf5y7HVPq1xibOV4/CNIrc1ZZc9aqdkE="
- secure: "am/rRca5akv7gSSMeNQfHnWiTHhk8fQhOZvZ0Ut+PezkQlLgKp7bzmMFkkuQ4L5hpJU40kFzuWmIPgO33dacgq69Vx/Xct1bEnxGBGjriI5qOhMizmzLYPs5uWiRjtJnBqb4JOUh5K7JBlwrgvD72fY5ZK2lwtzTksfWo8N+ahU="
- secure: "mh/WDQapGJb6MAFvgCjiMAAv1aa8gUaIs2Ohtx7yPrDBwsD8UqlyEM7ktGLZGQ1q/7OJ/Z6QfDMfJQwDKzxyUSY1yHZTNkP3QzkTt2D1Qyvi++O6EkGqSdSS6Lb3aID3IsEaye/yasJ+rxiRSp05O9+OYvhJlqRZnzaimiAv5KI="
- secure: "OGNJ6Cj3trq4nASgm4BK331aij+FZ11St7/YF9rfxeQBwg4MCPH2+D0jvAULBHvJR7K2RmepX/FG5d4S+rtwKNGngg3ovPdd1MbwFltHpn5/KM+hxe7kCZx2+V9/FN+4YSyO0zSUDra6AXHOs72mfyrZoB3a36SS4lg2sAp33gU="
- GH_REF=github.com/openvenues/libpostal
- DICTIONARIES_CHANGED=$(git diff --name-only $TRAVIS_COMMIT_RANGE | grep "resources/dictionaries/.*/.*.txt\|src/gazetteer_data.c" | wc -l)
- NUMEX_CHANGED=$(git diff --name-only $TRAVIS_COMMIT_RANGE | grep "resources/numex\|src/numex_table_builder.c" | wc -l)
- TRANSLIT_CHANGED=$(git diff --name-only $TRAVIS_COMMIT_RANGE | grep "src/transliteration_data.c" | wc -l)
- TAG_VERSION=$(cat ./versions/base).$TRAVIS_BUILD_NUMBER
- SRC_TARBALL_FILENAME=libpostal-$(cat ./versions/base).tar.gz
- LIBPOSTAL_DATA_DIR=$(pwd)/data
- LIBPOSTAL_DATA_FILENAME=libpostal_data.tar.gz
compiler:
- clang
- gcc
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- gcc-4.8
- pkg-config
before_script:
- ./bootstrap.sh
- if [[ $DICTIONARIES_CHANGED -ne 0 || $NUMEX_CHANGED -ne 0 ]]; then git clone https://github.com/pypa/virtualenv; cd virtualenv; git checkout master; python virtualenv.py ../env; cd ..; env/bin/pip install -r scripts/requirements-simple.txt; fi;
- if [ $NUMEX_CHANGED -ne 0 ]; then env/bin/python scripts/geodata/numbers/numex.py; fi;
- if [ $DICTIONARIES_CHANGED -ne 0 ]; then env/bin/python scripts/geodata/address_expansions/address_dictionaries.py; fi;
install:
- if [ "$CC" = "gcc" ]; then export CC="gcc-4.8"; fi
script:
- ./configure --datadir=$LIBPOSTAL_DATA_DIR
- make -j4
- if [[ $DICTIONARIES_CHANGED -ne 0 ]]; then ./src/build_address_dictionary; fi;
- if [[ $NUMEX_CHANGED -ne 0 ]]; then ./src/build_numex_table; fi;
- if [[ $TRANSLIT_CHANGED -ne 0 ]]; then ./src/build_trans_table; fi;
- make check

after_success:
- |
if [[ "$CC" == "gcc" && "$TRAVIS_PULL_REQUEST" = "false" && "$TRAVIS_BRANCH" = "master" ]]; then
if [[ ( $DICTIONARIES_CHANGED -ne 0 || $NUMEX_CHANGED -ne 0 || $TRANSLIT_CHANGED -ne 0 ) ]]; then
export PATH=$PATH:env/bin/;
git clone -b master "https://${GH_TOKEN}@${GH_REF}" _travis > /dev/null 2>&1 || exit 1
cp src/*_data.c _travis/src
echo "$TAG_VERSION" > _travis/versions/base_data
cd _travis
git config user.name "$GIT_COMMITTER_NAME";
git config user.email "$GIT_COMMITTER_EMAIL";
git commit -a -m "[auto][ci skip] Adding data files from Travis build #$TRAVIS_BUILD_NUMBER";
git push --quiet origin master;
tar -C $LIBPOSTAL_DATA_DIR -cvzf $LIBPOSTAL_DATA_DIR/$LIBPOSTAL_DATA_FILENAME $BASIC_MODULE_DIRS
fi
git tag $TAG_VERSION -a -m "[auto][ci skip] Generating tag for Travis build #$TRAVIS_BUILD_NUMBER";
git push --tags --quiet origin master;
fi;
before_deploy:
- make dist

deploy:
- provider: releases
file:
- "$SRC_TARBALL_FILENAME"
on:
tags: true
branch: master
skip_cleanup: true
- provider: releases
file:
- "$LIBPOSTAL_DATA_DIR/$LIBPOSTAL_DATA_FILENAME"
on:
tags: true
branch: master
condition: "$CC = gcc && ( $DICTIONARIES_CHANGED -ne 0 || $NUMEX_CHANGED -ne 0 || $TRANSLIT_CHANGED -ne 0 )"
skip_cleanup: true
- provider: releases
api_key:
secure: ${GITHUB_TOKEN}
file_glob: true
file: /output/release/**/*
skip_cleanup: true
4 changes: 2 additions & 2 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
ARG UBUNTU_VERSION="18.04"
FROM ubuntu:${UBUNTU_VERSION}
ARG BASE_IMAGE="ubuntu:18.04"
FROM ${BASE_IMAGE}
ARG DEB_PACKAGE_NAME="libpostal"
ARG DEB_PACKAGE_DESC="Debian wrapping for https://github.com/openvenues/libpostal"
ARG DEB_PACKAGE_VERSION="0.0.1"
Expand Down

0 comments on commit 1779c5a

Please sign in to comment.