merge buildman scripts, automate deploy and release (#130)

* add OpenBangla/buildman (commit:a81fbcd6728647cd3e49b4475ab770a1a7e4e98f)

* integrate buildman workflows

* update build process

* get build version from cmake config

* update buildman readme

* keep version info centrally in version.txt

* skip ci on non-essential file change

* removed unnecessary condition from workflow

* implement github release generation

if deployment succeeds, the build workflow will create a github
release, along with a matching a git tag.

the placeholder release description should be updated, and the
attached installers.zip replaced with separate installer packages
contained within itself.

the workflow has to attach all artifacts as a single zip because
the relevant action cannot upload multiple files to a release yet
(not without insane workflow step duplication.)

* workflow name: build -> deploy. update buildman readme

* skip running workflows on tag push

* update install instructions with repo config.

(connectivity note removed because it should be obvious)

* add branch filter to PR ci

* simplify ubuntu/debian install cmd + fix target paths

* add back a missing cmd

* support minor ubuntu versions

* get bintray username from secrets

* checking $DEPLOY not needed now in deploy workflow

* drop ubuntu 17 (unmaint'd)

* don't overwrite dist deploys

* drop ubuntu16. readme typo fix

* fix ubuntu deployment

* fix typo in readme cmd

* revert readme

* uploads separate installers to release

+ updated build candidates
+ ubuntu package upload optimized
+ publish.sh sanitized a bit

* added deployment distro units note

the deploy workflow's distro list needs to be updated periodiclly.

particularly, the ubuntu versions list must be kept up-to-date
with their current latest short term and long term versions.
otherwise it will break deploy workflow, because canonical moves
repos into archives fairly often. if our deploy workflow tries to
build for a ubuntu version that has been archived, it will fail.

for example, as of this commit, ubuntu19.04 repo has already been
archived, so I had to use 19.10 in the build list instead.

* fixed typo in tools/publish.sh

(thanks @mominul)

Co-Authored-By: Muhammad Mominul Huque <mominul2082@gmail.com>

* add archlinux packager metadata, update pkg fmt

* package metadata consistency

Co-authored-by: Muhammad Mominul Huque <mominul2082@gmail.com>

.github/workflows/ci.yml

@@ -1,18 +1,27 @@
 name: CI
-on: [push, pull_request]
+on: 
+  push:
+    branches: [ '**' ]
+    tags-ignore: [ '**' ]
+    paths-ignore: [ '**.md', '**.adoc' ]
+  pull_request:
+    branches: [ '**' ]
+    paths-ignore: [ '**.md', '**.adoc' ]
+
 jobs:
   build:
-    runs-on: [ "ubuntu-latest" ]
+    if: contains(github.event.head_commit.message, 'deploy+') == false
+    runs-on: "ubuntu-latest"
     strategy:
       matrix:
         env:
           - { CXX: g++, CC: gcc, CXX_FOR_BUILD: g++, CC_FOR_BUILD: gcc }
           - { CXX: clang++, CC: clang, CXX_FOR_BUILD: clang++, CC_FOR_BUILD: clang }
     steps:
       - name: checkout-source
-        uses: actions/checkout@v2
-      - name: checkout-submodules
-        run: git submodule update --init --recursive
+        uses: actions/checkout@v1
+        with:
+          submodules: recursive
       - name: update-system
         run: sudo apt-get update -qq
       - name: install-deps
@@ -24,3 +33,4 @@ jobs:
           cd build
           cmake ..
           make
+

.github/workflows/deploy.yml

@@ -0,0 +1,97 @@
+name: DEPLOY
+
+on:
+  push:
+    branches: [ '**' ]
+    tags-ignore: [ '**' ]
+
+jobs:
+
+  build:
+    if: contains(github.event.head_commit.message, 'deploy+')
+    strategy:
+      matrix:
+        container: [ "ubuntu:18.04", "ubuntu:19.10", "ubuntu:20.04", "debian:9", "debian:10", "fedora:29", "fedora:30", "fedora:31", "fedora:32", "archlinux:latest" ]
+        # this list should be updated from time to time by consulting these pages:
+        # https://releases.ubuntu.com/
+        # https://wiki.debian.org/DebianReleases#Production_Releases
+        # https://fedoraproject.org/wiki/Releases
+    runs-on: "ubuntu-latest"
+    container:
+      image: ${{ matrix.container }}
+      env:
+        DIST: ${{ matrix.container }}
+        DEBIAN_FRONTEND: noninteractive
+    steps:
+      - name: checkout-source
+        uses: actions/checkout@v1
+        with:
+          submodules: recursive
+      - name: run-builder
+        shell: bash
+        run: ./tools/build.sh
+      - name: upload-artifacts
+        uses: actions/upload-artifact@v1
+        with:
+          name: package-archives
+          path: artifact
+
+  deploy:
+    needs: build
+    runs-on: "ubuntu-latest"
+    container:
+      image: "archlinux:latest"
+    env:
+      BINTRAY_USER: ${{ secrets.BINTRAY_USER }}
+      BINTRAY_APIKEY: ${{ secrets.BINTRAY_APIKEY }}
+      BINTRAY_GPG_DATA: ${{ secrets.GPG_DATA }}
+      BINTRAY_GPG_ID: ${{ secrets.GPG_ID }}
+      JFROG_CLI_OFFER_CONFIG: false
+    steps:
+      - name: checkout-source
+        uses: actions/checkout@v1
+        with:
+          submodules: recursive
+      - name: install-jfrog-cli
+        uses: jfrog/setup-jfrog-cli@v1
+      - name: download-artifacts
+        uses: actions/download-artifact@v1
+        with:
+          name: package-archives
+          path: artifact
+      - name: bintray-publish
+        shell: bash
+        run: ./tools/publish.sh
+
+  release:
+    needs: deploy
+    runs-on: "ubuntu-latest"
+    steps:
+      - name: checkout-source
+        uses: actions/checkout@v1
+      - name: get-version
+        id: get_version
+        shell: bash
+        run: |
+          RELEASE_VERSION=$(cat version.txt | head -n1)
+          echo ::set-output name=RELEASE_VERSION::$RELEASE_VERSION
+      - name: download-artifacts
+        uses: actions/download-artifact@v1
+        with:
+          name: package-archives
+          path: artifact
+      - name: create-release
+        id: create_release
+        uses: ncipollo/release-action@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          tag: ${{ steps.get_version.outputs.RELEASE_VERSION }}
+          name: ${{ steps.get_version.outputs.RELEASE_VERSION }}
+          artifacts: "artifact/*"
+          allowUpdates: true
+          draft: true
+          body: |
+            [![Github Releases (by Release)](https://img.shields.io/github/downloads/${{ github.repository }}/${{ steps.get_version.outputs.RELEASE_VERSION }}/total.svg)]()
+            ### Changes in this release:
+            - ...
+

CMakeLists.txt

@@ -1,7 +1,8 @@
 cmake_minimum_required(VERSION 3.0.2)
 set(CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake")
 
-project(OpenBangla-Keyboard VERSION 2.0.0)
+file(STRINGS "${CMAKE_SOURCE_DIR}/version.txt" VERSION_STRING LIMIT_COUNT 1 REGEX "^.+")
+project("OpenBangla-Keyboard" VERSION ${VERSION_STRING})
 set(APP_NAME "OpenBangla-Keyboard")
 
 ## Set versions
@@ -75,21 +76,21 @@ endif ()
 
 
 set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "OpenSource Bengali input method")
-set(CPACK_PACKAGE_VENDOR "Muhammad Mominul Huque")
-set(CPACK_DEBIAN_PACKAGE_MAINTAINER "Muhammad Mominul Huque <mominul2082@gmail.com>")
+set(CPACK_PACKAGE_VENDOR "OpenBangla Team")
+set(CPACK_DEBIAN_PACKAGE_MAINTAINER "OpenBangla Team <openbanglateam@gmail.com>")
 set(CPACK_PACKAGE_DESCRIPTION_FILE "${CMAKE_SOURCE_DIR}/data/Readme.txt")
 set(CPACK_RESOURCE_FILE_LICENSE "${CMAKE_SOURCE_DIR}/LICENSE")
 set(CPACK_PACKAGE_VERSION_MAJOR "${PROJECT_VERSION_MAJOR}")
 set(CPACK_PACKAGE_VERSION_MINOR "${PROJECT_VERSION_MINOR}")
 set(CPACK_PACKAGE_VERSION_PATCH "${PROJECT_VERSION_PATCH}")
 set(CPACK_PACKAGE_FILE_NAME "${PROJECT_NAME}_${PROJECT_VERSION}-$ENV{DIST}")
 set(CPACK_STRIP_FILES TRUE)
-set(CPACK_DEBIAN_PACKAGE_HOMEPAGE "http://openbangla.github.io/")
+set(CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://openbangla.github.io/")
 set(CPACK_DEBIAN_PACKAGE_SECTION "utils")
 set(CPACK_DEBIAN_PACKAGE_DEPENDS "ibus (>= 1.5.1)")
 set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON)
 set(CPACK_RPM_PACKAGE_LICENSE "GPLv3")
-set(CPACK_RPM_PACKAGE_URL "http://openbangla.github.io/")
+set(CPACK_RPM_PACKAGE_URL "https://openbangla.github.io/")
 set(CPACK_RPM_PACKAGE_REQUIRES "qt5-qtbase >= 5.9.0, ibus >= 1.5.1, ibus-libs >= 1.5.1")
 set(CPACK_RPM_PACKAGE_RELEASE_DIST ON)
 # Prevents CPack from generating file conflicts

tools/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2018 OpenBangla
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

tools/PKGBUILD.stub

@@ -0,0 +1,20 @@
+# Maintainer: OpenBangla Team <openbanglateam@gmail.com>
+# THIS IS A *PARTIAL* PKGBUILD TEMPLATE FOR CUSTOM AUTOBUILD PROCESS ONLY
+pkgname="openbangla-keyboard"
+pkgrel=1
+pkgdesc="An OpenSource, Unicode compliant Bengali Input Method"
+arch=('x86_64')
+url="https://openbangla.github.io"
+license=('GPL3')
+depends=('ibus' 'qt5-base')
+build() {
+    cd "$srcdir"
+    mkdir build && cd build
+    cmake ..
+    make
+}
+package() {
+    cd "$srcdir"
+    make -C build DESTDIR="$pkgdir" install
+}
+

tools/README.md

@@ -0,0 +1,7 @@
+# buildman
+
+Builds OpenBangla Keyboard packages for various distributions.
+
+- A lightweight C/I workflow runs by default on each commit.
+- if commit message has the string `deploy+`, then a deploy workflow runs instead of the default one.
+  - if build passes, this workflow will deploy packages to repositories, and create a github release.

tools/build.sh

@@ -0,0 +1,64 @@
+#! /bin/bash
+RELEASE_VERSION=$(cat version.txt | head -n1)
+# follow cmake PACKAGE_FILE_NAME directive in main repo
+RELEASE_STUB="OpenBangla-Keyboard_${RELEASE_VERSION}-"
+
+makeDeb () {
+    RELEASE_FILENAME="${RELEASE_STUB}${DIST}.deb"
+    apt-get -y install build-essential cmake libibus-1.0-dev qt5-default rustc cargo ninja-build curl
+    cmake -H"$GITHUB_WORKSPACE" -B/build -GNinja -DCPACK_GENERATOR=DEB
+    ninja package -C /build
+    RELEASE_FILE="/build/${RELEASE_FILENAME}"
+}
+
+makeRpm () {
+    RELEASE_FILENAME="${RELEASE_STUB}${DIST}.rpm"
+    dnf install -y --allowerasing @buildsys-build cmake ibus-devel qt5-qtdeclarative-devel rust cargo ninja-build
+    cmake -H"$GITHUB_WORKSPACE" -B/build -GNinja -DCPACK_GENERATOR=RPM
+    ninja package -C /build
+    RELEASE_FILE="/build/${RELEASE_FILENAME}"
+}
+
+makeArch () {
+    DIST=archlinux
+    echo "PACKAGER='OpenBangla Team <openbanglateam@gmail.com>'" >> /etc/makepkg.conf
+    PKGEXT=".pkg.tar.zst"
+    echo "PKGEXT='$PKGEXT'" >> /etc/makepkg.conf
+    RELEASE_FILENAME="${RELEASE_STUB}${DIST}${PKGEXT}"
+    pacman -S --noconfirm --needed base-devel cmake libibus qt5-base rust curl
+    mkdir /build
+    cd /build
+    cp -fpr "$GITHUB_WORKSPACE" /build/src
+    cp "${GITHUB_WORKSPACE}/tools/PKGBUILD.stub" /build/PKGBUILD
+    sed -i "/pkgname=/a pkgver=\"${RELEASE_VERSION}\"" /build/PKGBUILD
+    useradd -m builder
+    chown -R builder:builder /build
+    sudo -u builder makepkg -fd --skipinteg
+    mv openbangla-keyboard-*${PKGEXT} ${RELEASE_FILENAME}
+    RELEASE_FILE=$(realpath "${RELEASE_FILENAME}")
+}
+
+if [[ $DIST =~ ^(ubuntu|debian) ]]; then
+    apt-get -qq update
+    apt-get -y install git
+    # this is to read distro codename from filename during deployment
+    CODENAME=$(cat /etc/os-release | grep "VERSION_CODENAME" | cut -d= -f2)
+    DIST="${DIST}-${CODENAME}"
+    BUILDFUNC=makeDeb
+elif [[ $DIST =~ ^fedora ]]; then
+    dnf -y --allowerasing distro-sync
+    dnf -y install git
+    BUILDFUNC=makeRpm
+elif [[ $DIST =~ ^archlinux ]]; then
+    pacman -Syyu --noconfirm --needed
+    pacman -S --noconfirm --needed base git
+    BUILDFUNC=makeArch
+fi
+
+DIST=${DIST/:/}
+${BUILDFUNC}
+
+DEPLOY_PATH="${GITHUB_WORKSPACE}/artifact"
+mkdir "$DEPLOY_PATH"
+mv "$RELEASE_FILE" "${DEPLOY_PATH}/"
+

tools/publish.sh

@@ -0,0 +1,92 @@
+#!/bin/bash
+ORG='openbangla'
+PACKAGE='openbangla-keyboard'
+DISTLIST=(ubuntu debian fedora archlinux)
+RELEASE_VERSION=$(cat version.txt | head -n1)
+jfrog bt config --user "$BINTRAY_USER" --key "$BINTRAY_APIKEY" --licenses 'GPL-3.0'
+pacman -Syyuu --noconfirm --needed jq rpm
+cd artifact
+
+gpgImport () {
+    echo -e "-----BEGIN PGP PRIVATE KEY BLOCK-----\n\n${BINTRAY_GPG_DATA}\n-----END PGP PRIVATE KEY BLOCK-----" \
+    | gpg --import 2> /dev/null
+}
+
+pubDeb () {
+    # prepare an ordered list of ubuntu release {version:codename} dict
+    curl https://api.launchpad.net/devel/ubuntu/series \
+    | jq -M '.entries|sort_by(.version|tonumber)|map({(.version): .name})' \
+    | sed -r 's/[][ ,"\{\}]//g' \
+    | grep -v '^\s*$' \
+    > dists.txt
+    for PKG in $(ls ./*${REPO}* 2>/dev/null); do
+        # read distro code from filename
+        CODENAME=$(echo "$PKG" | grep -oP '[^-]+.deb$' | cut -d. -f1)
+        jfrog bt upload --publish --override --deb "${CODENAME}/main/amd64" "$PKG" "$VERSION_PATH"
+        if [ $REPO == ubuntu ]; then
+            # we only build for major/first yearly releases.
+            # this block uploads the package for the other version in the same range
+            VERSION_STR=$(grep "$CODENAME" dists.txt)
+            VERSION_MAJOR=$(echo "$VERSION_STR" | cut -d. -f1)
+            OTHER_VERSION_STR=$(grep "^${VERSION_MAJOR}\." dists.txt | grep -v "$CODENAME")
+            if [ -n "$OTHER_VERSION_STR" ]; then
+                OTHER_CODENAME=$(echo "$OTHER_VERSION_STR" | cut -d: -f2)
+                OTHER_VERSION_STR=${OTHER_VERSION_STR/:/-}
+                VERSION_STR=${VERSION_STR/:/-}
+                ln -frs "$PKG" "${PKG/$VERSION_STR/$OTHER_VERSION_STR}"
+                jfrog bt upload --publish --override --deb "${OTHER_CODENAME}/main/amd64" ./*${OTHER_CODENAME}.deb "$VERSION_PATH"
+            fi
+        fi
+    done
+}
+
+pubRpm () {
+    gpgImport
+    ARCH='x86_64'
+    gpg --export -a "${BINTRAY_GPG_ID}" > pubkey.asc
+    rpm --import pubkey.asc
+    rm pubkey.asc
+    echo "%_signature gpg" > "${HOME}/.rpmmacros"
+    echo "%_gpg_name ${BINTRAY_GPG_ID}" >> "${HOME}/.rpmmacros"
+    for PKG in $(ls ./*${REPO}* 2>/dev/null); do
+        rpm --addsign "$PKG"
+        VERSION_ID=$(echo "$PKG" | grep -oP '[\d]+.rpm$' | cut -d. -f1)
+        jfrog bt upload --publish --override "$PKG" "$VERSION_PATH" "${VERSION_ID}/${ARCH}/"
+    done
+}
+
+pubArch () {
+    gpgImport
+    ARCH='x86_64'
+    # only one file should be listed anyway
+    for PKG in $(ls -1 ./*${REPO}* 2>/dev/null); do
+        # get repo metadata from the "meta" version, creating it if necessary
+        [[ $(jfrog bt version-show "${PACKAGE_PATH}/meta" 2> /dev/null) ]] \
+        || jfrog bt version-create --vcs-tag "0.0.0" --desc "archlinux repo metadata" "${PACKAGE_PATH}/meta"
+        jfrog bt download-ver "${PACKAGE_PATH}/meta"
+        if [ -d "$ARCH" ]; then
+            mv "$ARCH" meta
+        fi
+        mkdir -p meta
+        gpg --detach-sign --no-armor -u "$BINTRAY_GPG_ID" "$PKG"
+        repo-add -p -s -k "$BINTRAY_GPG_ID" "meta/${ORG}.db.tar.gz" "$PKG"
+        jfrog bt upload --publish --override "meta/*" "${PACKAGE_PATH}/meta" "${ARCH}/"
+        jfrog bt upload --publish --override "${PKG}*" "${VERSION_PATH}" "${ARCH}/"
+    done
+}
+
+for REPO in ${DISTLIST[*]}; do
+    PACKAGE_PATH="${ORG}/${REPO}/${PACKAGE}"
+    VERSION_PATH="${PACKAGE_PATH}/${RELEASE_VERSION}"
+    # create version
+    [[ $(jfrog bt version-show "$VERSION_PATH" 2>/dev/null) ]] && op=version-update || op=version-create
+    jfrog bt $op --vcs-tag "$RELEASE_VERSION" --released "$(date +'%Y-%m-%dT%H:%M:%S.000Z')" "$VERSION_PATH"
+    # upload files
+    if [[ $REPO =~ ubuntu|debian ]]; then
+        pubDeb
+    elif [ $REPO == fedora ]; then
+        pubRpm
+    elif [ $REPO == archlinux ]; then
+        pubArch
+    fi
+done

version.txt

@@ -0,0 +1 @@
+2.0.0