From 43e078bbcc23b29bbe2876d20bad06cea5c25fed Mon Sep 17 00:00:00 2001 From: Adeel Date: Sun, 3 Feb 2019 02:35:06 +0200 Subject: [PATCH 1/6] Add .cirrus.yml for FreeBSD (amd64, i386) CI/CD For each pull request, it triggers a job consists of 'node.js version x architecture' matrix. When a tag is pushed, it creates a release idempotently, and publishes artifacts to the release. The versions of node.js and npm packages are selected from package management. This is because node.js official distribution does not (yet) offer FreeBSD binaries and `nvm` spends too much time compiling node.js and npm from source. `pkg` does not support installation of old version of packages with `pkg-install` command, so the `pkg-add` approach has been elected to install older node.js. For that matter, the exact version of node.js and npm packages are resolved using pattern matching. `cbsd` is used to create an i386 jail environment. Co-authored-by: Oleg Ginzburg --- .cirrus.yml | 65 +++++++++++++++++ appveyor.yml | 2 +- package.json | 4 +- scripts/configure_freebsd_ci_jail.sh | 99 ++++++++++++++++++++++++++ scripts/upload-github-release-asset.sh | 60 ++++++++++++++++ 5 files changed, 227 insertions(+), 3 deletions(-) create mode 100644 .cirrus.yml create mode 100755 scripts/configure_freebsd_ci_jail.sh create mode 100755 scripts/upload-github-release-asset.sh diff --git a/.cirrus.yml b/.cirrus.yml new file mode 100644 index 000000000..c37c02457 --- /dev/null +++ b/.cirrus.yml @@ -0,0 +1,65 @@ +freebsd_instance: + image: freebsd-11-2-release-amd64 + +freebsd_build_task: + + env: + matrix: + - node_js: "11" + nodePattern: "^node-11.*V8" + npmPattern: "^npm-6\\." + - node_js: "10" + nodePattern: "^node10-.*V8" + npmPattern: "^npm-node10-" + - node_js: "8" + nodePattern: "^node8-.*V8" + npmPattern: "^npm-node8-" + - node_js: "6" + nodePattern: "^node6-.*V8" + npmPattern: "^npm-node6-" + + env: + matrix: + - abi: freebsd:11:x86:64 + - abi: freebsd:11:x86:32 + jailName: j11i386 + execPrefix: cbsd jexec jname=j11i386 + + env: + GH_API_TOKEN: ENCRYPTED[5e482f417304528148bb96eca8d030eacd6ab9972d482485fc4d42907283b995f658b351e0676e9493a37d815398f541] + + prepare_script: + - sed -i '' 's/quarterly/latest/g' /etc/pkg/FreeBSD.conf + - | + if test "$abi" = "freebsd:11:x86:32"; then + ./scripts/configure_freebsd_ci_jail.sh $jailName $CIRRUS_WORKING_DIR; + $execPrefix sed -i -- '' 's/quarterly/latest/g' /etc/pkg/FreeBSD.conf; + $execPrefix pkg update -f; + $execPrefix pkg install -y python2 > /dev/null; + fi + - $execPrefix pkg install -y c-ares gmake icu libnghttp2 libuv git > /dev/null + - $execPrefix pkg add http://pkg.freebsd.org/$abi/latest/All/$(pkg search node | grep $nodePattern | cut -d' ' -f 1).txz + - $execPrefix pkg add http://pkg.freebsd.org/$abi/latest/All/$(pkg search npm | grep $npmPattern | cut -d' ' -f 1).txz + - $execPrefix node --version + - $execPrefix npm --version + - $execPrefix clang++ --version + + build_script: + - | + if test "$abi" = "freebsd:11:x86:32"; then + changeDir="cd /etc/skel &&" + fi + echo "$changeDir npm install --unsafe-perm" | $execPrefix /bin/sh + + publish_script: + - | + if test "$CIRRUS_TAG" != ""; then + for file in `ls vendor/**/*.node`; do + parent=${file%/*}; + name=${parent##*/}; + fullyQualifiedName="$(pwd)/$parent/${name}_binding.node"; + mv "$file" "$parent/${name}_binding.node"; + echo -e "New filename\072 $fullyQualifiedName"; + ./scripts/upload-github-release-asset.sh github_api_token=$GH_API_TOKEN owner=am11 repo=node-sass tag=$CIRRUS_TAG filename=$fullyQualifiedName; + done + fi diff --git a/appveyor.yml b/appveyor.yml index ed7c2f8ff..1003c125b 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -166,7 +166,7 @@ - nodejs_version: 11 GYP_MSVS_VERSION: 2015 APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015 - + install: - ps: Install-Product node $env:nodejs_version $env:platform - node --version diff --git a/package.json b/package.json index 23bc0502a..b8285dd69 100644 --- a/package.json +++ b/package.json @@ -76,8 +76,8 @@ "eslint": "^3.4.0", "fs-extra": "^0.30.0", "istanbul": "^0.4.2", - "mocha": "^3.1.2", - "mocha-lcov-reporter": "^1.2.0", + "mocha": "^5.2.0", + "mocha-lcov-reporter": "^1.3.0", "object-merge": "^2.5.1", "read-yaml": "^1.0.0", "rimraf": "^2.5.2", diff --git a/scripts/configure_freebsd_ci_jail.sh b/scripts/configure_freebsd_ci_jail.sh new file mode 100755 index 000000000..4d0c6756e --- /dev/null +++ b/scripts/configure_freebsd_ci_jail.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env sh + +set -o xtrace + +jailName=$1 +skelDirectory=$2 +cbsd_workdir=/usr/jails +jail_arch="i386" +jail_ver="11.2" + +echo "Installing build dependencies for cbsd" +pkg install -y libssh2 rsync sqlite3 git pkgconf + +echo "Clone and setup cbsd" +git clone https://github.com/cbsd/cbsd.git /usr/local/cbsd --single-branch --branch v12.0.4 --depth 1 + +cd /usr/local/etc/rc.d +ln -sf /usr/local/cbsd/rc.d/cbsdd +mkdir -p /usr/local/libexec/bsdconfig +cd /usr/local/libexec/bsdconfig +ln -s /usr/local/cbsd/share/bsdconfig/cbsd +pw useradd cbsd -s /bin/sh -d ${cbsd_workdir} -c "cbsd user" + +# determine uplink ip address +# determine uplink iface +auto_iface=$( /sbin/route -n get 0.0.0.0 |/usr/bin/awk '/interface/{print $2}' ) +my_ipv4=$( /sbin/ifconfig ${auto_iface} | /usr/bin/awk '/inet [0-9]+/{print $2}' ) + +if [ -z "${my_ipv4}" ]; then + echo "IPv4 not detected" + exit 1 +fi + +echo "Writing '${jailName}' configuration file" +cat > /tmp/${jailName}.jconf << EOF +jname="${jailName}" +path="${cbsd_workdir}/${jailName}" +host_hostname="${jailName}.my.domain" +ip4_addr="${my_ipv4}" +mount_devfs="1" +allow_mount="1" +allow_devfs="1" +allow_nullfs="1" +allow_raw_sockets="1" +mount_fstab="${cbsd_workdir}/jails-fstab/fstab.${jailName}" +arch="${jail_arch}" +mkhostsfile="1" +devfs_ruleset="4" +ver="${jail_ver}" +basename="" +baserw="0" +mount_src="0" +mount_obj="0" +mount_kernel="0" +mount_ports="1" +astart="1" +data="${cbsd_workdir}/jails-data/${jailName}-data" +vnet="0" +applytpl="1" +mdsize="0" +rcconf="${cbsd_workdir}/jails-rcconf/rc.conf_${jailName}" +floatresolv="1" +exec_poststart="0" +exec_poststop="" +exec_prestart="0" +exec_prestop="0" +exec_master_poststart="0" +exec_master_poststop="0" +exec_master_prestart="0" +exec_master_prestop="0" +pkg_bootstrap="1" +interface="0" +jailskeldir="$skelDirectory" +exec_start="/bin/sh /etc/rc" +exec_stop="/bin/sh /etc/rc.shutdown" +EOF + +echo "Initializing cbsd environment" +env workdir=${cbsd_workdir} /usr/local/cbsd/sudoexec/initenv /usr/local/cbsd/share/initenv.conf + +echo "Writing 'FreeBSD-bases' configuration file" +cat > ${cbsd_workdir}/etc/FreeBSD-bases.conf << EOF +auto_baseupdate=0 +default_obtain_base_method="extract repo" +default_obtain_base_extract_source="/usr/freebsd-dist/base.txz" +default_obtain_base_repo_sources="https://bintray.com/am11/freebsd-dist/download_file?file_path=base-${jail_ver}-${jail_arch}.txz" +EOF + +echo "Creating ${jailName}" +cbsd jcreate jconf=/tmp/${jailName}.jconf inter=0 +cbsd jailscp /etc/resolv.conf ${jailName}:/etc/resolv.conf + +cat > ~cbsd/jails-fstab/fstab.${jailName}.local < /dev/null + +# Read asset tags. +response=$(curl -sH "$AUTH" $GH_TAGS) + +# Get ID of the asset based on given filename. +eval "$(echo "$response" | grep -m 1 "id.:" | grep -w id | tr : = | tr -cd '[[:alnum:]]=')" +[ -n "$id" ] || { echo "Error: Failed to get release id for tag: $tag"; echo "$response" | awk 'length($0)<100' >&2; exit 1; } + +# Upload asset +echo "Uploading asset... " + +# Construct url +GH_ASSET="https://uploads.github.com/repos/$owner/$repo/releases/$id/assets?name=$(basename $filename)" + +curl "$GITHUB_OAUTH_BASIC" --data-binary @"$filename" -H "$AUTH" -H "Content-Type: application/octet-stream" $GH_ASSET From cf5f4774a2e3e29ccdbb44c0837cb40659d87b66 Mon Sep 17 00:00:00 2001 From: Adeel Date: Mon, 4 Feb 2019 02:00:24 +0200 Subject: [PATCH 2/6] Use scraping to get latest version of node/npm In FreeBSD VM hosted on Google GCE, with `pkg search` command, it fails to retrieve the latest list even after `pkg update -f` with `latest` repository selected (checked with `pkg -vv search`). There seems to be some caching mechanism involved, which I couldn't find a way to avoid. --- .cirrus.yml | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index c37c02457..849515ab8 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -5,18 +5,18 @@ freebsd_build_task: env: matrix: - - node_js: "11" - nodePattern: "^node-11.*V8" - npmPattern: "^npm-6\\." - - node_js: "10" - nodePattern: "^node10-.*V8" - npmPattern: "^npm-node10-" - - node_js: "8" - nodePattern: "^node8-.*V8" - npmPattern: "^npm-node8-" - - node_js: "6" - nodePattern: "^node6-.*V8" - npmPattern: "^npm-node6-" + - node_js: 11 + nodeTxzKey: node-11 + npmTxzKey: npm-6 + - node_js: 10 + nodeTxzKey: node10 + npmTxzKey: npm-node10 + - node_js: 8 + nodeTxzKey: node8 + npmTxzKey: npm-node8 + - node_js: 6 + nodeTxzKey: node6 + npmTxzKey: npm-node6 env: matrix: @@ -38,8 +38,12 @@ freebsd_build_task: $execPrefix pkg install -y python2 > /dev/null; fi - $execPrefix pkg install -y c-ares gmake icu libnghttp2 libuv git > /dev/null - - $execPrefix pkg add http://pkg.freebsd.org/$abi/latest/All/$(pkg search node | grep $nodePattern | cut -d' ' -f 1).txz - - $execPrefix pkg add http://pkg.freebsd.org/$abi/latest/All/$(pkg search npm | grep $npmPattern | cut -d' ' -f 1).txz + - | + fetch http://pkg.freebsd.org/$abi/latest/All/ -o list + nodeTxz=$(sed -n "/\"$nodeTxzKey/s/.*\"\($nodeTxzKey[^\"]*\).*/\1/p" list) + npmTxz=$(sed -n "/\"$npmTxzKey/s/.*\"\($npmTxzKey[^\"]*\).*/\1/p" list) + $execPrefix pkg add http://pkg.freebsd.org/$abi/latest/All/$nodeTxz + $execPrefix pkg add http://pkg.freebsd.org/$abi/latest/All/$npmTxz - $execPrefix node --version - $execPrefix npm --version - $execPrefix clang++ --version From eb5801f875f910451f672a2cc4ee013ec4108a96 Mon Sep 17 00:00:00 2001 From: Adeel Date: Mon, 4 Feb 2019 03:07:13 +0200 Subject: [PATCH 3/6] Switch to Sass org account --- .cirrus.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cirrus.yml b/.cirrus.yml index 849515ab8..b99ddc4e9 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -64,6 +64,6 @@ freebsd_build_task: fullyQualifiedName="$(pwd)/$parent/${name}_binding.node"; mv "$file" "$parent/${name}_binding.node"; echo -e "New filename\072 $fullyQualifiedName"; - ./scripts/upload-github-release-asset.sh github_api_token=$GH_API_TOKEN owner=am11 repo=node-sass tag=$CIRRUS_TAG filename=$fullyQualifiedName; + ./scripts/upload-github-release-asset.sh github_api_token=$GH_API_TOKEN owner=sass repo=node-sass tag=$CIRRUS_TAG filename=$fullyQualifiedName; done fi From fb333067656d004b97cabdc4f71d4b064504c72e Mon Sep 17 00:00:00 2001 From: Adeel Date: Wed, 6 Feb 2019 00:42:11 +0200 Subject: [PATCH 4/6] Use last version of mocha that supports node 0.x --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index b8285dd69..74606ea9d 100644 --- a/package.json +++ b/package.json @@ -76,7 +76,7 @@ "eslint": "^3.4.0", "fs-extra": "^0.30.0", "istanbul": "^0.4.2", - "mocha": "^5.2.0", + "mocha": "^3.5.3", "mocha-lcov-reporter": "^1.3.0", "object-merge": "^2.5.1", "read-yaml": "^1.0.0", From 4f6b9d81d4b2bf750f223137a642d69702455faa Mon Sep 17 00:00:00 2001 From: Adeel Date: Wed, 20 Feb 2019 07:16:30 +0200 Subject: [PATCH 5/6] Use cbsd v12.0.4 from ports --- scripts/configure_freebsd_ci_jail.sh | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/scripts/configure_freebsd_ci_jail.sh b/scripts/configure_freebsd_ci_jail.sh index 4d0c6756e..9fd21589c 100755 --- a/scripts/configure_freebsd_ci_jail.sh +++ b/scripts/configure_freebsd_ci_jail.sh @@ -9,17 +9,7 @@ jail_arch="i386" jail_ver="11.2" echo "Installing build dependencies for cbsd" -pkg install -y libssh2 rsync sqlite3 git pkgconf - -echo "Clone and setup cbsd" -git clone https://github.com/cbsd/cbsd.git /usr/local/cbsd --single-branch --branch v12.0.4 --depth 1 - -cd /usr/local/etc/rc.d -ln -sf /usr/local/cbsd/rc.d/cbsdd -mkdir -p /usr/local/libexec/bsdconfig -cd /usr/local/libexec/bsdconfig -ln -s /usr/local/cbsd/share/bsdconfig/cbsd -pw useradd cbsd -s /bin/sh -d ${cbsd_workdir} -c "cbsd user" +pkg install -y cbsd # determine uplink ip address # determine uplink iface @@ -78,14 +68,6 @@ EOF echo "Initializing cbsd environment" env workdir=${cbsd_workdir} /usr/local/cbsd/sudoexec/initenv /usr/local/cbsd/share/initenv.conf -echo "Writing 'FreeBSD-bases' configuration file" -cat > ${cbsd_workdir}/etc/FreeBSD-bases.conf << EOF -auto_baseupdate=0 -default_obtain_base_method="extract repo" -default_obtain_base_extract_source="/usr/freebsd-dist/base.txz" -default_obtain_base_repo_sources="https://bintray.com/am11/freebsd-dist/download_file?file_path=base-${jail_ver}-${jail_arch}.txz" -EOF - echo "Creating ${jailName}" cbsd jcreate jconf=/tmp/${jailName}.jconf inter=0 cbsd jailscp /etc/resolv.conf ${jailName}:/etc/resolv.conf From 0aa9518068ed969300dfb3e6925449bbbff49c00 Mon Sep 17 00:00:00 2001 From: Adeel Date: Wed, 20 Feb 2019 08:17:34 +0200 Subject: [PATCH 6/6] Remove obsolete comment --- scripts/configure_freebsd_ci_jail.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/configure_freebsd_ci_jail.sh b/scripts/configure_freebsd_ci_jail.sh index 9fd21589c..b6f83a9e5 100755 --- a/scripts/configure_freebsd_ci_jail.sh +++ b/scripts/configure_freebsd_ci_jail.sh @@ -8,7 +8,6 @@ cbsd_workdir=/usr/jails jail_arch="i386" jail_ver="11.2" -echo "Installing build dependencies for cbsd" pkg install -y cbsd # determine uplink ip address