From f86d1ba5b4bba389e8e8969a9cae5d6de9f7d8a8 Mon Sep 17 00:00:00 2001 From: TJC Date: Mon, 25 Jan 2021 09:37:04 -0700 Subject: [PATCH] Setup GitHub actions (#617) * intial commit to test one job for Github actions * fix envs * exclude this branch while testing github actions * Copy the travis CI scripts and re-use for github actions * Overwite ruby with custom version. Add new checks in bash * run the unit tests now that setup is complete * fix old travis envs * add in step to print the logs * print logs on failure * Change the env BUILD_TYPE to GITHUB_JOB * Add in the linux integration steps and use the BUILD_TYPE vs the GITHUB_JOB env to id the build * use proper syntax for assigning envs via github actions * add docker integration tests * add and enable docker-compose COMPOSE_INTERACTIVE_NO_CLI for CI * Add the docker-compose setup script as the built-on github action docker-compose is not working * fix bash check for platform and build_type * add option to docker-compose to hopefully remove error on Github actions * Add and test out macos builds. Comment out linux builds for now as that's all working fine * For mac builds, use the tar.gz openstudio files as mac 10.15 is having errors trying to headless install the OpenStudio.dmg. Error msg: "Unable to create basic Accelerated OpenGL renderer." * install core utils for macos for building nokogiri * fix the tar * fix the curl output file name of OpenStudio * almost there. adding suffix * Setup is now working for osx. update the test.sh for osx * adding integration tests for macos * Use one job to string together the build, unit, integration and finally uploading gem packages. This should save some time with the uploading as we don't have to call the setup.sh again * fix yml * change -uses to uses * fixup mac export script * export to non repo dir and then cp back to repo dir * forgot to wrap var using bash * Adding in all final steps for docker, linux and mac builds and publishing * update push to docker hub * clean up and another condition for docker hub uploads for good measure to prevent accidental uploads * rename the github action file and add in checks for docker upload * Run on push and pull_request events * Restore the travis and appveyor CI jobs and fix export of osx on travis * remove all travis scripts and travis.yaml. Disabled project on travis-ci.org --- .github/workflows/openstudio-server-tests.yml | 129 ++++++++++++++++++ .travis.yml | 125 ----------------- bin/resources/local/rspec_command | 2 +- ci/{travis => github-actions}/cipher.rb | 0 ci/github-actions/export_build_linux.sh | 20 +++ ci/github-actions/export_build_osx.sh | 17 +++ ci/{travis => github-actions}/install-mac.qs | 0 .../install_openstudio.sh | 0 ci/{travis => github-actions}/print_logs.sh | 16 +-- ci/{travis => github-actions}/setup.sh | 76 ++++++----- ci/{travis => github-actions}/test.sh | 26 ++-- .../verify_openstudio.rb | 0 ci/travis/export_build_linux.sh | 25 ---- ci/travis/export_build_osx.sh | 22 --- .../scripts/deploy_docker_github_actions.sh | 40 ++++++ 15 files changed, 267 insertions(+), 231 deletions(-) create mode 100644 .github/workflows/openstudio-server-tests.yml delete mode 100644 .travis.yml rename ci/{travis => github-actions}/cipher.rb (100%) create mode 100755 ci/github-actions/export_build_linux.sh create mode 100755 ci/github-actions/export_build_osx.sh rename ci/{travis => github-actions}/install-mac.qs (100%) rename ci/{travis => github-actions}/install_openstudio.sh (100%) rename ci/{travis => github-actions}/print_logs.sh (57%) rename ci/{travis => github-actions}/setup.sh (61%) rename ci/{travis => github-actions}/test.sh (73%) rename ci/{travis => github-actions}/verify_openstudio.rb (100%) delete mode 100755 ci/travis/export_build_linux.sh delete mode 100755 ci/travis/export_build_osx.sh create mode 100755 docker/deployment/scripts/deploy_docker_github_actions.sh diff --git a/.github/workflows/openstudio-server-tests.yml b/.github/workflows/openstudio-server-tests.yml new file mode 100644 index 000000000..effea320e --- /dev/null +++ b/.github/workflows/openstudio-server-tests.yml @@ -0,0 +1,129 @@ +name: openstudio-server + +on: [push, pull_request] + +# example of how to restrict to one branch and push event +#on: +# push: +# branches: +# - test_branch + +env: + USE_TESTING_TIMEOUTS: "true" + OPENSTUDIO_VERSION: 3.1.0 + OPENSTUDIO_VERSION_SHA: e165090621 + OPENSTUDIO_VERSION_EXT: "" + DOCKER_COMPOSE_VERSION: 1.21.1 + BUNDLE_WITHOUT: native_ext + + +jobs: + linux-test: + runs-on: ubuntu-18.04 + steps: + - name: Check out repository + uses: actions/checkout@v2 + - name: setup + shell: bash + run: ./ci/github-actions/setup.sh + - name: unit-test + shell: bash + run: ./ci/github-actions/test.sh + env: + BUILD_TYPE: test + - name: integration + shell: bash + run: ./ci/github-actions/test.sh + env: + BUILD_TYPE: integration + - name: logs + shell: bash + run: ./ci/github-actions/print_logs.sh + - name: build gem package + if: ${{ success() }} + shell: bash + run: ./ci/github-actions/export_build_linux.sh + - name: upload gem package + if: ${{ success() }} + uses: actions/upload-artifact@v2 + with: + name: openstudio-server-gems-linux + path: build/NREL/export/*.tar.gz + macos-test: + runs-on: macos-10.15 + steps: + - name: Check out repository + uses: actions/checkout@v2 + - name: setup + shell: bash + run: ./ci/github-actions/setup.sh + - name: unit-test + shell: bash + run: ./ci/github-actions/test.sh + env: + BUILD_TYPE: test + - name: integration + shell: bash + run: ./ci/github-actions/test.sh + env: + BUILD_TYPE: integration + - name: logs + shell: bash + run: ./ci/github-actions/print_logs.sh + - name: build gem package + if: ${{ success() }} + shell: bash + run: ./ci/github-actions/export_build_osx.sh + - name: upload gem package + if: ${{ success() }} + uses: actions/upload-artifact@v2 + with: + name: openstudio-server-gems-darwin + path: build/NREL/export/*.tar.gz + docker: + runs-on: ubuntu-18.04 + steps: + - name: Check out repository + uses: actions/checkout@v2 + - name: setup + shell: bash + run: ./ci/github-actions/setup.sh + env: + BUILD_TYPE: docker + - name: docker + shell: bash + run: | + export OPENSTUDIO_TAG=develop + sed -i -E "s/.git//g" .dockerignore + docker volume create --name=osdata + docker images --all + docker --version + docker-compose --version + docker-compose -f docker-compose.test.yml pull + docker-compose -f docker-compose.test.yml build --build-arg OPENSTUDIO_VERSION=$OPENSTUDIO_TAG + docker-compose -f docker-compose.test.yml up -d + docker-compose exec -T web /usr/local/bin/run-server-tests + docker-compose stop + git checkout -- .dockerignore && git checkout -- Dockerfile + env: + CI: true + OS_SERVER_NUMBER_OF_WORKERS: 4 + BUILD_TYPE: docker + COMPOSE_INTERACTIVE_NO_CLI: 1 + - name: logs + shell: bash + run: ./ci/github-actions/print_logs.sh + - name: docker-upload + if: | + github.ref == 'refs/heads/master' || + github.ref == 'refs/heads/develop' || + github.ref == 'refs/heads/setup_github_actions' + shell: bash + run: ./docker/deployment/scripts/deploy_docker_github_actions.sh + env: + CI: true + BUILD_TYPE: docker + # On forked Pull Requests secrets will not be sent to the runner. See more info here: + # https://docs.github.com/en/actions/reference/encrypted-secrets + DOCKER_PASS: ${{ secrets.DOCKER_PASS }} + DOCKER_USER: ${{ secrets.DOCKER_USER }} diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index eba8ddf06..000000000 --- a/.travis.yml +++ /dev/null @@ -1,125 +0,0 @@ -language: minimal - -env: - global: - - USE_TESTING_TIMEOUTS: "true" - - OPENSTUDIO_VERSION=3.1.0 OPENSTUDIO_VERSION_SHA=e165090621 OPENSTUDIO_VERSION_EXT="" - - DOCKER_COMPOSE_VERSION=1.21.1 - - BUNDLE_WITHOUT=native_ext - -# Set the default scripts that are used for the majority of the tests in the matrix -# redis-server now wants to bind to ipv6 ::1. This changes this behavior. -before_install: - - ./ci/travis/setup.sh -before_script: - - sudo Xvfb :99 -ac -screen 0 1024x768x8 & - - sleep 3 -script: ./ci/travis/test.sh -after_failure: ./ci/travis/print_logs.sh - -# Services for linux -- all instances -services: - - docker - -jobs: - include: - - stage: test - os: linux - sudo: required - dist: bionic - env: BUILD_TYPE=test - - stage: test - os: linux - sudo: required - dist: bionic - env: BUILD_TYPE=integration - - stage: test - os: osx - osx_image: xcode10.2 - env: BUILD_TYPE=test - - stage: test - os: osx - osx_image: xcode11.4 - env: BUILD_TYPE=test - - stage: test - os: osx - osx_image: xcode10.2 - env: BUILD_TYPE=integration - - stage: test - os: osx - osx_image: xcode11.4 - env: BUILD_TYPE=integration - - stage: test - os: linux - sudo: required - dist: bionic - env: - - CI=true - - OS_SERVER_NUMBER_OF_WORKERS=4 - - BUILD_TYPE=docker - before_script: - # remove the .git in the .dockerignore so coverage can be reported correctly to coveralls - #- export OPENSTUDIO_TAG=$(ruby -e "load 'server/app/lib/openstudio_server/version.rb'; print OpenstudioServer::VERSION+OpenstudioServer::VERSION_EXT") - - export OPENSTUDIO_TAG=develop - - sed -i -E "s/.git//g" .dockerignore - - docker volume create --name=osdata - - docker images --all - - docker --version - - docker-compose --version - - docker-compose -f docker-compose.test.yml pull - - docker-compose -f docker-compose.test.yml build --build-arg OPENSTUDIO_VERSION=$OPENSTUDIO_TAG - - docker-compose -f docker-compose.test.yml up -d - script: - - docker-compose exec web /usr/local/bin/run-server-tests - - docker-compose stop - after_script: - # Reset the git checkout to remove the test-based data in the containers and rebuild - - git checkout -- .dockerignore && git checkout -- Dockerfile - - - stage: publish - # Include your branch name below to publish a custom image. - # This change must be made in combination with a modification to - # deployment/scripts/deploy_docker.sh, as indicated in comments for that file. - # Full documentation at https://github.com/NREL/OpenStudio-server/wiki/Contributor-Docs:-Building-and-Publishing-Docker-images - if: branch = master OR branch = develop - os: linux - sudo: required - dist: bionic - env: - - BUILD_TYPE=docker - before_script: - #- export OPENSTUDIO_TAG=$(ruby -e "load 'server/app/lib/openstudio_server/version.rb'; print OpenstudioServer::VERSION+OpenstudioServer::VERSION_EXT") - - export OPENSTUDIO_TAG=develop - - docker images --all - - docker --version - - docker-compose --version - - docker volume create --name=osdata - - docker-compose build --build-arg OPENSTUDIO_VERSION=$OPENSTUDIO_TAG web - - docker-compose build rserve - script: ./docker/deployment/scripts/deploy_docker.sh - - stage: publish - if: branch = master OR branch = develop - os: osx - osx_image: xcode10.2 - sudo: required - env: - - BUILD_TYPE=export - before_script: - script: ./ci/travis/export_build_osx.sh - - stage: publish - if: branch = master OR branch = develop - os: linux - dist: bionic - sudo: required - env: - - BUILD_TYPE=export - before_script: - script: ./ci/travis/export_build_linux.sh - -notifications: - slack: - on_pull_requests: true - on_success: change - on_failure: change - rooms: - secure: G5hoK8S1gFT/rZ3L1+SNHITDQSZaLIhEPb4/12TXoUuNK15SFWjr99updPBO8f04dS7GGS7MlzWvoXiLFujDWyWdb2v6wWw6A810ZU2u3KwFfevb1ewR+oxUdWfk0DI/PY2opdMyhC9YoHfSV4wo/IG659kXKKLOVBBqKKdhjMQJNHWIRiT7LgpYqlB1TRHfcxIlL2e/cIAIzVsj3o2vwgd7uFPdwVI9tzHSJlZyptOxWvQ0IVmNksSLNxaJLwHRs7cdrV5dcH+jLJaz2Kwdcu3b/0baaFGfQMKhQrTefLlESN7SjR5vzrh7Us0Y6fuOEa652bVy/gTkWGcw9F5x200vYuQSov3kPO8wxGSe9haMMkZ231jZW9Zi3VyhzDAe0eNnwxo42drOumrjK3ukvAXB4OTptxUlukvHT7DY5VcatTefI0zDt9aYA3/JLjqrxfcCweWs2IAiaapJRTSnpk+hKgDFxOh2FxohTOg/kH8f9omjg3RyJsy/wCggx12baUuyfKcOAseGtZQaXcKJ7An8epHUx8SV5ZmRUWSthDmiEcT7UmFfVqk+O0VM5gGvrgsiUvoRqs5/7fFEXAUP+djVAnd7Do4HTwG8ELgqmN0WnpNrujSenE9hQ1cou897/JTGHcqL+rwOcLEteo6VqJMVeRfsSB96hpU9faX5bE0= diff --git a/bin/resources/local/rspec_command b/bin/resources/local/rspec_command index 43aff0cfe..2f5290861 100644 --- a/bin/resources/local/rspec_command +++ b/bin/resources/local/rspec_command @@ -86,7 +86,7 @@ rspec_path = File.absolute_path(File.join(__FILE__, './../../../../gems/bin/rspe ::ENV['OS_SERVER_LOG_PATH'] = File.join(output_dir, 'logs', 'rails.log') ::ENV['RAILS_ENV'] = 'local-test' -#If this is a travis build and it's BUILD_TYPE=test we want to omit the openstudio_algo_spec.rb tests +#If this is a CI build (travis, github actions, etc..) and the BUILD_TYPE=test omit the openstudio_algo_spec.rb tests if ENV['BUILD_TYPE'] == 'test' sys_call = " \"#{ruby_exe}\" \"#{bundle_path}\" exec \"#{ruby_exe}\" \"#{rspec_path}\" --tag ~depends_r --tag ~depends_gecko --tag ~depends_resque --exclude-pattern "\ "\"**/models/cluster_spec.rb\" --exclude-pattern \"**/features/openstudio_algo_spec.rb\"" diff --git a/ci/travis/cipher.rb b/ci/github-actions/cipher.rb similarity index 100% rename from ci/travis/cipher.rb rename to ci/github-actions/cipher.rb diff --git a/ci/github-actions/export_build_linux.sh b/ci/github-actions/export_build_linux.sh new file mode 100755 index 000000000..9d31402f5 --- /dev/null +++ b/ci/github-actions/export_build_linux.sh @@ -0,0 +1,20 @@ +# empty dir for export +mkdir -p $GITHUB_WORKSPACE/build/NREL/export +mkdir -p $HOME/build/NREL/export + +export PATH="/usr/loca/bin/ruby:/usr/local/openstudio-${OPENSTUDIO_VERSION}${OPENSTUDIO_VERSION_EXT}/bin:${PATH}" +export GEM_HOME="$GITHUB_WORKSPACE/gems" +export GEM_PATH="$GITHUB_WORKSPACE/gems:$GITHUB_WORKSPACE/gems/bundler/gems" +# Dir containing openstudio +export RUBYLIB="/usr/local/openstudio-${OPENSTUDIO_VERSION}${OPENSTUDIO_VERSION_EXT}/Ruby" +export OPENSTUDIO_TEST_EXE="/usr/local/openstudio-${OPENSTUDIO_VERSION}${OPENSTUDIO_VERSION_EXT}/bin/openstudio" +mongo_dir="/usr/bin" +/usr/local/ruby/bin/ruby "${GITHUB_WORKSPACE}/bin/openstudio_meta" install_gems --export="${HOME}/build/NREL/export" +oss_filename="OpenStudio-server-$(git -C "${GITHUB_WORKSPACE}" rev-parse --short=10 HEAD)-linux.tar.gz" +ls -al $HOME/build/NREL/export/ +# Build the gems and output to tar gz file. You need to specify a dir outside of the repo to export or +# openstudio_meta will error. Then for sake of using relative dirs with GitHub upload-artifacts, cp the file to +# the repo working directory. +cp $HOME/build/NREL/export/$oss_filename $GITHUB_WORKSPACE/build/NREL/export/$oss_filename + + diff --git a/ci/github-actions/export_build_osx.sh b/ci/github-actions/export_build_osx.sh new file mode 100755 index 000000000..e58b54ca9 --- /dev/null +++ b/ci/github-actions/export_build_osx.sh @@ -0,0 +1,17 @@ +#empty dir for export +mkdir -p $GITHUB_WORKSPACE/build/NREL/export +mkdir -p $HOME/build/NREL/export + + +export OS_NAME_WITH_PLUS=OpenStudio-${OPENSTUDIO_VERSION}${OPENSTUDIO_VERSION_EXT}+${OPENSTUDIO_VERSION_SHA}-Darwin +export PATH="$GITHUB_WORKSPACE/gems/bin:/usr/local/ruby/bin:$HOME/$OS_NAME_WITH_PLUS/bin:$PATH" +export RUBYLIB="$HOME/$OS_NAME_WITH_PLUS/Ruby" +export GEM_HOME="$GITHUB_WORKSPACE/gems" +export GEM_PATH="$GITHUB_WORKSPACE/gems:$GITHUB_WORKSPACE/gems/bundler/gems" +oss_filename="OpenStudio-server-$(git -C "${GITHUB_WORKSPACE}" rev-parse --short=10 HEAD)-darwin.tar.gz" + +# Build the gems and output to tar gz file. You need to specify a dir outside of the repo to export or +# openstudio_meta will error. Then for sake of using relative dirs with GitHub upload-artifacts, cp the file to +# the repo working directory. +/usr/local/ruby/bin/ruby "${GITHUB_WORKSPACE}/bin/openstudio_meta" install_gems --export="${HOME}/build/NREL/export" +cp $HOME/build/NREL/export/$oss_filename $GITHUB_WORKSPACE/build/NREL/export/$oss_filename \ No newline at end of file diff --git a/ci/travis/install-mac.qs b/ci/github-actions/install-mac.qs similarity index 100% rename from ci/travis/install-mac.qs rename to ci/github-actions/install-mac.qs diff --git a/ci/travis/install_openstudio.sh b/ci/github-actions/install_openstudio.sh similarity index 100% rename from ci/travis/install_openstudio.sh rename to ci/github-actions/install_openstudio.sh diff --git a/ci/travis/print_logs.sh b/ci/github-actions/print_logs.sh similarity index 57% rename from ci/travis/print_logs.sh rename to ci/github-actions/print_logs.sh index 030f82974..cfc96a670 100755 --- a/ci/travis/print_logs.sh +++ b/ci/github-actions/print_logs.sh @@ -1,42 +1,42 @@ #!/usr/bin/env bash echo "Current directory is $(pwd)" -# echo "tree: ${TRAVIS_BUILD_DIR}/spec" -# tree "${TRAVIS_BUILD_DIR}/spec" +# echo "tree: ${GITHUB_WORKSPACE}/spec" +# tree "${GITHUB_WORKSPACE}/spec" echo "=== PRINTING ERROR LOG REPORTS ===" shopt -s nullglob echo "=== PRINTING spec/files/logs/* ===" -for F in "${TRAVIS_BUILD_DIR}/spec/files/logs/"* +for F in "${GITHUB_WORKSPACE}/spec/files/logs/"* do echo '======================================================' echo $F echo '======================================================' - # Limit the rate of printing the log (with pv) to keep travis happy. https://github.com/travis-ci/travis-ci/issues/6018 + # Limit the rate of printing the log (with pv) cat $F | pv -q -L 3k echo done echo "=== PRINTING /spec/unit-test/logs/* ===" -for F in "${TRAVIS_BUILD_DIR}/spec/unit-test/logs/"* +for F in "${GITHUB_WORKSPACE}/spec/unit-test/logs/"* do echo '======================================================' echo $F echo '======================================================' - # Limit the rate of printing the log (with pv) to keep travis happy. https://github.com/travis-ci/travis-ci/issues/6018 + # Limit the rate of printing the log (with pv) cat $F | pv -q -L 3k echo done echo "=== PRINTING /spec/unit-test/logs/rails.log/* ===" -for F in "${TRAVIS_BUILD_DIR}/spec/unit-test/logs/rails.log/"* +for F in "${GITHUB_WORKSPACE}/spec/unit-test/logs/rails.log/"* do echo '======================================================' echo $F echo '======================================================' - # Limit the rate of printing the log (with pv) to keep travis happy. https://github.com/travis-ci/travis-ci/issues/6018 + # Limit the rate of printing the log (with pv) cat $F | pv -q -L 3k echo done \ No newline at end of file diff --git a/ci/travis/setup.sh b/ci/github-actions/setup.sh similarity index 61% rename from ci/travis/setup.sh rename to ci/github-actions/setup.sh index 2afe88396..492a0f777 100755 --- a/ci/travis/setup.sh +++ b/ci/github-actions/setup.sh @@ -1,8 +1,8 @@ #!/bin/bash -x -echo "The build architecture is ${TRAVIS_OS_NAME}" +echo "The build architecture is ${ImageOS}" -if [ "${BUILD_TYPE}" == "docker" ]; then +if [ "${ImageOS}" == "ubuntu18" ] && [ "${BUILD_TYPE}" == "docker" ]; then echo "Installing docker compose" sudo rm /usr/local/bin/docker-compose curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose @@ -10,19 +10,19 @@ if [ "${BUILD_TYPE}" == "docker" ]; then sudo mv docker-compose /usr/local/bin # install pipeviewer sudo apt-get update - sudo apt-get install -y pv + sudo apt-get install -y pv ruby else - sudo rvm implode --force # rvm PATH rewriting interferes with portable Ruby. - if [ "${TRAVIS_OS_NAME}" == "osx" ]; then + # sudo rvm implode --force # rvm PATH rewriting interferes with portable Ruby. + if [ "${ImageOS}" == "macos1015" ]; then - brew update > /Users/travis/build/NREL/OpenStudio-server/spec/files/logs/brew-update.log - brew install pv tree + brew update > $GITHUB_WORKSPACE/spec/files/logs/brew-update.log + brew install pv tree coreutils # install portable ruby - required for build that will eventually be published # see https://github.com/NREL/OpenStudio-PAT/wiki/Pat-Build-Notes curl -SLO --insecure https://openstudio-resources.s3.amazonaws.com/pat-dependencies3/ruby-2.5.5-darwin.tar.gz - tar xvzf ruby-2.5.5-darwin.tar.gz + tar xzf ruby-2.5.5-darwin.tar.gz sudo mv ruby /usr/local/ otool -L /usr/local/ruby/bin/ruby rm ruby-2.5.5-darwin.tar.gz @@ -30,47 +30,48 @@ else # Install mongodb from a download. Brew is hanging and requires building mongo. This also speeds up the builds. curl -SLO https://fastdl.mongodb.org/osx/mongodb-macos-x86_64-4.4.2.tgz tar xvzf mongodb-macos-x86_64-4.4.2.tgz - cp mongodb-macos-x86_64-4.4.2/bin/* /usr/local/bin/ - rm -r mongodb-osx* + sudo cp mongodb-macos-x86_64-4.4.2/bin/* /usr/local/bin/ + rm -r mongodb-macos* # Install openstudio -- Use the install script that is in this repo now, the one on OpenStudio/develop has changed export OS_NAME=OpenStudio-${OPENSTUDIO_VERSION}${OPENSTUDIO_VERSION_EXT}%2B${OPENSTUDIO_VERSION_SHA}-Darwin export OS_NAME_WITH_PLUS=OpenStudio-${OPENSTUDIO_VERSION}${OPENSTUDIO_VERSION_EXT}+${OPENSTUDIO_VERSION_SHA}-Darwin - curl -SLO --insecure https://openstudio-builds.s3.amazonaws.com/${OPENSTUDIO_VERSION}/${OS_NAME}.dmg + curl -SL --insecure https://openstudio-builds.s3.amazonaws.com/${OPENSTUDIO_VERSION}/${OS_NAME}.tar.gz -o $OS_NAME_WITH_PLUS.tar.gz # OSX downloads with %2B but installs with + sign. These are the encoded chars in url strings. - hdiutil attach ${OS_NAME}.dmg - sed -i -e "s|REPLACEME|$HOME/openstudio|" ci/travis/install-mac.qs - rm -rf $HOME/openstudio + #hdiutil attach ${OS_NAME}.dmg + #sed -i -e "s|REPLACEME|$HOME/openstudio|" ci/github-actions/install-mac.qs # Will install into $HOME/openstudio and RUBYLIB will be $HOME/openstudio/Ruby - sudo /Volumes/${OS_NAME_WITH_PLUS}/${OS_NAME_WITH_PLUS}.app/Contents/MacOS/${OS_NAME_WITH_PLUS} --script ci/travis/install-mac.qs - hdiutil detach /Volumes/${OS_NAME_WITH_PLUS} -force - rm ${OS_NAME}.dmg - export PATH="/usr/local/ruby/bin:$TRAVIS_BUILD_DIR/gems/bin:$HOME/openstudio/bin:$PATH" - export RUBYLIB="$HOME/openstudio/Ruby" - export GEM_HOME="$TRAVIS_BUILD_DIR/gems" - export GEM_PATH="$TRAVIS_BUILD_DIR/gems:$TRAVIS_BUILD_DIR/gems/bundler/gems" + #sudo /Volumes/${OS_NAME_WITH_PLUS}/${OS_NAME_WITH_PLUS}.app/Contents/MacOS/${OS_NAME_WITH_PLUS} --script ci/github-actions/install-mac.qs + #hdiutil detach /Volumes/${OS_NAME_WITH_PLUS} -force + ls -l + tar xvzf $OS_NAME_WITH_PLUS.tar.gz -C $HOME + ls -l $HOME + rm -rf $OS_NAME_WITH_PLUS.tar.gz + export PATH="/usr/local/ruby/bin:$GITHUB_WORKSPACE/gems/bin:$HOME/$OS_NAME_WITH_PLUS/bin:$PATH" + export RUBYLIB="$HOME/$OS_NAME_WITH_PLUS/Ruby" + ls -l $RUBYLIB + export GEM_HOME="$GITHUB_WORKSPACE/gems" + export GEM_PATH="$GITHUB_WORKSPACE/gems:$GITHUB_WORKSPACE/gems/bundler/gems" # set the ulimit to be higher ulimit -a ulimit -n 4096 ulimit -a - elif [ "${TRAVIS_OS_NAME}" == "linux" ]; then + elif [ "${ImageOS}" == "ubuntu18" ]; then echo "Setting up Ubuntu for unit tests and Rubocop" # install pipe viewer to throttle printing logs to screen (not a big deal in linux, but it is in osx) sudo apt-get update && sudo apt-get install -y wget gnupg software-properties-common build-essential sudo wget -qO - https://www.mongodb.org/static/pgp/server-4.4.asc | sudo apt-key add - echo "deb http://repo.mongodb.org/apt/ubuntu xenial/mongodb-org/4.4 multiverse | tee /etc/apt/sources.list.d/mongodb-org-4.4.list" sudo apt-get update - sudo apt-get install -y pv tree mongodb libqdbm14 - # per travis docs, mongodb and redis should already be installed and started from services key in bionic, but this isn't working. explicitly install. - # the latest version of redis-server now binds to ipv6 which is not supported on travis (disabled). redis-server will fail to start due to the this so below - # is a work around to install it, configure to it only binds to ipv4. + sudo apt-get install -y pv tree mongodb libqdbm14 libxml2-dev + # explicitly install. the latest version of redis-server wget https://download.redis.io/releases/redis-6.0.9.tar.gz tar xzf redis-6.0.9.tar.gz && cd redis-6.0.9 make && sudo make install sudo cp utils/systemd-redis_server.service /etc/systemd/system/redis.service - cd $TRAVIS_BUILD_DIR + cd $GITHUB_WORKSPACE rm redis-6.0.9.tar.gz #sudo apt-get install redis-server || true #sudo systemctl stop redis-server.service @@ -84,33 +85,34 @@ else # see https://github.com/NREL/OpenStudio-PAT/wiki/Pat-Build-Notes curl -SLO --insecure https://openstudio-resources.s3.amazonaws.com/pat-dependencies3/ruby-2.5.5-linux.tar.gz tar xvzf ruby-2.5.5-linux.tar.gz + ls -l /usr/local/ + sudo rm -rf /usr/local/ruby sudo mv ruby /usr/local/ ldd /usr/local/ruby/bin/ruby rm ruby-2.5.5-linux.tar.gz mkdir -p reports/rspec - # AP: this appears to only be used for Travis/Linux so we should move it out of the docker/deployment/scripts dir - sudo ./ci/travis/install_openstudio.sh $OPENSTUDIO_VERSION $OPENSTUDIO_VERSION_SHA $OPENSTUDIO_VERSION_EXT + sudo ./ci/github-actions/install_openstudio.sh $OPENSTUDIO_VERSION $OPENSTUDIO_VERSION_SHA $OPENSTUDIO_VERSION_EXT export RUBYLIB=/usr/local/openstudio-${OPENSTUDIO_VERSION}${OPENSTUDIO_VERSION_EXT}/Ruby export ENERGYPLUS_EXE_PATH=/usr/local/openstudio-${OPENSTUDIO_VERSION}${OPENSTUDIO_VERSION_EXT}/EnergyPlus/energyplus export PATH=/usr/local/ruby/bin:/usr/local/bin:/usr/local/openstudio-${OPENSTUDIO_VERSION}${OPENSTUDIO_VERSION_EXT}/bin:${PATH} - export GEM_HOME="$TRAVIS_BUILD_DIR/gems" - export GEM_PATH="$TRAVIS_BUILD_DIR/gems:$TRAVIS_BUILD_DIR/gems/bundler/gems" + export GEM_HOME="$GITHUB_WORKSPACE/gems" + export GEM_PATH="$GITHUB_WORKSPACE/gems:$GITHUB_WORKSPACE/gems/bundler/gems" fi echo "verifying os installation" unset BUNDLE_GEMFILE && openstudio openstudio_version - cd ${TRAVIS_BUILD_DIR}/server - printenv + cd ${GITHUB_WORKSPACE}/server which ruby ruby -v # test openssl - # ruby ${TRAVIS_BUILD_DIR}/ci/travis/cipher.rb - ruby ${TRAVIS_BUILD_DIR}/ci/travis/verify_openstudio.rb + ruby ${GITHUB_WORKSPACE}/ci/github-actions/verify_openstudio.rb - ruby "${TRAVIS_BUILD_DIR}/bin/openstudio_meta" install_gems --with_test_develop --debug --verbose --use_cached_gems + ruby "${GITHUB_WORKSPACE}/bin/openstudio_meta" install_gems --with_test_develop --debug --verbose --use_cached_gems bundle -v # create dir for output files which will be generated in case of failure - mkdir "${TRAVIS_BUILD_DIR}/spec/unit-test" + if [ ! -d "${GITHUB_WORKSPACE}/spec/unit-test" ]; then + mkdir "${GITHUB_WORKSPACE}/spec/unit-test" + fi fi diff --git a/ci/travis/test.sh b/ci/github-actions/test.sh similarity index 73% rename from ci/travis/test.sh rename to ci/github-actions/test.sh index 175bc9895..454bd38a0 100755 --- a/ci/travis/test.sh +++ b/ci/github-actions/test.sh @@ -1,28 +1,29 @@ #!/usr/bin/env bash # platform-specific config here (also in setup.sh): -if [ "${TRAVIS_OS_NAME}" == "osx" ]; then +if [ "${ImageOS}" == "macos1015" ]; then # Dir containing openstudio - export RUBYLIB="${HOME}/openstudio/Ruby" - export OPENSTUDIO_TEST_EXE="${HOME}/openstudio/bin/openstudio" + export OS_NAME_WITH_PLUS=OpenStudio-${OPENSTUDIO_VERSION}${OPENSTUDIO_VERSION_EXT}+${OPENSTUDIO_VERSION_SHA}-Darwin + export RUBYLIB="$HOME/$OS_NAME_WITH_PLUS/Ruby" + export OPENSTUDIO_TEST_EXE="$HOME/$OS_NAME_WITH_PLUS/bin/openstudio" # re-export PATH, even though it's set in setup.sh. - export PATH="$TRAVIS_BUILD_DIR/gems/bin:/usr/local/ruby/bin:$HOME/openstudio/bin:$PATH" - export GEM_HOME="$TRAVIS_BUILD_DIR/gems" - export GEM_PATH="$TRAVIS_BUILD_DIR/gems:$TRAVIS_BUILD_DIR/gems/bundler/gems" + export PATH="$GITHUB_WORKSPACE/gems/bin:/usr/local/ruby/bin:$HOME/$OS_NAME_WITH_PLUS/bin:$PATH" + export GEM_HOME="$GITHUB_WORKSPACE/gems" + export GEM_PATH="$GITHUB_WORKSPACE/gems:$GITHUB_WORKSPACE/gems/bundler/gems" mongo_dir="/usr/local/bin" -elif [ "${TRAVIS_OS_NAME}" == "linux" ]; then +elif [ "${ImageOS}" == "ubuntu18" ]; then # Dir containing openstudio export ENERGYPLUS_EXE_PATH=/usr/local/openstudio-${OPENSTUDIO_VERSION}${OPENSTUDIO_VERSION_EXT}/EnergyPlus/energyplus export PATH=/usr/local/ruby/bin:/usr/bin:/usr/local/openstudio-${OPENSTUDIO_VERSION}${OPENSTUDIO_VERSION_EXT}/bin:${PATH} - export GEM_HOME="$TRAVIS_BUILD_DIR/gems" - export GEM_PATH="$TRAVIS_BUILD_DIR/gems:$TRAVIS_BUILD_DIR/gems/bundler/gems" + export GEM_HOME="$GITHUB_WORKSPACE/gems" + export GEM_PATH="$GITHUB_WORKSPACE/gems:$GITHUB_WORKSPACE/gems/bundler/gems" export RUBYLIB="/usr/local/openstudio-${OPENSTUDIO_VERSION}${OPENSTUDIO_VERSION_EXT}/Ruby" export OPENSTUDIO_TEST_EXE="/usr/local/openstudio-${OPENSTUDIO_VERSION}${OPENSTUDIO_VERSION_EXT}/bin/openstudio" mongo_dir="/usr/bin" fi # Env variables set in setup.sh do not seem to be available in test.sh -if [ "${BUILD_TYPE}" == "docker" ]; then +if [ "${ImageOS}" == "docker" ]; then echo "Skipping tests for docker builds" else # Do not report coverage from these build, use the build from docker with no excluded tags @@ -36,7 +37,7 @@ else # Threadsafe test requires higher ulimit to avoid EMFILE error ulimit -n ulimit -n 1024 - ruby "${TRAVIS_BUILD_DIR}/bin/openstudio_meta" run_rspec --debug --verbose --mongo-dir="$mongo_dir" --openstudio-exe="$OPENSTUDIO_TEST_EXE" "${TRAVIS_BUILD_DIR}/spec/unit-test" + ruby "${GITHUB_WORKSPACE}/bin/openstudio_meta" run_rspec --debug --verbose --mongo-dir="$mongo_dir" --openstudio-exe="$OPENSTUDIO_TEST_EXE" "${GITHUB_WORKSPACE}/spec/unit-test" exit_status=$? if [ $exit_status == 0 ];then echo "Completed unit tests successfully" @@ -51,8 +52,7 @@ else export RAILS_ENV=local # explicitly set directory. Probably unnecessary - cd $TRAVIS_BUILD_DIR - printenv + cd $GITHUB_WORKSPACE bundle install echo "Beginning integration tests. RUBYLIB=$RUBYLIB ; OPENSTUDIO_TEST_EXE=$OPENSTUDIO_TEST_EXE" bundle exec rspec; (( exit_status = exit_status || $? )) diff --git a/ci/travis/verify_openstudio.rb b/ci/github-actions/verify_openstudio.rb similarity index 100% rename from ci/travis/verify_openstudio.rb rename to ci/github-actions/verify_openstudio.rb diff --git a/ci/travis/export_build_linux.sh b/ci/travis/export_build_linux.sh deleted file mode 100755 index 30a7e3e9b..000000000 --- a/ci/travis/export_build_linux.sh +++ /dev/null @@ -1,25 +0,0 @@ -# empty dir for export -mkdir /home/travis/build/NREL/export - -export PATH="/usr/loca/bin/ruby:/usr/local/openstudio-${OPENSTUDIO_VERSION}${OPENSTUDIO_VERSION_EXT}/bin:${PATH}" -export GEM_HOME="$TRAVIS_BUILD_DIR/gems" -export GEM_PATH="$TRAVIS_BUILD_DIR/gems:$TRAVIS_BUILD_DIR/gems/bundler/gems" -# Dir containing openstudio -export RUBYLIB="/usr/local/openstudio-${OPENSTUDIO_VERSION}${OPENSTUDIO_VERSION_EXT}/Ruby" -export OPENSTUDIO_TEST_EXE="/usr/local/openstudio-${OPENSTUDIO_VERSION}${OPENSTUDIO_VERSION_EXT}/bin/openstudio" -mongo_dir="/usr/bin" -/usr/local/ruby/bin/ruby "${TRAVIS_BUILD_DIR}/bin/openstudio_meta" install_gems --export="/home/travis/build/NREL/export" -oss_filename="OpenStudio-server-$(git -C "${TRAVIS_BUILD_DIR}" rev-parse --short=10 HEAD)-linux.tar.gz" -ls -al /home/travis/build/NREL/export/ -#mv build package to root of travis build for artifact upload. -mv /home/travis/build/NREL/export/$oss_filename . -ls -al - -export ARTIFACTS_PATHS=$oss_filename -# set to 2GB instead of default of 1 -export ARTIFACTS_MAX_SIZE=2147483648 -# supposedly installed by default but not available -curl -sL https://raw.githubusercontent.com/travis-ci/artifacts/master/install | bash -artifacts upload -exit $? - diff --git a/ci/travis/export_build_osx.sh b/ci/travis/export_build_osx.sh deleted file mode 100755 index 266fe0545..000000000 --- a/ci/travis/export_build_osx.sh +++ /dev/null @@ -1,22 +0,0 @@ -#empty dir for export -mkdir /Users/travis/build/NREL/export - -export PATH="/usr/local/ruby/bin:$PATH" -export RUBYLIB="$HOME/openstudio/Ruby" -export GEM_HOME="$TRAVIS_BUILD_DIR/gems" -export GEM_PATH="$TRAVIS_BUILD_DIR/gems:$TRAVIS_BUILD_DIR/gems/bundler/gems" -/usr/local/ruby/bin/ruby "${TRAVIS_BUILD_DIR}/bin/openstudio_meta" install_gems --export="/Users/travis/build/NREL/export" -oss_filename="OpenStudio-server-$(git -C "${TRAVIS_BUILD_DIR}" rev-parse --short=10 HEAD)-darwin.tar.gz" - - -#mv build package to root of travis build for artifact upload. -mv /Users/travis/build/NREL/export/$oss_filename . - -export ARTIFACTS_PATHS=$oss_filename -# set to 2GB instead of default of 1 -export ARTIFACTS_MAX_SIZE=2147483648 -# supposedly installed by default but not available -curl -sL https://raw.githubusercontent.com/travis-ci/artifacts/master/install | bash -# TODO don't recreate entire path to file within the AWS bucket. -artifacts upload -exit $? diff --git a/docker/deployment/scripts/deploy_docker_github_actions.sh b/docker/deployment/scripts/deploy_docker_github_actions.sh new file mode 100755 index 000000000..da5978774 --- /dev/null +++ b/docker/deployment/scripts/deploy_docker_github_actions.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +# Default set IMAGETAG to skip. +IMAGETAG="skip" + +if [ "${GITHUB_REF}" == "refs/heads/develop" ]; then + IMAGETAG="develop" +elif [ "${GITHUB_REF}" == "refs/heads/2.9.X-LTS" ]; then + IMAGETAG="2.9.X-LTS" +elif [ "${GITHUB_REF}" == "refs/heads/master" ]; then + # Retrieve the version number from rails + IMAGETAG="$(ruby -e "load 'server/app/lib/openstudio_server/version.rb'; print OpenstudioServer::VERSION+OpenstudioServer::VERSION_EXT")" +# Uncomment and set branch name for custom builds. +# Currently setting this to setup_github_actions to test upload. +elif [ "${GITHUB_REF}" == "refs/heads/setup_github_actions" ]; then + IMAGETAG=experimental +fi + +if [ "${IMAGETAG}" != "skip" ]; then + echo "$DOCKER_PASS" | docker login -u "$DOCKER_USER" --password-stdin + + echo "Tagging image as $IMAGETAG" + docker tag nrel/openstudio-server nrel/openstudio-server:$IMAGETAG; (( exit_status = exit_status || $? )) + docker tag nrel/openstudio-rserve nrel/openstudio-rserve:$IMAGETAG; (( exit_status = exit_status || $? )) + docker push nrel/openstudio-server:$IMAGETAG; (( exit_status = exit_status || $? )) + docker push nrel/openstudio-rserve:$IMAGETAG; (( exit_status = exit_status || $? )) + + if [ "${GITHUB_REF}" == "refs/heads/master" ]; then + # Deploy master as the latest. + docker tag nrel/openstudio-server nrel/openstudio-server:latest; (( exit_status = exit_status || $? )) + docker tag nrel/openstudio-rserve nrel/openstudio-rserve:latest; (( exit_status = exit_status || $? )) + + docker push nrel/openstudio-server:latest; (( exit_status = exit_status || $? )) + docker push nrel/openstudio-rserve:latest; (( exit_status = exit_status || $? )) + fi + + exit $exit_status +else + echo "Not on a deployable branch [master/nrcan-master/develop] or this is a pull request" +fi