diff --git a/.github/workflows/pull_requests.yml b/.github/workflows/pull_requests.yml index bcb6ff8fc..efc0cf454 100644 --- a/.github/workflows/pull_requests.yml +++ b/.github/workflows/pull_requests.yml @@ -5,52 +5,137 @@ on: branches: [main] jobs: - test: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@master - with: - ref: ${{ github.ref }} - - # FROM: https://github.com/actions/cache/blob/main/examples.md#node---yarn - - name: Get yarn cache directory path - id: yarn-cache-dir-path - run: echo "::set-output name=dir::$(yarn cache dir)" - - - uses: actions/cache@v2 - id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`) - with: - path: ${{ steps.yarn-cache-dir-path.outputs.dir }} - key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - ${{ runner.os }}-yarn- - - - run: ./scripts/ci/prepare_linux.sh --integration-tests - - run: yarn format - - run: yarn eslint ui - - run: yarn tsc - - run: cd runner && gofmt -w . - - run: yarn build-language-definitions - - run: ./scripts/fail_on_diff.sh - - run: yarn test-licenses - # Needed so we can have ./build/desktop_runner.js and ./build/go_desktop_runner ready for tests - - run: yarn build-test-runner - - run: yarn test --detectOpenHandles --forceExit --verbose - env: - AIRTABLE_TOKEN: ${{ secrets.AIRTABLE_TOKEN }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - BIGQUERY_TOKEN: ${{ secrets.BIGQUERY_TOKEN }} - SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }} - SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} - SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} - - run: cd runner && go test -race -coverprofile ../coverage/gounit.cov - - run: ./runner/scripts/test_coverage.sh - # Test server builds - - run: yarn build-server - - e2e-linux: + # test: + # runs-on: ubuntu-latest + + # steps: + # - uses: actions/checkout@master + # with: + # ref: ${{ github.ref }} + + # # FROM: https://github.com/actions/cache/blob/main/examples.md#node---yarn + # - name: Get yarn cache directory path + # id: yarn-cache-dir-path + # run: echo "::set-output name=dir::$(yarn cache dir)" + + # - uses: actions/cache@v2 + # id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`) + # with: + # path: ${{ steps.yarn-cache-dir-path.outputs.dir }} + # key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} + # restore-keys: | + # ${{ runner.os }}-yarn- + # - run: ./scripts/ci/prepare_linux.sh --integration-tests + # - run: yarn format + # - run: yarn eslint ui + # - run: yarn tsc + # - run: cd runner && gofmt -w . + # - run: yarn build-language-definitions + # - run: ./scripts/fail_on_diff.sh + # - run: yarn test-licenses + # # Needed so we can have ./build/desktop_runner.js and ./build/go_desktop_runner ready for tests + # - run: yarn build-test-runner + # - run: yarn test --detectOpenHandles --forceExit --verbose + # env: + # AIRTABLE_TOKEN: ${{ secrets.AIRTABLE_TOKEN }} + # AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + # AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # BIGQUERY_TOKEN: ${{ secrets.BIGQUERY_TOKEN }} + # SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }} + # SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} + # SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} + # - run: cd runner && go test -race -coverprofile ../coverage/gounit.cov + # - run: ./runner/scripts/test_coverage.sh + + # e2e-linux: + # runs-on: ubuntu-latest + + # steps: + # - uses: actions/checkout@master + # with: + # ref: ${{ github.ref }} + + # # FROM: https://github.com/actions/cache/blob/main/examples.md#node---yarn + # - name: Get yarn cache directory path + # id: yarn-cache-dir-path + # run: echo "::set-output name=dir::$(yarn cache dir)" + + # - uses: actions/cache@v2 + # id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`) + # with: + # path: ${{ steps.yarn-cache-dir-path.outputs.dir }} + # key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} + # restore-keys: | + # ${{ runner.os }}-yarn- + # - run: ./scripts/ci/prepare_linux.sh + # - run: yarn release-desktop 0.0.0-e2etest + # - run: git reset --hard # release blows everything up + # # Set up a virtual framebuffer so Chrome will start + # # https://www.electronjs.org/docs/tutorial/testing-on-headless-ci + # # https://github.com/juliangruber/browser-run/issues/147 + # - run: xvfb-run --auto-servernum yarn e2e-test + + # e2e-macos: + # runs-on: macos-latest + + # steps: + # - uses: actions/checkout@master + # with: + # ref: ${{ github.ref }} + + # # FROM: https://github.com/actions/cache/blob/main/examples.md#node---yarn + # - name: Get yarn cache directory path + # id: yarn-cache-dir-path + # run: echo "::set-output name=dir::$(yarn cache dir)" + + # - uses: actions/cache@v2 + # id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`) + # with: + # path: ${{ steps.yarn-cache-dir-path.outputs.dir }} + # key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} + # restore-keys: | + # ${{ runner.os }}-yarn- + # - run: ./scripts/ci/prepare_macos.sh + # # Needed so we can have ./build/desktop_runner.js ready for tests + # - run: yarn build-test-runner + # # https://github.com/golang/go/issues/49138 bug in macos -race detector + # - run: MallocNanoZone=0 yarn test --detectOpenHandles --forceExit --verbose shared ui desktop + # - run: cd runner && MallocNanoZone=0 go test -race -cover + # - run: yarn release-desktop 0.0.0-e2etest + # - run: git reset --hard # release blows everything up + # - run: yarn e2e-test + + # e2e-windows: + # runs-on: windows-latest + + # steps: + # - uses: actions/checkout@master + # with: + # ref: ${{ github.ref }} + + # # FROM: https://github.com/actions/cache/blob/main/examples.md#node---yarn + # - name: Get yarn cache directory path + # id: yarn-cache-dir-path + # run: echo "::set-output name=dir::$(yarn cache dir)" + + # - uses: actions/cache@v2 + # id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`) + # with: + # path: ${{ steps.yarn-cache-dir-path.outputs.dir }} + # key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} + # restore-keys: | + # ${{ runner.os }}-yarn- + # - run: ./scripts/ci/prepare_windows.ps1 + # shell: pwsh + # # Needed so we can have ./build/desktop_runner.js ready for tests + # - run: yarn build-test-runner + # - run: yarn test --detectOpenHandles --forceExit --verbose shared ui desktop + # - run: cd runner && go test -race -cover + # - run: yarn release-desktop 0.0.0-e2etest + # - run: git reset --hard # release blows everything up + # - run: yarn e2e-test + + test-server: runs-on: ubuntu-latest steps: @@ -72,71 +157,6 @@ jobs: ${{ runner.os }}-yarn- - run: ./scripts/ci/prepare_linux.sh - - run: yarn release-desktop 0.0.0-e2etest - - run: git reset --hard # release blows everything up - # Set up a virtual framebuffer so Chrome will start - # https://www.electronjs.org/docs/tutorial/testing-on-headless-ci - # https://github.com/juliangruber/browser-run/issues/147 - - run: xvfb-run --auto-servernum yarn e2e-test - - e2e-macos: - runs-on: macos-latest - - steps: - - uses: actions/checkout@master - with: - ref: ${{ github.ref }} - - # FROM: https://github.com/actions/cache/blob/main/examples.md#node---yarn - - name: Get yarn cache directory path - id: yarn-cache-dir-path - run: echo "::set-output name=dir::$(yarn cache dir)" - - - uses: actions/cache@v2 - id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`) - with: - path: ${{ steps.yarn-cache-dir-path.outputs.dir }} - key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - ${{ runner.os }}-yarn- - - - run: ./scripts/ci/prepare_macos.sh - # Needed so we can have ./build/desktop_runner.js ready for tests - - run: yarn build-test-runner - # https://github.com/golang/go/issues/49138 bug in macos -race detector - - run: MallocNanoZone=0 yarn test --detectOpenHandles --forceExit --verbose shared ui desktop - - run: cd runner && MallocNanoZone=0 go test -race -cover - - run: yarn release-desktop 0.0.0-e2etest - - run: git reset --hard # release blows everything up - - run: yarn e2e-test - - e2e-windows: - runs-on: windows-latest - - steps: - - uses: actions/checkout@master - with: - ref: ${{ github.ref }} - - # FROM: https://github.com/actions/cache/blob/main/examples.md#node---yarn - - name: Get yarn cache directory path - id: yarn-cache-dir-path - run: echo "::set-output name=dir::$(yarn cache dir)" - - - uses: actions/cache@v2 - id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`) - with: - path: ${{ steps.yarn-cache-dir-path.outputs.dir }} - key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - ${{ runner.os }}-yarn- - - - run: ./scripts/ci/prepare_windows.ps1 - shell: pwsh - # Needed so we can have ./build/desktop_runner.js ready for tests - - run: yarn build-test-runner - - run: yarn test --detectOpenHandles --forceExit --verbose shared ui desktop - - run: cd runner && go test -race -cover - - run: yarn release-desktop 0.0.0-e2etest - - run: git reset --hard # release blows everything up - - run: yarn e2e-test + - run: yarn release-server 0.0.0-test + - run: sudo apt-get install -y podman + - run: ./server/scripts/test_release.sh datastation-server-x64-0.0.0-test.tar.gz diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index 170122ae0..326c21ca9 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -22,9 +22,9 @@ jobs: run: | curl --fail \ -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ - -H "Content-Type: application/zip" \ - --data-binary @./releases/datastation-server-x64-$GIT_TAG.zip \ - "https://uploads.github.com/repos/multiprocessio/datastation/releases/$RELEASE_ID/assets?name=datastation-server-x64-$GIT_TAG.zip" + -H "Content-Type: application/tar+gzip" \ + --data-binary @./releases/datastation-server-x64-$GIT_TAG.tar.gz \ + "https://uploads.github.com/repos/multiprocessio/datastation/releases/$RELEASE_ID/assets?name=datastation-server-x64-$GIT_TAG.tar.gz" build-linux-desktop: runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index d62e2b37e..1075f59a1 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ node_modules ui/build build +datastation yarn-error.log releases coverage diff --git a/desktop/scripts/release.build b/desktop/scripts/release.build index 624591882..1a22343f5 100644 --- a/desktop/scripts/release.build +++ b/desktop/scripts/release.build @@ -40,5 +40,5 @@ rm -rf releases yarn electron-rebuild # Build and package -yarn electron-packager --asar --overwrite --icon=build/icon.png --out=releases --build-version={arg0} --app-version={arg0} . "DataStation Community Edition" -zip -9 -r releases/datastation-{os}-{arch}-{arg0}.zip "releases/DataStation Community Edition-{os}-{arch}" \ No newline at end of file +yarn electron-packager --asar --overwrite --icon=build/icon.png --out=releases --build-version={arg0} --app-version={arg0} . "DataStation Desktop CE" +zip -9 -r releases/datastation-{os}-{arch}-{arg0}.zip "releases/DataStation Desktop CE-{os}-{arch}" \ No newline at end of file diff --git a/e2e/index.js b/e2e/index.js index d6df3196d..3bc5d254b 100644 --- a/e2e/index.js +++ b/e2e/index.js @@ -29,15 +29,14 @@ async function run() { }); const applicationPath = { - darwin: - 'DataStation Community Edition.app/Contents/MacOS/DataStation Community Edition', - win32: 'DataStation Community Edition.exe', - linux: 'DataStation Community Edition', + darwin: 'DataStation Desktop CE.app/Contents/MacOS/DataStation Desktop CE', + win32: 'DataStation Desktop CE.exe', + linux: 'DataStation Desktop CE', }[process.platform]; const directory = path.join( 'releases', - `DataStation Community Edition-${process.platform}-${process.arch}` + `DataStation Desktop CE-${process.platform}-${process.arch}` ); const driver = new webdriver.Builder() @@ -55,7 +54,7 @@ async function run() { try { await driver.wait(async () => { const title = await driver.getTitle(); - assert.equal(title, 'DataStation Community Edition'); + assert.equal(title, 'DataStation Desktop CE'); reached = true; return true; }, 10_000); diff --git a/package.json b/package.json index 2f06504e8..4e94ae411 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "build-server": "python3 ./scripts/build.py ./server/scripts/server.build", "docker-build-server": "bash ./server/scripts/build_image.sh", "build-language-definitions": "python3 ./scripts/build.py ./ui/scripts/languages.build && ./runner/scripts/generate_program_type_info.sh && yarn format", - "start-server": "yarn build-server && node ./build/server.js", + "start-server": "yarn build-server && cp node_modules/better-sqlite3/build/Release/better_sqlite3.node build && node ./build/server.js", "start-desktop": "yarn build-desktop && yarn electron --trace-warning --unhandled-rejection=warn build/desktop.js", "start-ui": "./ui/scripts/watch_and_serve.sh", "release-desktop": "python3 ./scripts/build.py ./desktop/scripts/release.build", diff --git a/server/release/datastation.service b/server/release/datastation.service index 518d9500a..7964d4fc0 100644 --- a/server/release/datastation.service +++ b/server/release/datastation.service @@ -9,7 +9,8 @@ Restart=always RestartSec=1 User=datastation ExecStartPre= -ExecStart=node /usr/share/datastation/server.js +WorkingDirectory=/usr/share/datastation +ExecStart=node /usr/share/datastation/build/server.js ExecStartPost= ExecStop= ExecReload= diff --git a/server/release/install.sh b/server/release/install.sh index 88109ee60..edbb59549 100755 --- a/server/release/install.sh +++ b/server/release/install.sh @@ -2,15 +2,15 @@ set -ex -ROOT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd)/.." +ROOT_DIR="$(readlink -f $(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd)/..)" -sudo mkdir -p /etc/datastation /usr/share/datastation/ -sudo cp -r $ROOT_DIR/* /usr/share/datastation/ +sudo mkdir -p /etc/datastation /usr/share /etc/systemd/system /home/datastation sudo cp $ROOT_DIR/release/config.yaml /etc/datastation/ sudo cp $ROOT_DIR/release/datastation.service /etc/systemd/system/ -sudo cp $ROOT_DIR/release/datastation-exporter.timer /etc/systemd/system/ +# sudo cp $ROOT_DIR/release/datastation-exporter.timer /etc/systemd/system/ +sudo mv $ROOT_DIR /usr/share/datastation sudo id -u datastation >/dev/null 2>&1 || sudo useradd -r -s /bin/false datastation -sudo chown -R datastation:datastation /etc/datastation /usr/share/datastation +sudo chown -R datastation:datastation /etc/datastation /usr/share/datastation /home/datastation sudo systemctl enable datastation diff --git a/server/scripts/release.build b/server/scripts/release.build index 061424db5..292f06f0b 100644 --- a/server/scripts/release.build +++ b/server/scripts/release.build @@ -7,10 +7,23 @@ prepend "window.DS_CONFIG_VERSION='{arg0}';" build/ui.js prepend "window.DS_CONFIG_VERSION='{arg0}';" build/server_runner.js prepend "global.DS_CONFIG_VERSION='{arg0}';" build/server.js -mv node_modules build +# Need to have a directory with a build directory and a node_modules directory +rm -rf datastation +mkdir datastation +mkdir datastation/node_modules +mv build datastation/ +# Bring in node_modules +cp -r node_modules/electron datastation/node_modules/electron +cp -r node_modules/better-sqlite3 datastation/node_modules/better-sqlite3 +# Only need stubs not full chrome install +rm -rf datastation/node_modules/electron/dist + +cp node_modules/better-sqlite3/build/Release/better_sqlite3.node datastation/build/ # Copy in install script and default configs -cp -r server/release build/release +cp -r server/release datastation/release + +mkdir releases +tar -zcvf releases/datastation-server-{arch}-{arg0}.tar.gz datastation -mkdir -p releases -zip -9 -r releases/datastation-server-{arch}-{arg0}.zip build +rm -rf datastation \ No newline at end of file diff --git a/server/scripts/server.build b/server/scripts/server.build index 77e74ad6d..ebb16020b 100644 --- a/server/scripts/server.build +++ b/server/scripts/server.build @@ -1,11 +1,15 @@ -setenv UI_CONFIG_OVERRIDES "window.DS_CONFIG_MODE = 'server';" +setenv UI_CONFIG_OVERRIDES "window.DS_CONFIG_MODE = 'server'; window.DS_CONFIG_UI_TITLE = 'DataStation Server CE';" + +setenv UI_TITLE "DataStation Server CE" setenv UI_ROOT "/" yarn build-ui cd runner && go build -trimpath -buildmode=pie -mod=readonly -modcacherw -ldflags="-s -w" -o ../build/go_server_runner{required_ext} ./cmd/main.go -yarn esbuild server/runner.ts --sourcemap --platform=node --bundle --target=node10.4 --external:sqlite3 --external:react-native-fs --external:react-native-fetch-blob --external:oracledb "--external:@elastic/elasticsearch" "--external:wasm-brotli" --external:prometheus-query --external:snowflake-sdk --external:ssh2 --external:ssh2-promise --external:ssh2-sftp-client --external:cpu-features --external:electron --external:pg-native --outfile=build/server_runner.js -yarn esbuild server/index.ts --sourcemap --platform=node --bundle --external:sqlite3 --external:react-native-fs --external:react-native-fetch-blob --external:oracledb "--external:@elastic/elasticsearch" "--external:wasm-brotli" --external:prometheus-query --external:snowflake-sdk --external:ssh2 --external:ssh2-promise --external:ssh2-sftp-client --external:cpu-features --external:electron --external:pg-native --outfile=build/server.js +yarn esbuild server/runner.ts --sourcemap --platform=node --bundle --target=node10.4 --external:sqlite3 --external:react-native-fs --external:react-native-fetch-blob --external:cpu-features --external:electron --outfile=build/server_runner.js +yarn esbuild server/index.ts --sourcemap --platform=node --bundle --external:sqlite3 --external:react-native-fs --external:react-native-fetch-blob --external:cpu-features --external:electron --outfile=build/server.js + +prepend "global.DS_UI_TITLE='DataStation Server CE';" build/server.js rm -rf build/migrations cp -r desktop/migrations build/ diff --git a/server/scripts/test_release.sh b/server/scripts/test_release.sh new file mode 100755 index 000000000..317d13aba --- /dev/null +++ b/server/scripts/test_release.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash + +set -e + +function docker() { + podman "$@" +} + +cp ./releases/$1 ./releases/datastation-docker-release-test.tar.gz +docker build . -q -f ./server/scripts/test_systemd_dockerfile -t datastation-server-test +cid=$(docker run -d datastation-server-test:latest) + +debug() { + rv=$? + if ! [[ "$rv" == "0" ]]; then + docker ps -a + docker logs $cid + docker rm -f $cid || echo "Container already exited." + fi + exit $rv +} +trap "debug" exit + +function c_run() { + docker exec $cid bash -c "$1" +} + +# TODO: test out systemd settings eventually +# # Copy in zip file +# docker cp ./releases/$1 $cid:/ +# +# c_run "tar xvzf /$1" +# c_run "/datastation/release/install.sh" +# c_run "truncate --size 0 /etc/datastation/config.yaml" +# c_run "systemctl restart datastation" + +# Wait for server to start +sleep 10 + +result="$(c_run 'curl localhost:8080')" + +expected='