diff --git a/.github/workflows/report_ram_log.yml b/.github/workflows/report_ram_log.yml index 6d7bc0dc5..3067e2e73 100644 --- a/.github/workflows/report_ram_log.yml +++ b/.github/workflows/report_ram_log.yml @@ -53,16 +53,9 @@ jobs: git clone -b arrow-3.0.0-oap https://github.com/oap-project/arrow.git cd arrow/java mvn clean install -B -P arrow-jni -am -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -Darrow.cpp.build.dir=/tmp/arrow/cpp/build/release/ -DskipTests -Dcheckstyle.skip - - name: Install Arrow datasource - run: | - cd /tmp - git clone https://github.com/oap-project/arrow-data-source.git - cd arrow-data-source - mvn clean install -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -DskipTests - name: Run Maven tests run: | - cd native-sql-engine/core/ - mvn test -B -DmembersOnlySuites=com.intel.oap.tpc.h -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -DtagsToInclude=com.intel.oap.tags.CommentOnContextPR -Dexec.skip=true + mvn test -B -pl native-sql-engine/core/ -am -DmembersOnlySuites=com.intel.oap.tpc.h -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -DtagsToInclude=com.intel.oap.tags.CommentOnContextPR -Dexec.skip=true env: MAVEN_OPTS: "-Xmx2048m" COMMENT_CONTENT_PATH: "/tmp/comment.md" diff --git a/.github/workflows/tpch.yml b/.github/workflows/tpch.yml index 903af77f0..b3a4bd19f 100644 --- a/.github/workflows/tpch.yml +++ b/.github/workflows/tpch.yml @@ -48,16 +48,9 @@ jobs: sudo make install cd ../../java mvn clean install -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -P arrow-jni -am -Darrow.cpp.build.dir=/tmp/arrow/cpp/build/release/ -DskipTests -Dcheckstyle.skip - - name: Install Arrow datasource - run: | - cd /tmp - git clone https://github.com/oap-project/arrow-data-source.git - cd arrow-data-source - mvn clean install -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn - name: Run Maven tests - BHJ run: | - cd native-sql-engine/core/ - mvn test -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -DmembersOnlySuites=com.intel.oap.tpc.h -DtagsToInclude=com.intel.oap.tags.BroadcastHashJoinMode -DargLine="-Xmx1G -XX:MaxDirectMemorySize=500M -Dio.netty.allocator.numDirectArena=1" + mvn test -B -pl native-sql-engine/core/ -am -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -DmembersOnlySuites=com.intel.oap.tpc.h -DtagsToInclude=com.intel.oap.tags.BroadcastHashJoinMode -DargLine="-Xmx1G -XX:MaxDirectMemorySize=500M -Dio.netty.allocator.numDirectArena=1" env: MALLOC_ARENA_MAX: "4" MAVEN_OPTS: "-Xmx1G" @@ -66,8 +59,7 @@ jobs: ENABLE_TPCH_TESTS: "true" - name: Run Maven tests - SMJ run: | - cd native-sql-engine/core/ - mvn test -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -DmembersOnlySuites=com.intel.oap.tpc.h -DtagsToInclude=com.intel.oap.tags.SortMergeJoinMode -DargLine="-Xmx1G -XX:MaxDirectMemorySize=500M -Dio.netty.allocator.numDirectArena=1" + mvn test -B -pl native-sql-engine/core/ -am -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -DmembersOnlySuites=com.intel.oap.tpc.h -DtagsToInclude=com.intel.oap.tags.SortMergeJoinMode -DargLine="-Xmx1G -XX:MaxDirectMemorySize=500M -Dio.netty.allocator.numDirectArena=1" env: MALLOC_ARENA_MAX: "4" MAVEN_OPTS: "-Xmx1G" diff --git a/arrow-data-source/.github/PULL_REQUEST_TEMPLATE b/arrow-data-source/.github/PULL_REQUEST_TEMPLATE deleted file mode 100644 index 989e95ccd..000000000 --- a/arrow-data-source/.github/PULL_REQUEST_TEMPLATE +++ /dev/null @@ -1,12 +0,0 @@ -## What changes were proposed in this pull request? - -(Please fill in changes proposed in this fix) - - -## How was this patch tested? - -(Please explain how this patch was tested. E.g. unit tests, integration tests, manual tests) - - -(If this patch involves UI changes, please attach a screenshot; otherwise, remove this) - diff --git a/arrow-data-source/.github/workflows/dev_cron.yml b/arrow-data-source/.github/workflows/dev_cron.yml deleted file mode 100644 index ab2840ebd..000000000 --- a/arrow-data-source/.github/workflows/dev_cron.yml +++ /dev/null @@ -1,58 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Dev PR - -on: - - pull_request_target: - types: - - opened - - edited - - synchronize - -jobs: - process: - name: Process - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - name: Comment Issues link - if: | - github.event_name == 'pull_request_target' && - (github.event.action == 'opened' || - github.event.action == 'edited') - uses: actions/github-script@v3 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const script = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/dev_cron/issues_link.js`); - script({github, context}); - - - name: Check title - if: | - github.event_name == 'pull_request_target' && - (github.event.action == 'opened' || - github.event.action == 'edited') - uses: actions/github-script@v3 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const script = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/dev_cron/title_check.js`); - script({github, context}); - diff --git a/arrow-data-source/.github/workflows/dev_cron/issues_link.js b/arrow-data-source/.github/workflows/dev_cron/issues_link.js deleted file mode 100644 index 1794b2566..000000000 --- a/arrow-data-source/.github/workflows/dev_cron/issues_link.js +++ /dev/null @@ -1,70 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -function detectISSUESID(title) { - if (!title) { - return null; - } - const matched = /^\[NSE-\d+\]/.exec(title); - if (!matched) { - return null; - } - const issues_number = matched[0].replace(/[^0-9]/ig,""); - return issues_number; -} - -async function haveComment(github, context, pullRequestNumber, body) { - const options = { - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: pullRequestNumber, - page: 1 - }; - while (true) { - const response = await github.issues.listComments(options); - if (response.data.some(comment => comment.body === body)) { - return true; - } - if (!/;\s*rel="next"/.test(response.headers.link || "")) { - break; - } - options.page++; - } - return false; -} - -async function commentISSUESURL(github, context, pullRequestNumber, issuesID) { - const issuesURL = `https://github.com/oap-project/native-sql-engine/issues/${issuesID}`; - if (await haveComment(github, context, pullRequestNumber, issuesURL)) { - return; - } - await github.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: pullRequestNumber, - body: issuesURL - }); -} - -module.exports = async ({github, context}) => { - const pullRequestNumber = context.payload.number; - const title = context.payload.pull_request.title; - const issuesID = detectISSUESID(title); - if (issuesID) { - await commentISSUESURL(github, context, pullRequestNumber, issuesID); - } -}; diff --git a/arrow-data-source/.github/workflows/dev_cron/title_check.js b/arrow-data-source/.github/workflows/dev_cron/title_check.js deleted file mode 100644 index 9380a4344..000000000 --- a/arrow-data-source/.github/workflows/dev_cron/title_check.js +++ /dev/null @@ -1,53 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -const fs = require("fs"); - -function haveISSUESID(title) { - if (!title) { - return false; - } - return /^\[NSE-\d+\]/.test(title); -} - -async function commentOpenISSUESIssue(github, context, pullRequestNumber) { - const {data: comments} = await github.issues.listComments({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: pullRequestNumber, - per_page: 1 - }); - if (comments.length > 0) { - return; - } - const commentPath = ".github/workflows/dev_cron/title_check.md"; - const comment = fs.readFileSync(commentPath).toString(); - await github.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: pullRequestNumber, - body: comment - }); -} - -module.exports = async ({github, context}) => { - const pullRequestNumber = context.payload.number; - const title = context.payload.pull_request.title; - if (!haveISSUESID(title)) { - await commentOpenISSUESIssue(github, context, pullRequestNumber); - } -}; diff --git a/arrow-data-source/.github/workflows/dev_cron/title_check.md b/arrow-data-source/.github/workflows/dev_cron/title_check.md deleted file mode 100644 index 928a8fc4b..000000000 --- a/arrow-data-source/.github/workflows/dev_cron/title_check.md +++ /dev/null @@ -1,33 +0,0 @@ - - -Thanks for opening a pull request! - -Could you open an issue for this pull request on Github Issues? - -https://github.com/oap-project/native-sql-engine/issues - -Then could you also rename ***commit message*** and ***pull request title*** in the following format? - - [NSE-${ISSUES_ID}] ${detailed message} - -See also: - - * [Other pull requests](https://github.com/oap-project/native-sql-engine/pulls/) - diff --git a/arrow-data-source/.github/workflows/report_ram_log.yml b/arrow-data-source/.github/workflows/report_ram_log.yml deleted file mode 100644 index 2fef9911c..000000000 --- a/arrow-data-source/.github/workflows/report_ram_log.yml +++ /dev/null @@ -1,72 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Report RAM Log - -on: - workflow_run: - workflows: ["Native SQL Engine TPC-H Suite"] - types: - - completed - -jobs: - comment-on-pr: - if: ${{ github.event.workflow_run.conclusion == 'success' }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up JDK 1.8 - uses: actions/setup-java@v1 - with: - java-version: 1.8 - - name: Download log - uses: dawidd6/action-download-artifact@v2 - with: - workflow: tpch.yml - run_id: ${{ github.event.workflow_run.id }} - name: comment_content - path: /tmp/ - - name: Download previous event payload - uses: dawidd6/action-download-artifact@v2 - with: - workflow: tpch.yml - run_id: ${{ github.event.workflow_run.id }} - name: pr_event - path: /tmp/ - - name: Install OAP optimized Arrow - run: | - cd /tmp - git clone -b arrow-3.0.0-oap https://github.com/oap-project/arrow.git - cd arrow/java - mvn clean install -B -P arrow-jni -am -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -Darrow.cpp.build.dir=/tmp/arrow/cpp/build/release/ -DskipTests -Dcheckstyle.skip - - name: Install Arrow datasource - run: | - cd /tmp - git clone https://github.com/oap-project/arrow-data-source.git - cd arrow-data-source - mvn clean install -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -DskipTests - - name: Run Maven tests - run: | - cd core/ - mvn test -B -DmembersOnlySuites=com.intel.oap.tpc.h -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -DtagsToInclude=com.intel.oap.tags.CommentOnContextPR -Dexec.skip=true - env: - MAVEN_OPTS: "-Xmx2048m" - COMMENT_CONTENT_PATH: "/tmp/comment.md" - PREVIOUS_EVENT_PATH: "/tmp/event.json" - GITHUB_TOKEN: ${{ github.token }} - ENABLE_TPCH_TESTS: "true" - diff --git a/arrow-data-source/.github/workflows/tpch.yml b/arrow-data-source/.github/workflows/tpch.yml deleted file mode 100644 index 8f6908d77..000000000 --- a/arrow-data-source/.github/workflows/tpch.yml +++ /dev/null @@ -1,90 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Native SQL Engine TPC-H Suite - -on: - pull_request - -jobs: - ram-usage-test: - if: ${{ !contains(github.event.pull_request.labels.*.name, 'No RAM Report') }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up JDK 1.8 - uses: actions/setup-java@v1 - with: - java-version: 1.8 - - uses: iterative/setup-cml@v1 - with: - version: latest - - run: sudo swapoff -a - - run: free - - run: sudo apt-get update - - run: sudo apt-get install cmake - - run: sudo apt-get install libboost-all-dev - - name: Install OAP optimized Arrow - run: | - cd /tmp - git clone https://github.com/oap-project/arrow.git - cd arrow && git checkout arrow-3.0.0-oap && cd cpp - mkdir build && cd build - cmake .. -DARROW_JNI=ON -DARROW_GANDIVA_JAVA=ON -DARROW_GANDIVA=ON -DARROW_PARQUET=ON -DARROW_HDFS=ON -DARROW_FILESYSTEM=ON -DARROW_WITH_SNAPPY=ON -DARROW_JSON=ON -DARROW_DATASET=ON -DARROW_WITH_LZ4=ON -DARROW_JEMALLOC=OFF && make -j2 - sudo make install - cd ../../java - mvn clean install -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -P arrow-jni -am -Darrow.cpp.build.dir=/tmp/arrow/cpp/build/release/ -DskipTests -Dcheckstyle.skip - - name: Install Arrow datasource - run: | - cd /tmp - git clone https://github.com/oap-project/arrow-data-source.git - cd arrow-data-source - mvn clean install -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn - - name: Run Maven tests - BHJ - run: | - cd core/ - mvn test -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -DmembersOnlySuites=com.intel.oap.tpc.h -DtagsToInclude=com.intel.oap.tags.BroadcastHashJoinMode -DargLine="-Xmx1G -XX:MaxDirectMemorySize=500M -Dio.netty.allocator.numDirectArena=1" - env: - MALLOC_ARENA_MAX: "4" - MAVEN_OPTS: "-Xmx1G" - COMMENT_TEXT_OUTPUT_PATH: "/tmp/comment_text_1.txt" - COMMENT_IMAGE_OUTPUT_PATH: "/tmp/comment_image_1.png" - ENABLE_TPCH_TESTS: "true" - - name: Run Maven tests - SMJ - run: | - cd core/ - mvn test -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -DmembersOnlySuites=com.intel.oap.tpc.h -DtagsToInclude=com.intel.oap.tags.SortMergeJoinMode -DargLine="-Xmx1G -XX:MaxDirectMemorySize=500M -Dio.netty.allocator.numDirectArena=1" - env: - MALLOC_ARENA_MAX: "4" - MAVEN_OPTS: "-Xmx1G" - COMMENT_TEXT_OUTPUT_PATH: "/tmp/comment_text_2.txt" - COMMENT_IMAGE_OUTPUT_PATH: "/tmp/comment_image_2.png" - ENABLE_TPCH_TESTS: "true" - - run: | - cml-publish /tmp/comment_image_1.png --md > /tmp/comment.md - cml-publish /tmp/comment_image_2.png --md >> /tmp/comment.md - - run: echo "::set-output name=event_path::${GITHUB_EVENT_PATH}" - id: output-envs - - uses: actions/upload-artifact@v2 - with: - name: comment_content - path: /tmp/comment.md - - uses: actions/upload-artifact@v2 - with: - name: pr_event - path: ${{steps.output-envs.outputs.event_path}} - diff --git a/arrow-data-source/.github/workflows/unittests.yml b/arrow-data-source/.github/workflows/unittests.yml deleted file mode 100644 index 0cdf3f470..000000000 --- a/arrow-data-source/.github/workflows/unittests.yml +++ /dev/null @@ -1,61 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Native SQL Engine Unit Tests Suite - -on: - pull_request - -jobs: - cpp-unit-test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up JDK 1.8 - uses: actions/setup-java@v1 - with: - java-version: 1.8 - - run: sudo swapoff -a - - run: free - - run: sudo apt-get update - - run: sudo apt-get install cmake - - run: sudo apt-get install libboost-all-dev - - name: Install Googletest - run: | - sudo apt-get install libgtest-dev - cd /usr/src/gtest - sudo cmake CMakeLists.txt -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=/usr/local - sudo make - sudo apt-get install google-mock - - name: Install OAP optimized Arrow (C++ libs) - run: | - cd /tmp - git clone https://github.com/oap-project/arrow.git - cd arrow && git checkout arrow-3.0.0-oap && cd cpp - mkdir build && cd build - cmake .. -DARROW_JNI=ON -DARROW_GANDIVA_JAVA=ON -DARROW_GANDIVA=ON -DARROW_PARQUET=ON -DARROW_HDFS=ON -DARROW_FILESYSTEM=ON -DARROW_WITH_SNAPPY=ON -DARROW_JSON=ON -DARROW_DATASET=ON -DARROW_WITH_LZ4=ON -DGTEST_ROOT=/usr/src/gtest && make -j2 - sudo make install - - name: Run unit tests - run: | - cd cpp/ - mkdir -p build - cd build - cmake .. -DTESTS=1 - make - cd src - ctest -R -