Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Improve][CI] improve ci and only build the changed modules #3179

Merged
merged 2 commits into from
Oct 26, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
157 changes: 151 additions & 6 deletions .github/workflows/backend.yml
Original file line number Diff line number Diff line change
Expand Up @@ -135,10 +135,140 @@ jobs:
- name: Check Dependencies Licenses
run: tools/dependencies/checkLicense.sh

changes:
runs-on: ubuntu-latest
# To prevent error when there's no base branch
if: github.repository == 'apache/incubator-seatunnel'
timeout-minutes: 10
outputs:
api: ${{ steps.filter.outputs.api }}
ut-modules: ${{ steps.ut-modules.outputs.modules }}
it-modules: ${{ steps.it-modules.outputs.modules }}
steps:
- uses: actions/checkout@v3 # required for push event
- uses: actions/setup-python@v4
with:
python-version: '2.7'
- name: Check for file changes
uses: dorny/paths-filter@v2
id: filter
with:
token: ${{ github.token }}
list-files: json
# The following is a single composite pattern that allows next CI steps,
# the pattern is in form of [not (foo or bar)] to be safe.
# When new files come in, the CI will NOT ignore them unless listed,
# so remember to extend here if they do not serve functional purposes.
# NOTE: careful with using ** in expression, keep !**/{old, new things}.
filters: |
cv2:
- "seatunnel-connectors-v2/**"
cv2-flink-e2e:
- "seatunnel-e2e/seatunnel-flink-connector-v2-e2e/**"
cv2-spark-e2e:
- "seatunnel-e2e/seatunnel-spark-connector-v2-e2e/**"
cv2-e2e:
- "seatunnel-e2e/seatunnel-connector-v2-e2e/**"
api:
- "seatunnel-api"
- "seatunnel-apis"
- "seatunnel-common"
- "seatunnel-config"
- "seatunnel-connectors"
- "seatunnel-core"
- "seatunnel-dist"
- "seatunnel-e2e/seatunnel-e2e-common"
- "seatunnel-formats"
- "seatunnel-plugin-discovery"
- "seatunnel-transforms"
- "seatunnel-translation"
- "seatunnel-e2e/seatunnel-flink-e2e"
- "seatunnel-e2e/seatunnel-spark-e2e"
- "seatunnel-connectors"
- "plugin-mapping.properties"
- "**/tools/**"
- "pom.xml"
- "**/workflows/**"
engine:
- "seatunnel-engine"
engine-e2e:
- "seatunnel-e2e/seatunnel-engine-e2e"
- name: Check Connector V2 Update
id: cv2-modules
if: ${{ steps.filter.outputs.cv2 == 'true' }}
run: |
update_files='${{ steps.filter.outputs.cv2_files }}'
modules=`python tools/update_modules_check/update_modules_check.py cv2 $update_files`
echo $modules
echo "modules=$modules" >> $GITHUB_OUTPUT
- name: Check Flink Connector V2 E2E Update
id: cv2-flink-e2e-modules
if: ${{ steps.filter.outputs.cv2-flink-e2e == 'true' }}
run: |
update_files='${{ steps.filter.outputs.cv2-flink-e2e_files }}'
modules=`python tools/update_modules_check/update_modules_check.py cv2-flink-e2e $update_files`
echo $modules
echo "modules=$modules" >> $GITHUB_OUTPUT
- name: Check Spark Connector V2 E2E Update
id: cv2-spark-e2e-modules
if: ${{ steps.filter.outputs.cv2-spark-e2e == 'true' }}
run: |
update_files='${{ steps.filter.outputs.cv2-spark-e2e_files }}'
modules=`python tools/update_modules_check/update_modules_check.py cv2-spark-e2e $update_files`
echo $modules
echo "modules=$modules" >> $GITHUB_OUTPUT
- name: Check Connector V2 E2E Update
id: cv2-e2e-modules
if: ${{ steps.filter.outputs.cv2-e2e == 'true' }}
run: |
update_files='${{ steps.filter.outputs.cv2-e2e_files }}'
modules=`python tools/update_modules_check/update_modules_check.py cv2-e2e $update_files`
echo $modules
echo "modules=$modules" >> $GITHUB_OUTPUT
- name: Check Engine Update
id: engine-modules
if: ${{ steps.filter.outputs.engine == 'true' }}
run: |
update_files='${{ steps.filter.outputs.engine_files }}'
modules=`python tools/update_modules_check/update_modules_check.py engine $update_files`
echo $modules
echo "modules=$modules" >> $GITHUB_OUTPUT
- name: Check Engine E2E Update
id: engine-e2e-modules
if: ${{ steps.filter.outputs.engine-e2e == 'true' }}
run: |
update_files='${{ steps.filter.outputs.engine-e2e_files }}'
modules=`python tools/update_modules_check/update_modules_check.py engine-e2e $update_files`
echo $modules
echo "modules=$modules" >> $GITHUB_OUTPUT
- name: Make unit test modules
id: ut-modules
if: ${{ steps.filter.outputs.api == 'false' }}
run: |
modules='${{ steps.engine-modules.outputs.modules }}${{ steps.cv2-modules.outputs.modules }}'
modules=${modules: 1}
echo $modules
echo "modules=$modules" >> $GITHUB_OUTPUT
- name: Make integration test modules
id: it-modules
if: ${{ steps.filter.outputs.api == 'false' }}
run: |
modules='${{ steps.cv2-e2e-modules.outputs.modules }}${{ steps.cv2-flink-e2e-modules.outputs.modules }}${{ steps.cv2-spark-e2e-modules.outputs.modules }}${{ steps.engine-e2e-modules.outputs.modules }}${{ steps.engine-modules.outputs.modules }}${{ steps.cv2-modules.outputs.modules }}'
modules=${modules: 1}
echo $modules
echo "modules=$modules" >> $GITHUB_OUTPUT
unit-test:
name: Unit Test
needs: [ changes, sanity-check ]
runs-on: ${{ matrix.os }}
needs: [ sanity-check ]
strategy:
matrix:
java: [ '8', '11' ]
Expand All @@ -152,16 +282,23 @@ jobs:
java-version: ${{ matrix.java }}
distribution: 'temurin'
cache: 'maven'
- name: Run Unit tests
- name: run all modules unit test
if: needs.changes.outputs.api == 'true'
run: |
./mvnw -B -T 1C clean verify -D"maven.test.skip"=false -D"checkstyle.skip"=true -D"scalastyle.skip"=true -D"license.skipAddThirdParty"=true --no-snapshot-updates
env:
MAVEN_OPTS: -Xmx2048m

- name: run updated modules unit test
if: needs.changes.outputs.api == 'false'
run: |
./mvnw -B -T 1C clean verify -D"maven.test.skip"=false -D"checkstyle.skip"=true -D"scalastyle.skip"=true -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl ${{needs.changes.outputs.ut-modules}} -am -amd -Pno_dist
env:
MAVEN_OPTS: -Xmx2048m

integration-test:
name: Integration Test
needs: [ changes, sanity-check ]
runs-on: ${{ matrix.os }}
needs: [ sanity-check ]
strategy:
matrix:
java: [ '8', '11' ]
Expand All @@ -175,8 +312,16 @@ jobs:
java-version: ${{ matrix.java }}
distribution: 'temurin'
cache: 'maven'
- name: Run Integration tests
- name: run all modules integration test
if: needs.changes.outputs.api == 'true'
run: |
./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -D"checkstyle.skip"=true -D"scalastyle.skip"=true -D"license.skipAddThirdParty"=true --no-snapshot-updates
env:
MAVEN_OPTS: -Xmx2048m

- name: run updated modules integration test
if: needs.changes.outputs.api == 'false'
run: |
./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -D"checkstyle.skip"=true -D"scalastyle.skip"=true -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl ${{needs.changes.outputs.it-modules}} -am -amd -Pno_dist
env:
MAVEN_OPTS: -Xmx2048m
14 changes: 13 additions & 1 deletion pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,6 @@
<module>seatunnel-translation</module>
<module>seatunnel-plugin-discovery</module>
<module>seatunnel-formats</module>
<module>seatunnel-dist</module>
<module>seatunnel-engine</module>
</modules>

Expand All @@ -104,6 +103,7 @@
<modules>
<module>seatunnel-examples</module>
<module>seatunnel-e2e</module>
<module>seatunnel-dist</module>
</modules>
</profile>
<profile>
Expand All @@ -114,6 +114,18 @@
<value>true</value>
</property>
</activation>
<modules>
<module>seatunnel-dist</module>
</modules>
</profile>
<profile>
<id>no_dist</id>
<activation>
<property>
<name>release</name>
<value>false</value>
</property>
</activation>
</profile>
</profiles>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
import java.io.IOException;

public class FakeSourceToFileIT extends FlinkContainer {

/**
* fake source -> local text file sink
*/
Expand Down
71 changes: 71 additions & 0 deletions tools/update_modules_check/update_modules_check.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

#!/usr/bin/python
import json
import sys

def get_cv2_modules(files):
get_modules(files, 1, "connector-", "seatunnel-connectors-v2")

def get_cv2_flink_e2e_modules(files):
get_modules(files, 2, "connector-", "seatunnel-e2e/seatunnel-flink-connector-v2-e2e")

def get_cv2_spark_e2e_modules(files):
get_modules(files, 2, "connector-", "seatunnel-e2e/seatunnel-spark-connector-v2-e2e")

def get_cv2_e2e_modules(files):
get_modules(files, 2, "connector-", "seatunnel-e2e/seatunnel-connector-v2-e2e")

def get_engine_modules(files):
get_modules(files, 1, "seatunnel-", "seatunnel-engine")

def get_engine_e2e_modules(files):
get_modules(files, 2, "connector-", "seatunnel-e2e/seatunnel-engine-e2e")

def get_modules(files, index, start_pre, root_module):
update_files = json.loads(files)
modules_name_set = set([])
for file in update_files:
module_name = file.split('/')[index]
if module_name.startswith(start_pre):
modules_name_set.add(root_module + "/" + module_name)

output_module = ""
if len(modules_name_set) > 0:
for module in modules_name_set:
output_module = output_module + "," + module

else:
output_module = root_module

print(output_module)

def main(argv):
if argv[1] == "cv2":
get_cv2_modules(argv[2])
elif argv[1] == "cv2-e2e":
get_cv2_e2e_modules(argv[2])
elif argv[1] == "cv2-flink-e2e":
get_cv2_flink_e2e_modules(argv[2])
elif argv[1] == "cv2-spark-e2e":
get_cv2_spark_e2e_modules(argv[2])
elif argv[1] == "engine":
get_engine_modules(argv[2])
elif argv[1] == "engine-e2e":
get_engine_e2e_modules(argv[2])

if __name__ == "__main__":
main(sys.argv)