From 3697d2ebfa29298b70c886bd02973bbbe1a204ab Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Tue, 21 Jan 2020 18:43:48 +0900 Subject: [PATCH 1/5] Add a fallback repository as GCS --- .github/workflows/master.yml | 8 -------- pom.xml | 15 +++++---------- 2 files changed, 5 insertions(+), 18 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index e4d2d8470cc34..d53119ad75599 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -66,14 +66,6 @@ jobs: export MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=1g -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN" export MAVEN_CLI_OPTS="--no-transfer-progress" mkdir -p ~/.m2 - # `Maven Central` is too flaky in terms of downloading artifacts in `GitHub Action` environment. - # `Google Maven Central Mirror` is too slow in terms of sycing upstream. To get the best combination, - # 1) we set `Google Maven Central` as a mirror of `central` in `GitHub Action` environment only. - # 2) we duplicates `Maven Central` in pom.xml with ID `central_without_mirror`. - # In other words, in GitHub Action environment, `central` is mirrored by `Google Maven Central` first. - # If `Google Maven Central` doesn't provide the artifact due to its slowness, `central_without_mirror` will be used. - # Note that we aim to achieve the above while keeping the existing behavior of non-`GitHub Action` environment unchanged. - echo "google-maven-centralGCS Maven Central mirrorhttps://maven-central.storage-download.googleapis.com/repos/central/data/central" > ~/.m2/settings.xml ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pmesos -Pkubernetes -Phive -P${{ matrix.hive }} -Phive-thriftserver -P${{ matrix.hadoop }} -Phadoop-cloud -Djava.version=${{ matrix.java }} install rm -rf ~/.m2/repository/org/apache/spark diff --git a/pom.xml b/pom.xml index d2b2b0205060f..04beb2e1ed846 100644 --- a/pom.xml +++ b/pom.xml @@ -246,7 +246,7 @@ - central + maven-central Maven Repository https://repo.maven.apache.org/maven2 @@ -258,14 +258,9 @@ - central_without_mirror - - Maven Repository - https://repo.maven.apache.org/maven2 + gcs-mirror + GCS Maven Mirror + https://maven-central.storage-download.googleapis.com/repos/central/data true @@ -276,7 +271,7 @@ - central + maven-central https://repo.maven.apache.org/maven2 true From e66ae98d9b32595ce8eb4b53bb5bd876301a9bc4 Mon Sep 17 00:00:00 2001 From: cody koeninger Date: Sun, 19 Jan 2020 12:27:01 -0600 Subject: [PATCH 2/5] SPARK-30570 scalafmt 1.0.3 onlyChangedFiles appears to work correctly on scala 2.12 --- dev/scalafmt | 5 +---- pom.xml | 5 +++-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/dev/scalafmt b/dev/scalafmt index 0541a4058467b..7471d90bbd65c 100755 --- a/dev/scalafmt +++ b/dev/scalafmt @@ -17,7 +17,4 @@ # limitations under the License. # -# by default, format only files that differ from git master -params="${@:---diff}" - -./build/mvn -Pscala-2.12 mvn-scalafmt_2.12:format -Dscalafmt.skip=false -Dscalafmt.parameters="$params" \ No newline at end of file +./build/mvn -Pscala-2.12 mvn-scalafmt_2.12:format -Dscalafmt.skip=false diff --git a/pom.xml b/pom.xml index 04beb2e1ed846..e83a9830676b9 100644 --- a/pom.xml +++ b/pom.xml @@ -162,7 +162,7 @@ 3.2.2 2.12.10 2.12 - --diff --test + --test true 1.9.13 @@ -3042,13 +3042,14 @@ org.antipathy mvn-scalafmt_2.12 - 0.12_1.5.1 + 1.0.3 ${scalafmt.parameters} ${scalafmt.skip} ${scalafmt.skip} ${scalafmt.skip} dev/.scalafmt.conf + true From 40892053dd85d89356281fb5d02687cd1f657150 Mon Sep 17 00:00:00 2001 From: cody koeninger Date: Sun, 19 Jan 2020 13:12:34 -0600 Subject: [PATCH 3/5] SPARK-30570 scalafmt 1.0.3 for scala 2.13 as well, cant think of a better way to handle profile changes than an argument to the dev/scalafmt script --- dev/scalafmt | 4 +++- pom.xml | 43 +++++++++++++++++++++---------------------- 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/dev/scalafmt b/dev/scalafmt index 7471d90bbd65c..3f69bc98f51c7 100755 --- a/dev/scalafmt +++ b/dev/scalafmt @@ -17,4 +17,6 @@ # limitations under the License. # -./build/mvn -Pscala-2.12 mvn-scalafmt_2.12:format -Dscalafmt.skip=false +VERSION="${@:-2.12}" +./build/mvn -Pscala-$VERSION mvn-scalafmt_$VERSION:format -Dscalafmt.skip=false + diff --git a/pom.xml b/pom.xml index e83a9830676b9..5c49a439e31b7 100644 --- a/pom.xml +++ b/pom.xml @@ -2874,6 +2874,27 @@ + + org.antipathy + mvn-scalafmt_${scala.binary.version} + 1.0.3 + + ${scalafmt.parameters} + ${scalafmt.skip} + ${scalafmt.skip} + ${scalafmt.skip} + dev/.scalafmt.conf + true + + + + validate + + format + + + + - - org.antipathy - mvn-scalafmt_2.12 - 1.0.3 - - ${scalafmt.parameters} - ${scalafmt.skip} - ${scalafmt.skip} - ${scalafmt.skip} - dev/.scalafmt.conf - true - - - - validate - - format - - - - From 9d252dcf33fc19e035856c85a6c5f92895261ba2 Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Tue, 21 Jan 2020 19:00:19 +0900 Subject: [PATCH 4/5] Debug --- .github/workflows/master.yml | 2 +- pom.xml | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index d53119ad75599..577ee7e905498 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -66,7 +66,7 @@ jobs: export MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=1g -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN" export MAVEN_CLI_OPTS="--no-transfer-progress" mkdir -p ~/.m2 - ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pmesos -Pkubernetes -Phive -P${{ matrix.hive }} -Phive-thriftserver -P${{ matrix.hadoop }} -Phadoop-cloud -Djava.version=${{ matrix.java }} install + ./build/mvn -X $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pmesos -Pkubernetes -Phive -P${{ matrix.hive }} -Phive-thriftserver -P${{ matrix.hadoop }} -Phadoop-cloud -Djava.version=${{ matrix.java }} install rm -rf ~/.m2/repository/org/apache/spark diff --git a/pom.xml b/pom.xml index 5c49a439e31b7..1a89277f2815b 100644 --- a/pom.xml +++ b/pom.xml @@ -246,10 +246,9 @@ - maven-central - - Maven Repository - https://repo.maven.apache.org/maven2 + gcs-mirror + GCS Maven Mirror + https://maven-central.storage-download.googleapis.com/repos/central/data true @@ -258,9 +257,10 @@ - gcs-mirror - GCS Maven Mirror - https://maven-central.storage-download.googleapis.com/repos/central/data + maven-central + + Maven Repository + https://repo.maven.apache.org/maven2 true From 992ad7a4929b719effacfff28b78d49ba4f0824e Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Tue, 21 Jan 2020 19:31:08 +0900 Subject: [PATCH 5/5] Update master.yml --- .github/workflows/master.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 577ee7e905498..e03e963da8e4b 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -10,7 +10,6 @@ on: jobs: build: - runs-on: ubuntu-latest strategy: matrix: