From 0e8a95ea99f3be0c31e8c76d5086d46062ef3bef Mon Sep 17 00:00:00 2001
From: yangjie01 <yangjie01@baidu.com>
Date: Tue, 4 Jul 2023 20:06:24 +0900
Subject: [PATCH 01/13] [SPARK-44295][BUILD] Upgrade `scala-parser-combinators`
 to 2.3.0

### What changes were proposed in this pull request?
This pr aims to upgrade `scala-parser-combinators` from 2.2.0 to 2.3.0

### Why are the changes needed?
The new version [dropped support for Scala 2.11](https://github.com/scala/scala-parser-combinators/pull/504) and bring a bug fix:
- https://github.com/scala/scala-parser-combinators/pull/507

The full release notes as follows:
- https://github.com/scala/scala-parser-combinators/releases/tag/v2.3.0

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Pass Github Actions

Closes #41848 from LuciferYang/scala-parser-combinators-23.

Authored-by: yangjie01 <yangjie01@baidu.com>
Signed-off-by: Hyukjin Kwon <gurwls223@apache.org>
---
 dev/deps/spark-deps-hadoop-3-hive-2.3 | 2 +-
 pom.xml                               | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3
index 3ac03fa64727f..910b1703c70c8 100644
--- a/dev/deps/spark-deps-hadoop-3-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-3-hive-2.3
@@ -227,7 +227,7 @@ rocksdbjni/8.3.2//rocksdbjni-8.3.2.jar
 scala-collection-compat_2.12/2.7.0//scala-collection-compat_2.12-2.7.0.jar
 scala-compiler/2.12.18//scala-compiler-2.12.18.jar
 scala-library/2.12.18//scala-library-2.12.18.jar
-scala-parser-combinators_2.12/2.2.0//scala-parser-combinators_2.12-2.2.0.jar
+scala-parser-combinators_2.12/2.3.0//scala-parser-combinators_2.12-2.3.0.jar
 scala-reflect/2.12.18//scala-reflect-2.12.18.jar
 scala-xml_2.12/2.1.0//scala-xml_2.12-2.1.0.jar
 shims/0.9.45//shims-0.9.45.jar
diff --git a/pom.xml b/pom.xml
index deccc904dd910..588f91155d6e9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1102,7 +1102,7 @@
       <dependency>
         <groupId>org.scala-lang.modules</groupId>
         <artifactId>scala-parser-combinators_${scala.binary.version}</artifactId>
-        <version>2.2.0</version>
+        <version>2.3.0</version>
       </dependency>
       <dependency>
         <groupId>jline</groupId>

From 16295a338220cc66fea5d91bcbd4213df0f2d4bd Mon Sep 17 00:00:00 2001
From: yangjie01 <yangjie01@baidu.com>
Date: Tue, 4 Jul 2023 22:37:04 +0800
Subject: [PATCH 02/13] [SPARK-44297][CORE][TESTS] Make
 `ClassLoaderIsolationSuite` test pass with Scala 2.13

### What changes were proposed in this pull request?
The main change of this pr as follows:

1. rename `TestHelloV2.jar` and `TestHelloV3.jar` added in https://github.com/apache/spark/pull/41789 to `TestHelloV2_2.12.jar` and `TestHelloV3_2.12.jar`
2. Add corresponding `TestHelloV2_2.13.jar` and `TestHelloV3_2.13.jar` which compiled with Scala 2.13
3. Make `ClassLoaderIsolationSuite` use the correct jar in testing

### Why are the changes needed?
Make `ClassLoaderIsolationSuite` test pass with Scala 2.13.

The Scala 2.13 daily test failed after https://github.com/apache/spark/pull/41789
- https://github.com/apache/spark/actions/runs/5447771717/jobs/9910185372

```
[info] - Executor classloader isolation with JobArtifactSet *** FAILED *** (83 milliseconds)
[info]   org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 0.0 failed 1 times, most recent failure: Lost task 0.0 in stage 0.0 (TID 0) (localhost executor driver): java.lang.NoClassDefFoundError: scala/Serializable
[info] 	at java.lang.ClassLoader.defineClass1(Native Method)
[info] 	at java.lang.ClassLoader.defineClass(ClassLoader.java:756)
[info] 	at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
[info] 	at java.net.URLClassLoader.defineClass(URLClassLoader.java:473)
[info] 	at java.net.URLClassLoader.access$100(URLClassLoader.java:74)
[info] 	at java.net.URLClassLoader$1.run(URLClassLoader.java:369)
[info] 	at java.net.URLClassLoader$1.run(URLClassLoader.java:363)
[info] 	at java.security.AccessController.doPrivileged(Native Method)
[info] 	at java.net.URLClassLoader.findClass(URLClassLoader.java:362)
[info] 	at java.lang.ClassLoader.loadClass(ClassLoader.java:418)
[info] 	at java.lang.ClassLoader.loadClass(ClassLoader.java:351)
[info] 	at java.lang.Class.forName0(Native Method)
[info] 	at java.lang.Class.forName(Class.java:348)
[info] 	at org.apache.spark.util.SparkClassUtils.classForName(SparkClassUtils.scala:35)
[info] 	at org.apache.spark.util.SparkClassUtils.classForName$(SparkClassUtils.scala:30)
[info] 	at org.apache.spark.util.Utils$.classForName(Utils.scala:94)
[info] 	at org.apache.spark.executor.ClassLoaderIsolationSuite.$anonfun$new$3(ClassLoaderIsolationSuite.scala:53)
[info] 	at scala.runtime.java8.JFunction1$mcVI$sp.apply(JFunction1$mcVI$sp.scala:18)
[info] 	at scala.collection.IterableOnceOps.foreach(IterableOnce.scala:576)
[info] 	at scala.collection.IterableOnceOps.foreach$(IterableOnce.scala:574)
[info] 	at org.apache.spark.InterruptibleIterator.foreach(InterruptibleIterator.scala:28)
[info] 	at org.apache.spark.rdd.RDD.$anonfun$foreach$2(RDD.scala:1028)
[info] 	at org.apache.spark.rdd.RDD.$anonfun$foreach$2$adapted(RDD.scala:1028)
[info] 	at org.apache.spark.SparkContext.$anonfun$runJob$5(SparkContext.scala:2406)
[info] 	at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
[info] 	at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:161)
[info] 	at org.apache.spark.scheduler.Task.run(Task.scala:141)
[info] 	at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:593)
[info] 	at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1478)
[info] 	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:596)
[info] 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[info] 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[info] 	at java.lang.Thread.run(Thread.java:750)
[info] Caused by: java.lang.ClassNotFoundException: scala.Serializable
[info] 	at java.net.URLClassLoader.findClass(URLClassLoader.java:387)
[info] 	at java.lang.ClassLoader.loadClass(ClassLoader.java:418)
[info] 	at java.lang.ClassLoader.loadClass(ClassLoader.java:351)
[info] 	... 33 more
```

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
- Pass GitHub Actions
- Manual check:

**Scala 2.12**
```
build/sbt "core/testOnly *ClassLoaderIsolationSuite"
```

```
[info] ClassLoaderIsolationSuite:
[info] - Executor classloader isolation with JobArtifactSet (1 second, 394 milliseconds)
[info] Run completed in 2 seconds, 437 milliseconds.
[info] Total number of tests run: 1
[info] Suites: completed 1, aborted 0
[info] Tests: succeeded 1, failed 0, canceled 0, ignored 0, pending 0
[info] All tests passed.
```

**Scala 2.13**

```
dev/change-scala-version.sh 2.13
build/sbt "core/testOnly *ClassLoaderIsolationSuite" -Pscala-2.13
```

```
[info] ClassLoaderIsolationSuite:
[info] - Executor classloader isolation with JobArtifactSet (1 second, 355 milliseconds)
[info] Run completed in 2 seconds, 264 milliseconds.
[info] Total number of tests run: 1
[info] Suites: completed 1, aborted 0
[info] Tests: succeeded 1, failed 0, canceled 0, ignored 0, pending 0
[info] All tests passed.
```

Closes #41852 from LuciferYang/SPARK-44297.

Authored-by: yangjie01 <yangjie01@baidu.com>
Signed-off-by: yangjie01 <yangjie01@baidu.com>
---
 .../{TestHelloV2.jar => TestHelloV2_2.12.jar}    | Bin
 core/src/test/resources/TestHelloV2_2.13.jar     | Bin 0 -> 4118 bytes
 .../{TestHelloV3.jar => TestHelloV3_2.12.jar}    | Bin
 core/src/test/resources/TestHelloV3_2.13.jar     | Bin 0 -> 3827 bytes
 .../executor/ClassLoaderIsolationSuite.scala     |  14 ++++++++++++--
 5 files changed, 12 insertions(+), 2 deletions(-)
 rename core/src/test/resources/{TestHelloV2.jar => TestHelloV2_2.12.jar} (100%)
 create mode 100644 core/src/test/resources/TestHelloV2_2.13.jar
 rename core/src/test/resources/{TestHelloV3.jar => TestHelloV3_2.12.jar} (100%)
 create mode 100644 core/src/test/resources/TestHelloV3_2.13.jar

diff --git a/core/src/test/resources/TestHelloV2.jar b/core/src/test/resources/TestHelloV2_2.12.jar
similarity index 100%
rename from core/src/test/resources/TestHelloV2.jar
rename to core/src/test/resources/TestHelloV2_2.12.jar
diff --git a/core/src/test/resources/TestHelloV2_2.13.jar b/core/src/test/resources/TestHelloV2_2.13.jar
new file mode 100644
index 0000000000000000000000000000000000000000..6dee8fcd9c95702fb7909c909c6faf22fe93766d
GIT binary patch
literal 4118
zcmZ{n2Q*w;7sp2#MvYFikeLa>Ac$xqA$s&Kf*^>_h{Pa?&Ql_K8AOZTTlC($L?_Bb
z@4ZC(czN$7dFy@q-n-Vl>;C`y>~q#SXYHTzBP?tR00;yE;QTSC3HT-O0N4NpNp&%p
ztfCaxc_#pX1yFv33%XKZ{v}iXzh;U{#Xn{RF-2J^Ni}s&1*u(y?k@R9FwOy@N3h$y
z-F?NX+=IL`^9W8k1rAw-0mm$CJng0H&2%U_w&wzx5eux?T2xV~3skGDP7Bag^Bu%C
z0^`pzG(1^6BrgZKdalaA2LLWhJN5p34%nA}jcu*Dekx3Sepe9wr7&?aw6?P{`BV90
zJs$i^N%JqIjER+%Eriq9%Fw~VTU}3yT#N$#MKzK$H~c0OCZ=j3@EL~6pynxnP~kBX
zm2B)0PdF*nr>dTp*)7v^SkH~fQAZZ&V4kU%ywoI#iM6%Ke1}cA!>c!E=jT}U`25$=
z&~z(5AucTrLo2*41Q{*O!!@tBGd?ZrVB^IZiEt4jF`rdAOCc$-@>`#}l}_u)XvGO@
zZof==jHWW{-w~#>MWFEf+Z}Jo@a{d6i!?_~(LUsk88=(%WxeA8a(2Wf2};DB<zD|z
zQz@ddA8w|J-)0)Y+{{k78g3{M(~~9?rcA7v?kn<LIFQzc#Eoz=3eR?hrh!$Eyi&9H
z^~~}dWkzP)O8;o4Q<||_DEgv?r?x~396T?-!Bf1f*h71kIqhr|5^XJMD_oYCQqe=w
zQ)>P>G+lz`OL<e`CqiJlye}fE-Bu}%HL7S#qBNPAvVjC$v|Iu5(XDb3GKJ!LcJx|2
z=76op82w>5BUc2zq4$M6U$O-a6^jYWRHXNb(G;`kIp{9p-sv^$A)_vabc{R3f6w&s
zB}*nU{E8mRGH3@gWIN@g5U9lZI4-{5*`Jr!wHoQfZTuR>E9)8Wz&awzVio$@goCTv
zc+oDZs+vP&_A|XC0V7vU->&~A8E3RK|Dgrv`o4#n!y|MA8_{c^j>~Xb1*OhO{6q3o
zKJ8@Yb<7}cI2d8EJh76dA<bpZ15pU`J>BOgW#zY9XE$1e@Q7&S1r7sp&-_j>cK0;L
z`^`)&4|P-SqBn8XdC>TtkdAiPZK#ZG%^xA?C2cHYvnUb|e8Zyod2stzlGX^l^bneK
zKeLi+xH1f2$w2u}++t!p9l9Q&3<}+6UI##4GX+S1LO0c7tzklfH%=f<WS@H3qJ;^Z
z0ZLfP%_$dMPq7hp=6%~qmGZ&8E(++iev|$BMN`S-QlTfXRxBcsib+FZ>9@j!?Ex*)
z+^#Pl789lRU69i6czQZza?lPh!xcx2Wg6M0h6>pb(Hx<M4-a;5uMPnQEfbQp%TvG@
z_+$6|_l}|duO0Ja!$fH7s?l{&io$av`6`)$z3RoEx_FU*NQ&xdJgr#3*FA^PZ+J(e
z85-PhAw*2(7!y?}mmTT@YXeo?h;dw&i2cgYtic7x3ze)QS;gRINHKi-&fyn_%U=xM
z?(PekRUVxiygjA^OnOX{zAp<Be<G|w;ERNb=Fac>5P{s;PgYngMDnQ)(GC@_t*flC
z<m&atHFqss&qqfy?a0CX7D6jhuCl_zL)GWvJ@a?zMnM7!CCm<Z?v`UK<Hpwag6+1U
zqK`ix`dd^Wa-etxlR7%=q+OQm#;7(#Cqwv>j-sOMs2pE7KImoyN0K^jSrL@w1zz}p
zpUO3deo~dtxf1a<3N}bpIUJnLiE5_;er=MtxXU)qoG$AkqjKOK9-Ry);O{hrqy2U9
zLvkQDG$@d&RIVz`*G46sTM^=Q@!zuFm<H>SGb|lyuY@YjF?!>nmcfo+Z-PW4U#7#5
z*Nm^r{^6I<tk>%)f)~UR7%ooF_pI;Z21;NKXQw1+7a1c&Nfw%EZU%d9n?L5+ZcrM1
zz!-z4h<utgzs$hGo}qvDE+VqlI)-tj0AH3$h9cpej^E7?#+F-w{wklR?`V$363JM!
za@{dEVoN=W<D1b|zUS+(I-DcVc$|TX7+=(qI(TNEM1g#ObCz65!Q6PRS(hONg3H*J
zkE=nr+sWLAVtNK9ou3`A$Un7;!WkSMOHYYVqHa#%PKuB&o?W(q`kfGdN(=7=Ihd71
znl;BNGS{stT6*WoNUNvHX)%3Ft3DLff_yYf7*IN!0;4|&bmzQOL`EmdkcWnEkF!|{
zzgO>&YoSh-%af<Kfko;*OdV>d#Bt5yPC(Hj;+y$tG+ZK|IS#NWJNC2pkvT&IAjp9J
zP!d{Akpan$I&L;dyLCrC3KoIM0z)01^21A3<aHT=l(%rg6PK>TnM8G<xNEE`OZ0C;
zLU0a^Ry#a4964TvjRJI!mtJ}i%H}*65r=F%i%><yDblJXH54|6k@j#U<Q|Y%+Bl$x
z)J5J;PRF))BA67bv^tG9Sir)T*0VM4_04ablO14EYr?vjYkU(lV~_GT<dO6Y;>zZ5
z>oJ*)CAR5rF+l^x7*wSnJ95vM=r$EX2AFSWuJbC{3Tp4!<cF^0KM&cF4IF(}ax!}t
z{9fKXWNQf3d~S1UR|Bi#{}QuTY##+zaP<`p5lK+@8eG`$1b^&e1`U?=XM@aQWZOde
zHHPkQJle1r-dbKuQxlK@4j%0b!(KS(+|m_o*nHz{f3GM-T7R8LW%lE&GNi3IMAdm)
z!#1uUey=(4{yB$=U*&!_8L9aK2YdF7lMHEBq{k`(m1IA*tIX^%pP^^??IUb+hT*Vy
zqcLT|(*-v29icSd*tFwg)Zpqo31jy!Kr=OlNg$Kv=hdw|{l1^|`Z}OeS|^jsNx+d`
z*EL4e3%ksb=JCvehBqIuIk3lfuN;znnrzGwo9ZPyY4GY=Sf+I$jgWvtgrVtk0tA`K
zO|7G7NMUz&z*yLr;y|q7dCQcJX_CW$jU>&H1Oww*i!;+xaX(cLNTzO#DQ~m2Yk0i>
ztG&&vro2LJ->}ctJULdZ45FWCzAlqkjWBdsdIKUk5&m_A=e?lO+micR+`%;)wN?+U
z6{IXp>^2H==p>(1o!8ZD3`=!U?$OkuYP4g+f+A0jf&p-%X~>Lq{k4fi%9!46xORu4
zPod8}RS=6_KhFBiquDHiJh!=eeOwF!@-poz%lyfys16B~Q+Xmmp1s#9Hi!0nS9>m3
zti3xSv(1LoOw)?A2~R#{i4js0Hq!>E{5~aHXgzoQ3TxiataH5ZG5<UJZ_OsV_cbjt
zU2~`p<ux9o*1T>K9+>pK{u&y!zUCq7C)9v{cIut53Z*VjlElU^;@nbqv5y<)Csv_a
z)vL*f+DuV=@<D@|u*nY#ZbSNfmT{KBS}S+1YZluVZ#%OvK&^>>Brdjo({7z|VcRmx
zc&jv-kZ99?B4%X*<JHR{P1ZK&>(@);M%>)$xX7&u!Mt8%*g_W58*+s?)Fn=x0z+~E
z5ojJ_t1ZBQL+f%;2wP++d_DTX1hOEOLDB`JMjY*BKDjA^R@H&6PC)5gy>c9z+RZuJ
z#XB*ApZd6^w;sfwUbIz=?7BbLYK(MMcfphQ0}__4K^aX&IsPaBb!EIdUQ?ckx!>0b
z%qDS6Y@)?DDSI3zl9hqXvjWF2;iQ@HR@4L)j@j{-ek=8yAWm?&SGp2NBr1j3H}rY8
zq%dV*bGvBKqslJ0t@j-PLd0Kvw51rfHG0RGOl@0m>B+`xksiI8c3Z?apHVrJK|Cwd
z^RWyYu?LgR)0;NkaX4k10$yNFV#WlGZ=Fv&bDD5iH11@($562l7XU(we2}eAHJ#}S
z1~!4!a}|g9<bicJM?ODD*PygbbUJyma4l84PA^%8_wjK>A#XloO0tu__G1y9nv5Pz
ze8KhdvicyE^4HucjKi((pecyzT=?KIQ-OAnzeY{!w;WXtS`pKro4!0?s9HDJD6QNw
zQFSx#llCmOD->gSe0zD`1T%k1UmsGi@7^B3u6#r&O}fW4I(~ueetwU{HRv_whi_jr
zN>kpll-+N-(e_=i%n5kN?A6ZBK`SVaIg4#2>9q{1b*J03AI~DcOI6*5Bq5ij=ywGY
z(GwBy#@^>;Bl|Ms^W2u{v{aC(-?(U=*q+{=dia70M;|r##$74fJoSEZ-_tGWJ(q!n
zyv-fPxjyc_eOxyZePR?x?Xh71xnpVCwi9dFyE=~o#<f!_DO6okP?i>v`{88Ti|RH>
z3Qm8SnU~}~wllrHcj#>^1ui{zzFGztm$nR^gmv}$3`d=p)NVet&aCI0P|wI$XVUmG
zSWl|OPE|c}jBDirurS0rD#cww^Hh6xTb@f*6V-Z;jojMe>Jdo4@XBsBwHz&?tt<S_
z+{~mGY|!mh!glfm&gVPLGKg=pf5Zb^TTFIfwAN0r5cnp5rYm-@^Xz{Tc$-(#to40&
zSU`6|qbbogQXxr3A!!Eao#*_a=D;SV(iPXjldthDm?V?jk1y<&*8NG~i$wzGnu%AQ
z89@g)2UCsiV|BZEVSK`CFbb{!w<%|An*D;RMAzkq;VjB|?B|qagcI@?^_ybXql1#R
z4BUElC<SUU<+cxjghZ~mjb7g5#M)ci&=-i_>JO)6dDutYEJKo1-yQWDxwYTcqs_4b
z-ru@#Te|kVt)WivA>D=T>|202r)}EhJ$8()$!q=4)zvpLa^n=o=euy7JL+t@@p@#t
z(#5E{zXfu8X#4g44%$**pJR)iT*k}5cvV6R_dE_tv48J*y)M3xy|^ho3fRXXV`_LO
zf<L99`xUQa&l2BiWsI09(H*5hjxVMa-qVO$dAEx{IgT7tZ3TD<Z{lQEog@Y(1>koY
z{Tnr}X!Q5^2QRPK^xxKB@$ia8e+K;W^&jgmWO|hWAMpP%@@IzT%hw-J`P1=VSozbj
z@R#E!Wd5A!6=we0PG4c>&mh70G1lMCU!n8YIbUV@bt$h}@1G%lxpr3-|CZ%PBd+`i
Sh;t>xxx8Ln4i-p!_4Pme8xa%$

literal 0
HcmV?d00001

diff --git a/core/src/test/resources/TestHelloV3.jar b/core/src/test/resources/TestHelloV3_2.12.jar
similarity index 100%
rename from core/src/test/resources/TestHelloV3.jar
rename to core/src/test/resources/TestHelloV3_2.12.jar
diff --git a/core/src/test/resources/TestHelloV3_2.13.jar b/core/src/test/resources/TestHelloV3_2.13.jar
new file mode 100644
index 0000000000000000000000000000000000000000..0c292e7d81ad741dbc11177224b9c30519b7a362
GIT binary patch
literal 3827
zcmZ{n2Q-{%7sm&K(R-LdMD#j(8=^#K^kI~U-i1+;SQ0f>L=8soC4v-0?;?Wey=O!T
z5jBa<d~9}CcF%tIJ?}Zsd*1*5KKFUfx#!%Uo)#XE761Z)0KYw6G6wt-L;xTFu41Gp
zprNA*J?jAg@Bn&RgrExr?w>Nf|7)f_SNv^;E9z*dsu&sx!d1V)dp~Gv2?!36X$kQ5
z_YPDV2#tzjmc0Zu;rtr#A>R@p5posW!I;j-<0NIAw89NExsqJ4a%G+S{tCys^RCy9
z7tSALAVRr9RL(cKcrVHz1^~`Wql@_G8Gz@1Z9UwfmkQ{=6r_JD?C)E<d%D^Gq5OFt
z{Qpui{7b2B@8;&gE@<m!?d|=@XiJwyk@CTeL5!t2j~?#<4SsG_7fxRB3#Sl>hM7Vs
z`)Mr`ACu))+B)J%U;>K*P5NZVE6y|h&Z&*Z@F&v+{xF#6R$p^!1B-miL%i1^?@9BE
z%*B-xOkAslvPfp`I*JK}FR=!`CrgJNcaCr73OyjR4o{S+vdHI`eKIpn*z8I|MirjY
zbE`Ogc7@c;dA)o6(+IeSdyi%b98O&1E{$&Kr0lo5$3Q^R6U7#j5<^LiB4vdX&I!E@
zf_`a^oQ>-YMmR=noAuqWN%=sFO*Gq+`!F@&c&za_K^&}l#02RVjmpNaV0)4}DjZBw
zU>FFvRk;8}Xgl`j^j=9~>NM`FvH1Oy36=%1`q?zU*VEe}UU^gbPJK*{;4NEq+6J9n
zgNilom!Uoru$T(-!n0lj7LlokOH#vKT_g_v?5l5N<E}lkCI3ch>sSH_+%j#x&*LH?
zsRGpvdMh-;^WIge89}GoJ?A?4$tPmzU5#9gUxBrTk`HUyOrs?Qaor1BFrN%TT#hjE
z-hPBft&_A(rT!e<JfCz_g`ps7!?T?eCZ&xI3&d=?d&inQg33a2l&3zg%uv6=XC11j
z+VX5!7_x*mkPXld@UJllewdbIf*{Km8d1)BqeN&Y8A63hEv$z0%kKtc!zFC0T%~q!
z!?2Tg(;$SI)IPYIMg+Y1Tu;?CVw@_9fau$Iqi3NkNv~j=H(u;m$rp90^F<|ah_YNS
z>&kQ2Hu4g%kln-q@df#CaRcHS#02B9I2&u?I&2!9_7=IGkB_b^z?YkE9)~f$cke}b
z-S(_}Wd5F5?3J;^clx4H9z$Y7<h4M>G2b7Oi?qVwR#w7#Rw{Wzh90(cA`q7aoh_C6
zq(QucxRX0TkZO)~SY)IBZtRkJ924hGf!bS$9CTy?dVL|WzC>D+it~xxefF(gl8eSp
ztDPunJ@0HK{GT25-(5ieUtRFC36e})4QE#AZoIPWu6CPJZ3e8=i*TlBZOgDP*r@ZS
zYiVnPD3z|gE^~X5(@`BI%;Pq*w$(*3*{#jRGOUw6=j<pm1KrS;-~a5Bo@HXJ9YF+7
zEuWw9>tB}sEJuw!8OX*G10)wXaO$kkG)K>kU!uQv$Tb;>WGaB($W2(fJbr5Iher0v
zPmuU>>EX^Vh8Pc1Q@A?pgt-#|mC!lXoD$#Da+9LA-UMje3rPriKP+)a5%hf-GMV;J
zPs#T&S@pC~7UIp323(hla@W!KG-aERTrs7TC*%Ce1zvf&OJ$+-UnbD3fup^sm{e|L
zZ?14N(^lSSgVW^k^MR9eioJ|bm8WHSb(x>0#JG;|hdy#q+~WaXS3?})Ctn%y(Yx#D
z)D9(y<a+CIgwM)Xnx7X4<sj4-1ZV1?M|e>q#i9dqcZFeQHJIs58Wm|xUN4U&BYh**
zCW*j;jc-i!w4>EgtvgL$EmoK!c|SP2%CB}j=$_x_F+J>>PZsT9t)Wlml_O6ZeaBYl
zDu@-kHEY%#cjrs=iB~`}Tci^((QlY1Yz*!(VCUP#_Y}WIU3Ayml#Y4!sZfSkO~P`o
zxSzY?QicyFJ<CQt4@uKuG&hldSlf@BlKGPGM;q4`?tK0blaZ%x4KW2XYs$jWl37|e
z=jVv}{okCpHNt%FAqR28w3ubhWATlk)261z5m1H9(DzzINh?)8<@xUM>3R9=C9{h+
zZJ<Q;(|Y|#W3j-XqcO~L4=#k*@UEFGCa-1ZDxaD8s9JY;UfJQG4(82pLOmnx^)@ju
zLrzmcW#a=_%XI%qq#sb4!Klju#$2E;&ah81L87F3>T-=65*iwIdt}DeoDYV(p-U((
zkRdallFF=EdK1sc=vGK*IwX>u#s0Z!2swx8X6o4;WQL&4!!6(95C(n1I?JtBnUBwM
z9=2#r@9q~6kfi5jeDnNJ(>bT1f!PzOc~`}>ytgG&;ab`t8Fkz|C*YABT5_j`I$AsR
zczxL5(^?{!cKJmHZ}aT8K95{1;S@8);)sPy@_6lhs>>c~;HkW<_iHcrv+f`nR+9Kx
zW22@uszkMv-@8vNZ|e+Qom;x@QrwozHo108u-IQ;MRA~PwTp4~f6%>!=~XgA|F{+G
zvZ-JY|ExK|e|e-huf$?Jll`=f(I=*(FY9r#)_UC|<q7TiS{kJWo~gP_7*jUaGi&{5
z3TibJRuxssq#^e)yPN0B8GjNXVOH5!4&A+Ej;9}W(z4!_>v+Wej!n9G4U9N**OPq$
zEe>$N>Y&ztv(x<USQu31WH-R-UFvUmL~{G<<64-0ojeVUX(~&WVUPSC6IELw)V7JM
z8;BT<@!EQEc-o32%zgf4@VDY`ZJT{UWLjwzqNgI>`JXKnn?5{Bbmk~8rjC*KQXLTv
z-ky4m)=5nj3-oCG$Tp~<Uw|7uY$TCkGFFW*?6<UOkLpb*=9J;4Yn;YwXTh(fOwE>?
zN3t+NHqrzg18lXx2r|BTJ2fwT%-YBrt2i^%t=je>D9#W!c-Aj_rg$Q%NA#<5e?p}3
z3v@q-w8L9qLs;DHUd5QpoDd(k{9#L>gZaj+BD-AlgJ_*MIkq~WJQlvvZ|QZ97$l~Q
z@yZi{)=u=SKxXQL+`i~aP%wt0*~+^L4-#1;BN9NQ6moUd9&!xYo<y&&x&}z`wVC9+
z-r$CmNX+Asa5R;fe4%(Fa@S)uIuO)og+}JT$IwjGTaJ8jAY<0B^gRQkm6DN42A!Yk
zNM7ZSBG4QRA+>}|LcO>$VuNFp^}{y0xX=S-3`VAhu9w$qB}JT9^^`n#+kQz7jd4}4
zr$U7W#j@I{oGQS0#%wi;+@hH0+D)D@ciarXfgt@tRNjyjfz<cn%|yRA4crs9aF9GM
ze-^!u%Kz=@eU}y!U0&b(vi#d-bKFcY9Of&eBfaa?C%6uCp7cg#hx|A#FR*x$STzH;
z5q_Sz>=gUb9|=8d7FIt5Z8(-*#=VZ*&jM)$kw5z4h-f<4&NHE&{k@hE-wSAg|DwXv
zCokzncko94cK^F4v+Uf1W>Nxf(|$g|`|WDFwlhSrgGae{15$*aM9}RId=5tMzFjZJ
z*G(Cii6}`YV<z>z!Qvz^t>xlLzn7Bc@%q5$%6>WUV|xkX{XzEotqx4IWqi&_+5I}?
zdQzRd;&)R$xbJ}o8UO?3uQxzmoWyeD+YMwbQ&-CgWoAw|Z{9Heew1q(P&^hV-AiH#
z3G6RkrO@f|YjVJAM4}$i^O=oVHc0j+m?_>Co4j5z-*slYIN54gWgu+&AdRl5rzFCy
zFbq<eghwNNysZx+Cg@rmz!6kXn=0&}Fa|4+uLUU)nM`bFU{;Utd}10yU)D%*>2*FE
zom8`sjG;S0MTnzlQz|5kLN~}Hq52~qONy*#!~F%MsugAJwsSau807;P!+l!;P>0pv
zcy=ot4VCJ{u=pA@6cHd(20w~t#Zi^t-G>O7+g2xea|DL%K=qX<cUjw|6EU8&SlrpY
zRSF?gUC3T|+>4+-BxY%`mFR3>@XIi)_S9YwF3~pOE<~n%8@~{~s-3Vc%-FNz@lwGH
zZb$6kesvF}g8I6iwSJgMw|$uHrZRNOR`Ag*Ee5F>d@LCT!h(yA(zJiDzDxBkTH!BR
zNvHE%Y-~pl_4XjB8U$(Ea5ooxYkp`iMuM-XVhLuK+azUa(z1wFd*ABHnTPtCFV;w~
zol3Cf#)665kO3#abb-Wxe$T)WPpo~S>>bV>8qK>eH495qd!F?j9Rgdh9EsH9H#b_b
z_CKx!RmBf&vBHipG9pLB87HouEd-i7`^0x*&w#U}1H^3Yi?*thXN2nJR0~@ef3i%f
z1?HvOisoS&w2W3+&CUi%_!9$@ZW*oK;N7dRaJ=rfbTj1+=?dEm0kPy>dsHV;RvC?s
z3zPl_0|(1?uf)488~8sQAZ!TJJL7Qz*zyBIcH&tu;Oad3mX^Gr0Qgw*IA`n4zK}@I
zsC+_-xr1*^Y;(S|NKVpsW#)yhKUhoMk>Ab5t>zroZOp2i#I@F&k<9Iym1D=4utN^d
z?b{b4)#=JG4>=J4P)>O<QsLmz0{)3we{s|WYW*|*W~U4A`nUC0_PGG9%aA)?{%!pQ
zyDl;i1O7iFU1sPwU;ZSfOUHjP)1_nOPsb%eT^{s;p?;k!FBs}FC~<!7^_TNkqWbmB
p7g>J2%8MEFGOnE8-9^R!$ntX{)ziW!xDXPYKR)N1Ig?#{{SQ1WZp#1w

literal 0
HcmV?d00001

diff --git a/core/src/test/scala/org/apache/spark/executor/ClassLoaderIsolationSuite.scala b/core/src/test/scala/org/apache/spark/executor/ClassLoaderIsolationSuite.scala
index 33c1baccd7298..72ee0e96fd014 100644
--- a/core/src/test/scala/org/apache/spark/executor/ClassLoaderIsolationSuite.scala
+++ b/core/src/test/scala/org/apache/spark/executor/ClassLoaderIsolationSuite.scala
@@ -17,21 +17,31 @@
 
 package org.apache.spark.executor
 
+import scala.util.Properties
+
 import org.apache.spark.{JobArtifactSet, LocalSparkContext, SparkConf, SparkContext, SparkFunSuite}
 import org.apache.spark.util.Utils
 
 class ClassLoaderIsolationSuite extends SparkFunSuite with LocalSparkContext  {
+
+  private val scalaVersion = Properties.versionNumberString
+    .split("\\.")
+    .take(2)
+    .mkString(".")
+
   val jar1 = Thread.currentThread().getContextClassLoader.getResource("TestUDTF.jar").toString
 
   // package com.example
   // object Hello { def test(): Int = 2 }
   // case class Hello(x: Int, y: Int)
-  val jar2 = Thread.currentThread().getContextClassLoader.getResource("TestHelloV2.jar").toString
+  val jar2 = Thread.currentThread().getContextClassLoader
+    .getResource(s"TestHelloV2_$scalaVersion.jar").toString
 
   // package com.example
   // object Hello { def test(): Int = 3 }
   // case class Hello(x: String)
-  val jar3 = Thread.currentThread().getContextClassLoader.getResource("TestHelloV3.jar").toString
+  val jar3 = Thread.currentThread().getContextClassLoader
+    .getResource(s"TestHelloV3_$scalaVersion.jar").toString
 
   test("Executor classloader isolation with JobArtifactSet") {
     sc = new SparkContext(new SparkConf().setAppName("test").setMaster("local"))

From 68862589a0cc0c450434d3a1675bb241a5917f1b Mon Sep 17 00:00:00 2001
From: yangjie01 <yangjie01@baidu.com>
Date: Tue, 4 Jul 2023 22:38:43 +0800
Subject: [PATCH 03/13] [SPARK-44296][BUILD] Upgrade dropwizard metrics 4.2.19

### What changes were proposed in this pull request?
This pr aims upgrade dropwizard metrics to 4.2.19.

### Why are the changes needed?
The new version bring a bug fix related to metrics-jetty module:
- https://github.com/dropwizard/metrics/pull/3379

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Pass Github Actions

Closes #41849 from LuciferYang/SPARK-44296.

Authored-by: yangjie01 <yangjie01@baidu.com>
Signed-off-by: yangjie01 <yangjie01@baidu.com>
---
 dev/deps/spark-deps-hadoop-3-hive-2.3 | 10 +++++-----
 pom.xml                               |  2 +-
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3
index 910b1703c70c8..1cdf08f321eb6 100644
--- a/dev/deps/spark-deps-hadoop-3-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-3-hive-2.3
@@ -177,11 +177,11 @@ log4j-slf4j2-impl/2.20.0//log4j-slf4j2-impl-2.20.0.jar
 logging-interceptor/3.12.12//logging-interceptor-3.12.12.jar
 lz4-java/1.8.0//lz4-java-1.8.0.jar
 mesos/1.4.3/shaded-protobuf/mesos-1.4.3-shaded-protobuf.jar
-metrics-core/4.2.18//metrics-core-4.2.18.jar
-metrics-graphite/4.2.18//metrics-graphite-4.2.18.jar
-metrics-jmx/4.2.18//metrics-jmx-4.2.18.jar
-metrics-json/4.2.18//metrics-json-4.2.18.jar
-metrics-jvm/4.2.18//metrics-jvm-4.2.18.jar
+metrics-core/4.2.19//metrics-core-4.2.19.jar
+metrics-graphite/4.2.19//metrics-graphite-4.2.19.jar
+metrics-jmx/4.2.19//metrics-jmx-4.2.19.jar
+metrics-json/4.2.19//metrics-json-4.2.19.jar
+metrics-jvm/4.2.19//metrics-jvm-4.2.19.jar
 minlog/1.3.0//minlog-1.3.0.jar
 netty-all/4.1.93.Final//netty-all-4.1.93.Final.jar
 netty-buffer/4.1.93.Final//netty-buffer-4.1.93.Final.jar
diff --git a/pom.xml b/pom.xml
index 588f91155d6e9..2e29d1de0c986 100644
--- a/pom.xml
+++ b/pom.xml
@@ -152,7 +152,7 @@
     If you changes codahale.metrics.version, you also need to change
     the link to metrics.dropwizard.io in docs/monitoring.md.
     -->
-    <codahale.metrics.version>4.2.18</codahale.metrics.version>
+    <codahale.metrics.version>4.2.19</codahale.metrics.version>
     <!-- Should be consistent with SparkBuild.scala and docs -->
     <avro.version>1.11.1</avro.version>
     <aws.kinesis.client.version>1.12.0</aws.kinesis.client.version>

From d53585c91b2e29fc4ef3d35a7cf78440c8d37938 Mon Sep 17 00:00:00 2001
From: Jiaan Geng <beliefer@163.com>
Date: Tue, 4 Jul 2023 20:08:47 +0300
Subject: [PATCH 04/13] [SPARK-44292][SQL] Assign names to the error class
 _LEGACY_ERROR_TEMP_[2315-2319]

### What changes were proposed in this pull request?
The pr aims to assign names to the error class _LEGACY_ERROR_TEMP_[2315-2319].

### Why are the changes needed?
Improve the error framework.

### Does this PR introduce _any_ user-facing change?
'No'.

### How was this patch tested?
Exists test cases updated and added new test cases.

Closes #41850 from beliefer/SPARK-44292.

Authored-by: Jiaan Geng <beliefer@163.com>
Signed-off-by: Max Gekk <max.gekk@gmail.com>
---
 .../main/resources/error/error-classes.json   | 57 +++++++++++--------
 ...onditions-datatype-mismatch-error-class.md |  4 ++
 ...ns-invalid-observed-metrics-error-class.md | 42 ++++++++++++++
 .../sql/catalyst/analysis/CheckAnalysis.scala | 24 +++++---
 .../sql/catalyst/analysis/AnalysisSuite.scala | 40 +++++++++----
 5 files changed, 122 insertions(+), 45 deletions(-)
 create mode 100644 docs/sql-error-conditions-invalid-observed-metrics-error-class.md

diff --git a/common/utils/src/main/resources/error/error-classes.json b/common/utils/src/main/resources/error/error-classes.json
index 6a72fc5449eed..3126fb9519b16 100644
--- a/common/utils/src/main/resources/error/error-classes.json
+++ b/common/utils/src/main/resources/error/error-classes.json
@@ -558,6 +558,11 @@
           "The data type of the column (<columnIndex>) do not have the same type: <leftType> (<leftParamIndex>) <> <rightType> (<rightParamIndex>)."
         ]
       },
+      "TYPE_CHECK_FAILURE_WITH_HINT" : {
+        "message" : [
+          "<msg><hint>."
+        ]
+      },
       "UNEXPECTED_CLASS_TYPE" : {
         "message" : [
           "class <className> not found."
@@ -1331,6 +1336,33 @@
       "Numeric literal <rawStrippedQualifier> is outside the valid range for <typeName> with minimum value of <minValue> and maximum value of <maxValue>. Please adjust the value accordingly."
     ]
   },
+  "INVALID_OBSERVED_METRICS" : {
+    "message" : [
+      "Invalid observed metrics."
+    ],
+    "subClass" : {
+      "MISSING_NAME" : {
+        "message" : [
+          "The observed metrics should be named: <operator>."
+        ]
+      },
+      "NESTED_AGGREGATES_UNSUPPORTED" : {
+        "message" : [
+          "Nested aggregates are not allowed in observed metrics, but found: <expr>."
+        ]
+      },
+      "NON_AGGREGATE_FUNC_ARG_IS_NON_DETERMINISTIC" : {
+        "message" : [
+          "Non-deterministic expression <expr> can only be used as an argument to an aggregate function."
+        ]
+      },
+      "WINDOW_EXPRESSIONS_UNSUPPORTED" : {
+        "message" : [
+          "Window expressions are not allowed in observed metrics, but found: <expr>."
+        ]
+      }
+    }
+  },
   "INVALID_OPTIONS" : {
     "message" : [
       "Invalid options:"
@@ -5614,31 +5646,6 @@
       "The input <valueType> '<input>' does not match the given number format: '<format>'."
     ]
   },
-  "_LEGACY_ERROR_TEMP_2315" : {
-    "message" : [
-      "cannot resolve '<sqlExpr>' due to data type mismatch: <msg><hint>."
-    ]
-  },
-  "_LEGACY_ERROR_TEMP_2316" : {
-    "message" : [
-      "observed metrics should be named: <operator>."
-    ]
-  },
-  "_LEGACY_ERROR_TEMP_2317" : {
-    "message" : [
-      "window expressions are not allowed in observed metrics, but found: <sqlExpr>."
-    ]
-  },
-  "_LEGACY_ERROR_TEMP_2318" : {
-    "message" : [
-      "non-deterministic expression <sqlExpr> can only be used as an argument to an aggregate function."
-    ]
-  },
-  "_LEGACY_ERROR_TEMP_2319" : {
-    "message" : [
-      "nested aggregates are not allowed in observed metrics, but found: <sqlExpr>."
-    ]
-  },
   "_LEGACY_ERROR_TEMP_2320" : {
     "message" : [
       "distinct aggregates are not allowed in observed metrics, but found: <sqlExpr>."
diff --git a/docs/sql-error-conditions-datatype-mismatch-error-class.md b/docs/sql-error-conditions-datatype-mismatch-error-class.md
index 7d203432562d3..6ec4f07895029 100644
--- a/docs/sql-error-conditions-datatype-mismatch-error-class.md
+++ b/docs/sql-error-conditions-datatype-mismatch-error-class.md
@@ -207,6 +207,10 @@ The lower bound of a window frame must be `<comparison>` to the upper bound.
 
 The data type of the column (`<columnIndex>`) do not have the same type: `<leftType>` (`<leftParamIndex>`) <> `<rightType>` (`<rightParamIndex>`).
 
+## TYPE_CHECK_FAILURE_WITH_HINT
+
+`<msg>``<hint>`.
+
 ## UNEXPECTED_CLASS_TYPE
 
 class `<className>` not found.
diff --git a/docs/sql-error-conditions-invalid-observed-metrics-error-class.md b/docs/sql-error-conditions-invalid-observed-metrics-error-class.md
new file mode 100644
index 0000000000000..10c144e06fa79
--- /dev/null
+++ b/docs/sql-error-conditions-invalid-observed-metrics-error-class.md
@@ -0,0 +1,42 @@
+---
+layout: global
+title: INVALID_OBSERVED_METRICS error class
+displayTitle: INVALID_OBSERVED_METRICS error class
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+Invalid observed metrics.
+
+This error class has the following derived error classes:
+
+## MISSING_NAME
+
+The observed metrics should be named: `<operator>`.
+
+## NESTED_AGGREGATES_UNSUPPORTED
+
+Nested aggregates are not allowed in observed metrics, but found: `<expr>`.
+
+## NON_AGGREGATE_FUNC_ARG_IS_NON_DETERMINISTIC
+
+Non-deterministic expression `<expr>` can only be used as an argument to an aggregate function.
+
+## WINDOW_EXPRESSIONS_UNSUPPORTED
+
+Window expressions are not allowed in observed metrics, but found: `<expr>`.
+
+
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
index 11387fde37e52..5507fa28bc4e2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
@@ -277,13 +277,13 @@ trait CheckAnalysis extends PredicateHelper with LookupCatalog with QueryErrorsB
                 e.dataTypeMismatch(e, checkRes)
               case TypeCheckResult.TypeCheckFailure(message) =>
                 e.setTagValue(DATA_TYPE_MISMATCH_ERROR, true)
-                extraHintForAnsiTypeCoercionExpression(operator)
+                val extraHint = extraHintForAnsiTypeCoercionExpression(operator)
                 e.failAnalysis(
-                  errorClass = "_LEGACY_ERROR_TEMP_2315",
+                  errorClass = "TYPE_CHECK_FAILURE_WITH_HINT",
                   messageParameters = Map(
-                    "sqlExpr" -> e.sql,
+                    "expr" -> toSQLExpr(e),
                     "msg" -> message,
-                    "hint" -> extraHintForAnsiTypeCoercionExpression(operator)))
+                    "hint" -> extraHint))
               case checkRes: TypeCheckResult.InvalidFormat =>
                 e.setTagValue(INVALID_FORMAT_ERROR, true)
                 e.invalidFormat(checkRes)
@@ -486,8 +486,8 @@ trait CheckAnalysis extends PredicateHelper with LookupCatalog with QueryErrorsB
           case CollectMetrics(name, metrics, _) =>
             if (name == null || name.isEmpty) {
               operator.failAnalysis(
-                errorClass = "_LEGACY_ERROR_TEMP_2316",
-                messageParameters = Map("operator" -> operator.toString))
+                errorClass = "INVALID_OBSERVED_METRICS.MISSING_NAME",
+                messageParameters = Map("operator" -> planToString(operator)))
             }
             // Check if an expression is a valid metric. A metric must meet the following criteria:
             // - Is not a window function;
@@ -498,11 +498,17 @@ trait CheckAnalysis extends PredicateHelper with LookupCatalog with QueryErrorsB
             def checkMetric(s: Expression, e: Expression, seenAggregate: Boolean = false): Unit = {
               e match {
                 case _: WindowExpression =>
-                  e.failAnalysis("_LEGACY_ERROR_TEMP_2317", Map("sqlExpr" -> s.sql))
+                  e.failAnalysis(
+                    "INVALID_OBSERVED_METRICS.WINDOW_EXPRESSIONS_UNSUPPORTED",
+                    Map("expr" -> toSQLExpr(s)))
                 case _ if !e.deterministic && !seenAggregate =>
-                  e.failAnalysis("_LEGACY_ERROR_TEMP_2318", Map("sqlExpr" -> s.sql))
+                  e.failAnalysis(
+                    "INVALID_OBSERVED_METRICS.NON_AGGREGATE_FUNC_ARG_IS_NON_DETERMINISTIC",
+                    Map("expr" -> toSQLExpr(s)))
                 case a: AggregateExpression if seenAggregate =>
-                  e.failAnalysis("_LEGACY_ERROR_TEMP_2319", Map("sqlExpr" -> s.sql))
+                  e.failAnalysis(
+                    "INVALID_OBSERVED_METRICS.NESTED_AGGREGATES_UNSUPPORTED",
+                    Map("expr" -> toSQLExpr(s)))
                 case a: AggregateExpression if a.isDistinct =>
                   e.failAnalysis("_LEGACY_ERROR_TEMP_2320", Map("sqlExpr" -> s.sql))
                 case a: AggregateExpression if a.filter.isDefined =>
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
index 55005d87cdc03..61cb3e81c98a6 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
@@ -770,8 +770,12 @@ class AnalysisSuite extends AnalysisTest with Matchers {
 
     // Bad name
     assert(!CollectMetrics("", sum :: Nil, testRelation).resolved)
-    assertAnalysisError(CollectMetrics("", sum :: Nil, testRelation),
-      "observed metrics should be named" :: Nil)
+    assertAnalysisErrorClass(
+      CollectMetrics("", sum :: Nil, testRelation),
+      expectedErrorClass = "INVALID_OBSERVED_METRICS.MISSING_NAME",
+      expectedMessageParameters = Map(
+        "operator" -> "'CollectMetrics , [sum(a#x) AS sum#xL]\n+- LocalRelation <empty>, [a#x]\n")
+    )
 
     // No columns
     assert(!CollectMetrics("evt", Nil, testRelation).resolved)
@@ -786,9 +790,11 @@ class AnalysisSuite extends AnalysisTest with Matchers {
       "Attribute", "can only be used as an argument to an aggregate function")
 
     // Unwrapped non-deterministic expression
-    checkAnalysisError(
-      Rand(10).as("rnd") :: Nil,
-      "non-deterministic expression", "can only be used as an argument to an aggregate function")
+    assertAnalysisErrorClass(
+      CollectMetrics("event", Rand(10).as("rnd") :: Nil, testRelation),
+      expectedErrorClass = "INVALID_OBSERVED_METRICS.NON_AGGREGATE_FUNC_ARG_IS_NON_DETERMINISTIC",
+      expectedMessageParameters = Map("expr" -> "\"rand(10) AS rnd\"")
+    )
 
     // Distinct aggregate
     checkAnalysisError(
@@ -796,18 +802,30 @@ class AnalysisSuite extends AnalysisTest with Matchers {
     "distinct aggregates are not allowed in observed metrics, but found")
 
     // Nested aggregate
-    checkAnalysisError(
-      Sum(Sum(a).toAggregateExpression()).toAggregateExpression().as("sum") :: Nil,
-      "nested aggregates are not allowed in observed metrics, but found")
+    assertAnalysisErrorClass(
+      CollectMetrics(
+        "event",
+        Sum(Sum(a).toAggregateExpression()).toAggregateExpression().as("sum") :: Nil,
+        testRelation),
+      expectedErrorClass = "INVALID_OBSERVED_METRICS.NESTED_AGGREGATES_UNSUPPORTED",
+      expectedMessageParameters = Map("expr" -> "\"sum(sum(a)) AS sum\"")
+    )
 
     // Windowed aggregate
     val windowExpr = WindowExpression(
       RowNumber(),
       WindowSpecDefinition(Nil, a.asc :: Nil,
         SpecifiedWindowFrame(RowFrame, UnboundedPreceding, CurrentRow)))
-    checkAnalysisError(
-      windowExpr.as("rn") :: Nil,
-      "window expressions are not allowed in observed metrics, but found")
+    assertAnalysisErrorClass(
+      CollectMetrics("event", windowExpr.as("rn") :: Nil, testRelation),
+      expectedErrorClass = "INVALID_OBSERVED_METRICS.WINDOW_EXPRESSIONS_UNSUPPORTED",
+      expectedMessageParameters = Map(
+        "expr" ->
+          """
+            |"row_number() OVER (ORDER BY a ASC NULLS FIRST ROWS
+            | BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS rn"
+            |""".stripMargin.replace("\n", ""))
+    )
   }
 
   test("check CollectMetrics duplicates") {

From c08700f05f96e083dd8dec12fb2ca90a49d16a52 Mon Sep 17 00:00:00 2001
From: vicennial <venkata.gudesa@databricks.com>
Date: Wed, 5 Jul 2023 08:41:47 +0900
Subject: [PATCH 05/13] [SPARK-44293][CONNECT] Fix invalid URI for custom JARs
 in Spark Connect

### What changes were proposed in this pull request?

This PR fixs the bug where invalid JAR URIs were being generated because the URI was stored as `artifactURI + "/" + target.toString` (here, `target` is the absolute path of the file) instead of `artifactURI + "/" + remoteRelativePath.toString` (here, the `remoteRelativePath` is in the form of `jars/...`)

### Why are the changes needed?

Without this change, Spark Connect users attempting to use a custom JAR (such as in UDFs) will hit task failure issue as an exception would be thrown during the JAR file fetch operation.
Example stacktrace:
```
23/07/03 17:00:15 INFO Executor: Fetching spark://ip-10-110-22-170.us-west-2.compute.internal:43743/artifacts/d9548b02-ff3b-4278-ab52-aef5d1fc724e//home/venkata.gudesa/spark/artifacts/spark-d6141194-c487-40fd-ba40-444d922808ea/d9548b02-ff3b-4278-ab52-aef5d1fc724e/jars/TestHelloV2.jar with timestamp 0
23/07/03 17:00:15 ERROR Executor: Exception in task 6.0 in stage 4.0 (TID 55)
java.lang.RuntimeException: Stream '/artifacts/d9548b02-ff3b-4278-ab52-aef5d1fc724e//home/venkata.gudesa/spark/artifacts/spark-d6141194-c487-40fd-ba40-444d922808ea/d9548b02-ff3b-4278-ab52-aef5d1fc724e/jars/TestHelloV2.jar' was not found.
	at org.apache.spark.network.client.TransportResponseHandler.handle(TransportResponseHandler.java:260)
	at org.apache.spark.network.server.TransportChannelHandler.channelRead0(TransportChannelHandler.java:142)
	at org.apache.spark.network.server.TransportChannelHandler.channelRead0(TransportChannelHandler.java:53)
	at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
	at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
	at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
	at org.apache.spark.network.util.TransportFrameDecoder.channelRead(TransportFrameDecoder.java:102)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
	at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
	at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	at java.lang.Thread.run(Thread.java:748)
```

### Does this PR introduce _any_ user-facing change?

No (the bug-fix is consistent with what users expect)

### How was this patch tested?

New E2E test in `ReplE2ESuite`.

Closes #41844 from vicennial/SPARK-44293.

Authored-by: vicennial <venkata.gudesa@databricks.com>
Signed-off-by: Hyukjin Kwon <gurwls223@apache.org>
---
 .../src/test/resources/TestHelloV2_2.12.jar   | Bin 0 -> 3784 bytes
 .../src/test/resources/TestHelloV2_2.13.jar   | Bin 0 -> 4118 bytes
 .../spark/sql/application/ReplE2ESuite.scala  |  44 +++++++++++++++++-
 .../SparkConnectArtifactManager.scala         |   2 +-
 4 files changed, 43 insertions(+), 3 deletions(-)
 create mode 100644 connector/connect/client/jvm/src/test/resources/TestHelloV2_2.12.jar
 create mode 100644 connector/connect/client/jvm/src/test/resources/TestHelloV2_2.13.jar

diff --git a/connector/connect/client/jvm/src/test/resources/TestHelloV2_2.12.jar b/connector/connect/client/jvm/src/test/resources/TestHelloV2_2.12.jar
new file mode 100644
index 0000000000000000000000000000000000000000..d89cf6543a20090abd953af39ac97608aa02050e
GIT binary patch
literal 3784
zcmaJ^1yB^+{#{}T=|(_BWTh6Q5fD&;1tbJ^iKUSR0qJI?8zf|rkZvRei4|#<l8_Lj
zl~}q%I-dN$H>2;p@B5!Scjn%?=l<q*@64GwM_m~YpB6wwL<G3z6Q~aO1GE5qfT}#~
z5wDUO<eusyH6@5V6vn3tx#$J}_EdX&RFrx72Fa9pxxe)GS8DPP3(YKC<5N`qqSB+H
z2F8h42|*-abM?4_03)8ci&$(<59TM+^7Uuk3wc-uI9G0NJ}6=F&9;;0B0V;DF@G_B
zMvz$*r)ZWDb)*{FwT8@R<(gf<1*j_%5{;z*sjYu?_Vd?N|JCm`0DuVaKmFqWIyJMm
zz4y-y(tk6|JxpyKY|Q_U`S)*U{4b`Wxs8oIJD-`2sk3u1O#dm(BSxu8TeusSCiX5p
ze(p08MJm{^c0F@4*aZA~-%$^uJGCjgk&x-z#c&HilQPz(L*x5qCuKa!igQrZ`IPVZ
z7x(ewpQoo3fWz&8S?l&<i+1Ao-&kU&DWm7}T=+xUsALx5p>>4g8@sXZjwIL+)9)NY
zpTEksbMCxo>xmwqGFyhGD?jPUw6|-Ts5UBK3@NocVv)6PoEt0KcrPS3^j0b)y<Vb&
zN+E-o?3`4?SlBD?UKb{7+9Yg8ukW~vKY@={aC#u9SIzQuyGf#ECnt_#r<7-(2#pyd
zhuP!N6g#2^DUJ9dW<U-5QLjno71YpW>F%smff7BB=h`z-=V_Gd3ACQ0(7gNkOv6HL
zK(|2OfW51tljHtezW?=J!6Z|qdhYl+`n>wRIaswOXD*Y-4fH%C<S{{7xEcExZ<1mU
zOJr`{I6hoYP#_eH8OZw_?G7n13-%ZV{_y2sN_Pf_J(2iI2Cii$7&-C}f5*MAY@ZQH
z2%e<V@3V#vyCaM>D&aS0w;WeRBtl!;^0z4(&e8Q>idz|6S0gU6`+07Odh!YjL>&0{
zoLi(FSPZ`S#xobiONS@_89k%96&`T<J;z6#dI}_=q0T(S&n|N!fwUAIR6r@XFlB39
z?2A^>m!%fs-Daa=0uOPkR*f-;&sPMzMlm`GcjEC9(A@_ryiVSo$}GN9m24XskD}o<
zkZJIv)FArmuNB7Xj#1=pPy(u=6w(hu?yiA(REmgIP;vFSD(Xz7LL@S#sds)}pSSIe
zpHNJsZ!}Hsct+x`rz@>>C>5LaSjgSA#Et4Y+3O4Hw=J~huT2dH8?!72QuyU*g^`HO
z(Cd2FglMHYC)$Z5ooUJZMUEMR8jG{u3;bxs9KY9<JSFs_!j;>G!B;#WYwS|i0>p(*
z{p<o-GV!30B|((Tm7+w=>*kwr0f6hkzZCWNDq#3K75pWFD4j_t6P)Stixrb4J{=WZ
z3nE#WEn6^CAhRR)kuWFw_l}@m_DWvi2?EQq>wR}6wN8zDC+ltUtY6&xpfCA(SZ&1U
zru#*<chdOymN2hmP6Bi3rt@i8)7DVvt&6?UCY<I6UP{gI;UbL%xw3COPxc!%3oYqc
z@d;uU7@VruM#*z1D5B$^P2#fPC=_$*V3)O_Jg#H(Pfgy{RTm3u)n+%oq2P`Hg7HsW
z;FTv_{x53z;7~3dbzPUlQnwLI^3<kqJP7x!#Y*3b3fio4RIj2LbZ0bm65(E^r#nt#
z-a3eLxH=Yj4?dkrT_Yc!AZ@%&^>!&th-;v#O|t&1W;wC9>fRQ!s~3Ic4*E41WL+B7
zl@(Y}vv!+9$$6ODu3j(0lYT4gu{NoL%NTE0Mc2`sqeOb$aRrCb^uF>Wi;}G-!#zTF
zC#KmrmBvmG;+!ko?!h!1L`dIC=D1i=5BW%-O4Staa3}vYYbn*lt}+>o5-}IwFcIa6
zCN)MaJy$6tMk(%lTgA!Ht;EW*lx$X~IquoiP5uQlMDL1r-F#46cy`;@*QzLzaL9{J
znjV{IBD@TCaRxRjy0}S@eu6p_bu$;D<R{x8E?D09jET9bFMe-v<@9{iV28+;NjGl2
zLZRN|N!(6Q;vqh?t*$w^RY9b`boXR^ADNzmo<6G)+EO0Ya&{<LR37v2e86duQ002L
zzp+~2+#TuZVe>7H)Al4TTdSLzdDKCvc;}M=dH(Z`C5rUUz(XPO@5m?kEi3M24(P7j
z#gvy7jb@*yZe+;Vt&@C+FE|VW`Xj`X=lZgPcB7~2J5`rkAC08awBL?(#^S|k*w1x|
z$6{SCNdn|J=-0-*e;|izJxUUEn6qE0N4~l*UqWUMI#S#Nm(oGFV|E)sVlrRn1k^ke
zGeh{*3JVnPQG{|XfTR56o>snv$-1na`G*b9|4>&5>164rE+yI$U{J#DEC-Y~d>&K?
zQ!ZJpGn`(R!GN3sb7j-|$WA{k>eCXyzO2xG#n!+h`aGy#w0oZ-o8C${q&P{O$9Qgj
zc~g1YmSZ)T>eSSCqDtiXA~429mT^Wk@@a1pNKjuF#%mdB7S(Q8#oq+Alg#R;ez^+S
zEEoN*VEpQHh%AcoaQNMu<QLLniFOp3>4v@^Dt6>rA8@*~4dX6hu^O=<-N8pZ%E=$C
z$<eGRz@VK3>{Z56--`Ytf0g~g02wgEJ1K@SbUCh_eAW9w&-Qn;{i}3DjyidVx#=<f
zo}lzVU(e?K*o~nZlgx9~ZT_j;QRHQVI;p{J@|&B9VW)_Qq#A766a$49?iR9PiZqui
z&$K}0Adx9J)|7^xdUU{%F`Hhz3qGWqgkDb)W2xQ)3Y@U#2SL0z&Knn;bSYt|tiz9M
z<4K&my8Z=OX9doncxqOTBgBq4$Yf<0i);v}Y({K87Sg}QoVH}zP--o*E*yOMwA!ZR
zKthqq_)FSWurh<Y_I3uqo%)PXe-3#3h=epJ?>MwL11tCl7sH@R#5Zn=4*@!0l5P;R
zSYgKRi$XB9gP(0gy}}pVwp;LX8X(O<yj4-!Dj}6_M?`Wp8|Xm16JRDK0$;)UmEW4C
zIG*7LC3YTbSD2NN_6zHmKwFbfhZ~gsR?M2cl*`X)TTq$_lJRc*9WG((%kz5*j(2rr
zk}>@X4X3kXt^Ixxw^`CjFL6IlyFp8De6aVoi>0%pDQ9NUgxcoFpLYmCHHw~)8M;2T
zz%T@65wbx{rjC|ftZl1{<e(GbLlY~*2M(tGlAPxcCC9&hsIGB(Crli9rk7L^(=`?R
zQdAd*7`mR=30lCK7Mz!-4oOsN@;Gb4_SY*OCi_*Fx}I0a642Y21c+E{$h+5l{Elwn
zHhC<0?;X1Nu0IZD%SPX-+m_5WE$M;sc(%4_1`Ng`B?$pM(@8T%4T4`bG=(T%LYN&)
zaqY8)rs!yq7n?-oWOa<=2Udn$wrOZ2Qt+^txot>GZETii44qZ4K=fRr%$(^)trNK%
z=46|u<WtXXGm;%0J*1~J>(eYrA(-Ku$%`I&)`Q#6Z(alj{e*R{9+P#69<h+~D|K5(
zP20RV<?JTU+w_0@@aPQOfm{>c8VgpOoTZFTlHpEBXic>m&jLKvb%gB~jX<+HQ1Y@U
z)?#V!L>9@1*fN|xX?{HV%JACIZ~Jyo56>Q+jXR}PAhCs_00*{_hiau<6cQtNd9|IJ
zmZbN{@-}-s>@n39D-Y!&`uCisbtm1b)hgeW6xz&=ue8>@uYo;?w-$+-Ef`7il4AH!
zeXZ#8w{FX(pX<?tY!a5Oout}c6cs)}=>FT^92{H%nkQ~J477~!Yf`e5`V#aHOKvv;
z4-Ro}bsPhuJ(gT<2R5JFv=h85CJ1E#BA5;v1J?}sAiU}@v9KaW!Wuc+FZT^TQ3FVv
z>I0j_P6EpLEGXq15$tEf#;p5MWf%BAEWG0ap<a`9_zrk>tG#%Mf}t{zo<&Q&-fa<>
z!2_v*fYHO77C(%*XLI=CZ~ON#mdRxZAXYpti6>kKTiq#7`AOyz1^ay&_jq;rlb(}W
z*l{}68oQbKbR@5@q>1Ws`NnHb%o_xO@V@@I1C)I+{<8B>fdvDAw83`)2G9Au2n)q#
z%%oQ;v$jmDXWCaJb$uzAeSM49JPX$Z&WX%CroB!4r+_?=eVD;!yZQA?xIyHU;kP@n
zp@bn;JC~rdo2x!vbe!6<mS~onb2n=!WYws3R0M7va}4zCtmt(dg69X5p7!e>z$rFK
zQ9=Y;17FSU88Wk_-;Q#exvMvFlYcj25p0;ra6UMaO$BY&FJYS;u#QtH5ob6~v4ld)
zYdR(()1G+V_I~pYSTEz2$JkpB8B}{MVu+26-7(#Zd&Zc|&@@;R?yD8Dq>q(5DMnt2
z%_lT&L?lVe{koBNNz_wmyZ}ByaD(r6E(uO(F4#I6GTu@cQXgQBodyn(nOlYg5=JGY
z6lh1p+tuV<ACfGtA85*SB*o9dt<3WRo(aEVjvpg0@XC57{fK>vNvLP&qP-QZF$M2~
z%bPl)$PgQ&WJ7VvTi1~rYrIclg46FH4Oc<=9F<11$lRZIi^OD)Zd6jw|1`d`T*1zX
z&-jP|00+t|%Y}nW3;1oi{tP%*vFktaPa*88$~C}0hoRq9%KvTT{=XYn;pjJpel4%U
z)bFVDpB?{cAg{vGZ=C&F{(1Ay==9(3{_{_+%;s<4{`!Rfsq&ZgR96NPToHl4h7=9}
J;7WG2`xl*1oPhuU

literal 0
HcmV?d00001

diff --git a/connector/connect/client/jvm/src/test/resources/TestHelloV2_2.13.jar b/connector/connect/client/jvm/src/test/resources/TestHelloV2_2.13.jar
new file mode 100644
index 0000000000000000000000000000000000000000..6dee8fcd9c95702fb7909c909c6faf22fe93766d
GIT binary patch
literal 4118
zcmZ{n2Q*w;7sp2#MvYFikeLa>Ac$xqA$s&Kf*^>_h{Pa?&Ql_K8AOZTTlC($L?_Bb
z@4ZC(czN$7dFy@q-n-Vl>;C`y>~q#SXYHTzBP?tR00;yE;QTSC3HT-O0N4NpNp&%p
ztfCaxc_#pX1yFv33%XKZ{v}iXzh;U{#Xn{RF-2J^Ni}s&1*u(y?k@R9FwOy@N3h$y
z-F?NX+=IL`^9W8k1rAw-0mm$CJng0H&2%U_w&wzx5eux?T2xV~3skGDP7Bag^Bu%C
z0^`pzG(1^6BrgZKdalaA2LLWhJN5p34%nA}jcu*Dekx3Sepe9wr7&?aw6?P{`BV90
zJs$i^N%JqIjER+%Eriq9%Fw~VTU}3yT#N$#MKzK$H~c0OCZ=j3@EL~6pynxnP~kBX
zm2B)0PdF*nr>dTp*)7v^SkH~fQAZZ&V4kU%ywoI#iM6%Ke1}cA!>c!E=jT}U`25$=
z&~z(5AucTrLo2*41Q{*O!!@tBGd?ZrVB^IZiEt4jF`rdAOCc$-@>`#}l}_u)XvGO@
zZof==jHWW{-w~#>MWFEf+Z}Jo@a{d6i!?_~(LUsk88=(%WxeA8a(2Wf2};DB<zD|z
zQz@ddA8w|J-)0)Y+{{k78g3{M(~~9?rcA7v?kn<LIFQzc#Eoz=3eR?hrh!$Eyi&9H
z^~~}dWkzP)O8;o4Q<||_DEgv?r?x~396T?-!Bf1f*h71kIqhr|5^XJMD_oYCQqe=w
zQ)>P>G+lz`OL<e`CqiJlye}fE-Bu}%HL7S#qBNPAvVjC$v|Iu5(XDb3GKJ!LcJx|2
z=76op82w>5BUc2zq4$M6U$O-a6^jYWRHXNb(G;`kIp{9p-sv^$A)_vabc{R3f6w&s
zB}*nU{E8mRGH3@gWIN@g5U9lZI4-{5*`Jr!wHoQfZTuR>E9)8Wz&awzVio$@goCTv
zc+oDZs+vP&_A|XC0V7vU->&~A8E3RK|Dgrv`o4#n!y|MA8_{c^j>~Xb1*OhO{6q3o
zKJ8@Yb<7}cI2d8EJh76dA<bpZ15pU`J>BOgW#zY9XE$1e@Q7&S1r7sp&-_j>cK0;L
z`^`)&4|P-SqBn8XdC>TtkdAiPZK#ZG%^xA?C2cHYvnUb|e8Zyod2stzlGX^l^bneK
zKeLi+xH1f2$w2u}++t!p9l9Q&3<}+6UI##4GX+S1LO0c7tzklfH%=f<WS@H3qJ;^Z
z0ZLfP%_$dMPq7hp=6%~qmGZ&8E(++iev|$BMN`S-QlTfXRxBcsib+FZ>9@j!?Ex*)
z+^#Pl789lRU69i6czQZza?lPh!xcx2Wg6M0h6>pb(Hx<M4-a;5uMPnQEfbQp%TvG@
z_+$6|_l}|duO0Ja!$fH7s?l{&io$av`6`)$z3RoEx_FU*NQ&xdJgr#3*FA^PZ+J(e
z85-PhAw*2(7!y?}mmTT@YXeo?h;dw&i2cgYtic7x3ze)QS;gRINHKi-&fyn_%U=xM
z?(PekRUVxiygjA^OnOX{zAp<Be<G|w;ERNb=Fac>5P{s;PgYngMDnQ)(GC@_t*flC
z<m&atHFqss&qqfy?a0CX7D6jhuCl_zL)GWvJ@a?zMnM7!CCm<Z?v`UK<Hpwag6+1U
zqK`ix`dd^Wa-etxlR7%=q+OQm#;7(#Cqwv>j-sOMs2pE7KImoyN0K^jSrL@w1zz}p
zpUO3deo~dtxf1a<3N}bpIUJnLiE5_;er=MtxXU)qoG$AkqjKOK9-Ry);O{hrqy2U9
zLvkQDG$@d&RIVz`*G46sTM^=Q@!zuFm<H>SGb|lyuY@YjF?!>nmcfo+Z-PW4U#7#5
z*Nm^r{^6I<tk>%)f)~UR7%ooF_pI;Z21;NKXQw1+7a1c&Nfw%EZU%d9n?L5+ZcrM1
zz!-z4h<utgzs$hGo}qvDE+VqlI)-tj0AH3$h9cpej^E7?#+F-w{wklR?`V$363JM!
za@{dEVoN=W<D1b|zUS+(I-DcVc$|TX7+=(qI(TNEM1g#ObCz65!Q6PRS(hONg3H*J
zkE=nr+sWLAVtNK9ou3`A$Un7;!WkSMOHYYVqHa#%PKuB&o?W(q`kfGdN(=7=Ihd71
znl;BNGS{stT6*WoNUNvHX)%3Ft3DLff_yYf7*IN!0;4|&bmzQOL`EmdkcWnEkF!|{
zzgO>&YoSh-%af<Kfko;*OdV>d#Bt5yPC(Hj;+y$tG+ZK|IS#NWJNC2pkvT&IAjp9J
zP!d{Akpan$I&L;dyLCrC3KoIM0z)01^21A3<aHT=l(%rg6PK>TnM8G<xNEE`OZ0C;
zLU0a^Ry#a4964TvjRJI!mtJ}i%H}*65r=F%i%><yDblJXH54|6k@j#U<Q|Y%+Bl$x
z)J5J;PRF))BA67bv^tG9Sir)T*0VM4_04ablO14EYr?vjYkU(lV~_GT<dO6Y;>zZ5
z>oJ*)CAR5rF+l^x7*wSnJ95vM=r$EX2AFSWuJbC{3Tp4!<cF^0KM&cF4IF(}ax!}t
z{9fKXWNQf3d~S1UR|Bi#{}QuTY##+zaP<`p5lK+@8eG`$1b^&e1`U?=XM@aQWZOde
zHHPkQJle1r-dbKuQxlK@4j%0b!(KS(+|m_o*nHz{f3GM-T7R8LW%lE&GNi3IMAdm)
z!#1uUey=(4{yB$=U*&!_8L9aK2YdF7lMHEBq{k`(m1IA*tIX^%pP^^??IUb+hT*Vy
zqcLT|(*-v29icSd*tFwg)Zpqo31jy!Kr=OlNg$Kv=hdw|{l1^|`Z}OeS|^jsNx+d`
z*EL4e3%ksb=JCvehBqIuIk3lfuN;znnrzGwo9ZPyY4GY=Sf+I$jgWvtgrVtk0tA`K
zO|7G7NMUz&z*yLr;y|q7dCQcJX_CW$jU>&H1Oww*i!;+xaX(cLNTzO#DQ~m2Yk0i>
ztG&&vro2LJ->}ctJULdZ45FWCzAlqkjWBdsdIKUk5&m_A=e?lO+micR+`%;)wN?+U
z6{IXp>^2H==p>(1o!8ZD3`=!U?$OkuYP4g+f+A0jf&p-%X~>Lq{k4fi%9!46xORu4
zPod8}RS=6_KhFBiquDHiJh!=eeOwF!@-poz%lyfys16B~Q+Xmmp1s#9Hi!0nS9>m3
zti3xSv(1LoOw)?A2~R#{i4js0Hq!>E{5~aHXgzoQ3TxiataH5ZG5<UJZ_OsV_cbjt
zU2~`p<ux9o*1T>K9+>pK{u&y!zUCq7C)9v{cIut53Z*VjlElU^;@nbqv5y<)Csv_a
z)vL*f+DuV=@<D@|u*nY#ZbSNfmT{KBS}S+1YZluVZ#%OvK&^>>Brdjo({7z|VcRmx
zc&jv-kZ99?B4%X*<JHR{P1ZK&>(@);M%>)$xX7&u!Mt8%*g_W58*+s?)Fn=x0z+~E
z5ojJ_t1ZBQL+f%;2wP++d_DTX1hOEOLDB`JMjY*BKDjA^R@H&6PC)5gy>c9z+RZuJ
z#XB*ApZd6^w;sfwUbIz=?7BbLYK(MMcfphQ0}__4K^aX&IsPaBb!EIdUQ?ckx!>0b
z%qDS6Y@)?DDSI3zl9hqXvjWF2;iQ@HR@4L)j@j{-ek=8yAWm?&SGp2NBr1j3H}rY8
zq%dV*bGvBKqslJ0t@j-PLd0Kvw51rfHG0RGOl@0m>B+`xksiI8c3Z?apHVrJK|Cwd
z^RWyYu?LgR)0;NkaX4k10$yNFV#WlGZ=Fv&bDD5iH11@($562l7XU(we2}eAHJ#}S
z1~!4!a}|g9<bicJM?ODD*PygbbUJyma4l84PA^%8_wjK>A#XloO0tu__G1y9nv5Pz
ze8KhdvicyE^4HucjKi((pecyzT=?KIQ-OAnzeY{!w;WXtS`pKro4!0?s9HDJD6QNw
zQFSx#llCmOD->gSe0zD`1T%k1UmsGi@7^B3u6#r&O}fW4I(~ueetwU{HRv_whi_jr
zN>kpll-+N-(e_=i%n5kN?A6ZBK`SVaIg4#2>9q{1b*J03AI~DcOI6*5Bq5ij=ywGY
z(GwBy#@^>;Bl|Ms^W2u{v{aC(-?(U=*q+{=dia70M;|r##$74fJoSEZ-_tGWJ(q!n
zyv-fPxjyc_eOxyZePR?x?Xh71xnpVCwi9dFyE=~o#<f!_DO6okP?i>v`{88Ti|RH>
z3Qm8SnU~}~wllrHcj#>^1ui{zzFGztm$nR^gmv}$3`d=p)NVet&aCI0P|wI$XVUmG
zSWl|OPE|c}jBDirurS0rD#cww^Hh6xTb@f*6V-Z;jojMe>Jdo4@XBsBwHz&?tt<S_
z+{~mGY|!mh!glfm&gVPLGKg=pf5Zb^TTFIfwAN0r5cnp5rYm-@^Xz{Tc$-(#to40&
zSU`6|qbbogQXxr3A!!Eao#*_a=D;SV(iPXjldthDm?V?jk1y<&*8NG~i$wzGnu%AQ
z89@g)2UCsiV|BZEVSK`CFbb{!w<%|An*D;RMAzkq;VjB|?B|qagcI@?^_ybXql1#R
z4BUElC<SUU<+cxjghZ~mjb7g5#M)ci&=-i_>JO)6dDutYEJKo1-yQWDxwYTcqs_4b
z-ru@#Te|kVt)WivA>D=T>|202r)}EhJ$8()$!q=4)zvpLa^n=o=euy7JL+t@@p@#t
z(#5E{zXfu8X#4g44%$**pJR)iT*k}5cvV6R_dE_tv48J*y)M3xy|^ho3fRXXV`_LO
zf<L99`xUQa&l2BiWsI09(H*5hjxVMa-qVO$dAEx{IgT7tZ3TD<Z{lQEog@Y(1>koY
z{Tnr}X!Q5^2QRPK^xxKB@$ia8e+K;W^&jgmWO|hWAMpP%@@IzT%hw-J`P1=VSozbj
z@R#E!Wd5A!6=we0PG4c>&mh70G1lMCU!n8YIbUV@bt$h}@1G%lxpr3-|CZ%PBd+`i
Sh;t>xxx8Ln4i-p!_4Pme8xa%$

literal 0
HcmV?d00001

diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/application/ReplE2ESuite.scala b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/application/ReplE2ESuite.scala
index 61959234c8790..720f66680ee15 100644
--- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/application/ReplE2ESuite.scala
+++ b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/application/ReplE2ESuite.scala
@@ -17,12 +17,15 @@
 package org.apache.spark.sql.application
 
 import java.io.{PipedInputStream, PipedOutputStream}
+import java.nio.file.Paths
 import java.util.concurrent.{Executors, Semaphore, TimeUnit}
 
+import scala.util.Properties
+
 import org.apache.commons.io.output.ByteArrayOutputStream
 import org.scalatest.BeforeAndAfterEach
 
-import org.apache.spark.sql.connect.client.util.RemoteSparkSession
+import org.apache.spark.sql.connect.client.util.{IntegrationTestUtils, RemoteSparkSession}
 
 class ReplE2ESuite extends RemoteSparkSession with BeforeAndAfterEach {
 
@@ -35,6 +38,11 @@ class ReplE2ESuite extends RemoteSparkSession with BeforeAndAfterEach {
   private var ammoniteIn: PipedInputStream = _
   private val semaphore: Semaphore = new Semaphore(0)
 
+  private val scalaVersion = Properties.versionNumberString
+    .split("\\.")
+    .take(2)
+    .mkString(".")
+
   private def getCleanString(out: ByteArrayOutputStream): String = {
     // Remove ANSI colour codes
     // Regex taken from https://stackoverflow.com/a/25189932
@@ -96,7 +104,10 @@ class ReplE2ESuite extends RemoteSparkSession with BeforeAndAfterEach {
 
   def assertContains(message: String, output: String): Unit = {
     val isContain = output.contains(message)
-    assert(isContain, "Ammonite output did not contain '" + message + "':\n" + output)
+    assert(
+      isContain,
+      "Ammonite output did not contain '" + message + "':\n" + output +
+        s"\nError Output: ${getCleanString(errorStream)}")
   }
 
   test("Simple query") {
@@ -151,4 +162,33 @@ class ReplE2ESuite extends RemoteSparkSession with BeforeAndAfterEach {
     assertContains("Array[java.lang.Long] = Array(0L, 2L, 4L, 6L, 8L)", output)
   }
 
+  test("Client-side JAR") {
+    // scalastyle:off classforname line.size.limit
+    val sparkHome = IntegrationTestUtils.sparkHome
+    val testJar = Paths
+      .get(
+        s"$sparkHome/connector/connect/client/jvm/src/test/resources/TestHelloV2_$scalaVersion.jar")
+      .toFile
+
+    assert(testJar.exists(), "Missing TestHelloV2 jar!")
+    val input = s"""
+        |import java.nio.file.Paths
+        |def classLoadingTest(x: Int): Int = {
+        |  val classloader =
+        |    Option(Thread.currentThread().getContextClassLoader).getOrElse(getClass.getClassLoader)
+        |  val cls = Class.forName("com.example.Hello$$", true, classloader)
+        |  val module = cls.getField("MODULE$$").get(null)
+        |  cls.getMethod("test").invoke(module).asInstanceOf[Int]
+        |}
+        |val classLoaderUdf = udf(classLoadingTest _)
+        |
+        |val jarPath = Paths.get("${testJar.toString}").toUri
+        |spark.addArtifact(jarPath)
+        |
+        |spark.range(5).select(classLoaderUdf(col("id"))).as[Int].collect()
+      """.stripMargin
+    val output = runCommandsInShell(input)
+    assertContains("Array[Int] = Array(2, 2, 2, 2, 2)", output)
+    // scalastyle:on classforname line.size.limit
+  }
 }
diff --git a/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/artifact/SparkConnectArtifactManager.scala b/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/artifact/SparkConnectArtifactManager.scala
index 0a91c6b955023..9fd8e367e4aa3 100644
--- a/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/artifact/SparkConnectArtifactManager.scala
+++ b/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/artifact/SparkConnectArtifactManager.scala
@@ -133,7 +133,7 @@ class SparkConnectArtifactManager(sessionHolder: SessionHolder) extends Logging
       Files.move(serverLocalStagingPath, target)
       if (remoteRelativePath.startsWith(s"jars${File.separator}")) {
         jarsList.add(target)
-        jarsURI.add(artifactURI + "/" + target.toString)
+        jarsURI.add(artifactURI + "/" + remoteRelativePath.toString)
       } else if (remoteRelativePath.startsWith(s"pyfiles${File.separator}")) {
         sessionHolder.session.sparkContext.addFile(target.toString)
         val stringRemotePath = remoteRelativePath.toString

From 3e72806bb421b103bf6e73518b80200ccdd58ce5 Mon Sep 17 00:00:00 2001
From: Chandni Singh <singh.chandni@gmail.com>
Date: Tue, 4 Jul 2023 18:46:18 -0500
Subject: [PATCH 06/13] [SPARK-44215][SHUFFLE] If num chunks are 0, then server
 should throw a RuntimeException

### What changes were proposed in this pull request?
The executor expects `numChunks` to be > 0. If it is zero, then we see that the executor fails with
```
23/06/20 19:07:37 ERROR task 2031.0 in stage 47.0 (TID 25018) Executor: Exception in task 2031.0 in stage 47.0 (TID 25018)
java.lang.ArithmeticException: / by zero
	at org.apache.spark.storage.PushBasedFetchHelper.createChunkBlockInfosFromMetaResponse(PushBasedFetchHelper.scala:128)
	at org.apache.spark.storage.ShuffleBlockFetcherIterator.next(ShuffleBlockFetcherIterator.scala:1047)
	at org.apache.spark.storage.ShuffleBlockFetcherIterator.next(ShuffleBlockFetcherIterator.scala:90)
	at org.apache.spark.util.CompletionIterator.next(CompletionIterator.scala:29)
	at scala.collection.Iterator$$anon$11.nextCur(Iterator.scala:484)
	at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:490)
	at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)
	at org.apache.spark.util.CompletionIterator.hasNext(CompletionIterator.scala:31)
	at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
	at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)
```
Because this is an `ArithmeticException`, the executor doesn't fallback. It's not a `FetchFailure` either, so the stage is not retried and the application fails.

### Why are the changes needed?
The executor should fallback to fetch original blocks and not fail because this suggests that there is an issue with push-merged block.

### Does this PR introduce _any_ user-facing change?
No.

### How was this patch tested?
Modified the existing UTs to validate that RuntimeException is thrown when numChunks are 0.

Closes #41762 from otterc/SPARK-44215.

Authored-by: Chandni Singh <singh.chandni@gmail.com>
Signed-off-by: Mridul Muralidharan <mridul<at>gmail.com>
---
 .../shuffle/RemoteBlockPushResolver.java      |  4 ++++
 .../shuffle/RemoteBlockPushResolverSuite.java | 24 ++++++++++++-------
 2 files changed, 20 insertions(+), 8 deletions(-)

diff --git a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/RemoteBlockPushResolver.java b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/RemoteBlockPushResolver.java
index 7f0862fcef435..b95a8700109f1 100644
--- a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/RemoteBlockPushResolver.java
+++ b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/RemoteBlockPushResolver.java
@@ -328,6 +328,10 @@ public MergedBlockMeta getMergedBlockMeta(
     int size = (int) indexFile.length();
     // First entry is the zero offset
     int numChunks = (size / Long.BYTES) - 1;
+    if (numChunks <= 0) {
+      throw new RuntimeException(String.format(
+          "Merged shuffle index file %s is empty", indexFile.getPath()));
+    }
     File metaFile = appShuffleInfo.getMergedShuffleMetaFile(shuffleId, shuffleMergeId, reduceId);
     if (!metaFile.exists()) {
       throw new RuntimeException(String.format("Merged shuffle meta file %s not found",
diff --git a/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/RemoteBlockPushResolverSuite.java b/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/RemoteBlockPushResolverSuite.java
index 2526a94f42940..0847121b0ccb0 100644
--- a/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/RemoteBlockPushResolverSuite.java
+++ b/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/RemoteBlockPushResolverSuite.java
@@ -281,7 +281,7 @@ public void testDuplicateBlocksAreIgnoredWhenPrevStreamIsInProgress() throws IOE
     verifyMetrics(4, 0, 0, 0, 0, 0, 4);
   }
 
-  @Test
+  @Test(expected = RuntimeException.class)
   public void testFailureAfterData() throws IOException {
     StreamCallbackWithID stream =
       pushResolver.receiveBlockDataAsStream(
@@ -289,12 +289,16 @@ public void testFailureAfterData() throws IOException {
     stream.onData(stream.getID(), ByteBuffer.wrap(new byte[4]));
     stream.onFailure(stream.getID(), new RuntimeException("Forced Failure"));
     pushResolver.finalizeShuffleMerge(new FinalizeShuffleMerge(TEST_APP, NO_ATTEMPT_ID, 0, 0));
-    MergedBlockMeta blockMeta = pushResolver.getMergedBlockMeta(TEST_APP, 0, 0, 0);
-    assertEquals("num-chunks", 0, blockMeta.getNumChunks());
-    verifyMetrics(4, 0, 0, 0, 0, 0, 4);
+    try {
+      pushResolver.getMergedBlockMeta(TEST_APP, 0, 0, 0);
+    } catch (RuntimeException e) {
+      assertTrue(e.getMessage().contains("is empty"));
+      verifyMetrics(4, 0, 0, 0, 0, 0, 4);
+      throw e;
+    }
   }
 
-  @Test
+  @Test(expected = RuntimeException.class)
   public void testFailureAfterMultipleDataBlocks() throws IOException {
     StreamCallbackWithID stream =
       pushResolver.receiveBlockDataAsStream(
@@ -304,9 +308,13 @@ public void testFailureAfterMultipleDataBlocks() throws IOException {
     stream.onData(stream.getID(), ByteBuffer.wrap(new byte[4]));
     stream.onFailure(stream.getID(), new RuntimeException("Forced Failure"));
     pushResolver.finalizeShuffleMerge(new FinalizeShuffleMerge(TEST_APP, NO_ATTEMPT_ID, 0, 0));
-    MergedBlockMeta blockMeta = pushResolver.getMergedBlockMeta(TEST_APP, 0, 0, 0);
-    assertEquals("num-chunks", 0, blockMeta.getNumChunks());
-    verifyMetrics(9, 0, 0, 0, 0, 0, 9);
+    try {
+      pushResolver.getMergedBlockMeta(TEST_APP, 0, 0, 0);
+    } catch (RuntimeException e) {
+      assertTrue(e.getMessage().contains("is empty"));
+      verifyMetrics(9, 0, 0, 0, 0, 0, 9);
+      throw e;
+    }
   }
 
   @Test

From e88f5ec54bf1fc5a9228178fe587c5400585a352 Mon Sep 17 00:00:00 2001
From: Alice Sayutina <alice.sayutina@databricks.com>
Date: Wed, 5 Jul 2023 08:49:01 +0900
Subject: [PATCH 07/13] [SPARK-44245][PYTHON] pyspark.sql.dataframe doctests
 behave differently

### What changes were proposed in this pull request?
Disable running example doctests in pyspark.sql.dataframe

### Why are the changes needed?
The doctest serves illustrative purpose and can be broken easily due to external factors.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Checked that doctest is ignoring those lines

Closes #41787 from cdkrot/doctest_bugfix.

Authored-by: Alice Sayutina <alice.sayutina@databricks.com>
Signed-off-by: Hyukjin Kwon <gurwls223@apache.org>
---
 python/pyspark/sql/dataframe.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/python/pyspark/sql/dataframe.py b/python/pyspark/sql/dataframe.py
index 046139f5952d3..99eb2a48bb21c 100644
--- a/python/pyspark/sql/dataframe.py
+++ b/python/pyspark/sql/dataframe.py
@@ -1873,11 +1873,11 @@ def sample(  # type: ignore[misc]
         Examples
         --------
         >>> df = spark.range(10)
-        >>> df.sample(0.5, 3).count()
+        >>> df.sample(0.5, 3).count() # doctest: +SKIP
         7
-        >>> df.sample(fraction=0.5, seed=3).count()
+        >>> df.sample(fraction=0.5, seed=3).count() # doctest: +SKIP
         7
-        >>> df.sample(withReplacement=True, fraction=0.5, seed=3).count()
+        >>> df.sample(withReplacement=True, fraction=0.5, seed=3).count() # doctest: +SKIP
         1
         >>> df.sample(1.0).count()
         10

From 82e091ef2707a349b322f011d8cab9bcae6d9638 Mon Sep 17 00:00:00 2001
From: Hyukjin Kwon <gurwls223@apache.org>
Date: Wed, 5 Jul 2023 09:03:28 +0900
Subject: [PATCH 08/13] Revert "[SPARK-43476][PYTHON][TESTS] Enable
 SeriesStringTests.test_string_replace for pandas 2.0.0"

This reverts commit 442fdb8be42789d9a3fac8361f339f4e2a304fb8.
---
 python/pyspark/pandas/tests/test_series_string.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/python/pyspark/pandas/tests/test_series_string.py b/python/pyspark/pandas/tests/test_series_string.py
index 956567bc5a4ed..3c2bd58da1a28 100644
--- a/python/pyspark/pandas/tests/test_series_string.py
+++ b/python/pyspark/pandas/tests/test_series_string.py
@@ -246,6 +246,10 @@ def test_string_repeat(self):
         with self.assertRaises(TypeError):
             self.check_func(lambda x: x.str.repeat(repeats=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]))
 
+    @unittest.skipIf(
+        LooseVersion(pd.__version__) >= LooseVersion("2.0.0"),
+        "TODO(SPARK-43476): Enable SeriesStringTests.test_string_replace for pandas 2.0.0.",
+    )
     def test_string_replace(self):
         self.check_func(lambda x: x.str.replace("a.", "xx", regex=True))
         self.check_func(lambda x: x.str.replace("a.", "xx", regex=False))
@@ -255,11 +259,10 @@ def test_string_replace(self):
         def repl(m):
             return m.group(0)[::-1]
 
-        regex_pat = re.compile(r"[a-z]+")
-        self.check_func(lambda x: x.str.replace(regex_pat, repl, regex=True))
+        self.check_func(lambda x: x.str.replace(r"[a-z]+", repl))
         # compiled regex with flags
         regex_pat = re.compile(r"WHITESPACE", flags=re.IGNORECASE)
-        self.check_func(lambda x: x.str.replace(regex_pat, "---", regex=True))
+        self.check_func(lambda x: x.str.replace(regex_pat, "---"))
 
     def test_string_rfind(self):
         self.check_func(lambda x: x.str.rfind("a"))

From 756b4ea7d6a5d658cb54922c5d3077c67a1e34c0 Mon Sep 17 00:00:00 2001
From: panbingkun <pbk1982@gmail.com>
Date: Wed, 5 Jul 2023 10:01:30 +0800
Subject: [PATCH 09/13] [SPARK-44298][BUILD] Disable PySpark test on the daily
 test of Java 21 before the new arrow version release

### What changes were proposed in this pull request?
The pr aims to disable PySpark test on the daily test of Java 21 before the new arrow version release.

### Why are the changes needed?
Make daily testing runs based on Java21 successful.

### Does this PR introduce _any_ user-facing change?
No.

### How was this patch tested?
Pass GA.

Closes #41826 from panbingkun/skip_python_test_java21.

Authored-by: panbingkun <pbk1982@gmail.com>
Signed-off-by: yangjie01 <yangjie01@baidu.com>
---
 .github/workflows/build_java21.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build_java21.yml b/.github/workflows/build_java21.yml
index 1e7c998027d7e..760d47ccb7928 100644
--- a/.github/workflows/build_java21.yml
+++ b/.github/workflows/build_java21.yml
@@ -42,7 +42,7 @@ jobs:
       jobs: >-
         {
           "build": "true",
-          "pyspark": "true",
+          "pyspark": "false",
           "sparkr": "true",
           "tpcds-1g": "true",
           "docker-integration-tests": "true"

From 531ec8bddc8dd22ca39486dbdd31e62e989ddc15 Mon Sep 17 00:00:00 2001
From: vicennial <venkata.gudesa@databricks.com>
Date: Wed, 5 Jul 2023 12:06:33 +0900
Subject: [PATCH 10/13] [SPARK-44300][CONNECT][BUG-FIX] Fix artifact cleanup to
 limit deletion scope to session specific artifacts

### What changes were proposed in this pull request?

Modify the directory deletion in `SparkConnectArtifactManager#cleanUpResources` to target the session-specific artifact directory instead of the root artifact directory.

### Why are the changes needed?

Currently, when `SparkConnectArtifactManager#cleanUpResources` is called, it would lead to the deletion of **all** artifacts instead of session-specific ones. This breaks resource isolation among sessions when the bug is triggered.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

New unit test in `ArtifactManagerSuite` that verifies that the correct directory is deleted as well as the existence of the root directory.

Closes #41854 from vicennial/SPARK-44300.

Authored-by: vicennial <venkata.gudesa@databricks.com>
Signed-off-by: Hyukjin Kwon <gurwls223@apache.org>
---
 .../artifact/SparkConnectArtifactManager.scala |  2 +-
 .../artifact/ArtifactManagerSuite.scala        | 18 ++++++++++++++++++
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/artifact/SparkConnectArtifactManager.scala b/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/artifact/SparkConnectArtifactManager.scala
index 9fd8e367e4aa3..449ba011c2196 100644
--- a/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/artifact/SparkConnectArtifactManager.scala
+++ b/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/artifact/SparkConnectArtifactManager.scala
@@ -188,7 +188,7 @@ class SparkConnectArtifactManager(sessionHolder: SessionHolder) extends Logging
     blockManager.removeCache(sessionHolder.userId, sessionHolder.sessionId)
 
     // Clean up artifacts folder
-    FileUtils.deleteDirectory(artifactRootPath.toFile)
+    FileUtils.deleteDirectory(artifactPath.toFile)
   }
 
   private[connect] def uploadArtifactToFs(
diff --git a/connector/connect/server/src/test/scala/org/apache/spark/sql/connect/artifact/ArtifactManagerSuite.scala b/connector/connect/server/src/test/scala/org/apache/spark/sql/connect/artifact/ArtifactManagerSuite.scala
index 612bf096b22bd..345e458cd2f04 100644
--- a/connector/connect/server/src/test/scala/org/apache/spark/sql/connect/artifact/ArtifactManagerSuite.scala
+++ b/connector/connect/server/src/test/scala/org/apache/spark/sql/connect/artifact/ArtifactManagerSuite.scala
@@ -274,6 +274,24 @@ class ArtifactManagerSuite extends SharedSparkSession with ResourceHelper {
       assert(result.forall(_.getString(0).contains("Ahri")))
     }
   }
+
+  test("SPARK-44300: Cleaning up resources only deletes session-specific resources") {
+    val copyDir = Utils.createTempDir().toPath
+    FileUtils.copyDirectory(artifactPath.toFile, copyDir.toFile)
+    val stagingPath = copyDir.resolve("Hello.class")
+    val remotePath = Paths.get("classes/Hello.class")
+
+    val sessionHolder = SparkConnectService.getOrCreateIsolatedSession("c1", "session")
+    sessionHolder.addArtifact(remotePath, stagingPath, None)
+
+    val sessionDirectory =
+      SparkConnectArtifactManager.getArtifactDirectoryAndUriForSession(sessionHolder)._1.toFile
+    assert(sessionDirectory.exists())
+
+    sessionHolder.artifactManager.cleanUpResources()
+    assert(!sessionDirectory.exists())
+    assert(SparkConnectArtifactManager.artifactRootPath.toFile.exists())
+  }
 }
 
 class ArtifactUriSuite extends SparkFunSuite with LocalSparkContext {

From f30ddff123d802dd6f2576ca1f69a8e712daa461 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= <iemejia@gmail.com>
Date: Wed, 5 Jul 2023 16:11:20 +0900
Subject: [PATCH 11/13] [SPARK-44277][BUILD] Upgrade to Avro 1.11.2
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?

Upgrade Avro dependency to version 1.11.2

### Why are the changes needed?

To keep up with upstream

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

Normal Spark build tests.

Closes #41830 from iemejia/SPARK-44277.

Authored-by: Ismaël Mejía <iemejia@gmail.com>
Signed-off-by: Hyukjin Kwon <gurwls223@apache.org>
---
 .../main/scala/org/apache/spark/sql/avro/AvroOptions.scala  | 4 ++--
 dev/deps/spark-deps-hadoop-3-hive-2.3                       | 6 +++---
 docs/sql-data-sources-avro.md                               | 4 ++--
 pom.xml                                                     | 2 +-
 project/SparkBuild.scala                                    | 2 +-
 .../org/apache/spark/sql/hive/client/HiveClientSuite.scala  | 2 +-
 6 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/connector/avro/src/main/scala/org/apache/spark/sql/avro/AvroOptions.scala b/connector/avro/src/main/scala/org/apache/spark/sql/avro/AvroOptions.scala
index c8057ca58798e..edaaa8835cc01 100644
--- a/connector/avro/src/main/scala/org/apache/spark/sql/avro/AvroOptions.scala
+++ b/connector/avro/src/main/scala/org/apache/spark/sql/avro/AvroOptions.scala
@@ -81,14 +81,14 @@ private[sql] class AvroOptions(
 
   /**
    * Top level record name in write result, which is required in Avro spec.
-   * See https://avro.apache.org/docs/1.11.1/specification/#schema-record .
+   * See https://avro.apache.org/docs/1.11.2/specification/#schema-record .
    * Default value is "topLevelRecord"
    */
   val recordName: String = parameters.getOrElse(RECORD_NAME, "topLevelRecord")
 
   /**
    * Record namespace in write result. Default value is "".
-   * See Avro spec for details: https://avro.apache.org/docs/1.11.1/specification/#schema-record .
+   * See Avro spec for details: https://avro.apache.org/docs/1.11.2/specification/#schema-record .
    */
   val recordNamespace: String = parameters.getOrElse(RECORD_NAMESPACE, "")
 
diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3
index 1cdf08f321eb6..1b91686ed4db0 100644
--- a/dev/deps/spark-deps-hadoop-3-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-3-hive-2.3
@@ -21,9 +21,9 @@ arrow-memory-core/12.0.1//arrow-memory-core-12.0.1.jar
 arrow-memory-netty/12.0.1//arrow-memory-netty-12.0.1.jar
 arrow-vector/12.0.1//arrow-vector-12.0.1.jar
 audience-annotations/0.5.0//audience-annotations-0.5.0.jar
-avro-ipc/1.11.1//avro-ipc-1.11.1.jar
-avro-mapred/1.11.1//avro-mapred-1.11.1.jar
-avro/1.11.1//avro-1.11.1.jar
+avro-ipc/1.11.2//avro-ipc-1.11.2.jar
+avro-mapred/1.11.2//avro-mapred-1.11.2.jar
+avro/1.11.2//avro-1.11.2.jar
 aws-java-sdk-bundle/1.12.367//aws-java-sdk-bundle-1.12.367.jar
 azure-data-lake-store-sdk/2.3.9//azure-data-lake-store-sdk-2.3.9.jar
 azure-keyvault-core/1.0.0//azure-keyvault-core-1.0.0.jar
diff --git a/docs/sql-data-sources-avro.md b/docs/sql-data-sources-avro.md
index 977886a6f34e2..b01174b918245 100644
--- a/docs/sql-data-sources-avro.md
+++ b/docs/sql-data-sources-avro.md
@@ -417,7 +417,7 @@ applications. Read the [Advanced Dependency Management](https://spark.apache
 Submission Guide for more details.
 
 ## Supported types for Avro -> Spark SQL conversion
-Currently Spark supports reading all [primitive types](https://avro.apache.org/docs/1.11.1/specification/#primitive-types) and [complex types](https://avro.apache.org/docs/1.11.1/specification/#complex-types) under records of Avro.
+Currently Spark supports reading all [primitive types](https://avro.apache.org/docs/1.11.2/specification/#primitive-types) and [complex types](https://avro.apache.org/docs/1.11.2/specification/#complex-types) under records of Avro.
 <table class="table table-striped">
   <thead><tr><th><b>Avro type</b></th><th><b>Spark SQL type</b></th></tr></thead>
   <tr>
@@ -481,7 +481,7 @@ In addition to the types listed above, it supports reading `union` types. The fo
 3. `union(something, null)`, where something is any supported Avro type. This will be mapped to the same Spark SQL type as that of something, with nullable set to true.
 All other union types are considered complex. They will be mapped to StructType where field names are member0, member1, etc., in accordance with members of the union. This is consistent with the behavior when converting between Avro and Parquet.
 
-It also supports reading the following Avro [logical types](https://avro.apache.org/docs/1.11.1/specification/#logical-types):
+It also supports reading the following Avro [logical types](https://avro.apache.org/docs/1.11.2/specification/#logical-types):
 
 <table class="table table-striped">
   <thead><tr><th><b>Avro logical type</b></th><th><b>Avro type</b></th><th><b>Spark SQL type</b></th></tr></thead>
diff --git a/pom.xml b/pom.xml
index 2e29d1de0c986..bc14cdd584e0e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -154,7 +154,7 @@
     -->
     <codahale.metrics.version>4.2.19</codahale.metrics.version>
     <!-- Should be consistent with SparkBuild.scala and docs -->
-    <avro.version>1.11.1</avro.version>
+    <avro.version>1.11.2</avro.version>
     <aws.kinesis.client.version>1.12.0</aws.kinesis.client.version>
     <!-- Should be consistent with Kinesis client dependency -->
     <aws.java.sdk.version>1.11.655</aws.java.sdk.version>
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 7f9da32224f70..8f2f5d7878794 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -1110,7 +1110,7 @@ object DependencyOverrides {
     dependencyOverrides += "com.google.guava" % "guava" % guavaVersion,
     dependencyOverrides += "xerces" % "xercesImpl" % "2.12.2",
     dependencyOverrides += "jline" % "jline" % "2.14.6",
-    dependencyOverrides += "org.apache.avro" % "avro" % "1.11.1")
+    dependencyOverrides += "org.apache.avro" % "avro" % "1.11.2")
 }
 
 /**
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientSuite.scala
index 6f799bbe7d3f5..bd1b5b557896b 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientSuite.scala
@@ -900,7 +900,7 @@ class HiveClientSuite(version: String, allVersions: Seq[String])
   test("Decimal support of Avro Hive serde") {
     val tableName = "tab1"
     // TODO: add the other logical types. For details, see the link:
-    // https://avro.apache.org/docs/1.11.1/specification/#logical-types
+    // https://avro.apache.org/docs/1.11.2/specification/#logical-types
     val avroSchema =
     """{
       |  "name": "test_record",

From 5b81de9d1c3fb736a49af0d7265438784af6fb56 Mon Sep 17 00:00:00 2001
From: yangjie01 <yangjie01@baidu.com>
Date: Wed, 5 Jul 2023 16:19:00 +0900
Subject: [PATCH 12/13] [SPARK-44259][CONNECT][TESTS][FOLLOWUP] No longer
 initializing `Ammonite` for Java 21
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?
This pr adds a check condition for `beforeAll` function of `ReplE2ESuite`, make the `beforeAll` function No longer initializing Ammonite test with Java 17+.

### Why are the changes needed?
Make `connect-client-jvm` module test pass with Java 21 on GA.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
- Pass GitHub Actions
- Checked with GA

**Before**

- https://github.com/apache/spark/actions/runs/5434602425/jobs/9883143909

```
at java.base/java.nio.channels.spi.AbstractInterruptibleChannel.end(AbstractInterruptibleChannel.java:200)
	at java.base/sun.nio.ch.FileChannelImpl.endBlocking(FileChannelImpl.java:172)
	at java.base/sun.nio.ch.FileChannelImpl.size(FileChannelImpl.java:430)
	at jdk.zipfs/jdk.nio.zipfs.ZipFileSystem.findEND(ZipFileSystem.java:1255)
	at jdk.zipfs/jdk.nio.zipfs.ZipFileSystem.initCEN(ZipFileSystem.java:1541)
	at jdk.zipfs/jdk.nio.zipfs.ZipFileSystem.<init>(ZipFileSystem.java:179)
	at jdk.zipfs/jdk.nio.zipfs.ZipFileSystemProvider.getZipFileSystem(ZipFileSystemProvider.java:125)
	at jdk.zipfs/jdk.nio.zipfs.ZipFileSystemProvider.newFileSystem(ZipFileSystemProvider.java:106)
	at java.base/java.nio.file.FileSystems.newFileSystem(FileSystems.java:339)
	at java.base/java.nio.file.FileSystems.newFileSystem(FileSystems.java:288)
	at io.github.retronym.java9rtexport.Export.rt(Export.java:60)
	at io.github.retronym.java9rtexport.Export.rtTo(Export.java:88)
	at io.github.retronym.java9rtexport.Export.rtAt(Export.java:100)
	at io.github.retronym.java9rtexport.Export.rtAt(Export.java:105)
	at ammonite.util.Classpath$.classpath(Classpath.scala:76)
	at ammonite.compiler.CompilerLifecycleManager.init(CompilerLifecycleManager.scala:92)
	at ammonite.compiler.CompilerLifecycleManager.preprocess(CompilerLifecycleManager.scala:64)
	at ammonite.interp.Interpreter.compileRunBlock$1(Interpreter.scala:526)
	at ammonite.interp.Interpreter.$anonfun$processAllScriptBlocks$15(Interpreter.scala:587)
	at ammonite.util.Res$Success.flatMap(Res.scala:62)
	at ammonite.interp.Interpreter.$anonfun$processAllScriptBlocks$14(Interpreter.scala:584)
	at ammonite.util.Res$Success.flatMap(Res.scala:62)
	at ammonite.interp.Interpreter.$anonfun$processAllScriptBlocks$12(Interpreter.scala:581)
	at scala.Option.getOrElse(Option.scala:189)
	at ammonite.interp.Interpreter.loop$1(Interpreter.scala:581)
	at ammonite.interp.Interpreter.processAllScriptBlocks(Interpreter.scala:619)
	at ammonite.interp.Interpreter.$anonfun$processModule$6(Interpreter.scala:414)
	at ammonite.util.Catching.flatMap(Res.scala:115)
	at ammonite.interp.Interpreter.$anonfun$processModule$5(Interpreter.scala:405)
	at ammonite.util.Res$Success.flatMap(Res.scala:62)
	at ammonite.interp.Interpreter.processModule(Interpreter.scala:395)
	at ammonite.interp.Interpreter.$anonfun$initializePredef$3(Interpreter.scala:148)
	at ammonite.interp.Interpreter.$anonfun$initializePredef$3$adapted(Interpreter.scala:148)
	at ammonite.interp.PredefInitialization$.$anonfun$apply$2(PredefInitialization.scala:79)
	at ammonite.util.Res$.$anonfun$fold$1(Res.scala:32)
	at scala.collection.LinearSeqOptimized.foldLeft(LinearSeqOptimized.scala:126)
	at scala.collection.LinearSeqOptimized.foldLeft$(LinearSeqOptimized.scala:122)
	at scala.collection.immutable.List.foldLeft(List.scala:91)
	at ammonite.util.Res$.fold(Res.scala:30)
	at ammonite.interp.PredefInitialization$.apply(PredefInitialization.scala:67)
	at ammonite.interp.Interpreter.initializePredef(Interpreter.scala:150)
	at ammonite.repl.Repl.initializePredef(Repl.scala:144)
	at ammonite.Main.run(Main.scala:224)
	at org.apache.spark.sql.application.ConnectRepl$.doMain(ConnectRepl.scala:104)
	at org.apache.spark.sql.application.ReplE2ESuite$$anon$1.run(ReplE2ESuite.scala:60)
	at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572)
	at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
	at java.base/java.lang.Thread.run(Thread.java:1583)
	...
[error] Error during tests:
[error] 	Running java with options -classpath /home/runner/work/spark/spark/connector/connect/client/jvm/target/scala-2.12/test-classes:/home/runner/work/spark/spark/connector/connect/client/jvm/target/scala-2.12/spark-connect-client-jvm_2.12-3.5.0-SNAPSHOT.jar:/home/runner/work/spark/spark/connector/connect/common/target/scala-2.12/spark-connect-common_2.12-3.5.0-
...
[error] (connect-client-jvm / Test / test) sbt.TestsFailedException: Tests unsuccessful
[error] Total time: 50 s, completed Jul 2, 2023, 4:55:33 AM
[error] running /home/runner/work/spark/spark/build/sbt -Phadoop-3 -Pyarn -Pmesos -Pconnect -Phadoop-cloud -Pkubernetes -Pspark-ganglia-lgpl -Pvolcano sql-kafka-0-10/test connect/test connect-client-jvm/test protobuf/test streaming/test streaming-kafka-0-10/test token-provider-kafka-0-10/test mllib-local/test mllib/test yarn/test network-yarn/test mesos/test kubernetes/test hadoop-cloud/test ; received return code 1
Error: Process completed with exit code 18.
```

The test result was judged as failed on GA.

**After**

- https://github.com/LuciferYang/spark/actions/runs/5439928518/jobs/9892364759

```
[info] Run completed in 10 seconds, 973 milliseconds.
[info] Total number of tests run: 858
[info] Suites: completed 22, aborted 0
[info] Tests: succeeded 858, failed 0, canceled 167, ignored 1, pending 0
[info] All tests passed.
```
<img width="1274" alt="image" src="https://github.com/apache/spark/assets/1475305/8f21a8dc-18b1-4663-9698-27513adbc38d">

Closes #41814 from LuciferYang/SPARK-44259-FOLLOWUP.

Lead-authored-by: yangjie01 <yangjie01@baidu.com>
Co-authored-by: YangJie <yangjie01@baidu.com>
Signed-off-by: Hyukjin Kwon <gurwls223@apache.org>
---
 .../spark/sql/application/ReplE2ESuite.scala  | 40 ++++++++++---------
 1 file changed, 22 insertions(+), 18 deletions(-)

diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/application/ReplE2ESuite.scala b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/application/ReplE2ESuite.scala
index 720f66680ee15..676ad6b090ed8 100644
--- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/application/ReplE2ESuite.scala
+++ b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/application/ReplE2ESuite.scala
@@ -23,6 +23,7 @@ import java.util.concurrent.{Executors, Semaphore, TimeUnit}
 import scala.util.Properties
 
 import org.apache.commons.io.output.ByteArrayOutputStream
+import org.apache.commons.lang3.{JavaVersion, SystemUtils}
 import org.scalatest.BeforeAndAfterEach
 
 import org.apache.spark.sql.connect.client.util.{IntegrationTestUtils, RemoteSparkSession}
@@ -50,26 +51,29 @@ class ReplE2ESuite extends RemoteSparkSession with BeforeAndAfterEach {
   }
 
   override def beforeAll(): Unit = {
-    super.beforeAll()
-    ammoniteOut = new ByteArrayOutputStream()
-    testSuiteOut = new PipedOutputStream()
-    // Connect the `testSuiteOut` and `ammoniteIn` pipes
-    ammoniteIn = new PipedInputStream(testSuiteOut)
-    errorStream = new ByteArrayOutputStream()
-
-    val args = Array("--port", serverPort.toString)
-    val task = new Runnable {
-      override def run(): Unit = {
-        ConnectRepl.doMain(
-          args = args,
-          semaphore = Some(semaphore),
-          inputStream = ammoniteIn,
-          outputStream = ammoniteOut,
-          errorStream = errorStream)
+    // TODO(SPARK-44121) Remove this check condition
+    if (SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) {
+      super.beforeAll()
+      ammoniteOut = new ByteArrayOutputStream()
+      testSuiteOut = new PipedOutputStream()
+      // Connect the `testSuiteOut` and `ammoniteIn` pipes
+      ammoniteIn = new PipedInputStream(testSuiteOut)
+      errorStream = new ByteArrayOutputStream()
+
+      val args = Array("--port", serverPort.toString)
+      val task = new Runnable {
+        override def run(): Unit = {
+          ConnectRepl.doMain(
+            args = args,
+            semaphore = Some(semaphore),
+            inputStream = ammoniteIn,
+            outputStream = ammoniteOut,
+            errorStream = errorStream)
+        }
       }
-    }
 
-    executorService.submit(task)
+      executorService.submit(task)
+    }
   }
 
   override def afterAll(): Unit = {

From 4b77aad824d6a5f2b1dce5b0813a6690b3a979bf Mon Sep 17 00:00:00 2001
From: Kent Yao <yao@apache.org>
Date: Wed, 5 Jul 2023 16:41:18 +0900
Subject: [PATCH 13/13] [SPARK-44294][UI] Fix HeapHistogram column shows
 unexpectedly w/ select-all-box

### What changes were proposed in this pull request?

This PR fixes the bug that the last element of execOptionalColumns does not point to the executor removeReason but HeapHistogram.

### Why are the changes needed?

bugfix, otherwise, after users've checked the select-all-box, an unexpected HeapHistogram will show with a dead link
### Does this PR introduce _any_ user-facing change?

no, bugfix and not released

### How was this patch tested?

locally built and tested to verify the HeapHistogram invisible

<img width="1768" alt="image" src="https://github.com/apache/spark/assets/8326978/1fa108a6-5598-4c90-855c-35104ed2c740">

Closes #41847 from yaooqinn/SPARK-44294.

Authored-by: Kent Yao <yao@apache.org>
Signed-off-by: Hyukjin Kwon <gurwls223@apache.org>
---
 .../org/apache/spark/ui/static/executorspage.js  | 16 +++-------------
 1 file changed, 3 insertions(+), 13 deletions(-)

diff --git a/core/src/main/resources/org/apache/spark/ui/static/executorspage.js b/core/src/main/resources/org/apache/spark/ui/static/executorspage.js
index b52ece87ba125..1ea0adf5467be 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/executorspage.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/executorspage.js
@@ -147,7 +147,7 @@ function totalDurationColor(totalGCTime, totalDuration) {
 }
 
 var sumOptionalColumns = [3, 4];
-var execOptionalColumns = [5, 6, 7, 8, 9, 10, 13, 14, 25];
+var execOptionalColumns = [5, 6, 7, 8, 9, 10, 13, 14, 26];
 var execDataTable;
 var sumDataTable;
 
@@ -572,15 +572,7 @@ $(document).ready(function () {
           ],
           "order": [[0, "asc"]],
           "columnDefs": [
-            {"visible": false, "targets": 5},
-            {"visible": false, "targets": 6},
-            {"visible": false, "targets": 7},
-            {"visible": false, "targets": 8},
-            {"visible": false, "targets": 9},
-            {"visible": false, "targets": 10},
-            {"visible": false, "targets": 13},
-            {"visible": false, "targets": 14},
-            {"visible": false, "targets": 26}
+            {"visible": false, "targets": execOptionalColumns}
           ],
           "deferRender": true
         };
@@ -712,10 +704,8 @@ $(document).ready(function () {
           "searching": false,
           "info": false,
           "columnDefs": [
-            {"visible": false, "targets": 3},
-            {"visible": false, "targets": 4}
+            {"visible": false, "targets": sumOptionalColumns}
           ]
-
         };
 
         sumDataTable = $(sumSelector).DataTable(sumConf);