Fix nightly builds of ShardingSphere Proxy Native

apache · Feb 4, 2023 · 2da5c0d · 2da5c0d
1 parent 88d04ad
commit 2da5c0d
Show file tree

Hide file tree

Showing 9 changed files with 80 additions and 79 deletions.
diff --git a/.github/workflows/nightly-build-artifact.yml b/.github/workflows/nightly-build-artifact.yml
@@ -125,4 +125,4 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: Push docker image
-        run: ./mvnw -am -pl distribution/proxy-native -B -DskipTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dspotless.apply.skip=true -Pnative,docker.buildx.push.native -Dproxy.image.repository=${{ env.PROXY_NATIVE }} -Dproxy.image.tag=${{ github.sha }} clean package
+        run: ./mvnw -am -pl distribution/proxy-native -Pnative,docker.buildx.push.native -B -DskipTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dspotless.apply.skip=true -Dproxy.image.repository=${{ env.PROXY_NATIVE }} -Dproxy.image.tag=${{ github.sha }} clean package
diff --git a/distribution/proxy-native/Dockerfile b/distribution/proxy-native/Dockerfile
@@ -14,23 +14,26 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+FROM alpine AS prepare
 
-FROM busybox AS prepare
+ARG APP_NAME
+ENV LOCAL_PATH /opt/shardingsphere-proxy-native
 
-RUN mkdir -p /conf/
+ADD target/${APP_NAME}.tar.gz /opt
+RUN mv /opt/${APP_NAME} ${LOCAL_PATH}
 
 FROM oraclelinux:9-slim
-
 MAINTAINER ShardingSphere "dev@shardingsphere.apache.org"
 
+ARG NATIVE_IMAGE_NAME
 ENV LOCAL_PATH /opt/shardingsphere-proxy-native
+ENV GRAALVM_HOME "/opt/graalvm-ce-java17-22.3.1"
+ENV PATH "$GRAALVM_HOME/bin:$PATH"
 
-bash <(curl -sL https://get.graalvm.org/jdk) -c espresso graalvm-ce-java17-22.3.1
-
-COPY --from=prepare /conf/ ${LOCAL_PATH}/conf
-
-ARG APP_NAME
-
-ADD target/${APP_NAME} ${LOCAL_PATH}/
+RUN microdnf install gzip -y && \
+    bash <(curl -sL https://get.graalvm.org/jdk) --to "/opt" -c espresso graalvm-ce-java17-22.3.1 && \
+    $GRAALVM_HOME/bin/gu remove native-image && \
+    microdnf clean all
 
-ENTRYPOINT ${LOCAL_PATH}/${APP_NAME} 3307 ${LOCAL_PATH}/conf
+COPY --from=prepare ${LOCAL_PATH} ${LOCAL_PATH}
+ENTRYPOINT ${LOCAL_PATH}/${NATIVE_IMAGE_NAME} 3307 ${LOCAL_PATH}/conf
diff --git a/distribution/proxy-native/pom.xml b/distribution/proxy-native/pom.xml
@@ -28,10 +28,11 @@
     <artifactId>shardingsphere-proxy-native-distribution</artifactId>
 
     <properties>
-        <native.image.name>apache-shardingsphere-proxy-native</native.image.name>
-        <native.image.repository>apache/shardingsphere-proxy-native</native.image.repository>
         <exec-maven-plugin.version>3.1.0</exec-maven-plugin.version>
         <native.maven.plugin.version>0.9.19</native.maven.plugin.version>
+        <imageName>apache-shardingsphere-proxy-native</imageName>
+        <mainClass>org.apache.shardingsphere.proxy.Bootstrap</mainClass>
+        <native.image.repository>apache/shardingsphere-proxy-native</native.image.repository>
     </properties>
 
     <dependencies>
@@ -65,37 +66,12 @@
             <scope>runtime</scope>
         </dependency>
     </dependencies>
+
+    <build>
+        <finalName>apache-shardingsphere-${project.version}</finalName>
+    </build>
 
     <profiles>
-        <profile>
-            <id>release</id>
-            <build>
-                <finalName>apache-shardingsphere-${project.version}</finalName>
-                <plugins>
-                    <plugin>
-                        <artifactId>maven-assembly-plugin</artifactId>
-                        <configuration>
-                            <descriptors>
-                                <descriptor>src/main/assembly/shardingsphere-proxy-native-binary-distribution.xml</descriptor>
-                            </descriptors>
-                        </configuration>
-                        <executions>
-                            <execution>
-                                <id>shardingsphere-proxy-native-bin</id>
-                                <goals>
-                                    <goal>single</goal>
-                                </goals>
-                                <phase>package</phase>
-                            </execution>
-                        </executions>
-                    </plugin>
-                    <plugin>
-                        <groupId>net.nicoulaj.maven.plugins</groupId>
-                        <artifactId>checksum-maven-plugin</artifactId>
-                    </plugin>
-                </plugins>
-            </build>
-        </profile>
         <profile>
             <id>native</id>
             <properties>
@@ -120,8 +96,6 @@
                         <version>${native.maven.plugin.version}</version>
                         <extensions>true</extensions>
                         <configuration>
-                            <imageName>${native.image.name}</imageName>
-                            <mainClass>org.apache.shardingsphere.proxy.Bootstrap</mainClass>
                             <fallback>false</fallback>
                             <verbose>true</verbose>
                             <buildArgs>
@@ -153,6 +127,27 @@
                             </execution>
                         </executions>
                     </plugin>
+                    <plugin>
+                        <artifactId>maven-assembly-plugin</artifactId>
+                        <configuration>
+                            <descriptors>
+                                <descriptor>src/main/assembly/shardingsphere-proxy-native-binary-distribution.xml</descriptor>
+                            </descriptors>
+                        </configuration>
+                        <executions>
+                            <execution>
+                                <id>shardingsphere-proxy-native-bin</id>
+                                <goals>
+                                    <goal>single</goal>
+                                </goals>
+                                <phase>package</phase>
+                            </execution>
+                        </executions>
+                    </plugin>
+                    <plugin>
+                        <groupId>net.nicoulaj.maven.plugins</groupId>
+                        <artifactId>checksum-maven-plugin</artifactId>
+                    </plugin>
                 </plugins>
             </build>
         </profile>
@@ -181,7 +176,9 @@
                                         <argument>build</argument>
                                         <argument>--pull</argument>
                                         <argument>--build-arg</argument>
-                                        <argument>APP_NAME=${native.image.name}</argument>
+                                        <argument>APP_NAME=${project.build.finalName}-shardingsphere-proxy-native-bin</argument>
+                                        <argument>--build-arg</argument>
+                                        <argument>NATIVE_IMAGE_NAME=${imageName}</argument>
                                         <argument>.</argument>
                                         <argument>-t</argument>
                                         <argument>${proxy.image.repository}:${proxy.image.tag}</argument>
@@ -245,7 +242,9 @@
                                         <argument>--platform</argument>
                                         <argument>${proxy.image.platform}</argument>
                                         <argument>--build-arg</argument>
-                                        <argument>APP_NAME=${native.image.name}</argument>
+                                        <argument>APP_NAME=${project.build.finalName}-shardingsphere-proxy-native-bin</argument>
+                                        <argument>--build-arg</argument>
+                                        <argument>NATIVE_IMAGE_NAME=${imageName}</argument>
                                         <argument>.</argument>
                                     </arguments>
                                 </configuration>
@@ -265,7 +264,9 @@
                                         <argument>--platform</argument>
                                         <argument>${proxy.image.platform}</argument>
                                         <argument>--build-arg</argument>
-                                        <argument>APP_NAME=${native.image.name}</argument>
+                                        <argument>APP_NAME=${project.build.finalName}-shardingsphere-proxy-native-bin</argument>
+                                        <argument>--build-arg</argument>
+                                        <argument>NATIVE_IMAGE_NAME=${imageName}</argument>
                                         <argument>.</argument>
                                         <argument>-t</argument>
                                         <argument>${proxy.image.repository}:${proxy.image.tag}</argument>

diff --git a/...bution/proxy-native/src/main/assembly/shardingsphere-proxy-native-binary-distribution.xml b/...bution/proxy-native/src/main/assembly/shardingsphere-proxy-native-binary-distribution.xml
@@ -26,15 +26,15 @@
 
     <fileSets>
         <fileSet>
-            <directory>../../proxy/bootstrap/src/main/resources/conf</directory>
-            <outputDirectory>conf</outputDirectory>
-            <fileMode>0644</fileMode>
-        </fileSet>
-        <fileSet>
-            <directory>../../proxy/bootstrap/src/main/resources</directory>
+            <directory>${project.build.directory}</directory>
             <includes>
-                <include>logback.xml</include>
+                <include>${imageName}</include>
             </includes>
+            <outputDirectory>/</outputDirectory>
+            <fileMode>0755</fileMode>
+        </fileSet>
+        <fileSet>
+            <directory>../../proxy/bootstrap/src/main/resources/conf</directory>
             <outputDirectory>conf</outputDirectory>
             <fileMode>0644</fileMode>
         </fileSet>

diff --git a/...ent/content/user-manual/shardingsphere-proxy/startup/graalvm-native-image.cn.md b/...ent/content/user-manual/shardingsphere-proxy/startup/graalvm-native-image.cn.md
@@ -58,7 +58,7 @@ services:
 
 3. 根据 https://www.graalvm.org/22.3/reference-manual/native-image/#prerequisites 的要求安装本地工具链。
 
-4. 如果需要构建 Docker Image， 确保 `docker-cli` 在系统环境变量内。
+4. 如果需要构建 Docker Image， 确保 `docker-ce` 已安装。
 
 ## 操作步骤
 
@@ -112,7 +112,7 @@ services:
 ./apache-shardingsphere-proxy-native 3307 ./custom/conf
 ```
 
-4. 如果需要构建 Docker Image, 在添加后存在 SPI 实现的依赖或第三方依赖后, 在命令行执行如下命令。
+4. 如果需要构建 Docker Image, 在添加存在 SPI 实现的依赖或第三方依赖后, 在命令行执行如下命令。
 
 ```shell
 ./mvnw -am -pl distribution/proxy-native -B -Pnative,docker.native -DskipTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dspotless.apply.skip=true -Drat.skip=true clean package
@@ -149,13 +149,13 @@ services:
 
 - 你可以使用 https://www.graalvm.org/22.3/tools/ 提供的一系列命令行工具或可视化工具观察 GraalVM Native Image
   的内部行为，并根据其要求使用 VSCode 完成调试工作。
-  如果你正在使用 IntelliJ IDEA 并且希望调试生成的 GraalVM Native Image，
-  你可以关注 https://blog.jetbrains.com/idea/2022/06/intellij-idea-2022-2-eap-5/#Experimental_GraalVM_Native_Debugger_for_Java
-  及其后继。
+  如果你正在使用 IntelliJ IDEA 并且希望调试生成的 GraalVM Native
+  Image，你可以关注 https://blog.jetbrains.com/idea/2022/06/intellij-idea-2022-2-eap-5/#Experimental_GraalVM_Native_Debugger_for_Java
+  及其后继。如果你使用的不是 Linux，则无法对 GraalVM Native Image 进行
+  Debug，请关注尚未关闭的 https://github.com/oracle/graal/issues/5648 。
 
-- 对于使用 `ShardingSphere Agent` 等 APM Java Agent 的情形，
-  GraalVM 的 `native-image` 组件尚未完全支持在构建 Native Image 时使用
-  javaagent，你需要关注尚未关闭的 https://github.com/oracle/graal/issues/1065。
+- 对于使用 `ShardingSphere Agent` 等 APM Java Agent 的情形， GraalVM 的 `native-image` 组件尚未完全支持在构建 Native
+  Image 时使用 javaagent，你需要关注尚未关闭的 https://github.com/oracle/graal/issues/1065。
 
 - 以下部分采用 `Apache SkyWalking Java Agent` 作为示例，可用于跟踪 GraalVM 社区的对应 issue。
 

diff --git a/...ent/content/user-manual/shardingsphere-proxy/startup/graalvm-native-image.en.md b/...ent/content/user-manual/shardingsphere-proxy/startup/graalvm-native-image.en.md
@@ -66,7 +66,7 @@ services:
 
 3. Install the local toolchain as required by https://www.graalvm.org/22.3/reference-manual/native-image/#prerequisites.
 
-4. If you need to build a Docker Image, make sure `docker-cli` is in the system environment variables.
+4. If you need to build a Docker Image, make sure `docker-ce` is installed.
 
 ## Steps
 
@@ -123,8 +123,8 @@ services:
 ./apache-shardingsphere-proxy-native 3307 ./custom/conf
 ````
 
-4. If you need to build a Docker Image, after adding the dependencies of the SPI implementation or third-party
-   dependencies, execute the following commands on the command line.
+4. If you need to build a Docker Image, execute the following command on the command line after adding dependencies that
+   exist for SPI implementation or third-party dependencies.
 
 ```shell
 ./mvnw -am -pl distribution/proxy-native -B -Pnative,docker.native -DskipTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dspotless.apply.skip=true -Drat .skip=true clean package
@@ -164,10 +164,10 @@ services:
 
 - You can observe GraalVM Native Image using a series of command line tools or visualization tools available
   at https://www.graalvm.org/22.3/tools/, and use VSCode to debug it according to its requirements.
-  If you are using IntelliJ IDEA and want to debug the generated GraalVM Native Image,
-  You can
+  If you are using IntelliJ IDEA and want to debug the generated GraalVM Native Image, You can
   follow https://blog.jetbrains.com/idea/2022/06/intellij-idea-2022-2-eap-5/#Experimental_GraalVM_Native_Debugger_for_Java
-  and its successors.
+  and its successors. If you are not using Linux, you cannot debug GraalVM Native Image, please pay attention
+  to https://github.com/oracle/graal/issues/5648 which has not been closed yet.
 
 - In the case of using APM Java Agent such as `ShardingSphere Agent`,
   GraalVM's `native-image` component is not yet fully supported when building Native Images

diff --git a/infra/util-groovy/pom.xml b/infra/util-groovy/pom.xml
@@ -32,9 +32,5 @@
             <groupId>org.apache.groovy</groupId>
             <artifactId>groovy</artifactId>
         </dependency>
-        <dependency>
-            <groupId>com.google.guava</groupId>
-            <artifactId>guava</artifactId>
-        </dependency>
     </dependencies>
 </project>
diff --git a/.../java/org/apache/shardingsphere/infra/util/groovy/expr/HotspotInlineExpressionParser.java b/.../java/org/apache/shardingsphere/infra/util/groovy/expr/HotspotInlineExpressionParser.java
@@ -36,7 +36,7 @@
 import java.util.stream.Collectors;
 
 /**
- * Inline expression parser.
+ * Hotspot Inline expression parser.
  */
 @RequiredArgsConstructor
 public final class HotspotInlineExpressionParser {

diff --git a/...c/main/java/org/apache/shardingsphere/infra/util/expr/EspressoInlineExpressionParser.java b/...c/main/java/org/apache/shardingsphere/infra/util/expr/EspressoInlineExpressionParser.java
@@ -19,6 +19,7 @@
 
 import groovy.lang.Closure;
 import org.graalvm.polyglot.Context;
+import org.graalvm.polyglot.TypeLiteral;
 import org.graalvm.polyglot.Value;
 
 import java.net.URL;
@@ -43,12 +44,12 @@ public class EspressoInlineExpressionParser {
         if (javaHome == null) {
             throw new RuntimeException("Failed to determine the system's environment variable GRAALVM_HOME or JAVA_HOME!");
         }
-        System.setProperty("org.graalvm.home", javaHome);
         URL resource = Thread.currentThread().getContextClassLoader().getResource("espresso-need-libs");
         assert null != resource;
         String dir = resource.getPath();
         String javaClasspath = String.join(":", dir + "/groovy.jar", dir + "/guava.jar", dir + "/shardingsphere-infra-util.jar");
         POLYGLOT = Context.newBuilder().allowAllAccess(true)
+                .option("java.Properties.org.graalvm.home", javaHome)
                 .option("java.MultiThreaded", "true")
                 .option("java.Classpath", javaClasspath)
                 .build();
@@ -70,18 +71,18 @@ public static String handlePlaceHolder(final String inlineExpression) {
         return POLYGLOT.getBindings("java")
                 .getMember("org.apache.shardingsphere.infra.util.expr.InlineExpressionParser")
                 .invokeMember("handlePlaceHolder", inlineExpression)
-                .as(String.class);
+                .asString();
     }
 
     /**
      * Split and evaluate inline expression.
      *
      * @return result list
      */
-    @SuppressWarnings("unchecked")
     public List<String> splitAndEvaluate() {
-        List<String> splitAndEvaluate = espressoInlineExpressionParser.invokeMember("splitAndEvaluate").as(List.class);
-        // GraalVM Truffle Espresso CE 22.3.1 has a different behavior for generic List than Hotspot.
+        List<String> splitAndEvaluate = espressoInlineExpressionParser.invokeMember("splitAndEvaluate").as(new TypeLiteral<List<String>>() {
+        });
+        // GraalVM Truffle Espresso 22.3.1 has a different behavior for generic List than Hotspot.
         return splitAndEvaluate.size() == 0 ? Collections.emptyList() : splitAndEvaluate;
     }