[featurn] support flyway database migration

.all-contributorsrc

@@ -1651,7 +1651,8 @@
       "avatar_url": "https://avatars.githubusercontent.com/u/71874373?v=4",
       "profile": "https://github.com/gjjjj0101",
       "contributions": [
-        "code"
+        "code",
+        "design"
       ]
     },
     {
@@ -1780,6 +1781,15 @@
       "contributions": [
         "code"
       ]
+    },
+    {
+      "login": "LLP2333",
+      "name": "llp2333",
+      "avatar_url": "https://avatars.githubusercontent.com/u/61670545?v=4",
+      "profile": "https://github.com/LLP2333",
+      "contributions": [
+        "code"
+      ]
     }
   ],
   "contributorsPerLine": 7,

README.md

@@ -416,7 +416,7 @@ Thanks to these wonderful people, welcome to join us:
       <td align="center" valign="top" width="14.28%"><a href="http://sharehoo.cn/"><img src="https://avatars.githubusercontent.com/u/45377370?v=4?s=100" width="100px;" alt="miki"/><br /><sub><b>miki</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=miki-hmt" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://codeflex.substack.com/"><img src="https://avatars.githubusercontent.com/u/85513042?v=4?s=100" width="100px;" alt="Keshav Carpenter"/><br /><sub><b>Keshav Carpenter</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=alpha951" title="Code">💻</a> <a href="https://github.com/apache/hertzbeat/commits?author=alpha951" title="Documentation">📖</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/makechoicenow"><img src="https://avatars.githubusercontent.com/u/9911918?v=4?s=100" width="100px;" alt="makechoicenow"/><br /><sub><b>makechoicenow</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=makechoicenow" title="Code">💻</a></td>
-      <td align="center" valign="top" width="14.28%"><a href="https://github.com/gjjjj0101"><img src="https://avatars.githubusercontent.com/u/71874373?v=4?s=100" width="100px;" alt="Gao Jian"/><br /><sub><b>Gao Jian</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=gjjjj0101" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/gjjjj0101"><img src="https://avatars.githubusercontent.com/u/71874373?v=4?s=100" width="100px;" alt="Gao Jian"/><br /><sub><b>Gao Jian</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=gjjjj0101" title="Code">💻</a> <a href="#design-gjjjj0101" title="Design">🎨</a></td>
     </tr>
     <tr>
       <td align="center" valign="top" width="14.28%"><a href="https://jangto.tistory.com/"><img src="https://avatars.githubusercontent.com/u/37864182?v=4?s=100" width="100px;" alt="Hyeon Sung"/><br /><sub><b>Hyeon Sung</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=dukbong" title="Code">💻</a></td>
@@ -436,6 +436,9 @@ Thanks to these wonderful people, welcome to join us:
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/zuobiao-zhou"><img src="https://avatars.githubusercontent.com/u/61108539?v=4?s=100" width="100px;" alt="Zhang Yuxuan"/><br /><sub><b>Zhang Yuxuan</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=zuobiao-zhou" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/Pzz-2021"><img src="https://avatars.githubusercontent.com/u/118056735?v=4?s=100" width="100px;" alt="P.P."/><br /><sub><b>P.P.</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=Pzz-2021" title="Code">💻</a></td>
     </tr>
+    <tr>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/LLP2333"><img src="https://avatars.githubusercontent.com/u/61670545?v=4?s=100" width="100px;" alt="llp2333"/><br /><sub><b>llp2333</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=LLP2333" title="Code">💻</a></td>
+    </tr>
   </tbody>
 </table>
 

README_CN.md

@@ -415,7 +415,7 @@ Thanks these wonderful people, welcome to join us:
       <td align="center" valign="top" width="14.28%"><a href="http://sharehoo.cn/"><img src="https://avatars.githubusercontent.com/u/45377370?v=4?s=100" width="100px;" alt="miki"/><br /><sub><b>miki</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=miki-hmt" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://codeflex.substack.com/"><img src="https://avatars.githubusercontent.com/u/85513042?v=4?s=100" width="100px;" alt="Keshav Carpenter"/><br /><sub><b>Keshav Carpenter</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=alpha951" title="Code">💻</a> <a href="https://github.com/apache/hertzbeat/commits?author=alpha951" title="Documentation">📖</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/makechoicenow"><img src="https://avatars.githubusercontent.com/u/9911918?v=4?s=100" width="100px;" alt="makechoicenow"/><br /><sub><b>makechoicenow</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=makechoicenow" title="Code">💻</a></td>
-      <td align="center" valign="top" width="14.28%"><a href="https://github.com/gjjjj0101"><img src="https://avatars.githubusercontent.com/u/71874373?v=4?s=100" width="100px;" alt="Gao Jian"/><br /><sub><b>Gao Jian</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=gjjjj0101" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/gjjjj0101"><img src="https://avatars.githubusercontent.com/u/71874373?v=4?s=100" width="100px;" alt="Gao Jian"/><br /><sub><b>Gao Jian</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=gjjjj0101" title="Code">💻</a> <a href="#design-gjjjj0101" title="Design">🎨</a></td>
     </tr>
     <tr>
       <td align="center" valign="top" width="14.28%"><a href="https://jangto.tistory.com/"><img src="https://avatars.githubusercontent.com/u/37864182?v=4?s=100" width="100px;" alt="Hyeon Sung"/><br /><sub><b>Hyeon Sung</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=dukbong" title="Code">💻</a></td>
@@ -435,6 +435,9 @@ Thanks these wonderful people, welcome to join us:
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/zuobiao-zhou"><img src="https://avatars.githubusercontent.com/u/61108539?v=4?s=100" width="100px;" alt="Zhang Yuxuan"/><br /><sub><b>Zhang Yuxuan</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=zuobiao-zhou" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/Pzz-2021"><img src="https://avatars.githubusercontent.com/u/118056735?v=4?s=100" width="100px;" alt="P.P."/><br /><sub><b>P.P.</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=Pzz-2021" title="Code">💻</a></td>
     </tr>
+    <tr>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/LLP2333"><img src="https://avatars.githubusercontent.com/u/61670545?v=4?s=100" width="100px;" alt="llp2333"/><br /><sub><b>llp2333</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=LLP2333" title="Code">💻</a></td>
+    </tr>
   </tbody>
 </table>
 

common/src/main/java/org/apache/hertzbeat/common/entity/manager/JsonStringListAttributeConverter.java

@@ -42,11 +42,16 @@ public List<String> convertToEntityAttribute(String dbData) {
         if (StringUtils.isBlank(dbData)) {
             return List.of();
         }
+        if (!JsonUtil.isJsonStr(dbData)) {
+            return List.of(dbData);
+        }
         TypeReference<List<String>> typeReference = new TypeReference<>() {
         };
         List<String> stringList = JsonUtil.fromJson(dbData, typeReference);
         if (stringList == null && !dbData.isEmpty()) {
             return List.of(dbData);
-        } else return stringList;
+        } else {
+            return stringList;
+        }
     }
 }

common/src/main/java/org/apache/hertzbeat/common/util/JsonUtil.java

@@ -93,4 +93,16 @@ public static JsonNode fromJson(String jsonStr) {
             return null;
         }
     }
+
+    public static boolean isJsonStr(String jsonStr) {
+        if (!StringUtils.hasText(jsonStr)) {
+            return false;
+        }
+        try {
+            OBJECT_MAPPER.readTree(jsonStr);
+            return true;
+        } catch (Exception ignored) {
+            return false;
+        }
+    }
 }

home/src/pages/team/index.jsx

@@ -950,7 +950,8 @@ export default function () {
                   <td align="center" valign="top" width="14.28%"><a href="https://github.com/gjjjj0101"><img
                     src="https://avatars.githubusercontent.com/u/71874373?v=4?s=100" width="100px;"
                     alt="Gao Jian"/><br/><sub><b>Gao Jian</b></sub></a><br/><a
-                    href="https://github.com/apache/hertzbeat/commits?author=gjjjj0101" title="Code">💻</a></td>
+                    href="https://github.com/apache/hertzbeat/commits?author=gjjjj0101" title="Code">💻</a> <a
+                    href="#design-gjjjj0101" title="Design">🎨</a></td>
                 </tr>
                 <tr>
                   <td align="center" valign="top" width="14.28%"><a href="https://jangto.tistory.com/"><img
@@ -1013,6 +1014,11 @@ export default function () {
                     alt="P.P."/><br/><sub><b>P.P.</b></sub></a><br/><a
                     href="https://github.com/apache/hertzbeat/commits?author=Pzz-2021" title="Code">💻</a></td>
                 </tr>
+                <tr>
+                  <td align="center" valign="top" width="14.28%"><a href="https://github.com/LLP2333"><img
+                    src="https://avatars.githubusercontent.com/u/61670545?v=4?s=100" width="100px;" alt="llp2333"/><br/><sub><b>llp2333</b></sub></a><br/><a
+                    href="https://github.com/apache/hertzbeat/commits?author=LLP2333" title="Code">💻</a></td>
+                </tr>
                 </tbody>
               </table>
 

manager/pom.xml

@@ -116,6 +116,19 @@
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-jdbc</artifactId>
         </dependency>
+        <!-- database migration -->
+        <dependency>
+            <groupId>org.flywaydb</groupId>
+            <artifactId>flyway-core</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.flywaydb</groupId>
+            <artifactId>flyway-mysql</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.flywaydb</groupId>
+            <artifactId>flyway-database-postgresql</artifactId>
+        </dependency>
         <!-- mysql -->
         <dependency>
             <groupId>mysql</groupId>
@@ -194,6 +207,7 @@
                     <include>*.xml</include>
                     <include>banner.txt</include>
                     <include>define/**</include>
+                    <include>db/**</include>
                     <include>templates/**</include>
                     <include>**/*.html</include>
                 </includes>

manager/src/main/java/org/apache/hertzbeat/manager/config/FlywayConfiguration.java

@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hertzbeat.manager.config;
+
+import org.flywaydb.core.Flyway;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.boot.autoconfigure.flyway.FlywayMigrationInitializer;
+import org.springframework.boot.autoconfigure.flyway.FlywayProperties;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.DependsOn;
+
+/**
+ * flyway database migration config
+ */
+@Configuration
+@ConditionalOnProperty(prefix = "spring.flyway", name = "enabled", havingValue = "true")
+public class FlywayConfiguration {
+
+    @Bean
+    public FlywayMigrationInitializer flywayInitializer(Flyway flyway) {
+        return new FlywayMigrationInitializer(flyway, (f) -> {
+        });
+    }
+
+    static class Dummy {
+    }
+
+    @Bean
+    @DependsOn("entityManagerFactory")
+    Dummy delayedFlywayInitializer(Flyway flyway, FlywayProperties flywayProperties) {
+        if (flywayProperties.isEnabled()) {
+            flyway.migrate();
+        }
+        return new Dummy();
+    }
+}

manager/src/main/resources/application-test.yml

@@ -30,6 +30,8 @@ spring:
       eclipselink:
         logging:
           level: SEVERE
+  flyway:
+    enabled: false
   mail:
     host: smtp.exmail.qq.com
     username: example@tancloud.cn

manager/src/main/resources/application.yml

@@ -82,6 +82,14 @@ spring:
         logging:
           level: SEVERE
 
+  flyway:
+    enabled: true
+    clean-disabled: true
+    baseline-on-migrate: true
+    baseline-version: 1
+    locations:
+      - classpath:db/migration/{vendor}
+
   mail:
     # Attention: this is mail server address. 
     # 请注意此为邮件服务器地址：qq邮箱为 smtp.qq.com qq 企业邮箱为 smtp.exmail.qq.com

manager/src/main/resources/db/migration/h2/V160__update_column.sql

@@ -0,0 +1,30 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- ensure every sql can rerun without error
+
+ALTER TABLE HZB_PARAM ADD COLUMN IF NOT EXISTS "value" VARCHAR(255);
+UPDATE HZB_PARAM SET param_value = "value" WHERE "value" IS NOT NULL AND param_value IS NULL;
+ALTER TABLE HZB_PARAM DROP COLUMN "value";
+
+ALTER TABLE HZB_TAG ADD COLUMN IF NOT EXISTS "value" VARCHAR(255);
+UPDATE HZB_TAG SET tag_value = "value" WHERE "value" IS NOT NULL AND tag_value IS NULL;
+ALTER TABLE HZB_TAG DROP COLUMN "value";
+
+ALTER TABLE HZB_STATUS_PAGE_HISTORY ADD COLUMN IF NOT EXISTS "unknown" integer;
+UPDATE HZB_STATUS_PAGE_HISTORY SET unknowing = "unknown" WHERE "unknown" IS NOT NULL AND unknowing IS NULL;
+ALTER TABLE HZB_STATUS_PAGE_HISTORY DROP COLUMN "unknown";

manager/src/main/resources/db/migration/mysql/V160__update_column.sql

@@ -0,0 +1,32 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- ensure every sql can rerun without error
+
+-- for to hertzbeat v1.6.0 upgrade in mysql, user need run this by themself, ignore for non-upgrading -> 1.6.0 users
+
+# UPDATE hzb_param SET param_value = `value` WHERE `value` IS NOT NULL AND param_value IS NULL;
+# ALTER TABLE hzb_param DROP COLUMN `value`;
+# commit;
+# 
+# UPDATE hzb_tag SET tag_value = `value` WHERE `value` IS NOT NULL AND tag_value IS NULL;
+# ALTER TABLE hzb_tag DROP COLUMN `value`;
+# commit;
+# 
+# UPDATE hzb_status_page_history SET unknowing = `unknown` WHERE `unknown` IS NOT NULL AND unknowing IS NULL;
+# ALTER TABLE hzb_status_page_history DROP COLUMN `unknown`;
+# commit;

manager/src/main/resources/db/migration/postgresql/V160__update_column.sql

@@ -0,0 +1,32 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- ensure every sql can rerun without error
+
+ALTER TABLE HZB_PARAM ADD COLUMN IF NOT EXISTS value VARCHAR(255);
+UPDATE HZB_PARAM SET param_value = value WHERE value IS NOT NULL AND param_value IS NULL;
+ALTER TABLE HZB_PARAM DROP COLUMN value;
+
+ALTER TABLE HZB_TAG ADD COLUMN IF NOT EXISTS value VARCHAR(255);
+UPDATE HZB_TAG SET tag_value = value WHERE value IS NOT NULL AND tag_value IS NULL;
+ALTER TABLE HZB_TAG DROP COLUMN value;
+
+ALTER TABLE HZB_STATUS_PAGE_HISTORY ADD COLUMN IF NOT EXISTS unknown integer;
+UPDATE HZB_STATUS_PAGE_HISTORY SET unknowing = unknown WHERE unknown IS NOT NULL AND unknowing IS NULL;
+ALTER TABLE HZB_STATUS_PAGE_HISTORY DROP COLUMN unknown;
+
+commit;

manager/src/main/resources/logback-spring.xml

@@ -130,4 +130,13 @@
         </root>
     </springProfile>
 
+    <!-- Development environment configuration -->
+    <springProfile name="pg">
+        <root level="INFO">
+            <appender-ref ref="ConsoleAppender"/>
+            <appender-ref ref="SystemOutFileAppender"/>
+            <appender-ref ref="ErrOutFileAppender"/>
+        </root>
+    </springProfile>
+
 </configuration>

material/licenses/backend/LICENSE

@@ -410,6 +410,9 @@ The text of each license is the standard Apache 2.0 license.
     https://mvnrepository.com/artifact/org.webjars/swagger-ui/5.10.3
     https://mvnrepository.com/artifact/com.google.flatbuffers/flatbuffers-java/1.12.0
     https://mvnrepository.com/artifact/org.apache.commons/commons-jexl3/3.2.1
+    https://mvnrepository.com/artifact/org.flywaydb/flyway-core/10.11.1
+    https://mvnrepository.com/artifact/org.flywaydb/flyway-mysql/10.11.1
+    https://mvnrepository.com/artifact/org.flywaydb/flyway-database-postgresql/10.11.1
 
 
 ========================================================================