perf($Redis): support Redis master-slave replication

https://redis.io/topics/replication https://docs.spring.io/spring-data/redis/docs/current/reference/html/#redis:write-to-master-read-from-replica
johnnymillergh · Jul 31, 2021 · d414b02 · d414b02
1 parent 560fb9b
commit d414b02
Show file tree

Hide file tree

Showing 33 changed files with 2,686 additions and 220 deletions.
diff --git a/api-gateway/src/main/resources/application-development-docker.yml b/api-gateway/src/main/resources/application-development-docker.yml
@@ -3,7 +3,13 @@ spring:
     base-url: http://maf-zipkin:9411
   devtools:
     add-properties: true
-  redis:
-    host: maf-redis
+
+redis:
+  master:
+    host: maf-redis-master
     port: 6379
     password: maf@redis
+  slaves:
+    - host: maf-redis-slave-1
+      port: 6379
+      password: maf@redis
diff --git a/api-gateway/src/main/resources/application-development-local.yml b/api-gateway/src/main/resources/application-development-local.yml
@@ -3,11 +3,17 @@ spring:
     add-properties: true
   zipkin:
     base-url: http://localhost:9411
-  redis:
+
+redis:
+  master:
     host: localhost
     port: 6379
     password: maf@redis
+  slaves:
+    - host: localhost
+      port: 6380
+      password: maf@redis
 
 maf:
   configuration:
-    web-security-enabled: false
+    web-security-enabled: true
diff --git a/api-gateway/src/main/resources/application-production.yml b/api-gateway/src/main/resources/application-production.yml
@@ -3,7 +3,13 @@ spring:
     add-properties: false
   zipkin:
     base-url: http://maf-zipkin:9411
-  redis:
-    host: maf-redis
+
+redis:
+  master:
+    host: maf-redis-master
     port: 6379
     password: maf@redis
+  slaves:
+    - host: maf-redis-slave-1
+      port: 6379
+      password: maf@redis
diff --git a/api-gateway/src/main/resources/application-stage.yml b/api-gateway/src/main/resources/application-stage.yml
@@ -3,7 +3,13 @@ spring:
     add-properties: false
   zipkin:
     base-url: http://maf-zipkin:9411
-  redis:
-    host: maf-redis
+
+redis:
+  master:
+    host: maf-redis-master
     port: 6379
     password: maf@redis
+  slaves:
+    - host: maf-redis-slave-1
+      port: 6379
+      password: maf@redis
diff --git a/api-gateway/src/main/resources/application-test.yml b/api-gateway/src/main/resources/application-test.yml
@@ -3,7 +3,13 @@ spring:
     add-properties: false
   zipkin:
     base-url: http://maf-zipkin:9411
-  redis:
-    host: maf-redis
+
+redis:
+  master:
+    host: maf-redis-master
     port: 6379
     password: maf@redis
+  slaves:
+    - host: maf-redis-slave-1
+      port: 6379
+      password: maf@redis
diff --git a/auth-center/src/main/resources/application-development-docker.yml b/auth-center/src/main/resources/application-development-docker.yml
@@ -21,16 +21,22 @@ spring:
           username: maf_mysql_rw
           password: maf@mysql
           driver-class-name: com.mysql.cj.jdbc.Driver
-  redis:
-    host: maf-redis
-    port: 6379
-    password: maf@redis
   rabbitmq:
     host: maf-rabbitmq
     port: 5672
     username: maf_rabbitmq_su
     password: maf@rabbitmq
 
+redis:
+  master:
+    host: maf-redis-master
+    port: 6379
+    password: maf@redis
+  slaves:
+    - host: maf-redis-slave-1
+      port: 6379
+      password: maf@redis
+
 maf:
   configuration:
     swagger-enabled: true
diff --git a/auth-center/src/main/resources/application-development-local.yml b/auth-center/src/main/resources/application-development-local.yml
@@ -21,17 +21,23 @@ spring:
           username: maf_mysql_rw
           password: maf@mysql
           driver-class-name: com.mysql.cj.jdbc.Driver
-  redis:
-    host: localhost
-    port: 6379
-    password: maf@redis
   rabbitmq:
     host: localhost
     port: 5672
     username: maf_rabbitmq_su
     password: maf@rabbitmq
 
-# Configure logging level for the environment
+redis:
+  master:
+    host: localhost
+    port: 6379
+    password: maf@redis
+  slaves:
+    - host: localhost
+      port: 6380
+      password: maf@redis
+
+    # Configure logging level for the environment
 logging:
   level:
     com.baomidou: INFO

diff --git a/auth-center/src/main/resources/application-production.yml b/auth-center/src/main/resources/application-production.yml
@@ -21,16 +21,22 @@ spring:
           username: maf_mysql_rw
           password: maf@mysql
           driver-class-name: com.mysql.cj.jdbc.Driver
-  redis:
-    host: maf-redis
-    port: 6379
-    password: maf@redis
   rabbitmq:
     host: maf-rabbitmq
     port: 5672
     username: maf_rabbitmq_su
     password: maf@rabbitmq
 
+redis:
+  master:
+    host: maf-redis-master
+    port: 6379
+    password: maf@redis
+  slaves:
+    - host: maf-redis-slave-1
+      port: 6379
+      password: maf@redis
+
 maf:
   configuration:
     swagger-enabled: false
diff --git a/auth-center/src/main/resources/application-stage.yml b/auth-center/src/main/resources/application-stage.yml
@@ -21,16 +21,22 @@ spring:
           username: maf_mysql_rw
           password: maf@mysql
           driver-class-name: com.mysql.cj.jdbc.Driver
-  redis:
-    host: maf-redis
-    port: 6379
-    password: maf@redis
   rabbitmq:
     host: maf-rabbitmq
     port: 5672
     username: maf_rabbitmq_su
     password: maf@rabbitmq
 
+redis:
+  master:
+    host: maf-redis-master
+    port: 6379
+    password: maf@redis
+  slaves:
+    - host: maf-redis-slave-1
+      port: 6379
+      password: maf@redis
+
 maf:
   configuration:
     swagger-enabled: false
diff --git a/auth-center/src/main/resources/application-test.yml b/auth-center/src/main/resources/application-test.yml
@@ -21,16 +21,22 @@ spring:
           username: maf_mysql_rw
           password: maf@mysql
           driver-class-name: com.mysql.cj.jdbc.Driver
-  redis:
-    host: maf-redis
-    port: 6379
-    password: maf@redis
   rabbitmq:
     host: maf-rabbitmq
     port: 5672
     username: maf_rabbitmq_su
     password: maf@rabbitmq
 
+redis:
+  master:
+    host: maf-redis-master
+    port: 6379
+    password: maf@redis
+  slaves:
+    - host: maf-redis-slave-1
+      port: 6379
+      password: maf@redis
+
 maf:
   configuration:
     swagger-enabled: true
diff --git a/docker/.env b/docker/.env
@@ -8,6 +8,7 @@ MAF_TAG=0.0.2
 ##################################################
 MYSQL_SERVER_TAG=8.0.23-1.1.19
 REDIS_TAG=6.2.5-alpine
+REDIS_CLUSTER_TAG=6.2.5
 ATMOZ_SFTP_TAG=alpine
 MINIO_TAG=RELEASE.2021-06-17T00-10-46Z
 RABBITMQ_TAG=3.8.14-management-alpine
@@ -35,7 +36,14 @@ SPRING_ENVIRONMENT=development-docker
 ############# Infrastructure Services ############
 MYSQL_SERVER_MASTER_CONTAINER_NAME=maf-mysql-server-master
 MYSQL_SERVER_SLAVE_CONTAINER_NAME=maf-mysql-server-slave
-REDIS_CONTAINER_NAME=maf-redis
+REDIS_MASTER_CONTAINER_NAME=maf-redis-master
+REDIS_SLAVE_1_CONTAINER_NAME=maf-redis-slave-1
+REDIS_NODE_1_CONTAINER_NAME=maf-redis-node-1
+REDIS_NODE_2_CONTAINER_NAME=maf-redis-node-2
+REDIS_NODE_3_CONTAINER_NAME=maf-redis-node-3
+REDIS_NODE_4_CONTAINER_NAME=maf-redis-node-4
+REDIS_NODE_5_CONTAINER_NAME=maf-redis-node-5
+REDIS_NODE_6_CONTAINER_NAME=maf-redis-node-6
 ATMOZ_SFTP_CONTAINER_NAME=maf-atmoz-sftp
 MINIO_CONTAINER_NAME=maf-minio
 RABBITMQ_CONTAINER_NAME=maf-rabbitmq

diff --git a/docker/config/redis/redis.conf → docker/config/redis/master/redis.conf b/docker/config/redis/redis.conf → docker/config/redis/master/redis.conf
@@ -299,7 +299,7 @@ loglevel notice
 # Specify the log file name. Also the empty string can be used to force
 # Redis to log on the standard output. Note that if you use standard
 # output for logging but daemonize, logs will be sent to /dev/null
-logfile "redis.log"
+logfile ""
 
 # To enable logging to the system logger, just set 'syslog-enabled' to yes,
 # and optionally update the other syslog parameters to suit your needs.
@@ -481,15 +481,15 @@ dir ./
 # starting the replication synchronization process, otherwise the master will
 # refuse the replica request.
 #
-# masterauth <master-password>
+# masterauth maf@redis
 #
 # However this is not enough if you are using Redis ACLs (for Redis version
 # 6 or greater), and the default user is not capable of running the PSYNC
 # command and/or other commands needed for replication. In this case it's
 # better to configure a special user to use with replication, and specify the
 # masteruser configuration as such:
 #
-# masteruser <username>
+# masteruser default
 #
 # When masteruser is specified, the replica will authenticate against its
 # master using the new AUTH form: AUTH <username> <password>.