feat($ELK): integrate ELK stack

Elasticsearch 7.10.1 Logstash 7.10.1 Kibana 7.10.1 BREAKING CHANGE: aggregate logs from all applications
johnnymillergh · Feb 21, 2021 · a42e87a · a42e87a
1 parent 7a2ce2b
commit a42e87a
Show file tree

Hide file tree

Showing 36 changed files with 180 additions and 2 deletions.
diff --git a/api-gateway/src/main/resources/logback-configuration/logback-base.xml b/api-gateway/src/main/resources/logback-configuration/logback-base.xml
@@ -21,18 +21,23 @@
     <!-- Log for Multi Environment Configuration -->
     <springProfile name="development-local">
         <property name="ENVIRONMENT" value="development-local"/>
+        <property name="LOGSTASH_HOST" value="localhost"/>
     </springProfile>
     <springProfile name="development-docker">
         <property name="ENVIRONMENT" value="development-docker"/>
+        <property name="LOGSTASH_HOST" value="maf.logstash.${ENVIRONMENT}"/>
     </springProfile>
     <springProfile name="test">
         <property name="ENVIRONMENT" value="test"/>
+        <property name="LOGSTASH_HOST" value="maf.logstash.${ENVIRONMENT}"/>
     </springProfile>
     <springProfile name="stage">
         <property name="ENVIRONMENT" value="stage"/>
+        <property name="LOGSTASH_HOST" value="maf.logstash.${ENVIRONMENT}"/>
     </springProfile>
     <springProfile name="production">
         <property name="ENVIRONMENT" value="production"/>
+        <property name="LOGSTASH_HOST" value="maf.logstash.${ENVIRONMENT}"/>
     </springProfile>
 
     <!-- CONSOLE Log Configuration -->
@@ -67,4 +72,12 @@
             <totalSizeCap>1GB</totalSizeCap>
         </rollingPolicy>
     </appender>
+
+    <!-- LOGSTASH -->
+    <appender name="LOGSTASH" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
+        <destination>${LOGSTASH_HOST}:4560</destination>
+        <encoder charset="UTF-8" class="net.logstash.logback.encoder.LogstashEncoder">
+            <customFields>{"appname": "${APPLICATION}"}</customFields>
+        </encoder>
+    </appender>
 </included>
diff --git a/api-gateway/src/main/resources/logback-configuration/logback-development-docker.xml b/api-gateway/src/main/resources/logback-configuration/logback-development-docker.xml
@@ -6,5 +6,6 @@
     <root level="INFO">
         <appender-ref ref="CONSOLE"/>
         <appender-ref ref="FILE"/>
+        <appender-ref ref="LOGSTASH"/>
     </root>
 </configuration>
diff --git a/api-gateway/src/main/resources/logback-configuration/logback-development-local.xml b/api-gateway/src/main/resources/logback-configuration/logback-development-local.xml
@@ -6,5 +6,6 @@
     <root level="INFO">
         <appender-ref ref="CONSOLE"/>
         <appender-ref ref="FILE"/>
+        <appender-ref ref="LOGSTASH"/>
     </root>
 </configuration>
diff --git a/api-gateway/src/main/resources/logback-configuration/logback-production.xml b/api-gateway/src/main/resources/logback-configuration/logback-production.xml
@@ -6,5 +6,6 @@
     <root level="INFO">
         <appender-ref ref="CONSOLE"/>
         <appender-ref ref="FILE"/>
+        <appender-ref ref="LOGSTASH"/>
     </root>
 </configuration>
diff --git a/api-gateway/src/main/resources/logback-configuration/logback-stage.xml b/api-gateway/src/main/resources/logback-configuration/logback-stage.xml
@@ -6,5 +6,6 @@
     <root level="INFO">
         <appender-ref ref="CONSOLE"/>
         <appender-ref ref="FILE"/>
+        <appender-ref ref="LOGSTASH"/>
     </root>
 </configuration>
diff --git a/api-gateway/src/main/resources/logback-configuration/logback-test.xml b/api-gateway/src/main/resources/logback-configuration/logback-test.xml
@@ -6,5 +6,6 @@
     <root level="INFO">
         <appender-ref ref="CONSOLE"/>
         <appender-ref ref="FILE"/>
+        <appender-ref ref="LOGSTASH"/>
     </root>
 </configuration>
diff --git a/auth-center/src/main/resources/logback-configuration/logback-base.xml b/auth-center/src/main/resources/logback-configuration/logback-base.xml
@@ -21,18 +21,23 @@
     <!-- Log for Multi Environment Configuration -->
     <springProfile name="development-local">
         <property name="ENVIRONMENT" value="development-local"/>
+        <property name="LOGSTASH_HOST" value="localhost"/>
     </springProfile>
     <springProfile name="development-docker">
         <property name="ENVIRONMENT" value="development-docker"/>
+        <property name="LOGSTASH_HOST" value="maf.logstash.${ENVIRONMENT}"/>
     </springProfile>
     <springProfile name="test">
         <property name="ENVIRONMENT" value="test"/>
+        <property name="LOGSTASH_HOST" value="maf.logstash.${ENVIRONMENT}"/>
     </springProfile>
     <springProfile name="stage">
         <property name="ENVIRONMENT" value="stage"/>
+        <property name="LOGSTASH_HOST" value="maf.logstash.${ENVIRONMENT}"/>
     </springProfile>
     <springProfile name="production">
         <property name="ENVIRONMENT" value="production"/>
+        <property name="LOGSTASH_HOST" value="maf.logstash.${ENVIRONMENT}"/>
     </springProfile>
 
     <!-- CONSOLE Log Configuration -->
@@ -60,11 +65,20 @@
         <!-- in particular if post-processing tools impose size limits on the log files. -->
         <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
             <!-- rollover daily -->
-            <fileNamePattern>${LOG_HOME}/${APPLICATION}-${ENVIRONMENT}-${HOSTNAME}.%d{yyyy-MM-dd}.%i.log.gz</fileNamePattern>
+            <fileNamePattern>${LOG_HOME}/${APPLICATION}-${ENVIRONMENT}-${HOSTNAME}.%d{yyyy-MM-dd}.%i.log.gz
+            </fileNamePattern>
             <!-- each file should be at most 5MB, keep 7 days worth of history, but at most 1GB -->
             <maxFileSize>5MB</maxFileSize>
             <maxHistory>7</maxHistory>
             <totalSizeCap>1GB</totalSizeCap>
         </rollingPolicy>
     </appender>
+
+    <!-- LOGSTASH -->
+    <appender name="LOGSTASH" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
+        <destination>${LOGSTASH_HOST}:4560</destination>
+        <encoder charset="UTF-8" class="net.logstash.logback.encoder.LogstashEncoder">
+            <customFields>{"appname": "${APPLICATION}"}</customFields>
+        </encoder>
+    </appender>
 </included>
diff --git a/auth-center/src/main/resources/logback-configuration/logback-development-docker.xml b/auth-center/src/main/resources/logback-configuration/logback-development-docker.xml
@@ -6,5 +6,6 @@
     <root level="INFO">
         <appender-ref ref="CONSOLE"/>
         <appender-ref ref="FILE"/>
+        <appender-ref ref="LOGSTASH"/>
     </root>
 </configuration>
diff --git a/auth-center/src/main/resources/logback-configuration/logback-development-local.xml b/auth-center/src/main/resources/logback-configuration/logback-development-local.xml
@@ -6,5 +6,6 @@
     <root level="INFO">
         <appender-ref ref="CONSOLE"/>
         <appender-ref ref="FILE"/>
+        <appender-ref ref="LOGSTASH"/>
     </root>
 </configuration>
diff --git a/auth-center/src/main/resources/logback-configuration/logback-production.xml b/auth-center/src/main/resources/logback-configuration/logback-production.xml
@@ -6,5 +6,6 @@
     <root level="INFO">
         <appender-ref ref="CONSOLE"/>
         <appender-ref ref="FILE"/>
+        <appender-ref ref="LOGSTASH"/>
     </root>
 </configuration>
diff --git a/auth-center/src/main/resources/logback-configuration/logback-stage.xml b/auth-center/src/main/resources/logback-configuration/logback-stage.xml
@@ -6,5 +6,6 @@
     <root level="INFO">
         <appender-ref ref="CONSOLE"/>
         <appender-ref ref="FILE"/>
+        <appender-ref ref="LOGSTASH"/>
     </root>
 </configuration>
diff --git a/auth-center/src/main/resources/logback-configuration/logback-test.xml b/auth-center/src/main/resources/logback-configuration/logback-test.xml
@@ -6,5 +6,6 @@
     <root level="INFO">
         <appender-ref ref="CONSOLE"/>
         <appender-ref ref="FILE"/>
+        <appender-ref ref="LOGSTASH"/>
     </root>
 </configuration>
diff --git a/docker/.env b/docker/.env
@@ -13,6 +13,9 @@ ZIPKIN_MYSQL_TAG=2.22.2
 ZIPKIN_DEPENDENCIES_TAG=2.6.3
 ZIPKIN_TAG=2.23.2
 CONSUL_TAG=1.9.3
+ELASTICSEARCH_TAG=7.10.1
+LOGSTASH_TAG=7.10.1
+KIBANA_TAG=7.10.1
 
 ##################################################
 # SPRING_ENVIRONMENT available values:           #
@@ -35,6 +38,9 @@ ZIPKIN_MYSQL_CONTAINER_NAME="maf.zipkin-mysql.${SPRING_ENVIRONMENT}"
 ZIPKIN_DEPENDENCIES_CONTAINER_NAME="maf.zipkin-dependencies.${SPRING_ENVIRONMENT}"
 ZIPKIN_CONTAINER_NAME="maf.zipkin.${SPRING_ENVIRONMENT}"
 CONSUL_SERVICE_REGISTRY_CONTAINER_NAME="maf.consul-service-registry.${SPRING_ENVIRONMENT}"
+ELASTICSEARCH_CONTAINER_NAME="maf.elasticsearch.${SPRING_ENVIRONMENT}"
+LOGSTASH_CONTAINER_NAME="maf.logstash.${SPRING_ENVIRONMENT}"
+KIBANA_CONTAINER_NAME="maf.kibana.${SPRING_ENVIRONMENT}"
 SPRING_BOOT_ADMIN_CONTAINER_NAME="maf.spring-boot-admin.${SPRING_ENVIRONMENT}"
 ############### Business Services ################
 API_GATEWAY_CONTAINER_NAME="maf.api-gateway.${SPRING_ENVIRONMENT}"

diff --git a/docker/config/logstash/logstash-spring-boot.conf b/docker/config/logstash/logstash-spring-boot.conf
@@ -0,0 +1,19 @@
+input {
+  tcp {
+    mode => "server"
+    host => "0.0.0.0"
+    port => "4560"
+    codec => json_lines
+  }
+}
+output {
+  #stdout {
+  #  codec => rubydebug
+  #}
+  elasticsearch {
+    hosts => ["es:9200"]
+    index => "springboot-logstash-%{[appname]}-%{+YYYY.MM.dd}"
+    #user => "elastic"
+    #password => "changeme"
+  }
+}
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
@@ -127,6 +127,53 @@ services:
     networks:
       muscle_and_fitness_network:
 
+  elasticsearch:
+    container_name: "${ELASTICSEARCH_CONTAINER_NAME}"
+    image: elasticsearch:${ELASTICSEARCH_TAG}
+    ports:
+      - "9200:9200"
+    restart: always
+    volumes:
+      - "~/docker-file-mapping/${ELASTICSEARCH_CONTAINER_NAME}/plugins:/usr/share/elasticsearch/plugins"
+      - "~/docker-file-mapping/${ELASTICSEARCH_CONTAINER_NAME}/data:/usr/share/elasticsearch/data"
+    environment:
+      - "cluster.name=elasticsearch"
+      - "discovery.type=single-node"
+      - "ES_JAVA_OPTS=-Xms256m -Xmx256m"
+    networks:
+      muscle_and_fitness_network:
+
+  kibana:
+    container_name: "${KIBANA_CONTAINER_NAME}"
+    image: kibana:${KIBANA_TAG}
+    links:
+      - elasticsearch:es # Not sure what the fuck is this for
+    depends_on:
+      elasticsearch:
+        condition: service_started
+    environment:
+      elasticsearch.hosts: "http://${ELASTICSEARCH_CONTAINER_NAME}:9200"
+    ports:
+      - "5601:5601"
+    networks:
+      muscle_and_fitness_network:
+
+  logstash:
+    container_name: "${LOGSTASH_CONTAINER_NAME}"
+    image: logstash:${LOGSTASH_TAG}
+    links:
+      - elasticsearch:es # Not sure what the fuck is this for
+    depends_on:
+      elasticsearch:
+        condition: service_started
+    volumes:
+      - ./config/logstash/logstash-spring-boot.conf:/usr/share/logstash/pipeline/logstash.conf
+      # - "~/docker-file-mapping/${LOGSTASH_CONTAINER_NAME}:/usr/share/logstash/config/logstash.yml"
+    ports:
+      - "4560:4560"
+    networks:
+      muscle_and_fitness_network:
+
   spring-boot-admin:
     container_name: "${SPRING_BOOT_ADMIN_CONTAINER_NAME}"
     hostname: "${SPRING_BOOT_ADMIN_CONTAINER_NAME}"

diff --git a/exercise-mis/src/main/resources/logback-configuration/logback-base.xml b/exercise-mis/src/main/resources/logback-configuration/logback-base.xml
@@ -21,18 +21,23 @@
     <!-- Log for Multi Environment Configuration -->
     <springProfile name="development-local">
         <property name="ENVIRONMENT" value="development-local"/>
+        <property name="LOGSTASH_HOST" value="localhost"/>
     </springProfile>
     <springProfile name="development-docker">
         <property name="ENVIRONMENT" value="development-docker"/>
+        <property name="LOGSTASH_HOST" value="maf.logstash.${ENVIRONMENT}"/>
     </springProfile>
     <springProfile name="test">
         <property name="ENVIRONMENT" value="test"/>
+        <property name="LOGSTASH_HOST" value="maf.logstash.${ENVIRONMENT}"/>
     </springProfile>
     <springProfile name="stage">
         <property name="ENVIRONMENT" value="stage"/>
+        <property name="LOGSTASH_HOST" value="maf.logstash.${ENVIRONMENT}"/>
     </springProfile>
     <springProfile name="production">
         <property name="ENVIRONMENT" value="production"/>
+        <property name="LOGSTASH_HOST" value="maf.logstash.${ENVIRONMENT}"/>
     </springProfile>
 
     <!-- CONSOLE Log Configuration -->
@@ -67,4 +72,12 @@
             <totalSizeCap>1GB</totalSizeCap>
         </rollingPolicy>
     </appender>
+
+    <!-- LOGSTASH -->
+    <appender name="LOGSTASH" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
+        <destination>${LOGSTASH_HOST}:4560</destination>
+        <encoder charset="UTF-8" class="net.logstash.logback.encoder.LogstashEncoder">
+            <customFields>{"appname": "${APPLICATION}"}</customFields>
+        </encoder>
+    </appender>
 </included>
diff --git a/exercise-mis/src/main/resources/logback-configuration/logback-development-docker.xml b/exercise-mis/src/main/resources/logback-configuration/logback-development-docker.xml
@@ -6,5 +6,6 @@
     <root level="INFO">
         <appender-ref ref="CONSOLE"/>
         <appender-ref ref="FILE"/>
+        <appender-ref ref="LOGSTASH"/>
     </root>
 </configuration>
diff --git a/exercise-mis/src/main/resources/logback-configuration/logback-development-local.xml b/exercise-mis/src/main/resources/logback-configuration/logback-development-local.xml
@@ -6,5 +6,6 @@
     <root level="INFO">
         <appender-ref ref="CONSOLE"/>
         <appender-ref ref="FILE"/>
+        <appender-ref ref="LOGSTASH"/>
     </root>
 </configuration>
diff --git a/exercise-mis/src/main/resources/logback-configuration/logback-production.xml b/exercise-mis/src/main/resources/logback-configuration/logback-production.xml
@@ -6,5 +6,6 @@
     <root level="INFO">
         <appender-ref ref="CONSOLE"/>
         <appender-ref ref="FILE"/>
+        <appender-ref ref="LOGSTASH"/>
     </root>
 </configuration>
diff --git a/exercise-mis/src/main/resources/logback-configuration/logback-stage.xml b/exercise-mis/src/main/resources/logback-configuration/logback-stage.xml
@@ -6,5 +6,6 @@
     <root level="INFO">
         <appender-ref ref="CONSOLE"/>
         <appender-ref ref="FILE"/>
+        <appender-ref ref="LOGSTASH"/>
     </root>
 </configuration>
diff --git a/exercise-mis/src/main/resources/logback-configuration/logback-test.xml b/exercise-mis/src/main/resources/logback-configuration/logback-test.xml
@@ -6,5 +6,6 @@
     <root level="INFO">
         <appender-ref ref="CONSOLE"/>
         <appender-ref ref="FILE"/>
+        <appender-ref ref="LOGSTASH"/>
     </root>
 </configuration>
diff --git a/muscle-mis/src/main/resources/logback-configuration/logback-base.xml b/muscle-mis/src/main/resources/logback-configuration/logback-base.xml
@@ -21,18 +21,23 @@
     <!-- Log for Multi Environment Configuration -->
     <springProfile name="development-local">
         <property name="ENVIRONMENT" value="development-local"/>
+        <property name="LOGSTASH_HOST" value="localhost"/>
     </springProfile>
     <springProfile name="development-docker">
         <property name="ENVIRONMENT" value="development-docker"/>
+        <property name="LOGSTASH_HOST" value="maf.logstash.${ENVIRONMENT}"/>
     </springProfile>
     <springProfile name="test">
         <property name="ENVIRONMENT" value="test"/>
+        <property name="LOGSTASH_HOST" value="maf.logstash.${ENVIRONMENT}"/>
     </springProfile>
     <springProfile name="stage">
         <property name="ENVIRONMENT" value="stage"/>
+        <property name="LOGSTASH_HOST" value="maf.logstash.${ENVIRONMENT}"/>
     </springProfile>
     <springProfile name="production">
         <property name="ENVIRONMENT" value="production"/>
+        <property name="LOGSTASH_HOST" value="maf.logstash.${ENVIRONMENT}"/>
     </springProfile>
 
     <!-- CONSOLE Log Configuration -->
@@ -67,4 +72,12 @@
             <totalSizeCap>1GB</totalSizeCap>
         </rollingPolicy>
     </appender>
+
+    <!-- LOGSTASH -->
+    <appender name="LOGSTASH" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
+        <destination>${LOGSTASH_HOST}:4560</destination>
+        <encoder charset="UTF-8" class="net.logstash.logback.encoder.LogstashEncoder">
+            <customFields>{"appname": "${APPLICATION}"}</customFields>
+        </encoder>
+    </appender>
 </included>
diff --git a/muscle-mis/src/main/resources/logback-configuration/logback-development-docker.xml b/muscle-mis/src/main/resources/logback-configuration/logback-development-docker.xml
@@ -6,5 +6,6 @@
     <root level="INFO">
         <appender-ref ref="CONSOLE"/>
         <appender-ref ref="FILE"/>
+        <appender-ref ref="LOGSTASH"/>
     </root>
 </configuration>
diff --git a/muscle-mis/src/main/resources/logback-configuration/logback-development-local.xml b/muscle-mis/src/main/resources/logback-configuration/logback-development-local.xml
@@ -6,5 +6,6 @@
     <root level="INFO">
         <appender-ref ref="CONSOLE"/>
         <appender-ref ref="FILE"/>
+        <appender-ref ref="LOGSTASH"/>
     </root>
 </configuration>
diff --git a/muscle-mis/src/main/resources/logback-configuration/logback-production.xml b/muscle-mis/src/main/resources/logback-configuration/logback-production.xml
@@ -6,5 +6,6 @@
     <root level="INFO">
         <appender-ref ref="CONSOLE"/>
         <appender-ref ref="FILE"/>
+        <appender-ref ref="LOGSTASH"/>
     </root>
 </configuration>
diff --git a/muscle-mis/src/main/resources/logback-configuration/logback-stage.xml b/muscle-mis/src/main/resources/logback-configuration/logback-stage.xml
@@ -6,5 +6,6 @@
     <root level="INFO">
         <appender-ref ref="CONSOLE"/>
         <appender-ref ref="FILE"/>
+        <appender-ref ref="LOGSTASH"/>
     </root>
 </configuration>
diff --git a/muscle-mis/src/main/resources/logback-configuration/logback-test.xml b/muscle-mis/src/main/resources/logback-configuration/logback-test.xml
@@ -6,5 +6,6 @@
     <root level="INFO">
         <appender-ref ref="CONSOLE"/>
         <appender-ref ref="FILE"/>
+        <appender-ref ref="LOGSTASH"/>
     </root>
 </configuration>
diff --git a/pom.xml b/pom.xml
@@ -49,6 +49,7 @@
         <java-faker.version>1.0.2</java-faker.version>
         <easyexcel.version>2.2.6</easyexcel.version>
         <poi.version>5.0.0</poi.version>
+        <logstash-logback-encoder.version>6.6</logstash-logback-encoder.version>
     </properties>
 
     <!-- The modules (sometimes called subprojects) to build as a part of this project. -->