update feedback #238 (comment)

.gitignore

@@ -31,5 +31,7 @@ yarn-error.log
 # cypress test
 /cypress/screenshots/
 /cypress/videos/
-/swoole/.env
-/swoole/vendor/
+/log-receiver/.env
+/log-receiver/vendor
+/log-receiver/config/*
+!/log-receiver/config/syslog.yml

composer.json

@@ -2,7 +2,7 @@
     "type": "project",
     "license": "proprietary",
     "require": {
-        "php": ">=7.2.5",
+        "php": ">=8.1",
         "ext-ctype": "*",
         "ext-iconv": "*",
         "ext-json": "*",

docker-compose.yml

@@ -47,7 +47,7 @@ services:
             - "127.0.0.1:9506:9506/udp"
             - "9507:9507"
         volumes:
-            - ./swoole:/var/www/dev-php:cached
+            - ./log-receiver:/var/www/dev-php:cached
         command: "/usr/bin/bash -c 'composer install && php server.php'"
         links:
             - db

swoole/.env.dist → log-receiver/.env.dist

@@ -1,3 +1,6 @@
+#LOG LEVEL: debug, error //default: error
+LOG_LEVEL=debug
+
 SERVER_HOST=0.0.0.0
 UDP_SERVER_PORT=9506
 TCP_SERVER_PORT=9507

swoole/README.md → log-receiver/README.md

@@ -5,38 +5,42 @@
 ### 1.1 Server
 
 ```dotenv
+#Log level: debug, error //default: error
+LOG_LEVEL=error
+
+#Server
 SERVER_HOST=0.0.0.0
 UDP_SERVER_PORT=9506
 TCP_SERVER_PORT=9507
 
+#Database
 CLICKHOUSE_DB_HOST=
 CLICKHOUSE_DB_PORT=
 CLICKHOUSE_DB_USERNAME=
 CLICKHOUSE_DB_PASSWORD=
 CLICKHOUSE_DB_DBNAME=
 CLICKHOUSE_DB_POOL_SIZE=50
-
 CLICKHOUSE_DB_TIMEOUT=5 #in seconds
 CLICKHOUSE_DB_CONNECTION_TIMEOUT=5 #in seconds
 ```
 
 ### 1.2 Syslog pattern
 
 ```yaml
-// config/syslog.yml
+# config/syslog.yml
 pattern: '<%{POSINT:pri}>%{POSINT:version} %{TIMESTAMP_ISO8601:timestamp} %{HOSTNAME:hostname} %{USERNAME:table_name} %{USERNAME:proc_id} %{USERNAME:app_name} \- %{GREEDYDATA:message}'
 ```
 
 ### 1.3 Table
 
 ```yaml
-// config/{table_name}.yml
-// Ex: config/nginx_access.yml
+# config/{table_name}.yaml
+# Ex: config/nginx_access.yaml
 
 type: grok
 pattern: '%{TIMESTAMP_ISO8601:date} %{TIMESTAMP_ISO8601:timestamp} %{IP} %{GREEDYDATA:text}'
 
-// or
+# or
 
 type: json
 ```
@@ -54,11 +58,11 @@ php server.php
 ### 1. Send udp message with json format
 
 ```shell
-docker run --log-driver syslog --log-opt syslog-address=udp://{udp_server_address}:9506 --log-opt tag="{table_name}" --log-opt syslog-format=rfc5424 alpine echo '{"text":"testing json format","date":"2022-02-18 09:47:14","timestamp":"2022-02-18 09:47:00"}'
+docker-compose exec php logger --udp --server {udp_server_address} --port {udp_server_ports} --tag udp_logs '{"text":"testing json format","date":"2022-02-18 09:47:14","timestamp":"2022-02-18 09:47:00"}'
 ```
 
 ### 2. Send udp message with grok format
 
 ```shell
-docker run --log-driver syslog --log-opt syslog-address=udp://{udp_server_address}:9506 --log-opt tag="{table_name}" --log-opt syslog-format=rfc5424 alpine echo '2022-02-18 17:11:14 2022-02-18 17:11:14 10.0.0.219 testing grok message'
+docker-compose exec php logger --udp --server {udp_server_address} --port {udp_server_ports} --tag udp_logs '2022-02-18 17:11:14 2022-02-18 17:11:14 10.0.0.219 testing grok message'
 ```

swoole/composer.json → log-receiver/composer.json

@@ -3,7 +3,8 @@
     "smi2/phpclickhouse": "^1.4",
     "vlucas/phpdotenv": "^5.4",
     "symfony/yaml": "^6.0",
-    "symfony/uid": "^6.0"
+    "symfony/uid": "^6.0",
+    "php": ">=8.0"
   },
   "autoload": {
     "psr-4": {

swoole/config/syslog.yml → log-receiver/config/syslog.yml

@@ -1 +1 @@
-pattern: '<%{POSINT:pri}>%{POSINT:version} %{TIMESTAMP_ISO8601:timestamp} %{HOSTNAME:hostname} %{USERNAME:table_name} %{USERNAME:proc_id} %{USERNAME:app_name} \- %{GREEDYDATA:message}'
+pattern: '<%{POSINT:pri}>%{POSINT:version} %{TIMESTAMP_ISO8601:timestamp} %{HOSTNAME:hostname} %{USERNAME:table_name} %{USERNAME:proc_id} %{USERNAME:app_name} (\[%{DATA:structured_data}\]|\-) %{GREEDYDATA:message}'

swoole/server.php → log-receiver/server.php

@@ -2,6 +2,8 @@
 
 require_once dirname(__FILE__) . "/vendor/autoload.php";
 
+use App\Enum\LogLevel;
+use App\Logger\Logger;
 use App\Message\Process;
 use Swoole\Server;
 
@@ -18,38 +20,79 @@
     throw new Exception('Please provide at least TCP or UDP port');
 }
 
+/**
+ * Config log level
+ */
+$logLevel = LogLevel::tryFrom($_ENV['LOG_LEVEL'] ?? 'error');
+
+if ($logLevel === null) {
+    $logLevel = LogLevel::from('error');
+}
+
+$_ENV['LOG_LEVEL'] = $logLevel->value();
+
+/**
+ * Create new server instance
+ */
 $server = new Server($host, null, SWOOLE_BASE);
+
+/**
+ * Server configuration
+ */
+$server->set([
+    'log_level' => $_ENV['LOG_LEVEL']
+]);
+
+/**
+ * TCP request handler
+ */
 $server->on(
     'receive',
     function (Server $server, int $fd, int $reactorId, string $data) {
+        Logger::debug("[{$fd}] Received message {$data}");
         try {
             Process::process($data);
         } catch (\Exception $e) {
-            echo $e->getMessage() . "\r\n" . $e->getTraceAsString();
+            Logger::error($e->getMessage() . "\r\n" . $e->getTraceAsString());
         }
 
         $server->send($fd, "OK");
     }
 );
 
+/**
+ * Listening TCP request
+ */
+if ($tcpPort) {
+    $server->listen($host, $tcpPort, SWOOLE_SOCK_TCP);
+
+    Logger::info("TCP server listening on: \nHost {$host}\nPort {$tcpPort}");
+}
+
+/**
+ * UDP request handler
+ */
 if ($udpPort) {
     $server->listen($host, $udpPort, SWOOLE_SOCK_UDP);
     $server->on(
         'packet',
         function (Server $server, string $message, array $clientInfo) {
+            Logger::debug("Received message {$message} from " . $clientInfo['address']);
+
             try {
                 Process::process($message);
             } catch (\Exception $e) {
-                echo $e->getMessage() . "\r\n" . $e->getTraceAsString();
+                Logger::error($e->getMessage() . "\r\n" . $e->getTraceAsString());
             }
 
             $server->sendto($clientInfo['address'], $clientInfo['port'], "OK");
         }
     );
-}
 
-if ($tcpPort) {
-    $server->listen($host, $tcpPort, SWOOLE_SOCK_TCP);
+    Logger::info("UDP server listening on: \nHost {$host}\nPort {$udpPort}");
 }
 
+/**
+ * Starting server
+ */
 $server->start();

log-receiver/src/Config/Config.php

@@ -0,0 +1,36 @@
+<?php
+
+namespace App\Config;
+
+ use App\Exceptions\ConfigFileInvalid;
+use App\Exceptions\ConfigFileNotFound;
+use App\Logger\Logger;
+use Symfony\Component\Yaml\Yaml;
+
+class Config
+{
+    public static function read($filename, $key, $default = '')
+    {
+        $path = $_ENV['ROOT_DIR'] . "/config/{$filename}.y{ml,aml}";
+        $files = glob($path, GLOB_BRACE);
+
+        if (empty($files)) {
+            throw new ConfigFileNotFound("Config file not found. Please create a config file \"{$filename}.yaml\" in \/config directory");
+        }
+
+        $configFile = array_shift($files);
+        $config = Yaml::parseFile($configFile);
+
+        if (empty($config)) {
+            throw new ConfigFileInvalid("The file {$configFile} could not be read or the YAML is not valid");
+        }
+
+        if (!array_key_exists($key, $config)) {
+            Logger::debug("Key \"{$key}\" does not exist in config file, return default value \"{$default}\"");
+
+            return $default;
+        }
+
+        return $config[$key];
+    }
+}

log-receiver/src/Enum/LogLevel.php

@@ -0,0 +1,17 @@
+<?php
+
+namespace App\Enum;
+
+enum LogLevel: string
+{
+    case DEBUG = 'debug';
+    case ERROR = 'error';
+
+    public function value(): int
+    {
+        return match ($this) {
+            LogLevel::DEBUG => SWOOLE_LOG_DEBUG,
+            LogLevel::ERROR => SWOOLE_LOG_ERROR,
+        };
+    }
+}

log-receiver/src/Exceptions/ConfigFileInvalid.php

@@ -0,0 +1,8 @@
+<?php
+
+namespace App\Exceptions;
+
+class ConfigFileInvalid extends \Exception
+{
+    protected $message = 'The file could not be read or the YAML is not valid';
+}