diff --git a/.gitignore b/.gitignore index e110a134..3cb84138 100644 --- a/.gitignore +++ b/.gitignore @@ -31,5 +31,7 @@ yarn-error.log # cypress test /cypress/screenshots/ /cypress/videos/ -/swoole/.env -/swoole/vendor/ +/log-receiver/.env +/log-receiver/vendor +/log-receiver/config/* +!/log-receiver/config/syslog.yml diff --git a/composer.json b/composer.json index 8352b6f2..40527944 100644 --- a/composer.json +++ b/composer.json @@ -2,7 +2,7 @@ "type": "project", "license": "proprietary", "require": { - "php": ">=7.2.5", + "php": ">=8.1", "ext-ctype": "*", "ext-iconv": "*", "ext-json": "*", diff --git a/docker-compose.yml b/docker-compose.yml index 016f5463..6144aeea 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -47,7 +47,7 @@ services: - "127.0.0.1:9506:9506/udp" - "9507:9507" volumes: - - ./swoole:/var/www/dev-php:cached + - ./log-receiver:/var/www/dev-php:cached command: "/usr/bin/bash -c 'composer install && php server.php'" links: - db diff --git a/swoole/.env.dist b/log-receiver/.env.dist similarity index 82% rename from swoole/.env.dist rename to log-receiver/.env.dist index 069f39ca..1f590187 100644 --- a/swoole/.env.dist +++ b/log-receiver/.env.dist @@ -1,3 +1,6 @@ +#LOG LEVEL: debug, error //default: error +LOG_LEVEL=debug + SERVER_HOST=0.0.0.0 UDP_SERVER_PORT=9506 TCP_SERVER_PORT=9507 diff --git a/swoole/README.md b/log-receiver/README.md similarity index 61% rename from swoole/README.md rename to log-receiver/README.md index 23b1432b..2fd766e1 100644 --- a/swoole/README.md +++ b/log-receiver/README.md @@ -5,17 +5,21 @@ ### 1.1 Server ```dotenv +#Log level: debug, error //default: error +LOG_LEVEL=error + +#Server SERVER_HOST=0.0.0.0 UDP_SERVER_PORT=9506 TCP_SERVER_PORT=9507 +#Database CLICKHOUSE_DB_HOST= CLICKHOUSE_DB_PORT= CLICKHOUSE_DB_USERNAME= CLICKHOUSE_DB_PASSWORD= CLICKHOUSE_DB_DBNAME= CLICKHOUSE_DB_POOL_SIZE=50 - CLICKHOUSE_DB_TIMEOUT=5 #in seconds CLICKHOUSE_DB_CONNECTION_TIMEOUT=5 #in seconds ``` @@ -23,20 +27,20 @@ CLICKHOUSE_DB_CONNECTION_TIMEOUT=5 #in seconds ### 1.2 Syslog pattern ```yaml -// config/syslog.yml +# config/syslog.yml pattern: '<%{POSINT:pri}>%{POSINT:version} %{TIMESTAMP_ISO8601:timestamp} %{HOSTNAME:hostname} %{USERNAME:table_name} %{USERNAME:proc_id} %{USERNAME:app_name} \- %{GREEDYDATA:message}' ``` ### 1.3 Table ```yaml -// config/{table_name}.yml -// Ex: config/nginx_access.yml +# config/{table_name}.yaml +# Ex: config/nginx_access.yaml type: grok pattern: '%{TIMESTAMP_ISO8601:date} %{TIMESTAMP_ISO8601:timestamp} %{IP} %{GREEDYDATA:text}' -// or +# or type: json ``` @@ -54,11 +58,11 @@ php server.php ### 1. Send udp message with json format ```shell -docker run --log-driver syslog --log-opt syslog-address=udp://{udp_server_address}:9506 --log-opt tag="{table_name}" --log-opt syslog-format=rfc5424 alpine echo '{"text":"testing json format","date":"2022-02-18 09:47:14","timestamp":"2022-02-18 09:47:00"}' +docker-compose exec php logger --udp --server {udp_server_address} --port {udp_server_ports} --tag udp_logs '{"text":"testing json format","date":"2022-02-18 09:47:14","timestamp":"2022-02-18 09:47:00"}' ``` ### 2. Send udp message with grok format ```shell -docker run --log-driver syslog --log-opt syslog-address=udp://{udp_server_address}:9506 --log-opt tag="{table_name}" --log-opt syslog-format=rfc5424 alpine echo '2022-02-18 17:11:14 2022-02-18 17:11:14 10.0.0.219 testing grok message' +docker-compose exec php logger --udp --server {udp_server_address} --port {udp_server_ports} --tag udp_logs '2022-02-18 17:11:14 2022-02-18 17:11:14 10.0.0.219 testing grok message' ``` diff --git a/swoole/composer.json b/log-receiver/composer.json similarity index 79% rename from swoole/composer.json rename to log-receiver/composer.json index b3dc876d..54af697e 100644 --- a/swoole/composer.json +++ b/log-receiver/composer.json @@ -3,7 +3,8 @@ "smi2/phpclickhouse": "^1.4", "vlucas/phpdotenv": "^5.4", "symfony/yaml": "^6.0", - "symfony/uid": "^6.0" + "symfony/uid": "^6.0", + "php": ">=8.0" }, "autoload": { "psr-4": { diff --git a/swoole/composer.lock b/log-receiver/composer.lock similarity index 97% rename from swoole/composer.lock rename to log-receiver/composer.lock index f562e549..847d7b99 100644 --- a/swoole/composer.lock +++ b/log-receiver/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "f709ba75a5dcf55da5f9a42c0aa8cd9a", + "content-hash": "4bcd1e7451c517d18d770e8a183c64e4", "packages": [ { "name": "graham-campbell/result-type", @@ -200,7 +200,7 @@ }, { "name": "symfony/polyfill-ctype", - "version": "v1.24.0", + "version": "v1.25.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-ctype.git", @@ -232,12 +232,12 @@ } }, "autoload": { - "psr-4": { - "Symfony\\Polyfill\\Ctype\\": "" - }, "files": [ "bootstrap.php" - ] + ], + "psr-4": { + "Symfony\\Polyfill\\Ctype\\": "" + } }, "notification-url": "https://packagist.org/downloads/", "license": [ @@ -262,7 +262,7 @@ "portable" ], "support": { - "source": "https://github.com/symfony/polyfill-ctype/tree/v1.24.0" + "source": "https://github.com/symfony/polyfill-ctype/tree/v1.25.0" }, "funding": [ { @@ -282,7 +282,7 @@ }, { "name": "symfony/polyfill-mbstring", - "version": "v1.24.0", + "version": "v1.25.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-mbstring.git", @@ -345,7 +345,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.24.0" + "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.25.0" }, "funding": [ { @@ -365,16 +365,16 @@ }, { "name": "symfony/polyfill-php80", - "version": "v1.24.0", + "version": "v1.25.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-php80.git", - "reference": "57b712b08eddb97c762a8caa32c84e037892d2e9" + "reference": "4407588e0d3f1f52efb65fbe92babe41f37fe50c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-php80/zipball/57b712b08eddb97c762a8caa32c84e037892d2e9", - "reference": "57b712b08eddb97c762a8caa32c84e037892d2e9", + "url": "https://api.github.com/repos/symfony/polyfill-php80/zipball/4407588e0d3f1f52efb65fbe92babe41f37fe50c", + "reference": "4407588e0d3f1f52efb65fbe92babe41f37fe50c", "shasum": "" }, "require": { @@ -428,7 +428,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-php80/tree/v1.24.0" + "source": "https://github.com/symfony/polyfill-php80/tree/v1.25.0" }, "funding": [ { @@ -444,11 +444,11 @@ "type": "tidelift" } ], - "time": "2021-09-13T13:58:33+00:00" + "time": "2022-03-04T08:16:47+00:00" }, { "name": "symfony/polyfill-uuid", - "version": "v1.24.0", + "version": "v1.25.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-uuid.git", @@ -510,7 +510,7 @@ "uuid" ], "support": { - "source": "https://github.com/symfony/polyfill-uuid/tree/v1.24.0" + "source": "https://github.com/symfony/polyfill-uuid/tree/v1.25.0" }, "funding": [ { @@ -763,7 +763,9 @@ "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, - "platform": [], + "platform": { + "php": ">=8.0" + }, "platform-dev": [], "plugin-api-version": "2.2.0" } diff --git a/swoole/config/syslog.yml b/log-receiver/config/syslog.yml similarity index 59% rename from swoole/config/syslog.yml rename to log-receiver/config/syslog.yml index f837b429..0d65f31d 100644 --- a/swoole/config/syslog.yml +++ b/log-receiver/config/syslog.yml @@ -1 +1 @@ -pattern: '<%{POSINT:pri}>%{POSINT:version} %{TIMESTAMP_ISO8601:timestamp} %{HOSTNAME:hostname} %{USERNAME:table_name} %{USERNAME:proc_id} %{USERNAME:app_name} \- %{GREEDYDATA:message}' +pattern: '<%{POSINT:pri}>%{POSINT:version} %{TIMESTAMP_ISO8601:timestamp} %{HOSTNAME:hostname} %{USERNAME:table_name} %{USERNAME:proc_id} %{USERNAME:app_name} (\[%{DATA:structured_data}\]|\-) %{GREEDYDATA:message}' diff --git a/swoole/server.php b/log-receiver/server.php similarity index 55% rename from swoole/server.php rename to log-receiver/server.php index 378751da..32d309cb 100644 --- a/swoole/server.php +++ b/log-receiver/server.php @@ -2,6 +2,8 @@ require_once dirname(__FILE__) . "/vendor/autoload.php"; +use App\Enum\LogLevel; +use App\Logger\Logger; use App\Message\Process; use Swoole\Server; @@ -18,38 +20,79 @@ throw new Exception('Please provide at least TCP or UDP port'); } +/** + * Config log level + */ +$logLevel = LogLevel::tryFrom($_ENV['LOG_LEVEL'] ?? 'error'); + +if ($logLevel === null) { + $logLevel = LogLevel::from('error'); +} + +$_ENV['LOG_LEVEL'] = $logLevel->value(); + +/** + * Create new server instance + */ $server = new Server($host, null, SWOOLE_BASE); + +/** + * Server configuration + */ +$server->set([ + 'log_level' => $_ENV['LOG_LEVEL'] +]); + +/** + * TCP request handler + */ $server->on( 'receive', function (Server $server, int $fd, int $reactorId, string $data) { + Logger::debug("[{$fd}] Received message {$data}"); try { Process::process($data); } catch (\Exception $e) { - echo $e->getMessage() . "\r\n" . $e->getTraceAsString(); + Logger::error($e->getMessage() . "\r\n" . $e->getTraceAsString()); } $server->send($fd, "OK"); } ); +/** + * Listening TCP request + */ +if ($tcpPort) { + $server->listen($host, $tcpPort, SWOOLE_SOCK_TCP); + + Logger::info("TCP server listening on: \nHost {$host}\nPort {$tcpPort}"); +} + +/** + * UDP request handler + */ if ($udpPort) { $server->listen($host, $udpPort, SWOOLE_SOCK_UDP); $server->on( 'packet', function (Server $server, string $message, array $clientInfo) { + Logger::debug("Received message {$message} from " . $clientInfo['address']); + try { Process::process($message); } catch (\Exception $e) { - echo $e->getMessage() . "\r\n" . $e->getTraceAsString(); + Logger::error($e->getMessage() . "\r\n" . $e->getTraceAsString()); } $server->sendto($clientInfo['address'], $clientInfo['port'], "OK"); } ); -} -if ($tcpPort) { - $server->listen($host, $tcpPort, SWOOLE_SOCK_TCP); + Logger::info("UDP server listening on: \nHost {$host}\nPort {$udpPort}"); } +/** + * Starting server + */ $server->start(); diff --git a/swoole/src/Clickhouse/Client.php b/log-receiver/src/Clickhouse/Client.php similarity index 100% rename from swoole/src/Clickhouse/Client.php rename to log-receiver/src/Clickhouse/Client.php diff --git a/swoole/src/Clickhouse/Pool.php b/log-receiver/src/Clickhouse/Pool.php similarity index 100% rename from swoole/src/Clickhouse/Pool.php rename to log-receiver/src/Clickhouse/Pool.php diff --git a/log-receiver/src/Config/Config.php b/log-receiver/src/Config/Config.php new file mode 100644 index 00000000..72c44332 --- /dev/null +++ b/log-receiver/src/Config/Config.php @@ -0,0 +1,36 @@ + SWOOLE_LOG_DEBUG, + LogLevel::ERROR => SWOOLE_LOG_ERROR, + }; + } +} diff --git a/log-receiver/src/Exceptions/ConfigFileInvalid.php b/log-receiver/src/Exceptions/ConfigFileInvalid.php new file mode 100644 index 00000000..7680a7e7 --- /dev/null +++ b/log-receiver/src/Exceptions/ConfigFileInvalid.php @@ -0,0 +1,8 @@ +pattern_regex, $pattern, $matches, PREG_SET_ORDER)) { - //var_dump($matches); foreach ($matches as $match) { $subPattern = $this->resolve($this->patterns[$match['pattern']]); if (isset($match['subname']) && !empty($match['subname'])) { - //$this->fieldMap[$match['subname']] = ++$this->matchCount; //$subPattern; $this->fieldMap[++$this->matchCount] = $match['subname']; $subPattern = '(?<' . $match['subname'] . '>' . $subPattern . ')'; - //var_dump($subPattern); } $pattern = str_replace($match[0], $subPattern, $pattern, $replaced); } diff --git a/log-receiver/src/Logger/Logger.php b/log-receiver/src/Logger/Logger.php new file mode 100644 index 00000000..91067d10 --- /dev/null +++ b/log-receiver/src/Logger/Logger.php @@ -0,0 +1,41 @@ +parse($syslogPattern, $message); + Logger::debug("Syslog Message Parsed: " . json_encode($syslogParsed)); + /** * Then we will get the config of the table + * + * if table_name does not exist in syslog message, + * use the app_name instead */ - $table = $syslogParsed['table_name']; + $table = $syslogParsed['table_name'] ?? $syslogParsed['app_name']; $messageType = Config::read($table, 'type'); $parsedMessage = ''; @@ -41,12 +49,38 @@ public static function parse($message) */ if ($messageType == 'json') { $parsedMessage = json_decode($syslogParsed['message'], true); + + if ($parsedMessage === null) { + throw new InvalidJsonMessage(); + } } /** * Parse the grok message */ elseif ($messageType == 'grok') { $messagePattern = Config::read($table, 'pattern'); $parsedMessage = $grok->parse($messagePattern, $syslogParsed['message']); + + if (empty($parsedMessage)) { + throw new InvalidGrokMessage(); + } + } + + if (empty($parsedMessage['timestamp'])) { + Logger::debug("Syslog timestamp: {$syslogParsed['timestamp']}"); + + try { + $timestamp = \DateTime::createFromFormat('Y-m-d\TH:i:sP', $syslogParsed['timestamp']); + + if (empty($timestamp)) { + throw new \Exception('Invalid syslog timestamp format'); + } + + $timestamp = $timestamp->format('Y-m-d H:i:s'); + } catch (\Exception $e) { + $timestamp = date('Y-m-d H:i:s'); + } + + $parsedMessage['timestamp'] = $timestamp; } /** diff --git a/log-receiver/src/Message/Process.php b/log-receiver/src/Message/Process.php new file mode 100644 index 00000000..1cb50968 --- /dev/null +++ b/log-receiver/src/Message/Process.php @@ -0,0 +1,54 @@ +get(); + + if (!array_key_exists('_id', $data)) { + $data['_id'] = (Uuid::v4())->toRfc4122(); + } + + /** + * Fetch existing columns in clickhouse db table + */ + $table = $db->select("DESCRIBE {$tableName}"); + $existingColumns = array_column($table->rawData()['data'], 'name'); + Logger::debug("DESCRIBE TABLE \"{$tableName}\" in Clickhouse: " . json_encode($existingColumns)); + + /** + * Filtering data, remove data's column if not exist in $existingColumns + */ + $data = array_filter($data, function ($value, $key) use ($existingColumns) { + return in_array($key, $existingColumns); + }, ARRAY_FILTER_USE_BOTH); + + $values = array_values($data); + $columns = array_keys($data); + + Logger::debug("Filtered data: " . json_encode($data)); + + $db->insert($tableName, + [ + $values, + ], + $columns + ); + + $pool->put($db); + } +} diff --git a/swoole/src/Config/Config.php b/swoole/src/Config/Config.php deleted file mode 100644 index fb23f722..00000000 --- a/swoole/src/Config/Config.php +++ /dev/null @@ -1,25 +0,0 @@ -get(); - - if (!array_key_exists('_id', $data)) { - $data['_id'] = (Uuid::v4())->toRfc4122(); - } - - $values = array_values($data); - $columns = array_keys($data); - - $db->insert($table, - [ - $values, - ], - $columns - ); - - $pool->put($db); - } -}