PR3 - Persistent logging

Signed-off-by: jackyalbo <jacky.albo@gmail.com>

config.js

@@ -659,6 +659,9 @@ config.BUCKET_DIFF_FOR_REPLICATION = true;
 
 config.BUCKET_LOG_UPLOAD_ENABLED = true;
 config.BUCKET_LOG_UPLOADER_DELAY = 5 * 60 * 1000;
+config.BUCKET_LOG_TYPE = process.env.BUCKET_LOG_TYPE || 'BEST_EFFORT';
+config.PERSISTENT_BUCKET_LOG_DIR = '/var/run/noobaa-nsfs/bucket_log';
+config.PERSISTENT_BUCKET_LOG_NS = 'bucket_logging';
 
 ///////////////////////////
 //      KEY ROTATOR      //

src/api/bucket_api.js

@@ -828,7 +828,11 @@ module.exports = {
                 },
             },
             reply: {
-                $ref: 'common_api#/definitions/bucket_logging',
+                oneOf: [{
+                    $ref: 'common_api#/definitions/bucket_logging',
+                }, {
+                    type: 'null'
+                }]
             },
             auth: {
                 system: ['admin', 'user']

src/cmd/manage_nsfs.js

@@ -15,6 +15,7 @@ const SensitiveString = require('../util/sensitive_string');
 const ManageCLIError = require('../manage_nsfs/manage_nsfs_cli_errors').ManageCLIError;
 const ManageCLIResponse = require('../manage_nsfs/manage_nsfs_cli_responses').ManageCLIResponse;
 const manage_nsfs_glacier = require('../manage_nsfs/manage_nsfs_glacier');
+const manage_nsfs_logging = require('../manage_nsfs/manage_nsfs_logging');
 const nsfs_schema_utils = require('../manage_nsfs/nsfs_schema_utils');
 const { print_usage } = require('../manage_nsfs/manage_nsfs_help_utils');
 const { TYPES, ACTIONS, LIST_ACCOUNT_FILTERS, LIST_BUCKET_FILTERS,
@@ -104,6 +105,8 @@ async function main(argv = minimist(process.argv.slice(2))) {
             await whitelist_ips_management(argv);
         } else if (type === TYPES.GLACIER) {
             await glacier_management(argv);
+        } else if (type === TYPES.LOGGING) {
+            await logging_management(user_input);
         } else {
             // we should not get here (we check it before)
             throw_cli_error(ManageCLIError.InvalidType);
@@ -731,5 +734,9 @@ async function manage_glacier_operations(action, argv) {
     }
 }
 
+async function logging_management(user_input) {
+    await manage_nsfs_logging.export_bucket_logging(user_input);
+}
+
 exports.main = main;
 if (require.main === module) main();

src/endpoint/s3/s3_bucket_logging.js

@@ -5,6 +5,8 @@ const dbg = require('../../util/debug_module')(__filename);
 const http_utils = require('../../util/http_utils');
 const dgram = require('node:dgram');
 const { Buffer } = require('node:buffer');
+const { PersistentLogger } = require('../../util/persistent_logger');
+const config = require('../../../config');
 
 
 async function send_bucket_op_logs(req, res) {
@@ -14,7 +16,7 @@ async function send_bucket_op_logs(req, res) {
 
         if (is_bucket_logging_enabled(bucket_info)) {
             dbg.log2("Bucket logging is enabled for Bucket : ", req.params.bucket);
-            endpoint_bucket_op_logs(req.op_name, req, res, bucket_info);
+            await endpoint_bucket_op_logs(req.op_name, req, res, bucket_info);
         }
     }
 }
@@ -54,15 +56,30 @@ const create_syslog_udp_socket = (() => {
 })();
 
 
-function endpoint_bucket_op_logs(op_name, req, res, source_bucket) {
+async function endpoint_bucket_op_logs(op_name, req, res, source_bucket) {
 
     // 1 - Get all the information to be logged in a log message.
     // 2 - Format it and send it to log bucket/syslog.
     const s3_log = get_bucket_log_record(op_name, source_bucket, req, res);
     dbg.log1("Bucket operation logs = ", s3_log);
 
+    switch (config.BUCKET_LOG_TYPE) {
+        case 'PERSISTENT': {
+            const log = new PersistentLogger(config.PERSISTENT_BUCKET_LOG_DIR, req.params.bucket, { locking: 'SHARED' });
+            await log.append(JSON.stringify(s3_log));
+            await log.close();
+            break;
+        }
+        default: {
+            send_op_logs_to_syslog(req.object_sdk.rpc_client.rpc.router.syslog, s3_log);
+        }
+    }
+
+}
+
+function send_op_logs_to_syslog(syslog, s3_log) {
     const buffer = Buffer.from(JSON.stringify(s3_log));
-    const {client, port, hostname} = create_syslog_udp_socket(req.object_sdk.rpc_client.rpc.router.syslog);
+    const {client, port, hostname} = create_syslog_udp_socket(syslog);
     if (client && port && hostname) {
         client.send(buffer, port, hostname, err => {
             if (err) {
@@ -72,13 +89,12 @@ function endpoint_bucket_op_logs(op_name, req, res, source_bucket) {
     } else {
         dbg.log0(`Could not send bucket logs: client: ${client} port: ${port} hostname:${hostname}`);
     }
-
 }
 
 function get_bucket_log_record(op_name, source_bucket, req, res) {
 
     const client_ip = http_utils.parse_client_ip(req);
-    let status_code;
+    let status_code = 102;
     if (res && res.statusCode) {
         status_code = res.statusCode;
     }

src/endpoint/s3/s3_rest.js

@@ -12,6 +12,7 @@ const s3_logging = require('./s3_bucket_logging');
 const time_utils = require('../../util/time_utils');
 const http_utils = require('../../util/http_utils');
 const signature_utils = require('../../util/signature_utils');
+const config = require('../../../config');
 
 const S3_MAX_BODY_LEN = 4 * 1024 * 1024;
 
@@ -68,6 +69,11 @@ async function s3_rest(req, res) {
             await handle_website_error(req, res, err);
         } else {
             handle_error(req, res, err);
+            try {
+                await s3_logging.send_bucket_op_logs(req, res); // logging again with error
+            } catch (err1) {
+                dbg.error("Could not log bucket operation:", err1);
+            }
         }
     }
 }
@@ -118,7 +124,13 @@ async function handle_request(req, res) {
     usage_report.s3_usage_info.total_calls += 1;
     usage_report.s3_usage_info[op_name] = (usage_report.s3_usage_info[op_name] || 0) + 1;
 
-
+    if (config.BUCKET_LOG_TYPE === 'PERSISTENT') {
+        try {
+            await s3_logging.send_bucket_op_logs(req); // logging intension - no result
+        } catch (err) {
+            dbg.error("Could not log bucket operation:", err);
+        }
+    }
 
     if (req.query && req.query.versionId) {
         const caching = await req.object_sdk.read_bucket_sdk_caching_info(req.params.bucket);
@@ -151,7 +163,7 @@ async function handle_request(req, res) {
     http_utils.send_reply(req, res, reply, options);
     collect_bucket_usage(op, req, res);
     try {
-        await s3_logging.send_bucket_op_logs(req);
+        await s3_logging.send_bucket_op_logs(req, res); // logging again with result
     } catch (err) {
         dbg.error("Could not log bucket operation:", err);
     }

src/manage_nsfs/manage_nsfs_cli_errors.js

@@ -83,7 +83,7 @@ ManageCLIError.InvalidArgumentType = Object.freeze({
 
 ManageCLIError.InvalidType = Object.freeze({
     code: 'InvalidType',
-    message: 'Invalid type, available types are account, bucket or whitelist',
+    message: 'Invalid type, available types are account, bucket, logging or whitelist',
     http_code: 400,
 });
 
@@ -157,6 +157,16 @@ ManageCLIError.InvalidMasterKey = Object.freeze({
     http_code: 500,
 });
 
+//////////////////////////////
+//// BUCKET LOGGING ERRORS ///
+//////////////////////////////
+
+ManageCLIError.LoggingExportFailed = Object.freeze({
+    code: 'LoggingExportFailed',
+    message: 'Logging export attmept failed',
+    http_code: 500,
+});
+
 ////////////////////////
 //// ACCOUNT ERRORS ////
 ////////////////////////
@@ -426,6 +436,7 @@ const NSFS_CLI_ERROR_EVENT_MAP = {
     AccountDeleteForbiddenHasBuckets: NoobaaEvent.ACCOUNT_DELETE_FORBIDDEN,
     BucketAlreadyExists: NoobaaEvent.BUCKET_ALREADY_EXISTS,
     BucketSetForbiddenNoBucketOwner: NoobaaEvent.UNAUTHORIZED,
+    LoggingExportFailed: NoobaaEvent.LOGGING_FAILED,
 };
 
 exports.ManageCLIError = ManageCLIError;

src/manage_nsfs/manage_nsfs_cli_responses.js

@@ -102,12 +102,21 @@ ManageCLIResponse.BucketList = Object.freeze({
     list: {}
 });
 
+///////////////////////////////
+// LOGGING RESPONSES ///
+///////////////////////////////
+ManageCLIResponse.LoggingExported = Object.freeze({
+    code: 'LoggingExported',
+    status: {}
+});
+
 const NSFS_CLI_SUCCESS_EVENT_MAP = {
     AccountCreated: NoobaaEvent.ACCOUNT_CREATED,
     AccountDeleted: NoobaaEvent.ACCOUNT_DELETED,
     BucketCreated: NoobaaEvent.BUCKET_CREATED,
     BucketDeleted: NoobaaEvent.BUCKET_DELETE,
     WhiteListIPUpdated: NoobaaEvent.WHITELIST_UPDATED,
+    LoggingExported: NoobaaEvent.LOGGING_EXPORTED,
 };
 
 exports.ManageCLIResponse = ManageCLIResponse;

src/manage_nsfs/manage_nsfs_constants.js

@@ -6,7 +6,8 @@ const TYPES = {
     BUCKET: 'bucket',
     IP_WHITELIST: 'whitelist',
     GLACIER: 'glacier',
-    HEALTH: 'health'
+    HEALTH: 'health',
+    LOGGING: 'logging',
 };
 
 const ACTIONS = {

src/manage_nsfs/manage_nsfs_events_utils.js

@@ -325,4 +325,26 @@ NoobaaEvent.INVALID_BUCKET_STATE = Object.freeze({
     state: 'HEALTHY',
 });
 
+NoobaaEvent.LOGGING_EXPORTED = Object.freeze({
+    event_code: 'bucket_logging_exported',
+    entity_type: 'NODE',
+    event_type: 'INFO',
+    message: 'Bucket logs was exported to target buckets',
+    description: 'Bucket logs was successfully exported to target buckets',
+    scope: 'NODE',
+    severity: 'INFO',
+    state: 'HEALTHY',
+});
+
+NoobaaEvent.LOGGING_FAILED = Object.freeze({
+    event_code: 'bucket_logging_export_failed',
+    entity_type: 'NODE',
+    event_type: 'ERROR',
+    message: 'Bucket logging export failed.',
+    description: 'Bucket logging export failed due to error',
+    scope: 'NODE',
+    severity: 'ERROR',
+    state: 'DEGRADED',
+});
+
 exports.NoobaaEvent = NoobaaEvent;

src/manage_nsfs/manage_nsfs_help_utils.js

@@ -23,7 +23,7 @@ Usage:
 const ARGUMENTS = `
 Arguments:
 
-    <type>    Set the resource type: account, bucket, or whitelist
+    <type>    Set the resource type: account, bucket, logging or whitelist
     <action>  Action could be: add, update, list, status, and delete for accounts/buckets
 `;
 
@@ -59,6 +59,11 @@ Flags:
                                                                         in format: '["127.0.0.1", "192.0.10.0", "3002:0bd6:0000:0000:0000:ee00:0033:6778"]'
 `;
 
+const LOGGING_FLAGS = `
+logging                                                                 Use this to upload all the bucket logging collected in the system to their target buckets
+
+`;
+
 const GLOBAL_CONFIG_ROOT_ALL_FLAG = `
 --config_root <string>                                (optional)        Use configuration files path (default config.NSFS_NC_DEFAULT_CONF_DIR)
 --config_root_backend <none | GPFS | CEPH_FS | NFSv4> (optional)        Use the filesystem type in the configuration (default config.NSFS_NC_CONFIG_DIR_BACKEND)
@@ -220,6 +225,9 @@ function print_usage(type, action) {
         case TYPES.IP_WHITELIST:
             process.stdout.write(WHITELIST_FLAGS.trimStart());
             break;
+        case TYPES.LOGGING:
+            process.stdout.write(LOGGING_FLAGS.trimStart() + GLOBAL_CONFIG_ROOT_ALL_FLAG.trimStart());
+            break;
         case TYPES.GLACIER:
             print_help_glacier(action);
             break;