Skip to content

Commit

Permalink
Merge pull request #8052 from romayalon/romy-nc-health-fixes
Browse files Browse the repository at this point in the history 
NC | Health fix cli arguments validation and some small fixes
  • Loading branch information
@romayalon
romayalon authored Jun 16, 2024
2 parents 0f63797 + db26f04 commit 33d7471
Show file tree
Hide file tree
Showing 5 changed files with 181 additions and 43 deletions.
109 changes: 75 additions & 34 deletions src/cmd/health.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,9 @@ const native_fs_utils = require('../util/native_fs_utils');
const { read_stream_join } = require('../util/buffer_utils');
const { make_https_request } = require('../util/http_utils');
const { TYPES } = require('../manage_nsfs/manage_nsfs_constants');
const { validate_input_types } = require('../manage_nsfs/manage_nsfs_validations');
const { get_boolean_or_string_value } = require('../manage_nsfs/manage_nsfs_cli_utils');
const ManageCLIError = require('../manage_nsfs/manage_nsfs_cli_errors').ManageCLIError;

const HELP = `
Help:
Expand Down Expand Up @@ -120,22 +123,30 @@ class NSFSHealth {
this.all_account_details = options.all_account_details;
this.all_bucket_details = options.all_bucket_details;
}

/**
* nc_nsfs_health will execute the following checks and will result with a health report
* 1. get noobaa service state
* 2. get endpoint response
* 3. get service memory usage
* 5. if all_account_details flag provided check accounts status
* 6. if all_bucket_details flag provided check buckets status
* @returns {Promise<object>}
*/
async nc_nsfs_health() {
let endpoint_state;
let memory;
const { service_status, pid } = await this.get_service_state(NOOBAA_SERVICE);
const noobaa_service_state = await this.get_service_state(NOOBAA_SERVICE);
const { service_status, pid } = noobaa_service_state;
if (pid !== '0') {
endpoint_state = await this.get_endpoint_response();
memory = await this.get_service_memory_usage();
}
let bucket_details;
let account_details;
const response_code = endpoint_state ? endpoint_state.response.response_code : 'NOT_RUNNING';
let service_health = 'OK';
const service_health = service_status !== 'active' || pid === '0' || response_code !== 'RUNNING' ? 'NOTOK' : 'OK';

if (service_status !== 'active' || pid === '0' || response_code !== 'RUNNING') {
service_health = 'NOTOK';
}
const error_code = await this.get_error_code(service_status, pid, response_code);
if (this.all_bucket_details) bucket_details = await this.get_bucket_status(this.config_root);
if (this.all_account_details) account_details = await this.get_account_status(this.config_root);
Expand All @@ -145,13 +156,7 @@ class NSFSHealth {
memory: memory,
error: error_code,
checks: {
services: [{
name: NOOBAA_SERVICE,
service_status: service_status,
pid: pid,
error_type: health_errors_tyes.PERSISTENT,
}
],
services: [noobaa_service_state],
endpoint: {
endpoint_state,
error_type: health_errors_tyes.TEMPORARY,
Expand Down Expand Up @@ -188,9 +193,7 @@ class NSFSHealth {
});
} catch (err) {
console.log('Error while pinging endpoint host :' + HOSTNAME + ', port ' + this.https_port, err);
return {
response: fork_response_code.NOT_RUNNING.response_code,
};
endpoint_state = { response: fork_response_code.NOT_RUNNING };
}
return endpoint_state;
}
Expand All @@ -206,17 +209,35 @@ class NSFSHealth {
}

async get_service_state(service_name) {
const service_status = await os_util.exec('systemctl show -p ActiveState --value ' + service_name, {
ignore_rc: true,
return_stdout: true,
trim_stdout: true,
});
const pid = await os_util.exec('systemctl show --property MainPID --value ' + service_name, {
ignore_rc: true,
return_stdout: true,
trim_stdout: true,
});
return { service_status: service_status, pid: pid };
let service_status;
let pid;
try {
service_status = await os_util.exec('systemctl show -p ActiveState --value ' + service_name, {
ignore_rc: false,
return_stdout: true,
trim_stdout: true,
});
} catch (err) {
dbg.warn('could not receive service active state', service_name, err);
service_status = 'missing service status info';
}
try {
pid = await os_util.exec('systemctl show --property MainPID --value ' + service_name, {
ignore_rc: false,
return_stdout: true,
trim_stdout: true,
});
} catch (err) {
dbg.warn('could not receive service active state', service_name, err);
pid = 'missing pid info';
}
const service_health = { name: service_name, service_status, pid };
if (['inactive', 'missing service status info'].includes(service_status)) {
service_health.error_type = health_errors_tyes.PERSISTENT;
const service_error_name = _.upperCase(_.camelCase(service_name)) + '_SERVICE_FAILED';
service_health.error_code = health_errors[service_error_name];
}
return service_health;
}

async make_endpoint_health_request(url_path) {
Expand Down Expand Up @@ -284,11 +305,17 @@ class NSFSHealth {
}

async get_service_memory_usage() {
const memory_status = await os_util.exec('systemctl status ' + NOOBAA_SERVICE + ' | grep Memory ', {
ignore_rc: true,
return_stdout: true,
trim_stdout: true,
});
let memory_status;
try {
memory_status = await os_util.exec('systemctl status ' + NOOBAA_SERVICE + ' | grep Memory ', {
ignore_rc: false,
return_stdout: true,
trim_stdout: true,
});
} catch (err) {
dbg.warn('could not receive service active state', NOOBAA_SERVICE, err);
memory_status = 'Memory: missing memory info';
}
if (memory_status) {
const memory = memory_status.split('Memory: ')[1].trim();
return memory;
Expand Down Expand Up @@ -383,11 +410,18 @@ async function main(argv = minimist(process.argv.slice(2))) {
throw new Error('Root permissions required for NSFS Health execution.');
}
if (argv.help || argv.h) return print_usage();
await validate_input_types(TYPES.HEALTH, '', argv);
const config_root = argv.config_root ? String(argv.config_root) : config.NSFS_NC_CONF_DIR;
const https_port = Number(argv.https_port) || config.ENDPOINT_SSL_PORT;
const deployment_type = argv.deployment_type || 'nc';
const all_account_details = argv.all_account_details || false;
const all_bucket_details = argv.all_bucket_details || false;
const all_account_details = get_boolean_or_string_value(argv.all_account_details);
const all_bucket_details = get_boolean_or_string_value(argv.all_bucket_details);

if (argv.config_root) {
config.NSFS_NC_CONF_DIR = String(argv.config_root);
config.load_nsfs_nc_config();
config.reload_nsfs_nc_config();
}
if (deployment_type === 'nc') {
const health = new NSFSHealth({ https_port, config_root, all_account_details, all_bucket_details });
const health_status = await health.nc_nsfs_health();
Expand All @@ -399,7 +433,14 @@ async function main(argv = minimist(process.argv.slice(2))) {
}
} catch (err) {
dbg.error('Health: exit on error', err.stack || err);
process.exit(2);
const manage_err = ((err instanceof ManageCLIError) && err) ||
new ManageCLIError({
...(ManageCLIError.FS_ERRORS_TO_MANAGE[err.code] ||
ManageCLIError.RPC_ERROR_TO_MANAGE[err.rpc_code] ||
ManageCLIError.InternalError), cause: err });
process.stdout.write(manage_err.to_string() + '\n', () => {
process.exit(1);
});
}
}

Expand Down
14 changes: 13 additions & 1 deletion src/manage_nsfs/manage_nsfs_constants.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ const TYPES = {
BUCKET: 'bucket',
IP_WHITELIST: 'whitelist',
GLACIER: 'glacier',
HEALTH: 'health'
};

const ACTIONS = {
Expand Down Expand Up @@ -62,6 +63,8 @@ const VALID_OPTIONS_GLACIER = {
'expiry': new Set([ GLOBAL_CONFIG_ROOT]),
};

const VALID_OPTIONS_HEALTH = new Set(['https_port', 'deployment_type', 'all_account_details', 'all_bucket_details', ...GLOBAL_CONFIG_OPTIONS]);

const VALID_OPTIONS_WHITELIST = new Set(['ips', ...GLOBAL_CONFIG_OPTIONS]);

const VALID_OPTIONS_FROM_FILE = new Set(['from_file', ...GLOBAL_CONFIG_OPTIONS]);
Expand All @@ -73,6 +76,7 @@ const VALID_OPTIONS = {
whitelist_options: VALID_OPTIONS_WHITELIST,
from_file_options: VALID_OPTIONS_FROM_FILE,
anonymous_account_options: VALID_OPTIONS_ANONYMOUS_ACCOUNT,
health_options: VALID_OPTIONS_HEALTH
};

const OPTION_TYPE = {
Expand All @@ -98,10 +102,17 @@ const OPTION_TYPE = {
show_secrets: 'boolean',
ips: 'string',
force: 'boolean',
anonymous: 'boolean'
anonymous: 'boolean',
// health options
deployment_type: 'string',
all_account_details: 'boolean',
all_bucket_details: 'boolean',
https_port: 'number'
};

const BOOLEAN_STRING_VALUES = ['true', 'false'];
const BOOLEAN_STRING_OPTIONS = new Set(['allow_bucket_creation', 'regenerate', 'wide', 'show_secrets', 'force',
'force_md5_etag', 'all_account_details', 'all_bucket_details', 'anonymous']);

//options that can be unset using ''
const LIST_UNSETABLE_OPTIONS = ['fs_backend', 's3_policy', 'force_md5_etag'];
Expand All @@ -118,6 +129,7 @@ exports.VALID_OPTIONS = VALID_OPTIONS;
exports.OPTION_TYPE = OPTION_TYPE;
exports.FROM_FILE = FROM_FILE;
exports.BOOLEAN_STRING_VALUES = BOOLEAN_STRING_VALUES;
exports.BOOLEAN_STRING_OPTIONS = BOOLEAN_STRING_OPTIONS;
exports.LIST_UNSETABLE_OPTIONS = LIST_UNSETABLE_OPTIONS;

exports.LIST_ACCOUNT_FILTERS = LIST_ACCOUNT_FILTERS;
Expand Down
8 changes: 5 additions & 3 deletions src/manage_nsfs/manage_nsfs_validations.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ const ManageCLIError = require('../manage_nsfs/manage_nsfs_cli_errors').ManageCL
const bucket_policy_utils = require('../endpoint/s3/s3_bucket_policy_utils');
const { throw_cli_error, get_config_file_path, get_bucket_owner_account,
get_config_data, get_options_from_file, has_access_keys } = require('../manage_nsfs/manage_nsfs_cli_utils');
const { TYPES, ACTIONS, VALID_OPTIONS, OPTION_TYPE, FROM_FILE, BOOLEAN_STRING_VALUES,
const { TYPES, ACTIONS, VALID_OPTIONS, OPTION_TYPE, FROM_FILE, BOOLEAN_STRING_VALUES, BOOLEAN_STRING_OPTIONS,
GLACIER_ACTIONS, LIST_UNSETABLE_OPTIONS, ANONYMOUS } = require('../manage_nsfs/manage_nsfs_constants');

/////////////////////////////
Expand Down Expand Up @@ -68,7 +68,7 @@ function validate_type_and_action(type, action) {
if (!Object.values(TYPES).includes(type)) throw_cli_error(ManageCLIError.InvalidType);
if (type === TYPES.ACCOUNT || type === TYPES.BUCKET) {
if (!Object.values(ACTIONS).includes(action)) throw_cli_error(ManageCLIError.InvalidAction);
} else if (type === TYPES.IP_WHITELIST) {
} else if (type === TYPES.IP_WHITELIST || type === TYPES.HEALTH) {
if (action !== '') throw_cli_error(ManageCLIError.InvalidAction);
} else if (type === TYPES.GLACIER) {
if (!Object.values(GLACIER_ACTIONS).includes(action)) throw_cli_error(ManageCLIError.InvalidAction);
Expand Down Expand Up @@ -99,6 +99,8 @@ function validate_no_extra_options(type, action, input_options, is_options_from_
}
} else if (type === TYPES.GLACIER) {
valid_options = VALID_OPTIONS.glacier_options[action];
} else if (type === TYPES.HEALTH) {
valid_options = VALID_OPTIONS.health_options;
} else {
valid_options = VALID_OPTIONS.whitelist_options;
}
Expand Down Expand Up @@ -139,7 +141,7 @@ function validate_options_type_by_value(input_options_with_data) {
continue;
}
// special case for boolean values
if (['allow_bucket_creation', 'regenerate', 'wide', 'show_secrets', 'force', 'force_md5_etag', 'anonymous'].includes(option) && validate_boolean_string_value(value)) {
if (BOOLEAN_STRING_OPTIONS.has(option) && validate_boolean_string_value(value)) {
continue;
}
// special case for bucket_policy (from_file)
Expand Down
30 changes: 30 additions & 0 deletions src/test/system_tests/test_utils.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -266,6 +266,35 @@ async function exec_manage_cli(type, action, options) {
}
}

/**
* exec_health_cli runs the health cli
* @param {object} options
* @returns {Promise<string>}
*/
async function exec_health_cli(options) {
let flags = ``;
for (const key in options) {
if (options[key] !== undefined) {
const value = options[key];
if (typeof options[key] === 'boolean') {
flags += `--${key} `;
continue;
}
flags += `--${key} ${value} `;
}
}
flags = flags.trim();

const command = `node src/cmd/health ${flags}`;
try {
const res = await os_utils.exec(command, { return_stdout: true });
return res;
} catch (err) {
console.error('test_utils.exec_health_cli error', err);
throw err;
}
}

/**
* create_fs_user_by_platform creates a file system user by platform
* @param {string} new_user
Expand Down Expand Up @@ -368,6 +397,7 @@ exports.generate_s3_client = generate_s3_client;
exports.invalid_nsfs_root_permissions = invalid_nsfs_root_permissions;
exports.get_coretest_path = get_coretest_path;
exports.exec_manage_cli = exec_manage_cli;
exports.exec_health_cli = exec_health_cli;
exports.create_fs_user_by_platform = create_fs_user_by_platform;
exports.delete_fs_user_by_platform = delete_fs_user_by_platform;
exports.set_path_permissions_and_owner = set_path_permissions_and_owner;
Expand Down
Loading

0 comments on commit 33d7471

Please sign in to comment.