From 0796e790febb6cc51a6f252cbaf74c94ec4e0465 Mon Sep 17 00:00:00 2001 From: shirady <57721533+shirady@users.noreply.github.com> Date: Tue, 5 Mar 2024 13:59:11 +0200 Subject: [PATCH] NSFS | NC | from-file in account 1. Add the from_file option in the account that would include validation. The JSON file will include options (as key-value structure) that are the same as the flags we suggest. 2. Change the structure inside the main so that eventually it doesn't matter whether the parameters are from the flags arguments (argv) or file. 3. Move get_access_keys to a separate function. 4. Fix indentation in fetch_account_data in the part of access_key as SensitiveString. 5. Remove an unused parameter from validate_options_type_by_value. 6. In native_fs_utils.js rename the function read_config_file to read_file. 7. Update json in doc so _id will appear first (as fixed in this PR). Signed-off-by: shirady <57721533+shirady@users.noreply.github.com> --- docs/design/NonContainerizedNSFSDesign.md | 4 +- docs/non_containerized_NSFS.md | 25 ++- src/cmd/manage_nsfs.js | 209 +++++++++++------- src/manage_nsfs/manage_nsfs_cli_errors.js | 12 + src/manage_nsfs/manage_nsfs_constants.js | 25 ++- src/manage_nsfs/manage_nsfs_help_utils.js | 23 +- .../test_nc_nsfs_account_cli.test.js | 177 +++++++++++++++ src/test/unit_tests/test_bucketspace_fs.js | 6 +- src/util/native_fs_utils.js | 14 +- 9 files changed, 374 insertions(+), 121 deletions(-) diff --git a/docs/design/NonContainerizedNSFSDesign.md b/docs/design/NonContainerizedNSFSDesign.md index e5367a0183..27ba9bd832 100644 --- a/docs/design/NonContainerizedNSFSDesign.md +++ b/docs/design/NonContainerizedNSFSDesign.md @@ -27,6 +27,7 @@ node src/cmd/nsfs ../standalon/nsfs_root --config_dir ../standalon/fs_config ```json { + "_id": "65cb1e7c9e6ae40d499c0ae3", // _id automatically generated "name": "user1", "email": "user1", // the email will be internally (the account name), email will not be set by user "creation_date": "2024-01-11T08:24:14.937Z", @@ -41,8 +42,7 @@ node src/cmd/nsfs ../standalon/nsfs_root --config_dir ../standalon/fs_config "gid": 1001, // "new_buckets_path": "/", }, - "allow_bucket_creation": true, - "_id": "65cb1e7c9e6ae40d499c0ae3" // _id automatically generated + "allow_bucket_creation": true } ``` diff --git a/docs/non_containerized_NSFS.md b/docs/non_containerized_NSFS.md index 0f6b186847..296a58d083 100644 --- a/docs/non_containerized_NSFS.md +++ b/docs/non_containerized_NSFS.md @@ -468,13 +468,32 @@ sudo node src/cmd/manage_nsfs bucket delete --config_root ../standalon/config_ro sudo node src/cmd/manage_nsfs bucket list --config_root ../standalon/config_root 2>/dev/null ``` +NSFS management CLI run will create both accounts, access_keys, and buckets directories if they are missing under the config_root directory. **Important**: All the Account/Bucket commands end with `2>/dev/null` to make sure there are no unwanted logs. - +Using `from_file` flag: +- For account and bucket creation users can also pass account or bucket values in JSON file (hereinafter referred to as "options JSON file") instead of passing them in CLI as arguments using flags. +- General use: +``` +sudo node src/cmd/manage_nsfs account add --config_root ../standalon/config_root --from_file +sudo node src/cmd/manage_nsfs bucket add --config_root ../standalon/config_root --from_file +``` +- The options are key-value, where the key is the same as suggested flags. For example: +create JSON file for account: +```json +// JSON file of key-value options for creating an account +{ + "name": "account-1001", + "uid": 1001, + "gid": 1001, + "new_buckets_path": "/tmp/nsfs_root1/my-bucket" +} +``` ``` -sudo node src/cmd/manage_nsfs bucket add --config_root ../standalon/config_root --from_file /json_file/path +sudo node src/cmd/manage_nsfs account add --config_root ../standalon/config_root --from_file ``` -NSFS management CLI command will create both account and bucket dir if it's missing in the config_root path. +- When using `from_file` flag the details about the account/bucket should be only inside the options JSON file. +- The JSON config file and JSON options file of account are different! ## NSFS Certificate diff --git a/src/cmd/manage_nsfs.js b/src/cmd/manage_nsfs.js index 5819a84664..acb1adc97f 100644 --- a/src/cmd/manage_nsfs.js +++ b/src/cmd/manage_nsfs.js @@ -22,7 +22,7 @@ const manage_nsfs_glacier = require('../manage_nsfs/manage_nsfs_glacier'); const bucket_policy_utils = require('../endpoint/s3/s3_bucket_policy_utils'); const nsfs_schema_utils = require('../manage_nsfs/nsfs_schema_utils'); const { print_usage } = require('../manage_nsfs/manage_nsfs_help_utils'); -const { TYPES, ACTIONS, VALID_OPTIONS, OPTION_TYPE, BOOLEAN_STRING_VALUES, +const { TYPES, ACTIONS, VALID_OPTIONS, OPTION_TYPE, FROM_FILE, BOOLEAN_STRING_VALUES, LIST_ACCOUNT_FILTERS, LIST_BUCKET_FILTERS, GLACIER_ACTIONS } = require('../manage_nsfs/manage_nsfs_constants'); const NoobaaEvent = require('../manage_nsfs/manage_nsfs_events_utils').NoobaaEvent; @@ -89,7 +89,8 @@ async function main(argv = minimist(process.argv.slice(2))) { if (argv.help || argv.h) { return print_usage(type, action); } - validate_flags_arguments(type, action, argv); + const user_input_from_file = await validate_input_types(type, action, argv); + const user_input = user_input_from_file || argv; config_root = argv.config_root ? String(argv.config_root) : config.NSFS_NC_CONF_DIR; if (!config_root) throw_cli_error(ManageCLIError.MissingConfigDirPath); @@ -99,11 +100,12 @@ async function main(argv = minimist(process.argv.slice(2))) { config_root_backend = argv.config_root_backend ? String(argv.config_root_backend) : config.NSFS_NC_CONFIG_DIR_BACKEND; await check_and_create_config_dirs(); - const from_file = argv.from_file ? String(argv.from_file) : ''; if (type === TYPES.ACCOUNT) { - await account_management(argv, from_file); + await account_management(action, user_input); } else if (type === TYPES.BUCKET) { - await bucket_management(argv, from_file); + // GAP - from-file in bucket + const path_to_json_options = argv.from_file ? String(argv.from_file) : ''; + await bucket_management(argv, path_to_json_options); } else if (type === TYPES.IP_WHITELIST) { await whitelist_ips_management(argv); } else if (type === TYPES.GLACIER) { @@ -333,20 +335,19 @@ async function manage_bucket_operations(action, data, argv) { } } -async function account_management(argv, from_file) { - const action = argv._[1] || ''; - const show_secrets = get_boolean_or_string_value(argv.show_secrets); - const data = await fetch_account_data(argv, from_file); - await manage_account_operations(action, data, show_secrets, argv); +async function account_management(action, user_input) { + const show_secrets = get_boolean_or_string_value(user_input.show_secrets); + const data = await fetch_account_data(action, user_input); + await manage_account_operations(action, data, show_secrets, user_input); } /** - * set_access_keys will set the access keys either given as args or generated. - * @param {{ access_key: any; secret_key: any; }} argv + * set_access_keys will set the access keys either given or generated. + * @param {string} access_key + * @param {string} secret_key * @param {boolean} generate a flag for generating the access_keys automatically */ -function set_access_keys(argv, generate) { - const { access_key, secret_key } = argv; +function set_access_keys(access_key, secret_key, generate) { let generated_access_key; let generated_secret_key; if (generate) { @@ -362,50 +363,28 @@ function set_access_keys(argv, generate) { } // in name and new_name we allow type number, hence convert it to string -async function fetch_account_data(argv, from_file) { - let data; - let access_keys = [{ - access_key: undefined, - secret_key: undefined, - }]; - let new_access_key; - const action = argv._[1] || ''; - if (from_file) { - const fs_context = native_fs_utils.get_process_fs_context(); - const raw_data = (await nb_native().fs.readFile(fs_context, from_file)).data; - data = JSON.parse(raw_data.toString()); - // GAP - from-file is not validated - } - if (action !== ACTIONS.LIST && action !== ACTIONS.STATUS) _validate_access_keys(argv); - if (action === ACTIONS.ADD || action === ACTIONS.STATUS) { - const regenerate = action === ACTIONS.ADD; - access_keys = set_access_keys(argv, regenerate); - } else if (action === ACTIONS.UPDATE) { - const regenerate = get_boolean_or_string_value(argv.regenerate); - access_keys = set_access_keys(argv, regenerate); - new_access_key = access_keys[0].access_key; - access_keys[0].access_key = undefined; //Setting it as undefined so we can replace the symlink - } - - if (!data) { - data = _.omitBy({ - name: _.isUndefined(argv.name) ? undefined : String(argv.name), - email: _.isUndefined(argv.name) ? undefined : String(argv.name), // temp, keep the email internally - creation_date: action === ACTIONS.ADD ? new Date().toISOString() : undefined, - wide: _.isUndefined(argv.wide) ? undefined : get_boolean_or_string_value(argv.wide), - new_name: _.isUndefined(argv.new_name) ? undefined : String(argv.new_name), - new_access_key, - access_keys, - nsfs_account_config: { - distinguished_name: argv.user, - uid: argv.user ? undefined : argv.uid, - gid: argv.user ? undefined : argv.gid, - new_buckets_path: argv.new_buckets_path, - fs_backend: argv.fs_backend ? String(argv.fs_backend) : config.NSFS_NC_STORAGE_BACKEND - } - }, _.isUndefined); - } +async function fetch_account_data(action, user_input) { + const { access_keys, new_access_key } = get_access_keys(action, user_input); + let data = { + // added undefined values to keep the order the properties when printing the data object + _id: undefined, + name: _.isUndefined(user_input.name) ? undefined : String(user_input.name), + email: _.isUndefined(user_input.name) ? undefined : String(user_input.name), // temp, keep the email internally + creation_date: action === ACTIONS.ADD ? new Date().toISOString() : undefined, + wide: _.isUndefined(user_input.wide) ? undefined : get_boolean_or_string_value(user_input.wide), + new_name: _.isUndefined(user_input.new_name) ? undefined : String(user_input.new_name), + new_access_key, + access_keys, + nsfs_account_config: { + distinguished_name: user_input.user, + uid: user_input.user ? undefined : user_input.uid, + gid: user_input.user ? undefined : user_input.gid, + new_buckets_path: user_input.new_buckets_path, + fs_backend: user_input.fs_backend ? String(user_input.fs_backend) : config.NSFS_NC_STORAGE_BACKEND + } + }; if (action === ACTIONS.UPDATE || action === ACTIONS.DELETE) { + // @ts-ignore data = _.omitBy(data, _.isUndefined); data = await fetch_existing_account_data(data); } @@ -413,17 +392,17 @@ async function fetch_account_data(argv, from_file) { // override values // access_key as SensitiveString data.access_keys[0].access_key = _.isUndefined(data.access_keys[0].access_key) ? undefined : - new SensitiveString(String(data.access_keys[0].access_key)); + new SensitiveString(String(data.access_keys[0].access_key)); // secret_key as SensitiveString data.access_keys[0].secret_key = _.isUndefined(data.access_keys[0].secret_key) ? undefined : new SensitiveString(String(data.access_keys[0].secret_key)); // allow_bucket_creation either set by user or infer from new_buckets_path - if (_.isUndefined(argv.allow_bucket_creation)) { + if (_.isUndefined(user_input.allow_bucket_creation)) { data.allow_bucket_creation = !_.isUndefined(data.nsfs_account_config.new_buckets_path); - } else if (typeof argv.allow_bucket_creation === 'boolean') { - data.allow_bucket_creation = Boolean(argv.allow_bucket_creation); + } else if (typeof user_input.allow_bucket_creation === 'boolean') { + data.allow_bucket_creation = Boolean(user_input.allow_bucket_creation); } else { // string of true or false - data.allow_bucket_creation = argv.allow_bucket_creation.toLowerCase() === 'true'; + data.allow_bucket_creation = user_input.allow_bucket_creation.toLowerCase() === 'true'; } // fs_backend deletion specified with empty string '' (but it is not part of the schema) data.nsfs_account_config.fs_backend = data.nsfs_account_config.fs_backend || undefined; @@ -586,7 +565,7 @@ async function get_account_status(data, show_secrets) { } } -async function manage_account_operations(action, data, show_secrets, argv) { +async function manage_account_operations(action, data, show_secrets, user_input) { if (action === ACTIONS.ADD) { await add_account(data); } else if (action === ACTIONS.STATUS) { @@ -596,7 +575,7 @@ async function manage_account_operations(action, data, show_secrets, argv) { } else if (action === ACTIONS.DELETE) { await delete_account(data); } else if (action === ACTIONS.LIST) { - const account_filters = _.pick(argv, LIST_ACCOUNT_FILTERS); + const account_filters = _.pick(user_input, LIST_ACCOUNT_FILTERS); const accounts = await list_config_files(TYPES.ACCOUNT, accounts_dir_path, data.wide, show_secrets, account_filters); write_stdout_response(ManageCLIResponse.AccountList, accounts); } else { @@ -699,6 +678,49 @@ async function get_config_data(config_file_path, show_secrets = false) { return config_data; } +/** + * get_options_from_file will read a JSON file that include key-value of the options + * (instead of flags) and return its content + * @param {string} file_path + */ +async function get_options_from_file(file_path) { + const fs_context = native_fs_utils.get_process_fs_context(); + try { + const input_options_with_data = await native_fs_utils.read_file(fs_context, file_path); + return input_options_with_data; + } catch (err) { + if (err.code === 'ENOENT') throw_cli_error(ManageCLIError.InvalidFilePath, file_path); + if (err instanceof SyntaxError) throw_cli_error(ManageCLIError.InvalidJSONFile, file_path); + throw err; + } +} + +/** + * get_access_keys will return the access_keys and new_access_key according to the user input + * and action + * @param {string} action + * @param {object} user_input + */ +function get_access_keys(action, user_input) { + let access_keys = [{ + access_key: undefined, + secret_key: undefined, + }]; + let new_access_key; + if (action === ACTIONS.ADD || action === ACTIONS.UPDATE || action === ACTIONS.DELETE) { + _validate_access_keys(user_input.access_key, user_input.secret_key); + } + if (action === ACTIONS.ADD || action === ACTIONS.STATUS) { + const regenerate = action === ACTIONS.ADD; + access_keys = set_access_keys(user_input.access_key, user_input.secret_key, regenerate); + } else if (action === ACTIONS.UPDATE) { + const regenerate = get_boolean_or_string_value(user_input.regenerate); + access_keys = set_access_keys(user_input.access_key, user_input.secret_key, regenerate); + new_access_key = access_keys[0].access_key; + access_keys[0].access_key = undefined; //Setting it as undefined so we can replace the symlink + } + return { access_keys, new_access_key }; +} /////////////////////////// /// VALIDATIONS /// @@ -805,22 +827,37 @@ async function validate_account_args(data, action) { } } -/** validate_flags_arguments checks if input option are valid. +/** + * validate_input_types checks if input option are valid. + * if the the user uses from_file then the validation is on the file (in different iteration) * @param {string} type * @param {string} action * @param {object} argv */ -function validate_flags_arguments(type, action, argv) { +async function validate_input_types(type, action, argv) { validate_type_and_action(type, action); - // when we use validate_no_extra_options - // we don't care about the value, only the flags + // when we use validate_no_extra_options we don't care about the value, only the flags const input_options = Object.keys(argv); + const input_options_with_data = { ...argv }; // the first element is _ with the type and action, so we remove it input_options.shift(); - validate_no_extra_options(type, action, input_options); - const input_options_with_data = { ...argv }; delete input_options_with_data._; + validate_no_extra_options(type, action, input_options, false); validate_options_type_by_value(input_options_with_data); + + // currently we use from_file only in add action + const path_to_json_options = argv.from_file ? String(argv.from_file) : ''; + if (type === TYPES.ACCOUNT && action === ACTIONS.ADD && path_to_json_options) { + const input_options_with_data_from_file = await get_options_from_file(path_to_json_options); + const input_options_from_file = Object.keys(input_options_with_data_from_file); + if (input_options_from_file.includes(FROM_FILE)) { + const details = `${FROM_FILE} should not be passed inside json options`; + throw_cli_error(ManageCLIError.InvalidArgument, details); + } + validate_no_extra_options(type, action, input_options_from_file, true); + validate_options_type_by_value(input_options_with_data_from_file); + return input_options_with_data_from_file; + } } /** @@ -845,13 +882,23 @@ function validate_type_and_action(type, action) { * @param {string} type * @param {string} action * @param {string[]} input_options array with the names of the flags + * @param {boolean} is_options_from_file boolean to indicates that the validation is on values that origin from the file */ -function validate_no_extra_options(type, action, input_options) { +function validate_no_extra_options(type, action, input_options, is_options_from_file) { let valid_options; // for performance, we use Set as data structure - if (type === TYPES.BUCKET) { + const from_file_condition = (type === TYPES.ACCOUNT || type === TYPES.BUCKET) && + action === ACTIONS.ADD && input_options.includes(FROM_FILE); + if (from_file_condition) { + valid_options = VALID_OPTIONS.from_file_options; + } else if (type === TYPES.BUCKET) { valid_options = VALID_OPTIONS.bucket_options[action]; } else if (type === TYPES.ACCOUNT) { valid_options = VALID_OPTIONS.account_options[action]; + if (is_options_from_file) { + valid_options.delete('from_file'); + valid_options.delete('config_root'); + valid_options.delete('config_root_backend'); + } } else if (type === TYPES.GLACIER) { valid_options = VALID_OPTIONS.glacier_options[action]; } else { @@ -864,8 +911,9 @@ function validate_no_extra_options(type, action, input_options) { `${invalid_input_options[0]} is an invalid option` : `${invalid_input_options.join(', ')} are invalid options`; const supported_option_msg = `Supported options are: ${[...valid_options].join(', ')}`; - const err_msg = `${invalid_option_msg} for ${type_and_action}. ${supported_option_msg}`; - throw_cli_error(ManageCLIError.InvalidArgument, err_msg); + let details = `${invalid_option_msg} for ${type_and_action}. ${supported_option_msg}`; + if (from_file_condition) details += ` (when using ${FROM_FILE} flag only partial list of flags are supported)`; + throw_cli_error(ManageCLIError.InvalidArgument, details); } } /** @@ -873,7 +921,6 @@ function validate_no_extra_options(type, action, input_options) { * @param {object} input_options_with_data object with flag (key) and value */ function validate_options_type_by_value(input_options_with_data) { - let details; for (const [option, value] of Object.entries(input_options_with_data)) { const type_of_option = OPTION_TYPE[option]; const type_of_value = typeof value; @@ -886,7 +933,7 @@ function validate_options_type_by_value(input_options_with_data) { if (['allow_bucket_creation', 'regenerate', 'wide', 'show_secrets'].includes(option) && validate_boolean_string_value(value)) { continue; } - details = `type of flag ${option} should be ${type_of_option}`; + const details = `type of flag ${option} should be ${type_of_option}`; throw_cli_error(ManageCLIError.InvalidArgumentType, details); } } @@ -952,16 +999,16 @@ function verify_whitelist_ips(ips_to_validate) { } } -function _validate_access_keys(argv) { +function _validate_access_keys(access_key, secret_key) { // using the access_key flag requires also using the secret_key flag - if (!_.isUndefined(argv.access_key) && _.isUndefined(argv.secret_key)) { + if (!_.isUndefined(access_key) && _.isUndefined(secret_key)) { throw_cli_error(ManageCLIError.MissingAccountSecretKeyFlag); } - if (!_.isUndefined(argv.secret_key) && _.isUndefined(argv.access_key)) { + if (!_.isUndefined(secret_key) && _.isUndefined(access_key)) { throw_cli_error(ManageCLIError.MissingAccountAccessKeyFlag); } // checking the complexity of access_key - if (!_.isUndefined(argv.access_key) && !string_utils.validate_complexity(argv.access_key, { + if (!_.isUndefined(access_key) && !string_utils.validate_complexity(access_key, { require_length: 20, check_uppercase: true, check_lowercase: false, @@ -969,7 +1016,7 @@ function _validate_access_keys(argv) { check_symbols: false, })) throw_cli_error(ManageCLIError.AccountAccessKeyFlagComplexity); // checking the complexity of secret_key - if (!_.isUndefined(argv.secret_key) && !string_utils.validate_complexity(argv.secret_key, { + if (!_.isUndefined(secret_key) && !string_utils.validate_complexity(secret_key, { require_length: 40, check_uppercase: true, check_lowercase: true, diff --git a/src/manage_nsfs/manage_nsfs_cli_errors.js b/src/manage_nsfs/manage_nsfs_cli_errors.js index e3eebfba08..2b3a12ba3d 100644 --- a/src/manage_nsfs/manage_nsfs_cli_errors.js +++ b/src/manage_nsfs/manage_nsfs_cli_errors.js @@ -95,6 +95,18 @@ ManageCLIError.InvalidSchema = Object.freeze({ http_code: 400, }); +ManageCLIError.InvalidFilePath = Object.freeze({ + code: 'InvalidFilePath', + message: 'Invalid file path', + http_code: 400, +}); + +ManageCLIError.InvalidJSONFile = Object.freeze({ + code: 'InvalidJSONFile', + message: 'Invalid JSON file', + http_code: 400, +}); + ////////////////////////////// //// IP WHITE LIST ERRORS //// ////////////////////////////// diff --git a/src/manage_nsfs/manage_nsfs_constants.js b/src/manage_nsfs/manage_nsfs_constants.js index 13e9335265..c28a835084 100644 --- a/src/manage_nsfs/manage_nsfs_constants.js +++ b/src/manage_nsfs/manage_nsfs_constants.js @@ -29,22 +29,23 @@ const CONFIG_SUBDIRS = { }; const GLOBAL_CONFIG_ROOT = 'config_root'; -const GLOBAL_CONFIG_OPTIONS = new Set(['from_file', GLOBAL_CONFIG_ROOT, 'config_root_backend']); +const GLOBAL_CONFIG_OPTIONS = new Set([GLOBAL_CONFIG_ROOT, 'config_root_backend']); +const FROM_FILE = 'from_file'; const VALID_OPTIONS_ACCOUNT = { - 'add': new Set(['name', 'uid', 'gid', 'new_buckets_path', 'user', 'access_key', 'secret_key', 'fs_backend', 'allow_bucket_creation', ...GLOBAL_CONFIG_OPTIONS]), + 'add': new Set(['name', 'uid', 'gid', 'new_buckets_path', 'user', 'access_key', 'secret_key', 'fs_backend', 'allow_bucket_creation', FROM_FILE, ...GLOBAL_CONFIG_OPTIONS]), 'update': new Set(['name', 'uid', 'gid', 'new_buckets_path', 'user', 'access_key', 'secret_key', 'fs_backend', 'allow_bucket_creation', 'new_name', 'regenerate', ...GLOBAL_CONFIG_OPTIONS]), - 'delete': new Set(['name', GLOBAL_CONFIG_ROOT]), - 'list': new Set(['wide', 'show_secrets', GLOBAL_CONFIG_ROOT, 'gid', 'uid', 'user', 'name', 'access_key']), - 'status': new Set(['name', 'access_key', 'show_secrets', GLOBAL_CONFIG_ROOT]), + 'delete': new Set(['name', ...GLOBAL_CONFIG_OPTIONS]), + 'list': new Set(['wide', 'show_secrets', 'gid', 'uid', 'user', 'name', 'access_key', ...GLOBAL_CONFIG_OPTIONS]), + 'status': new Set(['name', 'access_key', 'show_secrets', ...GLOBAL_CONFIG_OPTIONS]), }; const VALID_OPTIONS_BUCKET = { - 'add': new Set(['name', 'owner', 'path', 'bucket_policy', 'fs_backend', ...GLOBAL_CONFIG_OPTIONS]), + 'add': new Set(['name', 'owner', 'path', 'bucket_policy', 'fs_backend', FROM_FILE, ...GLOBAL_CONFIG_OPTIONS]), 'update': new Set(['name', 'owner', 'path', 'bucket_policy', 'fs_backend', 'new_name', ...GLOBAL_CONFIG_OPTIONS]), - 'delete': new Set(['name', GLOBAL_CONFIG_ROOT]), - 'list': new Set(['wide', 'name', GLOBAL_CONFIG_ROOT]), - 'status': new Set(['name', GLOBAL_CONFIG_ROOT]), + 'delete': new Set(['name', ...GLOBAL_CONFIG_OPTIONS]), + 'list': new Set(['wide', 'name', ...GLOBAL_CONFIG_OPTIONS]), + 'status': new Set(['name', ...GLOBAL_CONFIG_OPTIONS]), }; const VALID_OPTIONS_GLACIER = { @@ -53,13 +54,16 @@ const VALID_OPTIONS_GLACIER = { 'expiry': new Set([ GLOBAL_CONFIG_ROOT]), }; -const VALID_OPTIONS_WHITELIST = new Set(['ips', GLOBAL_CONFIG_ROOT]); +const VALID_OPTIONS_WHITELIST = new Set(['ips', ...GLOBAL_CONFIG_OPTIONS]); + +const VALID_OPTIONS_FROM_FILE = new Set(['from_file', ...GLOBAL_CONFIG_OPTIONS]); const VALID_OPTIONS = { account_options: VALID_OPTIONS_ACCOUNT, bucket_options: VALID_OPTIONS_BUCKET, glacier_options: VALID_OPTIONS_GLACIER, whitelist_options: VALID_OPTIONS_WHITELIST, + from_file_options: VALID_OPTIONS_FROM_FILE, }; const OPTION_TYPE = { @@ -97,6 +101,7 @@ exports.GLACIER_ACTIONS = GLACIER_ACTIONS; exports.CONFIG_SUBDIRS = CONFIG_SUBDIRS; exports.VALID_OPTIONS = VALID_OPTIONS; exports.OPTION_TYPE = OPTION_TYPE; +exports.FROM_FILE = FROM_FILE; exports.BOOLEAN_STRING_VALUES = BOOLEAN_STRING_VALUES; exports.LIST_ACCOUNT_FILTERS = LIST_ACCOUNT_FILTERS; diff --git a/src/manage_nsfs/manage_nsfs_help_utils.js b/src/manage_nsfs/manage_nsfs_help_utils.js index 24c0368917..b28913fb5c 100644 --- a/src/manage_nsfs/manage_nsfs_help_utils.js +++ b/src/manage_nsfs/manage_nsfs_help_utils.js @@ -60,13 +60,8 @@ Flags: `; const GLOBAL_CONFIG_ROOT_ALL_FLAG = ` ---config_root (optional) Use configuration files path (default config.NSFS_NC_DEFAULT_CONF_DIR) -`; - -const GLOBAL_CONFIG_FLAGS_ADD_UPDATE_FLAGS = ` ---from_file (optional) Use details from the JSON file, there is no need to mention all the properties individually in the CLI ---config_root (optional) Use configuration files path (default config.NSFS_NC_DEFAULT_CONF_DIR) ---config_root_backend (optional) Use the filesystem type in the configuration (default config.NSFS_NC_CONFIG_DIR_BACKEND) +--config_root (optional) Use configuration files path (default config.NSFS_NC_DEFAULT_CONF_DIR) +--config_root_backend (optional) Use the filesystem type in the configuration (default config.NSFS_NC_CONFIG_DIR_BACKEND) `; const ACCOUNT_FLAGS_ADD = ` @@ -83,6 +78,7 @@ Flags: --secret_key (optional) Set the secret key for the account (default is generated) --fs_backend (optional) Set the filesystem type of new_buckets_path (default config.NSFS_NC_STORAGE_BACKEND) --allow_bucket_creation (optional) Set the account to explicitly allow or block bucket creation +--from_file (optional) Use details from the JSON file, there is no need to mention all the properties individually in the CLI `; const ACCOUNT_FLAGS_UPDATE = ` @@ -145,6 +141,7 @@ Flags: --path Set the bucket path --bucket_policy (optional) Set the bucket policy, type is a string of valid JSON policy --fs_backend (optional) Set the filesystem type (default config.NSFS_NC_STORAGE_BACKEND) +--from_file (optional) Use details from the JSON file, there is no need to mention all the properties individually in the CLI `; const BUCKET_FLAGS_UPDATE = ` @@ -239,12 +236,10 @@ function print_usage(type, action) { function print_help_account(action) { switch (action) { case ACTIONS.ADD: - process.stdout.write(ACCOUNT_FLAGS_ADD.trimStart() + - GLOBAL_CONFIG_FLAGS_ADD_UPDATE_FLAGS.trimStart()); + process.stdout.write(ACCOUNT_FLAGS_ADD.trimStart() + GLOBAL_CONFIG_ROOT_ALL_FLAG.trimStart()); break; case ACTIONS.UPDATE: - process.stdout.write(ACCOUNT_FLAGS_UPDATE.trimStart() + - GLOBAL_CONFIG_FLAGS_ADD_UPDATE_FLAGS.trimStart()); + process.stdout.write(ACCOUNT_FLAGS_UPDATE.trimStart() + GLOBAL_CONFIG_ROOT_ALL_FLAG.trimStart()); break; case ACTIONS.DELETE: process.stdout.write(ACCOUNT_FLAGS_DELETE.trimStart() + GLOBAL_CONFIG_ROOT_ALL_FLAG.trimStart()); @@ -268,12 +263,10 @@ function print_help_account(action) { function print_help_bucket(action) { switch (action) { case ACTIONS.ADD: - process.stdout.write(BUCKET_FLAGS_ADD.trimStart() + - GLOBAL_CONFIG_FLAGS_ADD_UPDATE_FLAGS.trimStart()); + process.stdout.write(BUCKET_FLAGS_ADD.trimStart() + GLOBAL_CONFIG_ROOT_ALL_FLAG.trimStart()); break; case ACTIONS.UPDATE: - process.stdout.write(BUCKET_FLAGS_UPDATE.trimStart() + - GLOBAL_CONFIG_FLAGS_ADD_UPDATE_FLAGS.trimStart()); + process.stdout.write(BUCKET_FLAGS_UPDATE.trimStart() + GLOBAL_CONFIG_ROOT_ALL_FLAG.trimStart()); break; case ACTIONS.DELETE: process.stdout.write(BUCKET_FLAGS_DELETE.trimStart() + GLOBAL_CONFIG_ROOT_ALL_FLAG.trimStart()); diff --git a/src/test/unit_tests/jest_tests/test_nc_nsfs_account_cli.test.js b/src/test/unit_tests/jest_tests/test_nc_nsfs_account_cli.test.js index 2393c39dd6..6ad7d22353 100644 --- a/src/test/unit_tests/jest_tests/test_nc_nsfs_account_cli.test.js +++ b/src/test/unit_tests/jest_tests/test_nc_nsfs_account_cli.test.js @@ -930,6 +930,170 @@ describe('manage nsfs cli account flow', () => { }); + describe('cli create account using from_file', () => { + const type = TYPES.ACCOUNT; + const config_root = path.join(tmp_fs_path, 'config_root_manage_nsfs'); + const root_path = path.join(tmp_fs_path, 'root_path_manage_nsfs/'); + const path_to_json_account_options_dir = path.join(tmp_fs_path, 'options'); + const defaults = { + name: 'account3', + new_buckets_path: `${root_path}new_buckets_path_user3/`, + uid: 1003, + gid: 1003, + access_key: 'GIGiFAnjaaE7OKD5N722', + secret_key: 'U2AYaMpU3zRDcRFWmvzgQr9MoHIAsD+22EXAMPLE', + }; + + beforeEach(async () => { + await P.all(_.map([CONFIG_SUBDIRS.ACCOUNTS, CONFIG_SUBDIRS.ACCESS_KEYS], async dir => + fs_utils.create_fresh_path(`${config_root}/${dir}`))); + await fs_utils.create_fresh_path(root_path); + await fs_utils.create_fresh_path(path_to_json_account_options_dir); + // create the new_buckets_path and set permissions + const { new_buckets_path, uid, gid } = defaults; + const owner_options = { uid, gid }; + await fs_utils.create_fresh_path(new_buckets_path); + await fs_utils.file_must_exist(new_buckets_path); + await set_path_permissions_and_owner(new_buckets_path, owner_options, 0o700); + }); + + afterEach(async () => { + await fs_utils.folder_delete(`${config_root}`); + await fs_utils.folder_delete(`${root_path}`); + await fs_utils.folder_delete(`${path_to_json_account_options_dir}`); + }); + + it('cli create account using from_file with required options (uid, gid)', async () => { + const action = ACTIONS.ADD; + const { name, new_buckets_path, uid, gid } = defaults; + const account_options = { name, new_buckets_path, uid, gid }; + // write the json_file_options + const path_to_option_json_file_name = await create_json_account_options(path_to_json_account_options_dir, account_options); + const command_flags = {config_root, from_file: path_to_option_json_file_name}; + // create tha account and check the details + await exec_manage_cli(type, action, command_flags); + // compare the details + const account = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + assert_account(account, account_options, false); + }); + + it('cli create account using from_file with access_key and secret_key', async () => { + const action = ACTIONS.ADD; + const { name, new_buckets_path, uid, gid, access_key, secret_key } = defaults; + const account_options = { name, new_buckets_path, uid, gid, access_key, secret_key}; + // write the json_file_options + const path_to_option_json_file_name = await create_json_account_options(path_to_json_account_options_dir, account_options); + const command_flags = {config_root, from_file: path_to_option_json_file_name}; + // create tha account and check the details + await exec_manage_cli(type, action, command_flags); + // compare the details + const account = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + assert_account(account, account_options, true); + expect(account.access_keys[0].access_key).toBe(access_key); + expect(account.access_keys[0].secret_key).toBe(secret_key); + }); + + it('should fail - cli create account using from_file with invalid access_key and secret_key', async () => { + const action = ACTIONS.ADD; + const { name, new_buckets_path, uid, gid } = defaults; + const access_key = 'abc'; // invalid access_key + const secret_key = '123'; // invalid secret_key + const account_options = { name, new_buckets_path, uid, gid, access_key, secret_key}; + // write the json_file_options + const path_to_option_json_file_name = await create_json_account_options(path_to_json_account_options_dir, account_options); + const command_flags = {config_root, from_file: path_to_option_json_file_name}; + // create tha account and check the details + const res = await exec_manage_cli(type, action, command_flags); + expect(JSON.parse(res.stdout).error.code).toBe(ManageCLIError.AccountAccessKeyFlagComplexity.code); + }); + + it('should fail - cli create account using from_file with additional flags (name)', async () => { + const action = ACTIONS.ADD; + const { name, new_buckets_path, uid, gid } = defaults; + const account_options = { name, new_buckets_path, uid, gid }; + // write the json_file_options + const path_to_option_json_file_name = await create_json_account_options(path_to_json_account_options_dir, account_options); + const command_flags = {config_root, from_file: path_to_option_json_file_name, name }; // name should be in file only + // create tha account and check the details + const res = await exec_manage_cli(type, action, command_flags); + expect(JSON.parse(res.stdout).error.code).toBe(ManageCLIError.InvalidArgument.code); + }); + + it('should fail - cli create account using from_file with invalid option (lala) in the file', async () => { + const action = ACTIONS.ADD; + const { name, new_buckets_path, uid, gid } = defaults; + const account_options = { name, new_buckets_path, uid, gid, lala: 'lala'}; // lala invalid option + // write the json_file_options + const path_to_option_json_file_name = await create_json_account_options(path_to_json_account_options_dir, account_options); + const command_flags = {config_root, from_file: path_to_option_json_file_name}; + // create tha account and check the details + const res = await exec_manage_cli(type, action, command_flags); + expect(JSON.parse(res.stdout).error.code).toBe(ManageCLIError.InvalidArgument.code); + }); + + it('should fail - cli create account using from_file with invalid option (creation_date) in the file', async () => { + const action = ACTIONS.ADD; + const { name, new_buckets_path, uid, gid } = defaults; + const account_options = { name, new_buckets_path, uid, gid, creation_date: new Date().toISOString()}; // creation_date invalid option (user cannot set it) + // write the json_file_options + const path_to_option_json_file_name = await create_json_account_options(path_to_json_account_options_dir, account_options); + const command_flags = {config_root, from_file: path_to_option_json_file_name}; + // create tha account and check the details + const res = await exec_manage_cli(type, action, command_flags); + expect(JSON.parse(res.stdout).error.code).toBe(ManageCLIError.InvalidArgument.code); + }); + + it('should fail - cli create account using from_file with from_file inside the JSON file', async () => { + const action = ACTIONS.ADD; + const { name, new_buckets_path, uid, gid } = defaults; + const account_options = { name, new_buckets_path, uid, gid, from_file: 'blabla' }; //from_file inside options JSON file + // write the json_file_options + const path_to_option_json_file_name = await create_json_account_options(path_to_json_account_options_dir, account_options); + const command_flags = {config_root, from_file: path_to_option_json_file_name}; + // create tha account and check the details + const res = await exec_manage_cli(type, action, command_flags); + expect(JSON.parse(res.stdout).error.code).toBe(ManageCLIError.InvalidArgument.code); + }); + + it('should fail - cli create account using from_file with invalid option type (in the file)', async () => { + const action = ACTIONS.ADD; + const { name, new_buckets_path, uid } = defaults; + const account_options = { name, new_buckets_path, uid, gid: 'lala'}; // gid should be number (not string) + // write the json_file_options + const path_to_option_json_file_name = await create_json_account_options(path_to_json_account_options_dir, account_options); + const command_flags = {config_root, from_file: path_to_option_json_file_name}; + // create tha account and check the details + const res = await exec_manage_cli(type, action, command_flags); + expect(JSON.parse(res.stdout).error.code).toBe(ManageCLIError.InvalidArgumentType.code); + }); + + it('should fail - cli create account using from_file with invalid path', async () => { + const action = ACTIONS.ADD; + const command_flags = {config_root, from_file: 'blabla'}; //invalid path + const res = await exec_manage_cli(type, action, command_flags); + expect(JSON.parse(res.stdout).error.code).toBe(ManageCLIError.InvalidFilePath.code); + }); + + it('should fail - cli create account using from_file with invalid JSON file', async () => { + const action = ACTIONS.ADD; + const { name, new_buckets_path, uid, gid } = defaults; + const account_options = { name, new_buckets_path, uid, gid }; + // write invalid json_file_options + const option_json_file_name = `${account_options.name}_options.json`; + const path_to_option_json_file_name = path.join(path_to_json_account_options_dir, option_json_file_name); + const content = JSON.stringify(account_options) + 'blabla'; // invalid JSON + await fs.promises.writeFile(path_to_option_json_file_name, content); + // write the json_file_options + const command_flags = {config_root, from_file: path_to_option_json_file_name}; + // create tha account + await exec_manage_cli(type, action, command_flags); + // compare the details + const res = await exec_manage_cli(type, action, command_flags); + expect(JSON.parse(res.stdout).error.code).toBe(ManageCLIError.InvalidJSONFile.code); + }); + + }); + }); @@ -1200,3 +1364,16 @@ function create_command(type, action, options) { const command = `node src/cmd/manage_nsfs ${type} ${action} ${account_flags}`; return command; } + +/** + * create_json_account_options would create a JSON file with the options (key-value) inside file + * @param {string} path_to_json_account_options_dir + * @param {object} account_options + */ +async function create_json_account_options(path_to_json_account_options_dir, account_options) { + const option_json_file_name = `${account_options.name}_options.json`; + const path_to_option_json_file_name = path.join(path_to_json_account_options_dir, option_json_file_name); + const content = JSON.stringify(account_options); + await fs.promises.writeFile(path_to_option_json_file_name, content); + return path_to_option_json_file_name; +} diff --git a/src/test/unit_tests/test_bucketspace_fs.js b/src/test/unit_tests/test_bucketspace_fs.js index 62efc19984..718f424269 100644 --- a/src/test/unit_tests/test_bucketspace_fs.js +++ b/src/test/unit_tests/test_bucketspace_fs.js @@ -10,7 +10,7 @@ const assert = require('assert'); const P = require('../../util/promise'); const config = require('../../../config'); const fs_utils = require('../../util/fs_utils'); -const { get_process_fs_context, read_config_file, get_user_by_distinguished_name} = require('../../util/native_fs_utils'); +const { get_process_fs_context, read_file, get_user_by_distinguished_name} = require('../../util/native_fs_utils'); const nb_native = require('../../util/nb_native'); const SensitiveString = require('../../util/sensitive_string'); const NamespaceFS = require('../../sdk/namespace_fs'); @@ -308,7 +308,7 @@ mocha.describe('bucketspace_fs', function() { assert.equal(objects.buckets.length, 1); assert.equal(objects.buckets[0].name.unwrap(), expected_bucket_name); const bucket_config_path = get_config_file_path(CONFIG_SUBDIRS.BUCKETS, expected_bucket_name); - const bucket_data = await read_config_file(process_fs_context, bucket_config_path); + const bucket_data = await read_file(process_fs_context, bucket_config_path); assert.equal(objects.buckets[0].creation_date, bucket_data.creation_date); }); }); @@ -384,7 +384,7 @@ mocha.describe('bucketspace_fs', function() { const param = {name: test_bucket, versioning: 'ENABLED'}; await bucketspace_fs.set_bucket_versioning(param, dummy_object_sdk); const bucket_config_path = get_config_file_path(CONFIG_SUBDIRS.BUCKETS, param.name); - const bucket = await read_config_file(process_fs_context, bucket_config_path); + const bucket = await read_file(process_fs_context, bucket_config_path); assert.equal(bucket.versioning, 'ENABLED'); }); diff --git a/src/util/native_fs_utils.js b/src/util/native_fs_utils.js index 825265ad13..9d7ad8d3b2 100644 --- a/src/util/native_fs_utils.js +++ b/src/util/native_fs_utils.js @@ -551,15 +551,15 @@ async function folder_delete(dir, fs_context, is_temp = false) { } /** - * read_config_file reads and returns the parsed config file data + * read_file reads file and returns the parsed file data as object * @param {nb.NativeFSContext} fs_context - * @param {string} config_path + * @param {string} _path * @return {Promise} */ -async function read_config_file(fs_context, config_path) { - const { data } = await nb_native().fs.readFile(fs_context, config_path); - const config_data = JSON.parse(data.toString()); - return config_data; +async function read_file(fs_context, _path) { + const { data } = await nb_native().fs.readFile(fs_context, _path); + const data_parsed = JSON.parse(data.toString()); + return data_parsed; } exports.get_umasked_mode = get_umasked_mode; @@ -587,7 +587,7 @@ exports.gpfs_unlink_retry_catch = gpfs_unlink_retry_catch; exports.create_config_file = create_config_file; exports.delete_config_file = delete_config_file; exports.update_config_file = update_config_file; -exports.read_config_file = read_config_file; +exports.read_file = read_file; exports.isDirectory = isDirectory; exports.get_process_fs_context = get_process_fs_context; exports.get_fs_context = get_fs_context;