From 19531bb03e7f49070e3ff54dba62c56ac27c18c7 Mon Sep 17 00:00:00 2001 From: Romy <35330373+romayalon@users.noreply.github.com> Date: Thu, 4 Jan 2024 19:42:06 +0200 Subject: [PATCH] add global storage backend type Signed-off-by: Romy <35330373+romayalon@users.noreply.github.com> --- config.js | 1 + ...NonContainerizedDeveloperCustomizations.md | 18 ++++++- src/cmd/manage_nsfs.js | 50 ++++++++++--------- src/test/unit_tests/test_nc_nsfs_cli.js | 4 +- 4 files changed, 45 insertions(+), 28 deletions(-) diff --git a/config.js b/config.js index f3acecb45e..533d1cf739 100644 --- a/config.js +++ b/config.js @@ -699,6 +699,7 @@ config.NSFS_NC_DEFAULT_CONF_DIR = '/etc/noobaa.conf.d'; config.NSFS_NC_CONF_DIR = process.env.NSFS_NC_CONF_DIR || ''; config.NSFS_TEMP_CONF_DIR_NAME = '.noobaa-config-nsfs'; config.NSFS_NC_CONFIG_DIR_BACKEND = ''; +config.NSFS_NC_STORAGE_BACKEND = ''; config.ENDPOINT_PORT = Number(process.env.ENDPOINT_PORT) || 6001; config.ENDPOINT_SSL_PORT = Number(process.env.ENDPOINT_SSL_PORT) || 6443; config.ENDPOINT_SSL_STS_PORT = Number(process.env.ENDPOINT_SSL_STS_PORT) || -1; diff --git a/docs/dev_guide/NonContainerizedDeveloperCustomizations.md b/docs/dev_guide/NonContainerizedDeveloperCustomizations.md index f75b98d9df..cdbfddad59 100644 --- a/docs/dev_guide/NonContainerizedDeveloperCustomizations.md +++ b/docs/dev_guide/NonContainerizedDeveloperCustomizations.md @@ -248,8 +248,8 @@ Example: node src/cmd/manage_nsfs whitelist --ips '["127.0.0.1", "192.000.10.000", "3002:0bd6:0000:0000:0000:ee00:0033:000"]' 2>/dev/null ``` -## 13. Config directory backend - -**Description -** Set custom config directory backend. Set to GPFS in order to increase performance of creation/deletion of config files under the config directory. +## 14. Config directory backend - +**Description -** Set custom file system backend type of the config directory. Set to GPFS in order to increase performance of creation/deletion of config files under the config directory. **Configuration Key -** NSFS_NC_CONFIG_DIR_BACKEND @@ -265,9 +265,23 @@ Example: "NSFS_NC_CONFIG_DIR_BACKEND": "GPFS" ``` +## 15. Storage Backend - +**Description -** Set custom file system backend type of the storage file system. Set to GPFS in order to increase performance of creation/deletion of objects stored in the underlying file system. +**Configuration Key -** NSFS_NC_STORAGE_BACKEND +**Type -** string + +**Default -** '' +**Steps -** +``` +1. Open /path/to/config_dir/config.json file. +2. Set the config key - +Example: +"NSFS_NC_STORAGE_BACKEND": "GPFS" +3. systemctl restart noobaa_nsfs +``` ## Config.json example ``` diff --git a/src/cmd/manage_nsfs.js b/src/cmd/manage_nsfs.js index a3be96ddf0..902348af3e 100644 --- a/src/cmd/manage_nsfs.js +++ b/src/cmd/manage_nsfs.js @@ -84,19 +84,18 @@ const ACCOUNT_OPTIONS = ` Account Options: # Read account details from the JSON file, there is no need to mention all the properties one by one in the CLI - --from_file (default none) Set account schema full path. - Get account details from JSON file - --config_root_backend (default none) Set the config_root FS type to be GPFS + --from_file (default none) Set account schema full path. Get account details from JSON file + --config_root_backend (default config.NSFS_NC_CONFIG_DIR_BACKEND) Set the config_root FS type to be GPFS # required for add, update - --name (default none) Set the name for the account. - --email (default none) Set the email for the account. - --new_name (default none) Set a new name for the account (update). - --uid (default none) Send requests to the Filesystem with uid. - --gid (default none) Send requests to the Filesystem with gid. - --secret_key (default none) The secret key pair for the access key. - --new_buckets_path (default none) Set the filesystem's root where each subdir is a bucket. - --fs_backend (default none) Set fs_backend of new_buckets_path to be GPFS + --name (default none) Set the name for the account. + --email (default none) Set the email for the account. + --new_name (default none) Set a new name for the account (update). + --uid (default none) Send requests to the Filesystem with uid. + --gid (default none) Send requests to the Filesystem with gid. + --secret_key (default none) The secret key pair for the access key. + --new_buckets_path (default none) Set the filesystem's root where each subdir is a bucket. + --fs_backend (default config.NSFS_NC_STORAGE_BACKEND) Set fs_backend of new_buckets_path to be GPFS # required for add, update, and delete --access_key (default none) Authenticate incoming requests for this access key only (default is no auth). @@ -115,14 +114,13 @@ const BUCKET_OPTIONS = ` Bucket Options: # Read Bucket details from JSON file, no need to mention all the properties one by one in CLI - --from_file (default none) Set bucket schema full path. - Get bucket details from the JSON file - --config_root_backend (default none) Set the config_root FS type to be GPFS + --from_file (default none) Set bucket schema full path. Get bucket details from the JSON file + --config_root_backend (default config.NSFS_NC_CONFIG_DIR_BACKEND) Set the config_root FS type to be GPFS # required for add, update - --email (default none) Set the email for the bucket. - --path (default none) Set the bucket path. - --fs_backend (default none) Set fs_backend to be GPFS + --email (default none) Set the email for the bucket. + --path (default none) Set the bucket path. + --fs_backend (default config.NSFS_NC_STORAGE_BACKEND) Set fs_backend to be GPFS # required for add, update, and delete --name (default none) Set the name for the bucket. @@ -233,7 +231,7 @@ async function fetch_bucket_data(argv, from_file) { path: argv.path, should_create_underlying_storage: false, new_name: argv.new_name, - fs_backend: argv.fs_backend === undefined ? undefined : String(argv.fs_backend) + fs_backend: argv.fs_backend ? String(argv.fs_backend) : config.NSFS_NC_STORAGE_BACKEND }; } @@ -256,7 +254,8 @@ async function fetch_bucket_data(argv, from_file) { system_owner: new SensitiveString(String(data.system_owner)), bucket_owner: new SensitiveString(String(data.bucket_owner)), // update bucket identifier - new_name: data.new_name && new SensitiveString(String(data.new_name)) + new_name: data.new_name && new SensitiveString(String(data.new_name)), + fs_backend: data.fs_backend || undefined }; return data; @@ -431,7 +430,7 @@ async function fetch_account_data(argv, from_file) { uid: !argv.user && argv.uid, gid: !argv.user && argv.gid, new_buckets_path: argv.new_buckets_path, - fs_backend: argv.fs_backend === undefined ? undefined : String(argv.fs_backend) + fs_backend: argv.fs_backend ? String(argv.fs_backend) : config.NSFS_NC_STORAGE_BACKEND } }, _.isUndefined); } @@ -454,7 +453,7 @@ async function fetch_account_data(argv, from_file) { uid: data.nsfs_account_config.uid && Number(data.nsfs_account_config.uid), gid: data.nsfs_account_config.gid && Number(data.nsfs_account_config.gid), new_buckets_path: data.nsfs_account_config.new_buckets_path, - fs_backend: data.nsfs_account_config.fs_backend + fs_backend: data.nsfs_account_config.fs_backend || undefined }, allow_bucket_creation: !is_undefined(data.nsfs_account_config.new_buckets_path), // updates of account identifiers @@ -693,7 +692,9 @@ async function validate_bucket_args(data, action) { throw_cli_error(ManageCLIError.InvalidStoragePath, data.path); } // fs_backend='' used for deletion of the fs_backend property - if (data.fs_backend !== undefined && data.fs_backend !== 'GPFS' && data.fs_backend !== '') throw_cli_error(ManageCLIError.InvalidFSBackend); + if (data.fs_backend !== undefined && !['GPFS', 'CEPH_FS', 'NFSv4'].includes(data.fs_backend)) { + throw_cli_error(ManageCLIError.InvalidFSBackend); + } if (data.s3_policy) { try { await bucket_policy_utils.validate_s3_policy(data.s3_policy, data.name.unwrap(), @@ -737,8 +738,9 @@ async function validate_account_args(data, action) { throw_cli_error(ManageCLIError.InvalidAccountNSFSConfig, data.nsfs_account_config); } // fs_backend='' used for deletion of the fs_backend property - if (data.nsfs_account_config.fs_backend !== undefined && data.nsfs_account_config.fs_backend !== 'GPFS' && - data.nsfs_account_config.fs_backend !== '') throw_cli_error(ManageCLIError.InvalidFSBackend); + if (data.nsfs_account_config.fs_backend !== undefined && !['GPFS', 'CEPH_FS', 'NFSv4'].includes(data.nsfs_account_config.fs_backend)) { + throw_cli_error(ManageCLIError.InvalidFSBackend); + } if (data.nsfs_account_config.uid && typeof data.nsfs_account_config.uid !== 'number') throw_cli_error(ManageCLIError.InvalidAccountUID); if (data.nsfs_account_config.gid && typeof data.nsfs_account_config.gid !== 'number') throw_cli_error(ManageCLIError.InvalidAccountGID); diff --git a/src/test/unit_tests/test_nc_nsfs_cli.js b/src/test/unit_tests/test_nc_nsfs_cli.js index 6f28852f96..e380e23321 100644 --- a/src/test/unit_tests/test_nc_nsfs_cli.js +++ b/src/test/unit_tests/test_nc_nsfs_cli.js @@ -868,7 +868,7 @@ function assert_bucket(bucket, bucket_options) { assert.strictEqual(bucket.path, bucket_options.bucket_path); assert.strictEqual(bucket.should_create_underlying_storage, false); assert.strictEqual(bucket.versioning, 'DISABLED'); - assert.strictEqual(bucket.fs_backend, bucket_options.fs_backend); + assert.strictEqual(bucket.fs_backend, bucket_options.fs_backend === '' ? undefined : bucket_options.fs_backend); assert.deepStrictEqual(bucket.s3_policy, bucket_options.bucket_policy); return true; } @@ -887,7 +887,7 @@ function assert_account(account, account_options, skip_secrets) { assert.equal(account.nsfs_account_config.gid, account_options.gid); } assert.equal(account.nsfs_account_config.new_buckets_path, account_options.new_buckets_path); - assert.equal(account.nsfs_account_config.fs_backend, account_options.fs_backend); + assert.equal(account.nsfs_account_config.fs_backend, account_options.fs_backend === '' ? undefined : account_options.fs_backend); return true; }