Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

NC | NSFS | CLI | Separate Bucket and Account List Functions #8747

Merged
merged 1 commit into from
Feb 10, 2025
Merged

NC | NSFS | CLI | Separate Bucket and Account List Functions #8747

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
242 changes: 146 additions & 96 deletions src/cmd/manage_nsfs.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,10 @@ const manage_nsfs_validations = require('../manage_nsfs/manage_nsfs_validations'
const nc_mkm = require('../manage_nsfs/nc_master_key_manager').get_instance();
const notifications_util = require('../util/notifications_util');

///////////////
//// GENERAL //
///////////////

let config_fs;

async function main(argv = minimist(process.argv.slice(2))) {
Expand Down Expand Up @@ -92,6 +96,10 @@ async function main(argv = minimist(process.argv.slice(2))) {
}
}

///////////////
//// BUCKETS //
///////////////

// in name and new_name we allow type number, hence convert it to string
async function fetch_bucket_data(action, user_input) {
let data = {
Expand Down Expand Up @@ -146,7 +154,6 @@ async function fetch_bucket_data(action, user_input) {
return data;
}


/**
* merge_new_and_existing_config_data returns the merged object of the existing bucket data and the user data
* @param {Object} user_input_bucket_data
Expand Down Expand Up @@ -259,6 +266,62 @@ async function delete_bucket(data, force) {
}
}


/**
* filter_bucket will return true or false based on whether a bucket meets the criteria defined by the supported flags.
* @param {object} bucket
* @param {object} [filters]
*/
function filter_bucket(bucket, filters) {
for (const [key, val] of Object.entries(filters)) {
if (bucket[key] !== val) { // We will never reach here if we will not add an appropriate field to the filter
return false;
}
}
return true;
}

/**
* list_bucket_config_files will list all the bucket config files (json) in a given config directory
* @param {boolean} [wide]
* @param {object} [filters]
*/
async function list_bucket_config_files(wide, filters = {}) {
let entry_names = [];
const should_filter = Object.keys(filters).length > 0;
const is_filter_by_name = filters.name !== undefined;

const options = {
silent_if_missing: true
};

// in case we have a filter by name, we don't need to read all the entries and iterate them
// instead we "mock" the entries array to have one entry and it is the name by the filter (we add it for performance)
if (is_filter_by_name) {
entry_names = [filters.name];
} else {
entry_names = await config_fs.list_buckets();
}

let config_files_list = await P.map_with_concurrency(10, entry_names, async entry_name => {
if (wide || should_filter) {
const data = await config_fs.get_bucket_by_name(entry_name, options);
if (!data) return undefined;
if (should_filter && !filter_bucket(data, filters)) return undefined;
if (!wide) return { name: entry_name };
await set_bucker_owner(data);
return data;
} else {
return { name: entry_name };
}
});
// it inserts undefined for the entry '.noobaa-config-nsfs' and we wish to remove it
// in case the entry was deleted during the list it also inserts undefined
config_files_list = config_files_list.filter(item => item);

return config_files_list;
}

/**
* bucket_management does the following -
* 1. fetches the bucket data if this is not a list operation
Expand Down Expand Up @@ -287,14 +350,18 @@ async function bucket_management(action, user_input) {
} else if (action === ACTIONS.LIST) {
const bucket_filters = _.pick(user_input, LIST_BUCKET_FILTERS);
const wide = get_boolean_or_string_value(user_input.wide);
const buckets = await list_config_files(TYPES.BUCKET, wide, undefined, bucket_filters);
const buckets = await list_bucket_config_files(wide, bucket_filters);
response = { code: ManageCLIResponse.BucketList, detail: buckets };
} else {
throw_cli_error(ManageCLIError.InvalidAction);
}
write_stdout_response(response.code, response.detail, response.event_arg);
}

////////////////
//// ACCOUNTS //
////////////////

/**
* set_access_keys will set the access keys either given or generated.
* @param {string} access_key
Expand Down Expand Up @@ -488,65 +555,10 @@ async function get_account_status(data, show_secrets) {
}
}

/**
* account_management does the following -
* 1. sets variables by the user input options
* 2. iniates nc_master_key_manager on UPDATE/ADD/show_secrets
* 2. validates account args - TODO - we should split it to validate_account_args
* and validations of the merged account (user_input + existing account config)
* 3. call account operation based on the action argument
* 4. write output to stdout
* @param {'add'|'update'|'delete'|'status'|'list'} action
* @param {Object} user_input
*/
async function account_management(action, user_input) {
const show_secrets = get_boolean_or_string_value(user_input.show_secrets);
const is_flag_iam_operate_on_root_account = get_boolean_or_string_value(user_input.iam_operate_on_root_account);
const account_filters = _.pick(user_input, LIST_ACCOUNT_FILTERS);
const wide = get_boolean_or_string_value(user_input.wide);
if (get_boolean_or_string_value(user_input.anonymous)) {
user_input.name = config.ANONYMOUS_ACCOUNT_NAME;
user_input.email = config.ANONYMOUS_ACCOUNT_NAME;
}
// init nc_mkm here to avoid concurrent initializations
// init if actions is add/update (require encryption) or show_secrets = true (require decryption)
if ([ACTIONS.ADD, ACTIONS.UPDATE].includes(action) || show_secrets) await nc_mkm.init();
const data = action === ACTIONS.LIST ? undefined : await fetch_account_data(action, user_input);
await manage_nsfs_validations.validate_account_args(config_fs, data, action, is_flag_iam_operate_on_root_account);

let response = {};
if (action === ACTIONS.ADD) {
response = await add_account(data);
} else if (action === ACTIONS.STATUS) {
response = await get_account_status(data, show_secrets);
} else if (action === ACTIONS.UPDATE) {
response = await update_account(data);
} else if (action === ACTIONS.DELETE) {
response = await delete_account(data);
} else if (action === ACTIONS.LIST) {
const accounts = await list_config_files(TYPES.ACCOUNT, wide, show_secrets, account_filters);
response = { code: ManageCLIResponse.AccountList, detail: accounts };
} else {
throw_cli_error(ManageCLIError.InvalidAction);
}
write_stdout_response(response.code, response.detail, response.event_arg);

}

/**
* filter_list_item will return an answer of filter_account() or filter_bucket() based on the entity type
* @param {string} type
* @param {object} entity
* @param {string[]} [filters]
*/
function filter_list_item(type, entity, filters) {
return type === TYPES.ACCOUNT ? filter_account(entity, filters) : filter_bucket(entity, filters);
}

/**
* filter_account will return true or false based on whether an account meets the criteria defined by the supported flags.
* @param {object} account
* @param {string[]} [filters]
* @param {object} [filters]
*/
function filter_account(account, filters) {
for (const [key, val] of Object.entries(filters)) {
Expand All @@ -570,32 +582,17 @@ function filter_account(account, filters) {
}

/**
* filter_bucket will return true or false based on whether a bucket meets the criteria defined by the supported flags.
* currently not implemented
* @param {object} bucket
* @param {string[]} [filters]
*/
function filter_bucket(bucket, filters) {
for (const [key, val] of Object.entries(filters)) {
if (bucket[key] !== val) { // We will never reach here if we will not add an appropriate field to the filter
return false;
}
}
return true;
}
/**
* list_config_files will list all the config files (json) in a given config directory
* @param {string} type
* list_account_config_files will list all the account config files (json) in a given config directory
* @param {boolean} [wide]
* @param {boolean} [show_secrets]
* @param {object} [filters]
*/
async function list_config_files(type, wide, show_secrets, filters = {}) {
async function list_account_config_files(wide, show_secrets, filters = {}) {
let entry_names = [];
const should_filter = Object.keys(filters).length > 0;
const is_filter_by_name = filters.name !== undefined;

// decryption causing mkm initalization
// decryption causing mkm initialization
// decrypt only if data has access_keys and show_secrets = true (no need to decrypt if show_secrets = false but should_filter = true)
const options = {
show_secrets: show_secrets || should_filter,
Expand All @@ -607,26 +604,18 @@ async function list_config_files(type, wide, show_secrets, filters = {}) {
// instead we "mock" the entries array to have one entry and it is the name by the filter (we add it for performance)
if (is_filter_by_name) {
entry_names = [filters.name];
} else if (type === TYPES.ACCOUNT) {
} else {
entry_names = await config_fs.list_accounts();
} else if (type === TYPES.BUCKET) {
entry_names = await config_fs.list_buckets();
}

let config_files_list = await P.map_with_concurrency(10, entry_names, async entry_name => {
if (wide || should_filter) {
const data = type === TYPES.ACCOUNT ?
await config_fs.get_account_by_name(entry_name, options) :
await config_fs.get_bucket_by_name(entry_name, options);
const data = await config_fs.get_account_by_name(entry_name, options);
if (!data) return undefined;
if (should_filter && !filter_list_item(type, data, filters)) return undefined;
if (should_filter && !filter_account(data, filters)) return undefined;
// remove secrets on !show_secrets && should filter
if (!wide) return { name: entry_name };
if (type === TYPES.ACCOUNT) return _.omit(data, show_secrets ? [] : ['access_keys']);
if (type === TYPES.BUCKET) {
await set_bucker_owner(data);
return data;
}
return _.omit(data, show_secrets ? [] : ['access_keys']);
} else {
return { name: entry_name };
}
Expand All @@ -639,16 +628,48 @@ async function list_config_files(type, wide, show_secrets, filters = {}) {
}

/**
* list_connections
* @returns An array with names of all connection files.
* account_management does the following -
* 1. sets variables by the user input options
* 2. iniates nc_master_key_manager on UPDATE/ADD/show_secrets
* 2. validates account args - TODO - we should split it to validate_account_args
* and validations of the merged account (user_input + existing account config)
* 3. call account operation based on the action argument
* 4. write output to stdout
* @param {'add'|'update'|'delete'|'status'|'list'} action
* @param {Object} user_input
*/
async function list_connections() {
let conns = await config_fs.list_connections();
// it inserts undefined for the entry '.noobaa-config-nsfs' and we wish to remove it
// in case the entry was deleted during the list it also inserts undefined
conns = conns.filter(item => item);
async function account_management(action, user_input) {
const show_secrets = get_boolean_or_string_value(user_input.show_secrets);
const is_flag_iam_operate_on_root_account = get_boolean_or_string_value(user_input.iam_operate_on_root_account);
const account_filters = _.pick(user_input, LIST_ACCOUNT_FILTERS);
const wide = get_boolean_or_string_value(user_input.wide);
if (get_boolean_or_string_value(user_input.anonymous)) {
user_input.name = config.ANONYMOUS_ACCOUNT_NAME;
user_input.email = config.ANONYMOUS_ACCOUNT_NAME;
}
// init nc_mkm here to avoid concurrent initializations
// init if actions is add/update (require encryption) or show_secrets = true (require decryption)
if ([ACTIONS.ADD, ACTIONS.UPDATE].includes(action) || show_secrets) await nc_mkm.init();
const data = action === ACTIONS.LIST ? undefined : await fetch_account_data(action, user_input);
await manage_nsfs_validations.validate_account_args(config_fs, data, action, is_flag_iam_operate_on_root_account);

let response = {};
if (action === ACTIONS.ADD) {
response = await add_account(data);
} else if (action === ACTIONS.STATUS) {
response = await get_account_status(data, show_secrets);
} else if (action === ACTIONS.UPDATE) {
response = await update_account(data);
} else if (action === ACTIONS.DELETE) {
response = await delete_account(data);
} else if (action === ACTIONS.LIST) {
const accounts = await list_account_config_files(wide, show_secrets, account_filters);
response = { code: ManageCLIResponse.AccountList, detail: accounts };
} else {
throw_cli_error(ManageCLIError.InvalidAction);
}
write_stdout_response(response.code, response.detail, response.event_arg);

return conns;
}

/**
Expand Down Expand Up @@ -689,6 +710,10 @@ async function set_bucker_owner(bucket_data) {
bucket_data.bucket_owner = account_data?.name;
}

////////////////////
//// IP WHITELIST //
////////////////////

async function whitelist_ips_management(args) {
const ips = args.ips;
manage_nsfs_validations.validate_whitelist_arg(ips);
Expand All @@ -708,6 +733,10 @@ async function whitelist_ips_management(args) {
write_stdout_response(ManageCLIResponse.WhiteListIPUpdated, ips);
}

///////////////
//// GLACIER //
///////////////

async function glacier_management(argv) {
const action = argv._[1] || '';
await manage_glacier_operations(action, argv);
Expand All @@ -729,10 +758,18 @@ async function manage_glacier_operations(action, argv) {
}
}

//////////////////////
//// BUCKET LOGGING //
//////////////////////

async function logging_management() {
await manage_nsfs_logging.export_bucket_logging(config_fs);
}

/////////////////////
//// NOTIFICATIONS //
////////////////////

async function notification_management() {
await new notifications_util.Notificator({
fs_context: config_fs.fs_context,
Expand Down Expand Up @@ -780,5 +817,18 @@ async function connection_management(action, user_input) {
write_stdout_response(response.code, response.detail, response.event_arg);
}

/**
* list_connections
* @returns An array with names of all connection files.
*/
async function list_connections() {
let conns = await config_fs.list_connections();
// it inserts undefined for the entry '.noobaa-config-nsfs' and we wish to remove it
// in case the entry was deleted during the list it also inserts undefined
conns = conns.filter(item => item);

return conns;
}

exports.main = main;
if (require.main === module) main();
Loading