Merge pull request #8326 from romayalon/romy-online-upgrade-process

NC | Online Upgrade Process

Author: romayalon

src/cmd/manage_nsfs.js

@@ -67,7 +67,7 @@ async function main(argv = minimist(process.argv.slice(2))) {
         } else if (type === TYPES.DIAGNOSE) {
             await noobaa_cli_diagnose.manage_diagnose_operations(action, user_input, config_fs);
         } else if (type === TYPES.UPGRADE) {
-            await noobaa_cli_upgrade.manage_upgrade_operations(action, config_fs);
+            await noobaa_cli_upgrade.manage_upgrade_operations(action, user_input, config_fs);
         } else {
             throw_cli_error(ManageCLIError.InvalidType);
         }

src/cmd/nsfs.js

@@ -23,6 +23,7 @@ const os = require('os');
 const fs = require('fs');
 const util = require('util');
 const minimist = require('minimist');
+const { ConfigFS } = require('../sdk/config_fs');
 
 if (process.env.LOCAL_MD_SERVER === 'true') {
     require('../server/system_services/system_store').get_instance({ standalone: true });
@@ -39,14 +40,13 @@ const BucketSpaceSimpleFS = require('../sdk/bucketspace_simple_fs');
 const BucketSpaceFS = require('../sdk/bucketspace_fs');
 const SensitiveString = require('../util/sensitive_string');
 const endpoint_stats_collector = require('../sdk/endpoint_stats_collector');
-const path = require('path');
-const json_utils = require('../util/json_utils');
 //const { RPC_BUFFERS } = require('../rpc');
 const pkg = require('../../package.json');
 const AccountSDK = require('../sdk/account_sdk');
 const AccountSpaceFS = require('../sdk/accountspace_fs');
 const NoobaaEvent = require('../manage_nsfs/manage_nsfs_events_utils').NoobaaEvent;
 const { set_debug_level } = require('../manage_nsfs/manage_nsfs_cli_utils');
+const { NCUpgradeManager } = require('../upgrade/nc_upgrade_manager');
 
 const HELP = `
 Help:
@@ -240,29 +240,39 @@ class NsfsAccountSDK extends AccountSDK {
     }
 }
 
-async function init_nsfs_system(config_root) {
-    const system_data_path = path.join(config_root, 'system.json');
-    const system_data = new json_utils.JsonFileWrapper(system_data_path);
+async function init_nc_system(config_root) {
+    const config_fs = new ConfigFS(config_root);
+    const system_data = await config_fs.get_system_config_file({silent_if_missing: true});
 
-    const data = await system_data.read();
     const hostname = os.hostname();
     // If the system data already exists, we should not create it again
-    if (data?.[hostname]?.current_version) return;
-
+    const updated_system_json = system_data || {};
+    if (updated_system_json[hostname]?.current_version && updated_system_json.config_directory) return;
+    if (!updated_system_json[hostname]?.current_version) {
+        updated_system_json[hostname] = {
+            current_version: pkg.version,
+            upgrade_history: { successful_upgrades: [], last_failure: undefined }
+        };
+    }
+    // If it's the first time a config_directory data is added to system.json
+    if (!updated_system_json.config_directory) {
+        updated_system_json.config_directory = {
+            config_dir_version: config_fs.config_dir_version,
+            upgrade_package_version: pkg.version,
+            phase: 'CONFIG_DIR_UNLOCKED',
+            upgrade_history: { successful_upgrades: [], last_failure: undefined }
+        };
+    }
     try {
-        await system_data.update({
-            ...data,
-            [hostname]: {
-                current_version: pkg.version,
-                upgrade_history: {
-                    successful_upgrades: [],
-                    last_failure: undefined
-                }
-            }
-        });
-        console.log('created NSFS system data with version: ', pkg.version);
+        if (system_data) {
+            await config_fs.update_system_config_file(JSON.stringify(updated_system_json));
+            console.log('updated NC system data with version: ', pkg.version);
+        } else {
+            await config_fs.create_system_config_file(JSON.stringify(updated_system_json));
+            console.log('created NC system data with version: ', pkg.version);
+        }
     } catch (err) {
-        const msg = 'failed to create NSFS system data due to - ' + err.message;
+        const msg = 'failed to create/update NC system data due to - ' + err.message;
         const error = new Error(msg);
         console.error(msg, err);
         throw error;
@@ -303,6 +313,11 @@ async function main(argv = minimist(process.argv.slice(2))) {
         const versioning = argv.versioning || 'DISABLED';
         const fs_root = argv._[0] || '';
 
+        // Do not move this function - we need to update RPM changes before starting the endpoint
+        const config_fs = new ConfigFS(nsfs_config_root);
+        const nc_upgrade_manager = new NCUpgradeManager(config_fs);
+        await nc_upgrade_manager.update_rpm_upgrade();
+
         const fs_config = {
             uid,
             gid,
@@ -348,7 +363,7 @@ async function main(argv = minimist(process.argv.slice(2))) {
             nsfs_config_root,
         });
 
-        if (!simple_mode) await init_nsfs_system(nsfs_config_root);
+        if (!simple_mode) await init_nc_system(nsfs_config_root);
 
         const endpoint = require('../endpoint/endpoint');
         await endpoint.main({

src/deploy/noobaa.service

@@ -7,7 +7,6 @@ Restart=always
 RestartSec=2
 User=root
 Group=root
-ExecStartPre=/usr/local/noobaa-core/bin/node /usr/local/noobaa-core/src/upgrade/upgrade_manager.js --nsfs true --upgrade_scripts_dir /usr/local/noobaa-core/src/upgrade/nsfs_upgrade_scripts
 ExecStart=/usr/local/noobaa-core/bin/node /usr/local/noobaa-core/src/cmd/nsfs.js
 EnvironmentFile=-/etc/sysconfig/noobaa
 ExecStop=/bin/kill $MAINPID

src/manage_nsfs/manage_nsfs_cli_errors.js

@@ -479,6 +479,12 @@ ManageCLIError.UpgradeHistoryFailed = Object.freeze({
     http_code: 500,
 });
 
+ManageCLIError.ConfigDirUpdateBlocked = Object.freeze({
+    code: 'ConfigDirUpdateBlocked',
+    message: 'Config directory updates are not allowed on mismatch of the config directory version mentioned in system.json and the config directory version of the source code',
+    http_code: 500,
+});
+
 ///////////////////////////////
 //       ERRORS MAPPING      //
 ///////////////////////////////
@@ -500,7 +506,8 @@ ManageCLIError.RPC_ERROR_TO_MANAGE = Object.freeze({
     INVALID_SCHEMA: ManageCLIError.InvalidSchema,
     NO_SUCH_USER: ManageCLIError.InvalidAccountDistinguishedName,
     INVALID_MASTER_KEY: ManageCLIError.InvalidMasterKey,
-    INVALID_BUCKET_NAME: ManageCLIError.InvalidBucketName
+    INVALID_BUCKET_NAME: ManageCLIError.InvalidBucketName,
+    CONFIG_DIR_VERSION_MISMATCH: ManageCLIError.ConfigDirUpdateBlocked
 });
 
 const NSFS_CLI_ERROR_EVENT_MAP = {

src/manage_nsfs/manage_nsfs_constants.js

@@ -77,6 +77,12 @@ const VALID_OPTIONS_DIAGNOSE = {
     'metrics': new Set([CONFIG_ROOT_FLAG])
 };
 
+const VALID_OPTIONS_UPGRADE = {
+    'start': new Set([ 'skip_verification', 'expected_version', 'expected_hosts', 'custom_upgrade_scripts_dir', ...CLI_MUTUAL_OPTIONS]),
+    'status': new Set([ ...CLI_MUTUAL_OPTIONS]),
+    'history': new Set([...CLI_MUTUAL_OPTIONS])
+};
+
 
 const VALID_OPTIONS_WHITELIST = new Set(['ips', ...CLI_MUTUAL_OPTIONS]);
 
@@ -89,7 +95,8 @@ const VALID_OPTIONS = {
     whitelist_options: VALID_OPTIONS_WHITELIST,
     from_file_options: VALID_OPTIONS_FROM_FILE,
     anonymous_account_options: VALID_OPTIONS_ANONYMOUS_ACCOUNT,
-    diagnose_options: VALID_OPTIONS_DIAGNOSE
+    diagnose_options: VALID_OPTIONS_DIAGNOSE,
+    upgrade_options: VALID_OPTIONS_UPGRADE
 };
 
 const OPTION_TYPE = {
@@ -123,6 +130,11 @@ const OPTION_TYPE = {
     all_bucket_details: 'boolean',
     https_port: 'number',
     debug: 'number',
+    // upgrade options
+    expected_version: 'string',
+    expected_hosts: 'string',
+    custom_upgrade_scripts_dir: 'string',
+    skip_verification: 'boolean'
 };
 
 const BOOLEAN_STRING_VALUES = ['true', 'false'];

src/manage_nsfs/manage_nsfs_help_utils.js

@@ -301,6 +301,19 @@ Run 'upgrade start' after upgrading NooBaa RPMs on all the cluster nodes, after
 S3 I/O, S3 Buckets getters and NooBaa CLI Account/Buckets/Whitelist getters operations will still be working
 But updates of the config directory will be blocked during the upgrade of the config directory.
 'upgrade start' should be executed on one node, the config directory changes will be available for all the nodes of the cluster.
+
+Usage:
+        
+    noobaa-cli upgrade start [flags]
+
+Flags:
+
+--expected_version                             <string>                             The expected target version of the upgrade
+--expected_hosts                               <string>                             The expected hosts running NooBaa NC, a string of hosts separated by , 
+--skip_verification                            <boolean>        (optional)          skip verification of the hosts package version
+                                                                                    WARNING: can cause corrupted config dir files created by hosts running old code
+--custom_upgrade_scripts_dir                   <string>         (optional)          custom upgrade scripts dir, use for running custom config dir upgrade scripts
+
 `;
 
 const UPGRADE_STATUS_OPTIONS = `

src/manage_nsfs/manage_nsfs_validations.js

@@ -125,6 +125,8 @@ function validate_no_extra_options(type, action, input_options, is_options_from_
         valid_options = VALID_OPTIONS.glacier_options[action];
     } else if (type === TYPES.DIAGNOSE) {
         valid_options = VALID_OPTIONS.diagnose_options[action];
+    } else if (type === TYPES.UPGRADE) {
+        valid_options = VALID_OPTIONS.upgrade_options[action];
     } else {
         valid_options = VALID_OPTIONS.whitelist_options;
     }

src/manage_nsfs/upgrade.js

@@ -1,21 +1,25 @@
 /* Copyright (C) 2024 NooBaa */
 'use strict';
 
+const _ = require('lodash');
 const dbg = require('../util/debug_module')(__filename);
 const { ManageCLIError } = require('./manage_nsfs_cli_errors');
 const { UPGRADE_ACTIONS } = require('./manage_nsfs_constants');
+const { NCUpgradeManager } = require('../upgrade/nc_upgrade_manager');
 const { ManageCLIResponse } = require('../manage_nsfs/manage_nsfs_cli_responses');
 const { throw_cli_error, write_stdout_response } = require('./manage_nsfs_cli_utils');
 
 /**
  * manage_upgrade_operations handles cli upgrade operations
  * @param {string} action 
+ * @param {string} user_input 
+ * @param {import('../sdk/config_fs').ConfigFS} config_fs
  * @returns {Promise<Void>}
  */
-async function manage_upgrade_operations(action, config_fs) {
+async function manage_upgrade_operations(action, user_input, config_fs) {
     switch (action) {
         case UPGRADE_ACTIONS.START:
-            await exec_config_dir_upgrade();
+            await start_config_dir_upgrade(user_input, config_fs);
             break;
         case UPGRADE_ACTIONS.STATUS:
             await get_upgrade_status(config_fs);
@@ -29,13 +33,25 @@ async function manage_upgrade_operations(action, config_fs) {
 }
 
 /**
- * exec_config_dir_upgrade handles cli upgrade operation
+ * start_config_dir_upgrade handles cli upgrade operation
+ * @param {Object} user_input
+ * @param {import('../sdk/config_fs').ConfigFS} config_fs
  * @returns {Promise<Void>}
  */
-async function exec_config_dir_upgrade() {
+async function start_config_dir_upgrade(user_input, config_fs) {
     try {
-        // TODO - add verifications and a call to the config directory upgrade
-        throw new Error('Upgrade Config Directory is not implemented yet');
+        const skip_verification = user_input.skip_verification;
+        const expected_version = user_input.expected_version;
+        const expected_hosts = user_input.expected_hosts && user_input.expected_hosts.split(',').filter(host => !_.isEmpty(host));
+        const custom_upgrade_scripts_dir = user_input.custom_upgrade_scripts_dir;
+
+        if (!expected_version) throw new Error('expected_version flag is required');
+        if (!expected_hosts) throw new Error('expected_hosts flag is required');
+
+        const nc_upgrade_manager = new NCUpgradeManager(config_fs, { custom_upgrade_scripts_dir });
+        const upgrade_res = await nc_upgrade_manager.upgrade_config_dir(expected_version, expected_hosts, { skip_verification });
+        if (!upgrade_res) throw new Error('Upgrade config directory failed', { cause: upgrade_res });
+        write_stdout_response(ManageCLIResponse.UpgradeSuccessful, upgrade_res);
     } catch (err) {
         dbg.error('could not upgrade config directory successfully - err', err);
         throw_cli_error({ ...ManageCLIError.UpgradeFailed, cause: err });