Merge pull request #8518 from shirady/nsfs-nc-changes-in-panic

NC | NSFS | Panic Printings Added + Try-Catch `memory_monitor` + Change Default Event Logs + Increase `LimitNOFILE`

Author: shirady

config.js

@@ -493,7 +493,7 @@ config.DEBUG_MODE_PERIOD = 10 * 60 * 1000; // 10 minutes for increased debug lev
 config.dbg_log_level = 0;
 config.DEBUG_FACILITY = 'LOG_LOCAL0';
 config.EVENT_FACILITY = 'LOG_LOCAL2';
-config.EVENT_LOGGING_ENABLED = true;
+config.EVENT_LOGGING_ENABLED = false; // should be changed in NC NSFS configuration
 config.EVENT_LEVEL = 5;
 
 config.LOG_TO_STDERR_ENABLED = true;

src/cmd/nsfs.js

@@ -244,6 +244,7 @@ class NsfsAccountSDK extends AccountSDK {
 async function main(argv = minimist(process.argv.slice(2))) {
     try {
         config.DB_TYPE = 'none';
+        config.EVENT_LOGGING_ENABLED = true;
         config.NSFS_VERSIONING_ENABLED = true;
         // when using data buckets on noobaa standalone we should set it to true
         config.ENABLE_OBJECT_IO_SEMAPHORE_MONITOR = false;

src/deploy/noobaa.service

@@ -11,7 +11,7 @@ ExecStart=/usr/local/noobaa-core/bin/node /usr/local/noobaa-core/src/cmd/nsfs.js
 EnvironmentFile=-/etc/sysconfig/noobaa
 ExecStop=/bin/kill $MAINPID
 WorkingDirectory=/usr/local/noobaa-core/
-LimitNOFILE=65536
+LimitNOFILE=600000
 
 [Install]
 WantedBy=multi-user.target

src/util/debug_module.js

@@ -509,7 +509,6 @@ function log_syslog_builder(syslevel) {
 */
 function log_event(event) {
     if (!config.EVENT_LOGGING_ENABLED) {
-        console.info('Event logging not enabled');
         return;
     }
     event.pid = process.pid;

src/util/panic.js

@@ -8,12 +8,16 @@ const child_process = require('child_process');
 process.on('uncaughtException', err => panic('process uncaughtException', err));
 
 function panic(message, err) {
+    // this printing is duplicated here in case LOOP_ON_PANIC is true (the process will not be exit)
     console.error('PANIC:', message, err.stack || err);
     while (process.env.LOOP_ON_PANIC === 'true') {
         console.warn('Encountered an error, holding the process on an infinite loop');
         child_process.execSync('sleep 10');
     }
-    process.exit(1);
+    // to avoid cases where the process can exit without printing the error
+    process.stderr.write('PANIC: ' + message + (err.stack || err) + '\n', () => {
+        process.exit(1);
+    });
 }
 
 // dump heap with kill -USR2 <pid>
@@ -37,25 +41,32 @@ function enable_heapdump(name, next_mb, step_mb) {
 }
 
 function memory_monitor() {
-    const m = process.memoryUsage();
-    const c = memory_monitor_config;
-    const h = c.heapdump;
-    const current = m.heapUsed;
-    if (c.logging_threshold &&
-        c.logging_threshold <= current) {
-        const usage = (m.heapUsed / 1024 / 1024).toFixed(0);
-        const total = (m.heapTotal / 1024 / 1024).toFixed(0);
-        const resident = (m.rss / 1024 / 1024).toFixed(0);
-        console.log(`memory_monitor: heap ${usage} MB | total ${total} MB | resident ${resident} MB`);
-    }
-    if (h && h.next && h.next <= current) {
-        const size_mb = (current / 1024 / 1024).toFixed(0);
-        const snapshot_name = `heapdump-${h.name}-${process.pid}-${new Date().toISOString()}-${size_mb}MB.heapsnapshot`;
-        console.log(`memory_monitor: writing ${snapshot_name}`);
-        heapdump.writeSnapshot(snapshot_name);
-        const increase = current - h.next;
-        const align = h.step - (increase % h.step);
-        h.next += increase + align;
+    try {
+        const m = process.memoryUsage();
+        const c = memory_monitor_config;
+        const h = c.heapdump;
+        const current = m.heapUsed;
+        if (c.logging_threshold &&
+            c.logging_threshold <= current) {
+            const usage = (m.heapUsed / 1024 / 1024).toFixed(0);
+            const total = (m.heapTotal / 1024 / 1024).toFixed(0);
+            const resident = (m.rss / 1024 / 1024).toFixed(0);
+            console.log(`memory_monitor: heap ${usage} MB | total ${total} MB | resident ${resident} MB`);
+        }
+        if (h && h.next && h.next <= current) {
+            const size_mb = (current / 1024 / 1024).toFixed(0);
+            const snapshot_name = `heapdump-${h.name}-${process.pid}-${new Date().toISOString()}-${size_mb}MB.heapsnapshot`;
+            console.log(`memory_monitor: writing ${snapshot_name}`);
+            heapdump.writeSnapshot(snapshot_name);
+            const increase = current - h.next;
+            const align = h.step - (increase % h.step);
+            h.next += increase + align;
+        }
+    } catch (err) {
+        console.error('memory_monitor got an error', err);
+        // we saw cases where the number of open files (file descriptors) was a reason for uncaught error during the memory_monitor
+        // we don't want the process to fail on that
+        if (err.code !== 'EMFILE') throw err;
     }
 }