NAS-133457 / 25.10 / TNC Support for HA (#16374)

Author: sonicaj

src/middlewared/middlewared/plugins/failover_/event.py

@@ -755,6 +755,10 @@ def vrrp_master(self, job, fobj, ifname, event):
         self.run_call('truecommand.start_truecommand_service')
         logger.info('Done starting truecommand service (if necessary)')
 
+        logger.info('Configuring TrueNAS Connect Service (if necessary)')
+        self.run_call('tn_connect.state.check')
+        logger.info('Configuring TrueNAS Connect Service (if necessary)')
+
         # The system, while it was in BACKUP state, might have failed to contact the remote node and reached a
         # conclusion that the other node needs to be rebooted. Let's clean this up.
         self.run_call('failover.reboot.discard_unbound_remote_reboot_reasons')

src/middlewared/middlewared/plugins/truenas_connect/acme.py

@@ -38,6 +38,16 @@ async def update_ui(self):
             # Let's restart UI now
             # TODO: Hash this out with everyone
             await self.middleware.call('system.general.ui_restart', 2)
+            if await self.middleware.call('failover.licensed'):
+                # We would like to make sure nginx is reloaded on the remote controller as well
+                logger.debug('Restarting UI on remote controller')
+                try:
+                    await self.middleware.call(
+                        'failover.call_remote', 'system.general.ui_restart', [2],
+                        {'raise_connect_error': False, 'timeout': 2, 'connect_timeout': 2}
+                    )
+                except Exception:
+                    logger.error('Failed to restart UI on remote controller', exc_info=True)
 
     async def update_ui_impl(self):
         config = await self.middleware.call('tn_connect.config')
@@ -139,18 +149,10 @@ async def revoke_cert(self):
 
 
 async def check_status(middleware):
-    tnc_config = await middleware.call('tn_connect.config')
-    if tnc_config['status'] == Status.CERT_GENERATION_IN_PROGRESS.name:
-        logger.debug('Middleware started and cert generation is in progress, initiating process')
-        middleware.create_task(middleware.call('tn_connect.acme.initiate_cert_generation'))
-    elif tnc_config['status'] in (
-            Status.CERT_GENERATION_SUCCESS.name, Status.CERT_RENEWAL_SUCCESS.name,
-    ):
-        logger.debug('Middleware started and cert generation is already successful, updating UI')
-        middleware.create_task(middleware.call('tn_connect.acme.update_ui'))
-    elif tnc_config['status'] == Status.CERT_RENEWAL_IN_PROGRESS.name:
-        logger.debug('Middleware started and cert renewal is in progress, initiating process')
-        middleware.create_task(middleware.call('tn_connect.acme.renew_cert'))
+    if not await middleware.call('failover.is_single_master_node'):
+        return
+
+    await middleware.call('tn_connect.state.check')
 
 
 async def _event_system_ready(middleware, event_type, args):

src/middlewared/middlewared/plugins/truenas_connect/finalize_registration.py

@@ -103,25 +103,3 @@ async def poll_once(self, claim_token, system_id, tnc_config):
             get_registration_finalization_uri(tnc_config), 'post',
             payload={'system_id': system_id, 'claim_token': claim_token},
         )
-
-
-async def check_status(middleware):
-    tn_config = await middleware.call('tn_connect.config')
-    if tn_config['status'] == Status.REGISTRATION_FINALIZATION_WAITING.name:
-        logger.debug(
-            'Registration finalization failed as middleware was restarted while waiting '
-            'for TNC registration finalization'
-        )
-        # This means middleware got restarted or the system was rebooted while we were waiting for
-        # registration to finalize, so in this case we set the state to registration failed
-        await middleware.call('tn_connect.finalize.status_update', Status.REGISTRATION_FINALIZATION_FAILED)
-
-
-async def _event_system_ready(middleware, event_type, args):
-    await check_status(middleware)
-
-
-async def setup(middleware):
-    middleware.event_subscribe('system.ready', _event_system_ready)
-    if await middleware.call('system.ready'):
-        await check_status(middleware)

src/middlewared/middlewared/plugins/truenas_connect/state.py

@@ -0,0 +1,37 @@
+import logging
+
+from truenas_connect_utils.status import Status
+
+from middlewared.service import Service
+
+
+logger = logging.getLogger('truenas_connect')
+
+
+class TrueNASConnectStateService(Service):
+
+    class Config:
+        namespace = 'tn_connect.state'
+        private = True
+
+    async def check(self):
+        tnc_config = await self.middleware.call('tn_connect.config')
+        if tnc_config['status'] == Status.REGISTRATION_FINALIZATION_WAITING.name:
+            logger.debug(
+                'Registration finalization failed as middleware was restarted while waiting '
+                'for TNC registration finalization'
+            )
+            # This means middleware got restarted or the system was rebooted while we were waiting for
+            # registration to finalize, so in this case we set the state to registration failed
+            await self.middleware.call('tn_connect.finalize.status_update', Status.REGISTRATION_FINALIZATION_FAILED)
+        elif tnc_config['status'] == Status.CERT_GENERATION_IN_PROGRESS.name:
+            logger.debug('Middleware started and cert generation is in progress, initiating process')
+            self.middleware.create_task(self.middleware.call('tn_connect.acme.initiate_cert_generation'))
+        elif tnc_config['status'] in (
+            Status.CERT_GENERATION_SUCCESS.name, Status.CERT_RENEWAL_SUCCESS.name,
+        ):
+            logger.debug('Middleware started and cert generation is already successful, updating UI')
+            self.middleware.create_task(self.middleware.call('tn_connect.acme.update_ui'))
+        elif tnc_config['status'] == Status.CERT_RENEWAL_IN_PROGRESS.name:
+            logger.debug('Middleware started and cert renewal is in progress, initiating process')
+            self.middleware.create_task(self.middleware.call('tn_connect.acme.renew_cert'))