mitigate the python proc mesh bug for KD controller service (#512)

Summary:
Pull Request resolved: #512

A bug introduced by D77250211, which messed up the lifecycle management of python proc mesh. The Alloc of the proc mesh can be destroyed earlier than expected, resulting in connection disconnecting.

This diff is a workaround, it keeps the proc meshes around, the same as the actor mesh.

Differential Revision: D78111736

fbshipit-source-id: 51c943ebf52824aa499a0d1384d45f755cc9dcea

Author: moonli

hyperactor_mesh/src/proc_mesh.rs

@@ -184,6 +184,7 @@ impl ProcMesh {
         let (router_channel_addr, router_rx) = channel::serve(ChannelAddr::any(alloc.transport()))
             .await
             .map_err(|err| AllocatorError::Other(err.into()))?;
+        tracing::info!("router channel started listening on addr: {router_channel_addr}");
         let router = DialMailboxRouter::new_with_default(global_router().boxed());
         for (rank, (addr, _agent)) in running.iter().enumerate() {
             let proc_id = proc_ids.get(rank).unwrap().clone();
@@ -202,6 +203,7 @@ impl ProcMesh {
         let (client_proc_addr, client_rx) = channel::serve(ChannelAddr::any(alloc.transport()))
             .await
             .map_err(|err| AllocatorError::Other(err.into()))?;
+        tracing::info!("client proc started listening on addr: {client_proc_addr}");
         let client_proc = Proc::new(
             client_proc_id.clone(),
             BoxedMailboxSender::new(router.clone()),