diff --git a/com.unity.netcode.gameobjects/Runtime/Components/NetworkAnimator.cs b/com.unity.netcode.gameobjects/Runtime/Components/NetworkAnimator.cs
index 583e7bda4a..f6209a98b3 100644
--- a/com.unity.netcode.gameobjects/Runtime/Components/NetworkAnimator.cs
+++ b/com.unity.netcode.gameobjects/Runtime/Components/NetworkAnimator.cs
@@ -996,7 +996,7 @@ internal void CheckForAnimatorChanges()
                 {
                     // Just notify all remote clients and not the local server
                     m_ClientSendList.Clear();
-                    foreach (var clientId in NetworkManager.ConnectedClientsIds)
+                    foreach (var clientId in NetworkManager.ConnectionManager.ConnectedClientIds)
                     {
                         if (clientId == NetworkManager.LocalClientId || !NetworkObject.Observers.Contains(clientId))
                         {
@@ -1315,7 +1315,7 @@ private unsafe void SendParametersUpdateServerRpc(ParametersUpdateMessage parame
                 if (NetworkManager.ConnectedClientsIds.Count > (IsHost ? 2 : 1))
                 {
                     m_ClientSendList.Clear();
-                    foreach (var clientId in NetworkManager.ConnectedClientsIds)
+                    foreach (var clientId in NetworkManager.ConnectionManager.ConnectedClientIds)
                     {
                         if (clientId == serverRpcParams.Receive.SenderClientId || clientId == NetworkManager.ServerClientId || !NetworkObject.Observers.Contains(clientId))
                         {
@@ -1378,7 +1378,7 @@ private void SendAnimStateServerRpc(AnimationMessage animationMessage, ServerRpc
                 if (NetworkManager.ConnectedClientsIds.Count > (IsHost ? 2 : 1))
                 {
                     m_ClientSendList.Clear();
-                    foreach (var clientId in NetworkManager.ConnectedClientsIds)
+                    foreach (var clientId in NetworkManager.ConnectionManager.ConnectedClientIds)
                     {
                         if (clientId == serverRpcParams.Receive.SenderClientId || clientId == NetworkManager.ServerClientId || !NetworkObject.Observers.Contains(clientId))
                         {
@@ -1452,7 +1452,7 @@ internal void SendAnimTriggerServerRpc(AnimationTriggerMessage animationTriggerM
             InternalSetTrigger(animationTriggerMessage.Hash, animationTriggerMessage.IsTriggerSet);
 
             m_ClientSendList.Clear();
-            foreach (var clientId in NetworkManager.ConnectedClientsIds)
+            foreach (var clientId in NetworkManager.ConnectionManager.ConnectedClientIds)
             {
                 if (clientId == NetworkManager.ServerClientId || !NetworkObject.Observers.Contains(clientId))
                 {
diff --git a/com.unity.netcode.gameobjects/Runtime/Connection/NetworkConnectionManager.cs b/com.unity.netcode.gameobjects/Runtime/Connection/NetworkConnectionManager.cs
index 96f0223ef3..6d3337342c 100644
--- a/com.unity.netcode.gameobjects/Runtime/Connection/NetworkConnectionManager.cs
+++ b/com.unity.netcode.gameobjects/Runtime/Connection/NetworkConnectionManager.cs
@@ -149,12 +149,12 @@ internal void InvokeOnClientConnectedCallback(ulong clientId)
             }
 
             // Invoking connection event on non-authority local client. Need to calculate PeerIds.
-            var peerClientIds = new NativeArray<ulong>(Math.Max(NetworkManager.ConnectedClientsIds.Count - 1, 0), Allocator.Temp);
+            var peerClientIds = new NativeArray<ulong>(Math.Max(ConnectedClientIds.Count - 1, 0), Allocator.Temp);
             // `using var peerClientIds` or `using(peerClientIds)` renders it immutable...
             using var sentinel = peerClientIds;
 
             var idx = 0;
-            foreach (var peerId in NetworkManager.ConnectedClientsIds)
+            foreach (var peerId in ConnectedClientIds)
             {
                 if (peerId == NetworkManager.LocalClientId)
                 {
diff --git a/com.unity.netcode.gameobjects/Runtime/Core/NetworkBehaviourUpdater.cs b/com.unity.netcode.gameobjects/Runtime/Core/NetworkBehaviourUpdater.cs
index 105f203c86..21dd550f48 100644
--- a/com.unity.netcode.gameobjects/Runtime/Core/NetworkBehaviourUpdater.cs
+++ b/com.unity.netcode.gameobjects/Runtime/Core/NetworkBehaviourUpdater.cs
@@ -34,8 +34,11 @@ internal void NetworkBehaviourUpdate(bool forceSend = false)
 #endif
             try
             {
-                m_DirtyNetworkObjects.UnionWith(m_PendingDirtyNetworkObjects);
-                m_PendingDirtyNetworkObjects.Clear();
+                if (m_PendingDirtyNetworkObjects.Count > 0)
+                {
+                    m_DirtyNetworkObjects.UnionWith(m_PendingDirtyNetworkObjects);
+                    m_PendingDirtyNetworkObjects.Clear();
+                }
 
                 // NetworkObject references can become null, when hidden or despawned. Once NUll, there is no point
                 // trying to process them, even if they were previously marked as dirty.
diff --git a/com.unity.netcode.gameobjects/Runtime/Core/NetworkObject.cs b/com.unity.netcode.gameobjects/Runtime/Core/NetworkObject.cs
index b8c5c5b551..104fbdad47 100644
--- a/com.unity.netcode.gameobjects/Runtime/Core/NetworkObject.cs
+++ b/com.unity.netcode.gameobjects/Runtime/Core/NetworkObject.cs
@@ -1616,7 +1616,7 @@ public void NetworkHide(ulong clientId)
                         // Send destroy call
                         size = NetworkManager.ConnectionManager.SendMessage(ref message, NetworkDelivery.ReliableSequenced, clientId);
                         // Broadcast the destroy to all clients so they can update their observers list
-                        foreach (var client in NetworkManager.ConnectedClientsIds)
+                        foreach (var client in NetworkManager.ConnectionManager.ConnectedClientIds)
                         {
                             if (client == clientId || client == NetworkManager.LocalClientId)
                             {
@@ -2363,7 +2363,7 @@ private void OnTransformParentChanged()
                 }
                 else
                 {
-                    foreach (var clientId in NetworkManager.ConnectedClientsIds)
+                    foreach (var clientId in NetworkManager.ConnectionManager.ConnectedClientIds)
                     {
                         if (clientId == NetworkManager.ServerClientId)
                         {
@@ -2382,7 +2382,7 @@ private void OnTransformParentChanged()
                     var maxCount = NetworkManager.ConnectedClientsIds.Count;
                     ulong* clientIds = stackalloc ulong[maxCount];
                     int idx = 0;
-                    foreach (var clientId in NetworkManager.ConnectedClientsIds)
+                    foreach (var clientId in NetworkManager.ConnectionManager.ConnectedClientIds)
                     {
                         if (clientId == NetworkManager.ServerClientId)
                         {
diff --git a/com.unity.netcode.gameobjects/Runtime/Messaging/Messages/ChangeOwnershipMessage.cs b/com.unity.netcode.gameobjects/Runtime/Messaging/Messages/ChangeOwnershipMessage.cs
index c9148712ef..7b18c362d1 100644
--- a/com.unity.netcode.gameobjects/Runtime/Messaging/Messages/ChangeOwnershipMessage.cs
+++ b/com.unity.netcode.gameobjects/Runtime/Messaging/Messages/ChangeOwnershipMessage.cs
@@ -250,8 +250,9 @@ public void Handle(ref NetworkContext context)
                 }
                 else
                 {
-                    foreach (var clientId in clientList)
+                    for (int i = 0; i < clientList.Count; i++)
                     {
+                        var clientId = clientList[i];
                         if (clientId == networkManager.LocalClientId)
                         {
                             continue;
diff --git a/com.unity.netcode.gameobjects/Runtime/Messaging/Messages/CreateObjectMessage.cs b/com.unity.netcode.gameobjects/Runtime/Messaging/Messages/CreateObjectMessage.cs
index 02f4263e9d..89fb526a61 100644
--- a/com.unity.netcode.gameobjects/Runtime/Messaging/Messages/CreateObjectMessage.cs
+++ b/com.unity.netcode.gameobjects/Runtime/Messaging/Messages/CreateObjectMessage.cs
@@ -247,9 +247,9 @@ internal static void CreateObject(ref NetworkManager networkManager, ulong sende
                     var clientList = hasObserverIdList && !networkObject.IsPlayerObject ? observerIds : networkManager.ConnectedClientsIds;
 
                     // Update the observers for this instance
-                    foreach (var clientId in clientList)
+                    for (int i = 0; i < clientList.Count; i++)
                     {
-                        networkObject.Observers.Add(clientId);
+                        networkObject.Observers.Add(clientList[i]);
                     }
 
                     // Mock CMB Service and forward to all clients
diff --git a/com.unity.netcode.gameobjects/Runtime/Messaging/Messages/DestroyObjectMessage.cs b/com.unity.netcode.gameobjects/Runtime/Messaging/Messages/DestroyObjectMessage.cs
index f37c9f95a4..be037e7b90 100644
--- a/com.unity.netcode.gameobjects/Runtime/Messaging/Messages/DestroyObjectMessage.cs
+++ b/com.unity.netcode.gameobjects/Runtime/Messaging/Messages/DestroyObjectMessage.cs
@@ -191,7 +191,7 @@ private void HandleDAHostForwardMessage(ulong senderId, ref NetworkManager netwo
                 DeferredDespawnTick = DeferredDespawnTick,
             };
             var ownerClientId = networkObject == null ? senderId : networkObject.OwnerClientId;
-            var clientIds = networkObject == null ? networkManager.ConnectedClientsIds.ToList() : networkObject.Observers.ToList();
+            var clientIds = networkObject == null ? networkManager.ConnectionManager.ConnectedClientIds : networkObject.Observers.ToList();
 
             foreach (var clientId in clientIds)
             {
diff --git a/com.unity.netcode.gameobjects/Runtime/Messaging/Messages/NetworkVariableDeltaMessage.cs b/com.unity.netcode.gameobjects/Runtime/Messaging/Messages/NetworkVariableDeltaMessage.cs
index af520a0468..0be9500c9f 100644
--- a/com.unity.netcode.gameobjects/Runtime/Messaging/Messages/NetworkVariableDeltaMessage.cs
+++ b/com.unity.netcode.gameobjects/Runtime/Messaging/Messages/NetworkVariableDeltaMessage.cs
@@ -228,7 +228,7 @@ public void Handle(ref NetworkContext context)
                     if (isServerAndDeltaForwarding)
                     {
                         m_ForwardUpdates = new Dictionary<ulong, List<int>>();
-                        foreach (var clientId in networkManager.ConnectedClientsIds)
+                        foreach (var clientId in networkManager.ConnectionManager.ConnectedClientIds)
                         {
                             if (clientId == context.SenderId || clientId == networkManager.LocalClientId || !networkObject.Observers.Contains(clientId))
                             {
diff --git a/com.unity.netcode.gameobjects/Runtime/Messaging/RpcTargets/BaseRpcTarget.cs b/com.unity.netcode.gameobjects/Runtime/Messaging/RpcTargets/BaseRpcTarget.cs
index 66c88b3eec..d9e1d2cbdf 100644
--- a/com.unity.netcode.gameobjects/Runtime/Messaging/RpcTargets/BaseRpcTarget.cs
+++ b/com.unity.netcode.gameobjects/Runtime/Messaging/RpcTargets/BaseRpcTarget.cs
@@ -11,6 +11,8 @@ public abstract class BaseRpcTarget : IDisposable
         /// The <see cref="NetworkManager"/> instance which can be used to handle sending and receiving the specific target(s)
         /// </summary>
         protected NetworkManager m_NetworkManager;
+
+        internal NetworkConnectionManager ConnectionManager;
         private bool m_Locked;
 
         internal void Lock()
@@ -26,6 +28,7 @@ internal void Unlock()
         internal BaseRpcTarget(NetworkManager manager)
         {
             m_NetworkManager = manager;
+            ConnectionManager = m_NetworkManager.ConnectionManager;
         }
 
         /// <summary>
diff --git a/com.unity.netcode.gameobjects/Runtime/Messaging/RpcTargets/NotAuthorityRpcTarget.cs b/com.unity.netcode.gameobjects/Runtime/Messaging/RpcTargets/NotAuthorityRpcTarget.cs
index 8ee736d6bf..0a0aa66725 100644
--- a/com.unity.netcode.gameobjects/Runtime/Messaging/RpcTargets/NotAuthorityRpcTarget.cs
+++ b/com.unity.netcode.gameobjects/Runtime/Messaging/RpcTargets/NotAuthorityRpcTarget.cs
@@ -40,7 +40,7 @@ internal override void Send(NetworkBehaviour behaviour, ref RpcMessage message,
                 }
                 else
                 {
-                    foreach (var clientId in m_NetworkManager.ConnectedClientsIds)
+                    foreach (var clientId in ConnectionManager.ConnectedClientIds)
                     {
                         if (clientId == behaviour.OwnerClientId || !networkObject.Observers.Contains(clientId))
                         {
diff --git a/com.unity.netcode.gameobjects/Runtime/Messaging/RpcTargets/NotMeRpcTarget.cs b/com.unity.netcode.gameobjects/Runtime/Messaging/RpcTargets/NotMeRpcTarget.cs
index 5ea687101c..adb579ee0d 100644
--- a/com.unity.netcode.gameobjects/Runtime/Messaging/RpcTargets/NotMeRpcTarget.cs
+++ b/com.unity.netcode.gameobjects/Runtime/Messaging/RpcTargets/NotMeRpcTarget.cs
@@ -43,7 +43,7 @@ internal override void Send(NetworkBehaviour behaviour, ref RpcMessage message,
             }
             else
             {
-                foreach (var clientId in m_NetworkManager.ConnectedClientsIds)
+                foreach (var clientId in ConnectionManager.ConnectedClientIds)
                 {
                     if (clientId == behaviour.NetworkManager.LocalClientId)
                     {
diff --git a/com.unity.netcode.gameobjects/Runtime/Messaging/RpcTargets/NotOwnerRpcTarget.cs b/com.unity.netcode.gameobjects/Runtime/Messaging/RpcTargets/NotOwnerRpcTarget.cs
index 348c1af313..7dbb13f1a2 100644
--- a/com.unity.netcode.gameobjects/Runtime/Messaging/RpcTargets/NotOwnerRpcTarget.cs
+++ b/com.unity.netcode.gameobjects/Runtime/Messaging/RpcTargets/NotOwnerRpcTarget.cs
@@ -64,7 +64,7 @@ internal override void Send(NetworkBehaviour behaviour, ref RpcMessage message,
             }
             else
             {
-                foreach (var clientId in m_NetworkManager.ConnectedClientsIds)
+                foreach (var clientId in ConnectionManager.ConnectedClientIds)
                 {
                     if (clientId == behaviour.OwnerClientId)
                     {
diff --git a/com.unity.netcode.gameobjects/Runtime/Messaging/RpcTargets/NotServerRpcTarget.cs b/com.unity.netcode.gameobjects/Runtime/Messaging/RpcTargets/NotServerRpcTarget.cs
index 3ac0b231a9..72a0f4b434 100644
--- a/com.unity.netcode.gameobjects/Runtime/Messaging/RpcTargets/NotServerRpcTarget.cs
+++ b/com.unity.netcode.gameobjects/Runtime/Messaging/RpcTargets/NotServerRpcTarget.cs
@@ -44,7 +44,7 @@ internal override void Send(NetworkBehaviour behaviour, ref RpcMessage message,
             }
             else
             {
-                foreach (var clientId in m_NetworkManager.ConnectedClientsIds)
+                foreach (var clientId in ConnectionManager.ConnectedClientIds)
                 {
                     if (clientId == NetworkManager.ServerClientId)
                     {
diff --git a/com.unity.netcode.gameobjects/Runtime/Messaging/RpcTargets/RpcTarget.cs b/com.unity.netcode.gameobjects/Runtime/Messaging/RpcTargets/RpcTarget.cs
index 7b651ebcd4..a19aecfe25 100644
--- a/com.unity.netcode.gameobjects/Runtime/Messaging/RpcTargets/RpcTarget.cs
+++ b/com.unity.netcode.gameobjects/Runtime/Messaging/RpcTargets/RpcTarget.cs
@@ -119,10 +119,11 @@ public enum RpcTargetUse
     public class RpcTarget
     {
         private NetworkManager m_NetworkManager;
+        private NetworkConnectionManager m_ConnectionManager;
         internal RpcTarget(NetworkManager manager)
         {
             m_NetworkManager = manager;
-
+            m_ConnectionManager = manager.ConnectionManager;
             Everyone = new EveryoneRpcTarget(manager);
             Owner = new OwnerRpcTarget(manager);
             NotOwner = new NotOwnerRpcTarget(manager);
@@ -312,7 +313,7 @@ public BaseRpcTarget Not(ulong excludedClientId, RpcTargetUse use)
                 }
             }
             target.Clear();
-            foreach (var clientId in m_NetworkManager.ConnectedClientsIds)
+            foreach (var clientId in m_NetworkManager.ConnectionManager.ConnectedClientIds)
             {
                 if (clientId != excludedClientId)
                 {
@@ -495,7 +496,7 @@ public BaseRpcTarget Not(NativeArray<ulong> excludedClientIds, RpcTargetUse use)
                 asASet.Add(clientId);
             }
 
-            foreach (var clientId in m_NetworkManager.ConnectedClientsIds)
+            foreach (var clientId in m_NetworkManager.ConnectionManager.ConnectedClientIds)
             {
                 if (!asASet.Contains(clientId))
                 {
@@ -590,7 +591,7 @@ public BaseRpcTarget Not<T>(T excludedClientIds, RpcTargetUse use) where T : IEn
                 asASet.Add(clientId);
             }
 
-            foreach (var clientId in m_NetworkManager.ConnectedClientsIds)
+            foreach (var clientId in m_ConnectionManager.ConnectedClientIds)
             {
                 if (!asASet.Contains(clientId))
                 {
diff --git a/com.unity.netcode.gameobjects/Runtime/SceneManagement/NetworkSceneManager.cs b/com.unity.netcode.gameobjects/Runtime/SceneManagement/NetworkSceneManager.cs
index 512e7ffdd7..134cc98ae8 100644
--- a/com.unity.netcode.gameobjects/Runtime/SceneManagement/NetworkSceneManager.cs
+++ b/com.unity.netcode.gameobjects/Runtime/SceneManagement/NetworkSceneManager.cs
@@ -2593,7 +2593,7 @@ internal void HandleSceneEvent(ulong clientId, FastBufferReader reader)
                                 EventData = sceneEventData,
                             };
                             // Forward synchronization to client then exit early because DAHost is not the current session owner
-                            foreach (var client in NetworkManager.ConnectedClientsIds)
+                            foreach (var client in NetworkManager.ConnectionManager.ConnectedClientIds)
                             {
                                 if (client == NetworkManager.LocalClientId)
                                 {
diff --git a/com.unity.netcode.gameobjects/Runtime/SceneManagement/SceneEventProgress.cs b/com.unity.netcode.gameobjects/Runtime/SceneManagement/SceneEventProgress.cs
index a649aec4bb..83fc0393ec 100644
--- a/com.unity.netcode.gameobjects/Runtime/SceneManagement/SceneEventProgress.cs
+++ b/com.unity.netcode.gameobjects/Runtime/SceneManagement/SceneEventProgress.cs
@@ -166,7 +166,7 @@ internal SceneEventProgress(NetworkManager networkManager, SceneEventProgressSta
                 {
                     m_NetworkManager.OnClientDisconnectCallback += OnClientDisconnectCallback;
                     // Track the clients that were connected when we started this event
-                    foreach (var connectedClientId in networkManager.ConnectedClientsIds)
+                    foreach (var connectedClientId in networkManager.ConnectionManager.ConnectedClientIds)
                     {
                         // Ignore the host or session owner
                         if ((!networkManager.DistributedAuthorityMode && NetworkManager.ServerClientId == connectedClientId) || (networkManager.DistributedAuthorityMode && networkManager.CurrentSessionOwner == connectedClientId))
diff --git a/com.unity.netcode.gameobjects/Runtime/Spawning/NetworkSpawnManager.cs b/com.unity.netcode.gameobjects/Runtime/Spawning/NetworkSpawnManager.cs
index 92b90972a7..4afad6a847 100644
--- a/com.unity.netcode.gameobjects/Runtime/Spawning/NetworkSpawnManager.cs
+++ b/com.unity.netcode.gameobjects/Runtime/Spawning/NetworkSpawnManager.cs
@@ -1099,7 +1099,7 @@ internal void SpawnNetworkObjectLocally(NetworkObject networkObject, ulong netwo
                 }
                 else
                 {
-                    foreach (var clientId in NetworkManager.ConnectedClientsIds)
+                    foreach (var clientId in NetworkManager.ConnectionManager.ConnectedClientIds)
                     {
                         // If SpawnWithObservers is enabled, then authority does take networkObject.CheckObjectVisibility into consideration
                         if (networkObject.CheckObjectVisibility != null && !networkObject.CheckObjectVisibility.Invoke(clientId))
@@ -1703,7 +1703,9 @@ internal void OnDespawnObject(NetworkObject networkObject, bool destroyGameObjec
                 // Otherwise send to the clients for which the object is visible
                 else
                 {
-                    foreach (var clientId in NetworkManager.ConnectedClientsIds)
+                    // We keep only the client for which the object is visible
+                    // as the other clients have them already despawned
+                    foreach (var clientId in NetworkManager.ConnectionManager.ConnectedClientIds)
                     {
                         if ((distributedAuthority && clientId == networkObject.OwnerClientId) || clientId == NetworkManager.LocalClientId)
                         {
diff --git a/com.unity.netcode.gameobjects/Runtime/Transports/UTP/UnityTransport.cs b/com.unity.netcode.gameobjects/Runtime/Transports/UTP/UnityTransport.cs
index 84513fde66..ad68e4060b 100644
--- a/com.unity.netcode.gameobjects/Runtime/Transports/UTP/UnityTransport.cs
+++ b/com.unity.netcode.gameobjects/Runtime/Transports/UTP/UnityTransport.cs
@@ -1146,9 +1146,9 @@ private void ExtractNetworkMetrics()
         {
             if (m_NetworkManager.IsServer)
             {
-                var ngoConnectionIds = m_NetworkManager.ConnectedClients.Keys;
-                foreach (var ngoConnectionId in ngoConnectionIds)
+                for (int i=0; i<m_NetworkManager.ConnectedClientsIds.Count; ++i)
                 {
+                    var ngoConnectionId =  m_NetworkManager.ConnectedClientsIds[i];
                     if (ngoConnectionId == 0 && m_NetworkManager.IsHost)
                     {
                         continue;