[ENH] Robustly handle errors on the compaction path of the log. (#4677)

## Description of changes

One path is timing out in staging.  Add tolerance for that and some
debugging.

## Test plan

Compiles

- [x] Tests pass locally with `pytest` for python, `yarn test` for js,
`cargo test` for rust

## Documentation Changes

_Are all docstrings for user-facing APIs updated if required? Do we need
to make documentation changes in the [docs
section](https://github.com/chroma-core/chroma/tree/main/docs/docs.trychroma.com)?_

Author: rescrv

rust/log/src/grpc_log.rs

@@ -554,6 +554,7 @@ impl GrpcLog {
         min_compaction_size: u64,
     ) -> Result<Vec<CollectionInfo>, GrpcGetCollectionsWithNewDataError> {
         let mut combined_response = Vec::new();
+        let mut error = None;
 
         if use_alt_log {
             // Iterate over all alt clients and gather collections
@@ -567,14 +568,25 @@ impl GrpcLog {
                 }
                 for mut alt_client in all_alt_clients.drain(..) {
                     // We error if any subrequest errors
-                    let response = alt_client
+                    match alt_client
                         .get_all_collection_info_to_compact(
                             chroma_proto::GetAllCollectionInfoToCompactRequest {
                                 min_compaction_size,
                             },
                         )
-                        .await?;
-                    combined_response.push(response.into_inner());
+                        .await
+                    {
+                        Ok(response) => {
+                            combined_response.push(response.into_inner());
+                        }
+                        Err(err) => {
+                            tracing::error!("could not get all collection info to compact: {err}");
+                            if error.is_none() {
+                                error = Some(err);
+                            }
+                            continue;
+                        }
+                    };
                 }
             } else {
                 tracing::warn!(
@@ -583,17 +595,33 @@ impl GrpcLog {
                 return Ok(vec![]);
             }
         } else {
-            let response = self
+            match self
                 .client
                 .get_all_collection_info_to_compact(
                     chroma_proto::GetAllCollectionInfoToCompactRequest {
                         min_compaction_size,
                     },
                 )
-                .await?;
-            combined_response.push(response.into_inner());
+                .await
+            {
+                Ok(response) => {
+                    combined_response.push(response.into_inner());
+                }
+                Err(err) => {
+                    tracing::error!("could not get all collection info to compact: {err}");
+                    if error.is_none() {
+                        error = Some(err);
+                    }
+                }
+            };
         };
 
+        if let Some(status) = error {
+            if combined_response.is_empty() {
+                return Err(status.into());
+            }
+        }
+
         let mut all_collections = Vec::new();
         for response in combined_response {
             let collections = response.all_collection_info;