Handle InternalError in Listing.__iter__ (#2697)

<!-- REMOVE IRRELEVANT COMMENTS BEFORE CREATING A PULL REQUEST -->
## Changes
<!-- Summary of your changes that are easy to understand. Add
screenshots when necessary -->

### Linked issues
<!-- DOC: Link issue with a keyword: close, closes, closed, fix, fixes,
fixed, resolve, resolves, resolved. See
https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword
-->

Resolves #2582 

### Tests
<!-- How is this tested? Please see the checklist below and also
describe any other relevant tests -->

- [x] manually tested
- [x] added unit tests

Author: aminmovahed-db

src/databricks/labs/ucx/workspace_access/generic.py

@@ -59,8 +59,8 @@ def __iter__(self):
         try:
             for item in self._func():
                 yield GenericPermissionsInfo(getattr(item, self._id_attribute), self._object_type)
-        except NotFound as e:
-            logger.warning(f"Listing {self._object_type} failed: {e}")
+        except (NotFound, InternalError) as e:
+            logger.warning(f"Listing {self._object_type} failed", exc_info=e)
         since = datetime.datetime.now() - started
         logger.info(f"Listed {self._object_type} in {since}")
 

tests/unit/workspace_access/test_generic.py

@@ -918,11 +918,21 @@ def test_models_page_listing():
         assert item.object_type == "registered-models"
 
 
-def test_serving_endpoints_not_enabled_raises_warning(caplog):
+def test_serving_endpoints_not_enabled_raises_warning(caplog) -> None:
     ws = create_autospec(WorkspaceClient)
     ws.serving_endpoints.list.side_effect = NotFound("Model serving is not enabled for your shard")
 
     sup = GenericPermissionsSupport(ws=ws, listings=[Listing(ws.serving_endpoints.list, "id", "serving-endpoints")])
-    with caplog.at_level('WARNING'):
+    with caplog.at_level('WARNING', logger="databricks.labs.ucx.workspace_access.generic"):
         list(sup.get_crawler_tasks())
-    assert "Listing serving-endpoints failed: Model serving is not enabled for your shard" in caplog.text
+    assert "Listing serving-endpoints failed" in caplog.text
+
+
+def test_internal_error_in_serving_endpoints_raises_warning(caplog) -> None:
+    ws = create_autospec(WorkspaceClient)
+    ws.serving_endpoints.list.side_effect = InternalError
+
+    sup = GenericPermissionsSupport(ws=ws, listings=[Listing(ws.serving_endpoints.list, "id", "serving-endpoints")])
+    with caplog.at_level('WARNING', logger="databricks.labs.ucx.workspace_access.generic"):
+        list(sup.get_crawler_tasks())
+    assert "Listing serving-endpoints failed" in caplog.text