bugfix/aqua.evaluation (#761)

Author: mingkang111

ads/aqua/evaluation.py

@@ -833,13 +833,13 @@ def get(self, eval_id) -> AquaEvaluationDetail:
         logger.info(f"Fetching evaluation: {eval_id} details ...")
 
         resource = utils.query_resource(eval_id)
-        model_provenance = self.ds_client.get_model_provenance(eval_id).data
-
         if not resource:
             raise AquaRuntimeError(
                 f"Failed to retrieve evalution {eval_id}."
                 "Please check if the OCID is correct."
             )
+        model_provenance = self.ds_client.get_model_provenance(eval_id).data
+
         jobrun_id = model_provenance.training_id
         job_run_details = self._fetch_jobrun(
             resource, use_rqs=False, jobrun_id=jobrun_id
@@ -1038,14 +1038,14 @@ def get_status(self, eval_id: str) -> dict:
         """
         eval = utils.query_resource(eval_id)
 
-        # TODO: add job_run_id as input param to skip the query below
-        model_provenance = self.ds_client.get_model_provenance(eval_id).data
-
         if not eval:
             raise AquaRuntimeError(
                 f"Failed to retrieve evalution {eval_id}."
                 "Please check if the OCID is correct."
             )
+
+        model_provenance = self.ds_client.get_model_provenance(eval_id).data
+
         jobrun_id = model_provenance.training_id
         job_run_details = self._fetch_jobrun(eval, use_rqs=False, jobrun_id=jobrun_id)
 
@@ -1295,7 +1295,10 @@ def cancel(self, eval_id) -> dict:
             raise AquaRuntimeError(
                 f"Failed to get evaluation details for model {eval_id}"
             )
-        job_run_id = model.provenance_metadata.training_id
+
+        job_run_id = (
+            model.provenance_metadata.training_id if model.provenance_metadata else None
+        )
         if not job_run_id:
             raise AquaMissingKeyError(
                 "Model provenance is missing job run training_id key"
@@ -1358,7 +1361,7 @@ def delete(self, eval_id):
             job_id = model.custom_metadata_list.get(
                 EvaluationCustomMetadata.EVALUATION_JOB_ID.value
             ).value
-        except ValueError:
+        except Exception:
             raise AquaMissingKeyError(
                 f"Custom metadata is missing {EvaluationCustomMetadata.EVALUATION_JOB_ID.value} key"
             )
@@ -1390,7 +1393,7 @@ def _delete_job_and_model(job, model):
             )
 
     def load_evaluation_config(self, eval_id):
-        # TODO
+        """Loads evaluation config."""
         return {
             "model_params": {
                 "max_tokens": 500,
@@ -1568,20 +1571,6 @@ def _build_resource_identifier(
             )
             return AquaResourceIdentifier()
 
-    def _get_jobrun(
-        self, model: oci.resource_search.models.ResourceSummary, mapping: dict = {}
-    ) -> Union[
-        oci.resource_search.models.ResourceSummary, oci.data_science.models.JobRun
-    ]:
-        jobrun_id = self._get_attribute_from_model_metadata(
-            model, EvaluationCustomMetadata.EVALUATION_JOB_RUN_ID.value
-        )
-        job_run = mapping.get(jobrun_id)
-
-        if not job_run:
-            job_run = self._fetch_jobrun(model, use_rqs=True, jobrun_id=jobrun_id)
-        return job_run
-
     def _fetch_jobrun(
         self,
         resource: oci.resource_search.models.ResourceSummary,
@@ -1758,7 +1747,7 @@ def _extract_job_lifecycle_details(self, lifecycle_details: str) -> str:
         Examples
         --------
         >>> _extract_job_lifecycle_details("Job run artifact execution failed with exit code 16")
-        'The evaluation configuration is invalid due to content validation errors.'
+        'Validation errors in the evaluation config. Exit code: 16.'
 
         >>> _extract_job_lifecycle_details("Job completed successfully.")
         'Job completed successfully.'

tests/unitary/with_extras/aqua/test_evaluation.py

@@ -569,7 +569,6 @@ def test_get(self, mock_get_model_provenance_response, mock_get_job_run_response
     def test_get_fail(self, mock_query_resource):
         """Tests get evaluation details failed because of invalid eval id."""
         mock_query_resource.return_value = None
-        self.app.ds_client.get_model_provenance = MagicMock()
         with self.assertRaises(AquaRuntimeError) as context:
             self.app.get(TestDataset.INVALID_EVAL_ID)
 
@@ -677,10 +676,10 @@ def test_cancel_evaluation(
     @parameterized.expand(
         [
             (None, AquaRuntimeError),
-            # (
-            #     DataScienceModel(),
-            #     AquaMissingKeyError,
-            # ),
+            (
+                DataScienceModel(),
+                AquaMissingKeyError,
+            ),
         ]
     )
     @patch.object(DataScienceModel, "from_id")
@@ -822,7 +821,6 @@ def test_get_status_when_missing_jobrun(
     def test_get_status_failed(self, mock_query_resource):
         """Tests when no correct evaluation found."""
         mock_query_resource.return_value = None
-        self.app.ds_client.get_model_provenance = MagicMock()
         with self.assertRaises(AquaRuntimeError) as context:
             self.app.get_status(TestDataset.INVALID_EVAL_ID)