Added a few more tests for assess_bias and also some slight optimizations

Author: djm21

src/sasctl/pzmm/write_json_files.py

@@ -21,7 +21,7 @@
 # Package Imports
 from sasctl.pzmm.write_score_code import ScoreCode as sc
 from ..core import current_session
-from ..utils.decorators import deprecated
+from ..utils.decorators import deprecated, experimental
 from ..utils.misc import check_if_jupyter
 
 try:
@@ -945,6 +945,20 @@ def assess_model_bias(
     def format_max_differences(
         maxdiff_dfs: List[DataFrame], datarole: str = "TEST"
     ) -> DataFrame:
+        """
+        Converts a list of max differences DataFrames into a singular DataFrame
+        Parameters
+        ----------
+        maxdiff_dfs: List[DataFrame]
+            A list of max_differences DataFrames returned by CAS
+        datarole : string, optional
+            The data being used to assess bias (i.e. 'TEST', 'VALIDATION', etc.). Default is 'TEST.'
+
+        Returns
+        -------
+        DataFrame
+            A singluar DataFrame containing all max differences data
+        """
         maxdiff_df = pd.concat(maxdiff_dfs)
         maxdiff_df = maxdiff_df.rename(
             columns={"Value": "maxdiff", "Base": "BASE", "Compare": "COMPARE"}
@@ -965,6 +979,28 @@ def format_group_metrics(
         pred_values: str = None,
         datarole: str = "TEST",
     ) -> DataFrame:
+        """
+        Converts list of group metrics DataFrames to a single DataFrame
+        Parameters
+        ----------
+        groupmetrics_dfs: List[DataFrame]
+            List of group metrics DataFrames generated by CASAction
+        pred_values : string, required for regression problems, otherwise not used
+            Variable name containing the predicted values in score_table. The variable name must follow SAS naming
+            conventions (no spaces and the name cannot begin with a number or symbol).Required for regression problems.
+            The default value is None.
+        prob_values : list of strings, required for classification problems, otherwise not used
+           A list of variable names containing the predicted probability values in the score table. The first element
+           should represent the predicted probability of the target class. Required for classification problems. Default
+           is None.
+        datarole : string, optional
+            The data being used to assess bias (i.e. 'TEST', 'VALIDATION', etc.). Default is 'TEST.'
+
+        Returns
+        -------
+        DataFrame
+            A singular DataFrame containing formatted data for group metrics
+        """
         # adding group metrics dataframes and adding values/ formatting
         groupmetrics_df = pd.concat(groupmetrics_dfs)
         groupmetrics_df = groupmetrics_df.rename(
@@ -1006,6 +1042,7 @@ def format_group_metrics(
         return groupmetrics_df
 
     # TODO: Add doc_string and unit tests
+    @experimental
     @classmethod
     def bias_dataframes_to_json(
         cls,
@@ -1018,6 +1055,42 @@ def bias_dataframes_to_json(
         pred_values: str = None,
         json_path: Union[str, Path, None] = None,
     ):
+        """
+        Properly formats data from FairAITools CAS Action Set into a JSON readable formats
+        Parameters
+        ----------
+        groupmetrics: DataFrame
+            A DataFrame containing the group metrics data
+        maxdifference: DataFrame
+            A DataFrame containing the max difference data
+        n_sensitivevariables: int
+            The total number of sensitive values
+        actual_values : String
+            Variable name containing the actual values in score_table. The variable name must follow SAS naming
+            conventions (no spaces and the name cannot begin with a number or symbol).
+        prob_values : list of strings, required for classification problems, otherwise not used
+           A list of variable names containing the predicted probability values in the score table. The first element
+           should represent the predicted probability of the target class. Required for classification problems. Default
+           is None.
+        levels: List of strings, required for classification problems, otherwise not used
+            List of classes of a nominal target in the order they were passed in prob_values. Levels must be passed as a
+            string. Default is None.
+        pred_values : string, required for regression problems, otherwise not used
+            Variable name containing the predicted values in score_table. The variable name must follow SAS naming
+            conventions (no spaces and the name cannot begin with a number or symbol).Required for regression problems.
+            The default value is None.
+        json_path : str or Path, optional
+            Location for the output JSON files. If a path is passed, the json files will populate in the directory and
+            the function will return None, unless return_dataframes is True. Otherwise, the function will return the json
+             strings in a dictionary (dict["maxDifferences.json"] and dict["groupMetrics.json"]). The default value is
+             None.
+
+        Returns
+        -------
+        dict
+            Dictionary containing a key-value pair representing the files name and json
+            dumps respectively.
+        """
         folder = "reg_jsons" if prob_values is None else "clf_jsons"
 
         dfs = (maxdifference, groupmetrics)
@@ -1061,20 +1134,20 @@ def bias_dataframes_to_json(
                     "type": "num",
                     "values": [prob_label],
                 }
-                json_dict[1]["parameterMap"] = cls.add_dict_key(
-                    dict=json_dict[1]["parameterMap"],
-                    pos=i + 3,
-                    new_key=prob_label,
-                    new_value=paramdict,
-                )
+                json_dict[1]["parameterMap"][prob_label] = paramdict
+                # cls.add_dict_key(
+                #     dict=json_dict[1]["parameterMap"],
+                #     pos=i + 3,
+                #     new_key=prob_label,
+                #     new_value=paramdict,]
+                # )
 
         else:
             json_dict[1]["parameterMap"]["predict"]["label"] = pred_values
             json_dict[1]["parameterMap"]["predict"]["parameter"] = pred_values
             json_dict[1]["parameterMap"]["predict"]["values"] = [pred_values]
-            json_dict[1]["parameterMap"] = cls.rename_dict_key(
-                json_dict[1]["parameterMap"], pred_values, "predict"
-            )
+            json_dict[1]["parameterMap"][pred_values] = json_dict[1]["parameterMap"]["predict"]
+            del json_dict[1]["parameterMap"]["predict"]
 
         if json_path:
             for i, name in enumerate([MAXDIFFERENCES, GROUPMETRICS]):
@@ -1091,34 +1164,7 @@ def bias_dataframes_to_json(
                 GROUPMETRICS: json.dumps(json_dict[1], indent=4, cls=NpEncoder),
             }
 
-    # TODO: Add doc_string and unit tests
-    @staticmethod
-    def add_dict_key(
-        dict: dict, pos: int, new_key: Union[str, int, float, bool], new_value
-    ):
-        result = {}
-        for i, k in enumerate(dict.keys()):
-            if i == pos:
-                result[new_key] = new_value
-                result[k] = dict[k]
-            else:
-                result[k] = dict[k]
-        return result
 
-    # TODO: Add doc_string and unit tests
-    @staticmethod
-    def rename_dict_key(
-        dict: dict,
-        new_key: Union[str, int, float, bool],
-        old_key: Union[str, int, float, bool],
-    ) -> dict:
-        result = {}
-        for k, v in dict.items():
-            if k == old_key:
-                result[new_key] = v
-            else:
-                result.update({k: v})
-        return result
 
     @classmethod
     def calculate_model_statistics(

tests/unit/test_write_json_files.py

@@ -22,6 +22,7 @@
 
 import sasctl.pzmm as pzmm
 from sasctl.pzmm.write_json_files import JSONFiles as jf
+from sasctl.core import Session
 
 # Example input variable list from hmeq dataset (generated by mlflow_model.py)
 input_dict = [
@@ -644,3 +645,150 @@ def test_create_requirements_json(change_dir):
     unittest.TestCase().assertCountEqual(
         json.loads(json_dict["requirements.json"]), expected
     )
+
+class TestAssessBiasHelpers(unittest.TestCase):
+    md_1 = pd.DataFrame({"Value": [0], "Base": ["A"], "Compare": ["C"]})
+    md_2 = pd.DataFrame({"Value": [1], "Base": ["B"], "Compare": ["C"]})
+    gm_1 = pd.DataFrame({
+        "Group": ['A'],
+        "N": [0],
+        "MISCEVENT": ["E"],
+        "MISCEVENTKS": ["F"],
+        "cutoffKS": [.5],
+        "PREDICTED": [1],
+        "maxKS": [100]
+    })
+    gm_2 = pd.DataFrame({
+        "Group": ['B'],
+        "N": [1],
+        "MISCEVENT": ["G"],
+        "MISCEVENTKS": ["H"],
+        "cutoffKS": [.2],
+        "PREDICTED": [0],
+        "maxKS": [500]
+    })
+
+    def test_max_differences(self):
+        md_2_copy = self.md_2.copy()
+        md_2_copy = md_2_copy.set_index(pd.Index([1]))
+        dfs = [self.md_1.copy(), md_2_copy]
+        datarole = "role"
+        return_table = jf.format_max_differences(dfs, datarole)
+        pd.testing.assert_frame_equal(
+            return_table,
+            pd.DataFrame({
+                "BASE": ["A", "B"],
+                "COMPARE": ["C", "C"],
+                "VLABEL": ["", ""],
+                "_DATAROLE_": ["role", "role"],
+                "maxdiff": [0, 1]
+            })
+        )
+
+        return_table = jf.format_max_differences(dfs)
+        pd.testing.assert_frame_equal(
+            return_table,
+            pd.DataFrame({
+                "BASE": ["A", "B"],
+                "COMPARE": ["C", "C"],
+                "VLABEL": ["", ""],
+                "_DATAROLE_": ["TEST", "TEST"],
+                "maxdiff": [0, 1]
+            })
+        )
+
+    def test_group_metrics(self):
+        gm_2_copy = self.gm_2.copy()
+        gm_2_copy = gm_2_copy.set_index(pd.Index([1]))
+        dfs = [self.gm_1.copy(), gm_2_copy.copy()]
+        prob_values = ['VarA', 'VarB']
+        for i in range(len(dfs)):
+            dfs[i][prob_values[0]] = [i]
+            dfs[i][prob_values[1]] = [i+2]
+
+        gm = jf.format_group_metrics(dfs, prob_values)
+
+        pd.testing.assert_frame_equal(
+            gm,
+            pd.DataFrame({
+                "LEVEL": ['A', 'B'],
+                "VLABEL": ["", ""],
+                "VarA": [0, 1],
+                "VarB": [2, 3],
+                "_DATAROLE_": ["TEST", "TEST"],
+                "_avgyhat_": [1, 0],
+                "_ks_": [100, 500],
+                "_kscut_": [.5, .2],
+                "_misccutoff_": ["E", "G"],
+                "_miscks_": ["F", "H"],
+                "_nobs_": [0, 1]
+            })
+        )
+
+        dfs_1 = [self.gm_1.copy(), gm_2_copy.copy()]
+        prob_values = ['VarA', 'VarB']
+        for i in range(len(dfs)):
+            dfs_1[i][prob_values[0]] = [i]
+            dfs_1[i][prob_values[1]] = [i + 2]
+
+        gm_1 = jf.format_group_metrics(dfs_1, prob_values, datarole="NEW")
+
+        pd.testing.assert_frame_equal(
+            gm_1,
+            pd.DataFrame({
+                "LEVEL": ['A', 'B'],
+                "VLABEL": ["", ""],
+                "VarA": [0, 1],
+                "VarB": [2, 3],
+                "_DATAROLE_": ["NEW", "NEW"],
+                "_avgyhat_": [1, 0],
+                "_ks_": [100, 500],
+                "_kscut_": [.5, .2],
+                "_misccutoff_": ["E", "G"],
+                "_miscks_": ["F", "H"],
+                "_nobs_": [0, 1]
+            })
+        )
+
+        dfs_2 = [self.gm_1.copy(), gm_2_copy.copy()]
+        pred_value = "Pred"
+        for i in range(len(dfs)):
+            dfs_2[i][pred_value] = [i]
+
+        gm_2 = jf.format_group_metrics(dfs_2, pred_values=pred_value, datarole="NEW")
+
+        pd.testing.assert_frame_equal(
+            gm_2,
+            pd.DataFrame({
+                "LEVEL": ['A', 'B'],
+                "Pred": [0,1],
+                "VLABEL": ["", ""],
+                "_DATAROLE_": ["NEW", "NEW"],
+                "_avgyhat_": [1, 0],
+                "_ks_": [100, 500],
+                "_kscut_": [.5, .2],
+                "_misccutoff_": ["E", "G"],
+                "_miscks_": ["F", "H"],
+                "_nobs_": [0, 1]
+            })
+        )
+
+class TestAssessBias(unittest.TestCase):
+
+    def test_errors(self):
+        with unittest.mock.patch("sasctl.core.Session._get_authorization_token"):
+            with unittest.mock.patch("sasctl.core.Session.as_swat") as swat:
+                with Session("host", "username", "password") as s:
+                    score_table = pd.DataFrame({"1nvalid": ["no."]})
+                    sensitive_values = "s"
+                    actual_values = "a"
+                    with pytest.raises(SyntaxError):
+                        jf.assess_model_bias(score_table, sensitive_values, actual_values)
+
+                    score_table = pd.DataFrame({"valid": ["yes"]})
+                    with pytest.raises(ValueError):
+                        jf.assess_model_bias(score_table, sensitive_values, actual_values)
+
+                    swat.side_effect = ImportError('oops')
+                    with pytest.raises(RuntimeError):
+                        jf.assess_model_bias(score_table, sensitive_values, actual_values)