Adding checks for missing data

Author: harsh97

ads/feature_store/statistics/charts/abstract_feature_stat.py

@@ -3,6 +3,7 @@
 # Copyright (c) 2023 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 from abc import abstractmethod
+from typing import Union
 
 from ads.common.decorator.runtime_dependency import OptionalDependency
 
@@ -16,13 +17,24 @@
 
 
 class AbsFeatureStat:
+    class ValidationFailedException(Exception):
+        def __init__(self):
+            pass
+
+    def __init__(self):
+        self.__validate__()
+
+    @abstractmethod
+    def __validate__(self):
+        pass
+
     @abstractmethod
     def add_to_figure(self, fig: Figure, xaxis: int, yaxis: int):
         pass
 
     @classmethod
     @abstractmethod
-    def from_json(cls, json_dict: dict):
+    def __from_json__(cls, json_dict: dict):
         pass
 
     @staticmethod
@@ -33,3 +45,15 @@ def get_x_y_str_axes(xaxis: int, yaxis: int) -> ():
             ("x" + str(xaxis + 1)),
             ("y" + str(yaxis + 1)),
         )
+
+    @classmethod
+    def from_json(
+        cls, json_dict: dict, ignore_errors: bool = False
+    ) -> Union["AbsFeatureStat", None]:
+        try:
+            return cls.__from_json__(json_dict=json_dict)
+        except Exception as e:
+            if ignore_errors:
+                return None
+            else:
+                raise e

ads/feature_store/statistics/charts/box_plot.py

@@ -28,7 +28,7 @@ class BoxPlot(AbsFeatureStat):
     CONST_MEAN = "Mean"
     CONST_BOX_PLOT_TITLE = "Box Plot"
     CONST_IQR = "IQR"
-    CONST_FREQUENCY_DISTRIBUTION = "FrequencyDistribution"
+    CONST_BOX_POINTS = "box_points"
 
     class Quartiles:
         CONST_Q1 = "q1"
@@ -42,14 +42,11 @@ def __init__(self, q1: float, q2: float, q3: float):
 
         @classmethod
         def from_json(cls, json_dict: dict) -> "BoxPlot.Quartiles":
-            if json_dict is not None:
-                return cls(
-                    json_dict.get(cls.CONST_Q1),
-                    json_dict.get(cls.CONST_Q2),
-                    json_dict.get(cls.CONST_Q3),
-                )
-            else:
-                return None
+            return cls(
+                json_dict.get(cls.CONST_Q1),
+                json_dict.get(cls.CONST_Q2),
+                json_dict.get(cls.CONST_Q3),
+            )
 
     def __init__(
         self,
@@ -58,25 +55,28 @@ def __init__(
         sd: float,
         q1: float,
         q3: float,
-        boxpoints: List[float],
+        box_points: List[float],
     ):
         self.mean = mean
         self.median = median
         self.q1 = q1
         self.q3 = q3
         self.sd = sd
         self.iqr = self.q3 - self.q1
-        self.boxpoints = boxpoints
+        self.box_points = box_points
+        super().__init__()
 
     def add_to_figure(self, fig: Figure, xaxis: int, yaxis: int):
         xaxis_str, yaxis_str, x_str, y_str = self.get_x_y_str_axes(xaxis, yaxis)
         fig.add_box(
+            notched=False,
+            boxmean=False,
             mean=[self.mean],
             median=[self.median],
             q1=[self.q1],
             q3=[self.q3],
             sd=[self.sd],
-            y=[self.boxpoints],
+            y=[self.box_points],
             upperfence=[self.q3 + 1.5 * self.iqr],
             lowerfence=[self.q1 - 1.5 * self.iqr],
             xaxis=x_str,
@@ -88,33 +88,33 @@ def add_to_figure(self, fig: Figure, xaxis: int, yaxis: int):
         fig.layout[yaxis_str]["title"] = "Values"
 
     @staticmethod
-    def get_boxpoints_from_frequency_distribution(
+    def get_box_points_from_frequency_distribution(
         frequency_distribution: FrequencyDistribution,
     ) -> List[float]:
-        boxpoints = []
-        if frequency_distribution is not None:
-            for frequency, bin in zip(
-                frequency_distribution.frequency, frequency_distribution.bins
-            ):
-                boxpoints.extend([bin] * frequency)
-
-        return boxpoints
+        # box_points = []
+        if (
+            frequency_distribution is not None
+            and frequency_distribution.frequency is not None
+            and frequency_distribution.bins is not None
+        ):
+            return [
+                bin_dist
+                for frequency, bin_dist in zip(
+                    frequency_distribution.frequency, frequency_distribution.bins
+                )
+                if frequency > 0
+            ]
+        else:
+            return []
 
     @classmethod
-    def from_json(cls, json_dict: dict) -> "BoxPlot":
-        if type(json_dict) is dict and json_dict.get(cls.CONST_QUARTILES) is not None:
-            quartiles = cls.Quartiles.from_json(json_dict.get(cls.CONST_QUARTILES))
-            return cls(
-                mean=GenericFeatureValue.from_json(json_dict.get(cls.CONST_MEAN)).val,
-                median=quartiles.q2,
-                sd=GenericFeatureValue.from_json(json_dict.get(cls.CONST_SD)).val,
-                q1=quartiles.q1,
-                q3=quartiles.q3,
-                boxpoints=cls.get_boxpoints_from_frequency_distribution(
-                    FrequencyDistribution.from_json(
-                        json_dict.get(cls.CONST_FREQUENCY_DISTRIBUTION)
-                    )
-                ),
-            )
-        else:
-            return None
+    def __from_json__(cls, json_dict: dict) -> "BoxPlot":
+        quartiles = cls.Quartiles.from_json(json_dict.get(cls.CONST_QUARTILES))
+        return cls(
+            mean=GenericFeatureValue.from_json(json_dict.get(cls.CONST_MEAN)).val,
+            median=quartiles.q2,
+            sd=GenericFeatureValue.from_json(json_dict.get(cls.CONST_SD)).val,
+            q1=quartiles.q1,
+            q3=quartiles.q3,
+            box_points=json_dict.get(cls.CONST_BOX_POINTS),
+        )

ads/feature_store/statistics/charts/frequency_distribution.py

@@ -21,19 +21,25 @@ class FrequencyDistribution(AbsFeatureStat):
     CONST_BINS = "bins"
     CONST_FREQUENCY_DISTRIBUTION_TITLE = "Frequency Distribution"
 
+    def __validate__(self):
+        if not (
+            type(self.frequency) == list
+            and type(self.bins) == list
+            and 0 < len(self.frequency) == len(self.bins) > 0
+        ):
+            raise self.ValidationFailedException()
+
     def __init__(self, frequency: List, bins: List):
         self.frequency = frequency
         self.bins = bins
+        super().__init__()
 
     @classmethod
-    def from_json(cls, json_dict: dict) -> "FrequencyDistribution":
-        if json_dict is not None:
-            return FrequencyDistribution(
-                frequency=json_dict.get(cls.CONST_FREQUENCY),
-                bins=json_dict.get(cls.CONST_BINS),
-            )
-        else:
-            return None
+    def __from_json__(cls, json_dict: dict) -> "FrequencyDistribution":
+        return FrequencyDistribution(
+            frequency=json_dict.get(cls.CONST_FREQUENCY),
+            bins=json_dict.get(cls.CONST_BINS),
+        )
 
     def add_to_figure(self, fig: Figure, xaxis: int, yaxis: int):
         xaxis_str, yaxis_str, x_str, y_str = self.get_x_y_str_axes(xaxis, yaxis)

ads/feature_store/statistics/charts/probability_distribution.py

@@ -17,23 +17,29 @@
 
 
 class ProbabilityDistribution(AbsFeatureStat):
+    def __validate__(self):
+        if not (
+            type(self.density) == list
+            and type(self.bins) == list
+            and 0 < len(self.density) == len(self.bins) > 0
+        ):
+            raise self.ValidationFailedException()
+
     CONST_DENSITY = "density"
     CONST_BINS = "bins"
     CONST_PROBABILITY_DISTRIBUTION_TITLE = "Probability Distribution"
 
     def __init__(self, density: List, bins: List):
         self.density = density
         self.bins = bins
+        super().__init__()
 
     @classmethod
-    def from_json(cls, json_dict: dict):
-        if json_dict is not None:
-            return cls(
-                density=json_dict.get(ProbabilityDistribution.CONST_DENSITY),
-                bins=json_dict.get(ProbabilityDistribution.CONST_BINS),
-            )
-        else:
-            return None
+    def __from_json__(cls, json_dict: dict) -> "ProbabilityDistribution":
+        return cls(
+            density=json_dict.get(ProbabilityDistribution.CONST_DENSITY),
+            bins=json_dict.get(ProbabilityDistribution.CONST_BINS),
+        )
 
     def add_to_figure(self, fig: Figure, xaxis: int, yaxis: int):
         xaxis_str, yaxis_str, x_str, y_str = self.get_x_y_str_axes(xaxis, yaxis)

ads/feature_store/statistics/charts/top_k_frequent_elements.py

@@ -3,7 +3,6 @@
 # Copyright (c) 2023 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 from typing import List
-
 from ads.common.decorator.runtime_dependency import OptionalDependency
 
 from ads.feature_store.statistics.charts.abstract_feature_stat import AbsFeatureStat
@@ -18,6 +17,10 @@
 
 
 class TopKFrequentElements(AbsFeatureStat):
+    def __validate__(self):
+        if not (type(self.elements) == list and len(self.elements) > 0):
+            raise self.ValidationFailedException
+
     CONST_VALUE = "value"
     CONST_TOP_K_FREQUENT_TITLE = "Top K Frequent Elements"
 
@@ -39,28 +42,21 @@ def __init__(
         def from_json(
             cls, json_dict: dict
         ) -> "TopKFrequentElements.TopKFrequentElement":
-            if json_dict is not None:
-                return cls(
-                    value=json_dict.get(cls.CONST_VALUE),
-                    estimate=json_dict.get(cls.CONST_ESTIMATE),
-                    lower_bound=json_dict.get(cls.CONST_LOWER_BOUND),
-                    upper_bound=json_dict.get(cls.CONST_UPPER_BOUND),
-                )
-            else:
-                return None
+            return cls(
+                value=json_dict.get(cls.CONST_VALUE),
+                estimate=json_dict.get(cls.CONST_ESTIMATE),
+                lower_bound=json_dict.get(cls.CONST_LOWER_BOUND),
+                upper_bound=json_dict.get(cls.CONST_UPPER_BOUND),
+            )
 
     def __init__(self, elements: List[TopKFrequentElement]):
         self.elements = elements
+        super().__init__()
 
     @classmethod
-    def from_json(cls, json_dict: dict) -> "TopKFrequentElements":
-        if json_dict is not None and json_dict.get(cls.CONST_VALUE) is not None:
-            elements = json_dict.get(cls.CONST_VALUE)
-            return cls(
-                [cls.TopKFrequentElement.from_json(element) for element in elements]
-            )
-        else:
-            return None
+    def __from_json__(cls, json_dict: dict) -> "TopKFrequentElements":
+        elements = json_dict.get(cls.CONST_VALUE)
+        return cls([cls.TopKFrequentElement.from_json(element) for element in elements])
 
     def add_to_figure(self, fig: Figure, xaxis: int, yaxis: int):
         xaxis_str, yaxis_str, x_str, y_str = self.get_x_y_str_axes(xaxis, yaxis)

ads/feature_store/statistics/feature_stat.py

@@ -53,16 +53,27 @@ def __init__(
     @classmethod
     def from_json(cls, feature_name: str, json_dict: dict) -> "FeatureStatistics":
         if json_dict is not None:
+            frequency_distribution = FrequencyDistribution.from_json(
+                json_dict.get(cls.CONST_FREQUENCY_DISTRIBUTION), ignore_errors=True
+            )
+
+            # inject box points for boxplot creation
+            json_dict[
+                BoxPlot.CONST_BOX_POINTS
+            ] = BoxPlot.get_box_points_from_frequency_distribution(
+                frequency_distribution
+            )
             return cls(
                 feature_name,
-                TopKFrequentElements.from_json(json_dict.get(cls.CONST_TOP_K_FREQUENT)),
-                FrequencyDistribution.from_json(
-                    json_dict.get(cls.CONST_FREQUENCY_DISTRIBUTION)
+                TopKFrequentElements.from_json(
+                    json_dict.get(cls.CONST_TOP_K_FREQUENT), ignore_errors=True
                 ),
+                frequency_distribution,
                 ProbabilityDistribution.from_json(
-                    json_dict.get(cls.CONST_PROBABILITY_DISTRIBUTION)
+                    json_dict.get(cls.CONST_PROBABILITY_DISTRIBUTION),
+                    ignore_errors=True,
                 ),
-                BoxPlot.from_json(json_dict),
+                BoxPlot.from_json(json_dict, ignore_errors=True),
             )
         else:
             return cls(feature_name)
@@ -72,10 +83,10 @@ def __feature_stat_objects__(self) -> List[AbsFeatureStat]:
         return [
             stat
             for stat in [
+                self.box_plot,
                 self.top_k_frequent_elements,
                 self.frequency_distribution,
                 self.probability_distribution,
-                self.box_plot,
             ]
             if stat is not None
         ]
@@ -87,8 +98,7 @@ def next_graph_position_generator():
             yield 0
             yield 2
 
-        graph_count = len(self.__feature_stat_objects__)
-        if graph_count > 0:
+        if len(self.__feature_stat_objects__) > 0:
             fig = make_subplots(cols=3, column_titles=[" "] * 3)
             for idx, stat in zip(
                 next_graph_position_generator(),
@@ -104,3 +114,7 @@ def next_graph_position_generator():
                 fig,
                 filename=self.CONST_PLOT_FORMAT.format(self.feature_name),
             )
+        else:
+            print(
+                f"No statistical information for feature {self.feature_name} can be visualised"
+            )

ads/feature_store/statistics/generic_feature_value.py

@@ -12,9 +12,6 @@ def __init__(self, val: any):
 
     @classmethod
     def from_json(cls, json_dict: dict) -> "GenericFeatureValue":
-        if json_dict is not None:
-            return GenericFeatureValue(
-                val=json_dict.get(cls.CONST_VALUE),
-            )
-        else:
-            return None
+        return GenericFeatureValue(
+            val=json_dict.get(cls.CONST_VALUE),
+        )

ads/feature_store/statistics/statistics.py

@@ -28,9 +28,8 @@ def kind(self) -> str:
 
     def to_viz(self, feature_list: List[str] = None):
         if self.content is not None:
-            stats: dict = json.loads(self.content)
             [
                 FeatureStatistics.from_json(feature, stat).to_viz()
-                for feature, stat in stats.items()
+                for feature, stat in json.loads(self.content).items()
                 if (feature_list is None or feature in feature_list)
             ]