refactor compute and compute_to_payload

user-friendly-metrics.py

@@ -17,7 +17,8 @@ import os
 
 import datasets
 import evaluate
-from seametrics.user_friendly.utils import calculate_from_payload
+from seametrics.user_friendly.utils import payload_to_uf_metrics, UFM
+from seametrics.payload import Payload
 
 import wandb
 
@@ -55,10 +56,22 @@ Args:
         Default is 0.5.
 """
 
-
 @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
 class UserFriendlyMetrics(evaluate.Metric):
     """TODO: Short description of my evaluation module."""
+    def __init__(
+        self,
+        iou_threshold: float = 1e-10,
+        recognition_thresholds=[0.3, 0.5, 0.8],
+        filter_dict={"name": "area", "ranges": [("all", [0, 1e5**2])]},
+        **kwargs):
+        super().__init__(**kwargs)
+
+        # save parameters for later
+        self.iou_threshold = iou_threshold
+        self.filter_dict = filter_dict
+        self.recognition_thresholds = recognition_thresholds
+
 
     def _info(self):
         # TODO: Specifies the evaluate.EvaluationModuleInfo object
@@ -89,30 +102,41 @@ class UserFriendlyMetrics(evaluate.Metric):
         # TODO: Download external resources if needed
         pass
 
-    # def compute_from_payload(self, payload, **kwargs):
+    def _compute(
+        self, 
+        predictions,
+        references,
+        ):
+
+        results = {}
+        filter_ranges = self.filter_dict["ranges"]
         
+        for filter_range in filter_ranges:
+            
+            filter_range_name = filter_range[0]
+            range_results = {}
+
+            for sequence_predictions, sequence_references in zip(predictions, references):
+                
+                ufm = UFM(
+                    iou_threshold=self.iou_threshold,
+                    recognition_thresholds=self.recognition_thresholds
+                )
+                
+                sequence_range_results = ufm.calculate(
+                    sequence_predictions,
+                    sequence_references[filter_range_name],
+                )
 
-    def _compute(self, 
-        payload: Payload,
-        iou_threshold: float = 1e-10,
-        filter={"name": "area", "ranges": [("all", [0, 1e5**2])]},
-        recognition_thresholds=[0.3, 0.5, 0.8],
-        **kwargs):
+                range_results = sum_dicts(range_results, sequence_range_results)
+
+            results[filter_range_name] = ufm.realize_metrics(range_results, self.recognition_thresholds)
         
-        return calculate_from_payload(
-            payload,
-            iou_threshold,
-            filter,
-            recognition_thresholds,
-            **kwargs
-        )
+        return results
 
     def compute_from_payload(self, 
         payload: Payload, 
-        iou_threshold: float = 1e-10,
-        filter={"name": "area", "ranges": [("all", [0, 1e5**2])]},
-        recognition_thresholds=[0.3, 0.5, 0.8],
-        **kwargs):
+        ):
 
         results = {}
 
@@ -128,28 +152,21 @@ class UserFriendlyMetrics(evaluate.Metric):
                     models=[model_name],
                     sequences={seq_name: sequence}
                 )
-                module = UserFriendlyMetrics(
-                    iou_threshold=iou_threshold,
-                    filter=filter,
-                    payload=sequence_payload
-                    recognition_thresholds=recognition_thresholds
-                )
-                results[model_name]["per_sequence"][seq_name] = module.compute()[model_name]["metrics"]
+                
+                predictions, references = payload_to_uf_metrics(payload, model_name=model_name, filter_dict=self.filter_dict)
 
-            # overall per-model loop
+                results[model_name]["per_sequence"][seq_name] = self._compute(predictions, references)
+
+            # overall 
             model_payload = Payload(
                     dataset=payload.dataset,
                     gt_field_name=payload.gt_field_name,
                     models=[model_name],
                     sequences=payload.sequences
                 )
-            module = UserFriendlyMetrics(
-                iou_threshold=iou_threshold,
-                filter=filter,
-                payload=model_payload
-                recognition_thresholds=recognition_thresholds
-            )
-            results[model_name]["overall"] = module.compute()[model_name]["metrics"]
+            predictions, references = payload_to_uf_metrics(payload, model_name=model_name, filter_dict=self.filter_dict)
+
+            results[model_name]["overall"] = self._compute(predictions, references)
 
         return results
 
@@ -285,3 +302,23 @@ class UserFriendlyMetrics(evaluate.Metric):
             print("All metrics have been logged.")
 
         run.finish()
+
+def sum_dicts(*dicts):
+    """
+    Sums multiple dictionaries with depth one. If keys overlap, their values are summed.
+    If keys are unique, they are simply included in the result.
+
+    Args:
+        *dicts: Any number of dictionaries to be summed.
+
+    Returns:
+        A single dictionary with the summed values.
+    """
+    result = {}
+    for d in dicts:
+        for key, value in d.items():
+            if key in result:
+                result[key] += value
+            else:
+                result[key] = value
+    return result