Merge branch 'master' into init_weights

Author: cleherny

examples/mineral-extract-sites-detection/config_trne.yaml

@@ -92,4 +92,5 @@ assess_detections.py:
       tst: tst_detections_at_0dot05_threshold.gpkg
   output_folder: .
   iou_threshold: 0.1
-  area_threshold: 50       # area under which the polygons are discarded from assessment
+  area_threshold: 50       # area under which the polygons are discarded from assessment
+  metrics_method: micro-average   # 1: macro-average ; 3: macro-weighted-average ; 2: micro-average

examples/mineral-extract-sites-detection/prepare_data.py

@@ -171,13 +171,14 @@ def bbox(bounds):
         labels_4326_gdf = labels_gdf.to_crs(epsg=4326).drop_duplicates(subset=['geometry', 'year'])
     else:
         labels_4326_gdf = labels_gdf.to_crs(epsg=4326).drop_duplicates(subset=['geometry'])
+    nb_labels = len(labels_gdf)
+    logger.info(f'There are {nb_labels} polygons in {SHPFILE}')
+
     labels_4326_gdf['CATEGORY'] = 'quarry'
     labels_4326_gdf['SUPERCATEGORY'] = 'land usage'
+    
     gt_labels_4326_gdf = labels_4326_gdf.copy()
 
-    nb_labels = len(labels_gdf)
-    logger.info(f'There are {nb_labels} polygons in {SHPFILE}')
-
     label_filename = 'labels.geojson'
     label_filepath = os.path.join(OUTPUT_DIR, label_filename)
     labels_4326_gdf.to_file(label_filepath, driver='GeoJSON')
@@ -206,14 +207,14 @@ def bbox(bounds):
         logger.success(f"{DONE_MSG} A file was written: {filepath}")
         labels_4326_gdf = pd.concat([labels_4326_gdf, fp_labels_4326_gdf], ignore_index=True)
 
-    # Get the label boundaries (minx, miny, maxx, maxy) 
+    # Tiling of the AoI
     logger.info("- Get the label boundaries")  
     boundaries_df = labels_4326_gdf.bounds
+    logger.info("- Tiling of the AoI")  
+    tiles_4326_aoi_gdf = aoi_tiling(boundaries_df)
+    tiles_4326_labels_gdf = gpd.sjoin(tiles_4326_aoi_gdf, labels_4326_gdf, how='inner', predicate='intersects')
 
-    # Get the global boundaries for all the labels (minx, miny, maxx, maxy) 
-    labels_bbox = bbox(labels_4326_gdf.iloc[0].geometry.bounds)
-
-    # Get tiles for a given AoI from which empty tiles will be selected
+    # Tiling of the AoI from which empty tiles will be selected
     if EPT_SHPFILE:
         EPT_aoi_gdf = gpd.read_file(EPT_SHPFILE)
         EPT_aoi_4326_gdf = EPT_aoi_gdf.to_crs(epsg=4326)
@@ -223,80 +224,55 @@ def bbox(bounds):
             logger.info("- Get AoI boundaries")  
             EPT_aoi_boundaries_df = EPT_aoi_4326_gdf.bounds
 
-            # Get the boundaries for all the AoI (minx, miny, maxx, maxy) 
-            aoi_bbox = bbox(EPT_aoi_4326_gdf.iloc[0].geometry.bounds)
-            aoi_bbox_contains = aoi_bbox.contains(labels_bbox)
-
-            if aoi_bbox_contains:
-                logger.info("- The surface area occupied by the bbox of the AoI used to find empty tiles is bigger than the label's one. The AoI boundaries will be used for tiling") 
-                boundaries_df = EPT_aoi_boundaries_df.copy()
-            else:
-                logger.info("- The surface area occupied by the bbox of the AoI used to find empty tiles is smaller than the label's one. Both the AoI and labels area will be used for tiling") 
-                # Get tiles coordinates and shapes
-                empty_tiles_4326_all_gdf = aoi_tiling(EPT_aoi_boundaries_df)
-                # Delete tiles outside of the AoI limits 
-                empty_tiles_4326_aoi_gdf = gpd.sjoin(empty_tiles_4326_all_gdf, EPT_aoi_4326_gdf, how='inner', lsuffix='ept_tiles', rsuffix='ept_aoi')
-                # Attribute a year to empty tiles if necessary
-                if 'year' in labels_4326_gdf.keys():
-                    if isinstance(EPT_YEAR, int):
-                        empty_tiles_4326_aoi_gdf['year'] = int(EPT_YEAR)
-                    else:
-                        empty_tiles_4326_aoi_gdf['year'] = np.random.randint(low=EPT_YEAR[0], high=EPT_YEAR[1], size=(len(empty_tiles_4326_aoi_gdf)))
-                elif EPT_SHPFILE and EPT_YEAR: 
-                    logger.warning("No year column in the label shapefile. The provided empty tile year will be ignored.")
+            # Get tile coordinates and shapes
+            logger.info("- Tiling of the empty tiles AoI")  
+            empty_tiles_4326_all_gdf = aoi_tiling(EPT_aoi_boundaries_df)
+            # Delete tiles outside of the AoI limits 
+            empty_tiles_4326_aoi_gdf = gpd.sjoin(empty_tiles_4326_all_gdf, EPT_aoi_4326_gdf, how='inner', lsuffix='ept_tiles', rsuffix='ept_aoi')
+            # Attribute a year to empty tiles if necessary
+            if 'year' in labels_4326_gdf.keys():
+                if isinstance(EPT_YEAR, int):
+                    empty_tiles_4326_aoi_gdf['year'] = int(EPT_YEAR)
+                else:
+                    empty_tiles_4326_aoi_gdf['year'] = np.random.randint(low=EPT_YEAR[0], high=EPT_YEAR[1], size=(len(empty_tiles_4326_aoi_gdf)))
+            elif EPT_SHPFILE and EPT_YEAR: 
+                logger.warning("No year column in the label shapefile. The provided empty tile year will be ignored.")
         elif EPT == 'shp':
             if EPT_YEAR:
                 logger.warning("A shapefile of selected empty tiles are provided. The year set for the empty tiles in the configuration file will be ignored")
                 EPT_YEAR = None
             empty_tiles_4326_aoi_gdf = EPT_aoi_4326_gdf.copy()
-            aoi_bbox = None
-            aoi_bbox_contains = False
-
-    logger.info("Creating tiles for the Area of Interest (AoI)...")   
-
-    # Get tiles coordinates and shapes
-    tiles_4326_aoi_gdf = aoi_tiling(boundaries_df)
-
-    # Compute labels intersecting tiles 
-    tiles_4326_gt_gdf = gpd.sjoin(tiles_4326_aoi_gdf, gt_labels_4326_gdf, how='inner', predicate='intersects')
-    tiles_4326_gt_gdf.drop_duplicates('title', inplace=True)
-    logger.info(f"- Number of tiles intersecting GT labels = {len(tiles_4326_gt_gdf)}")
-    
-    if FP_SHPFILE:
-        tiles_4326_fp_gdf = gpd.sjoin(tiles_4326_aoi_gdf, fp_labels_4326_gdf, how='inner', predicate='intersects')
-        tiles_4326_fp_gdf.drop_duplicates('title', inplace=True)
-        logger.info(f"- Number of tiles intersecting FP labels = {len(tiles_4326_fp_gdf)}")
-
-    if not EPT_SHPFILE or EPT_SHPFILE and aoi_bbox_contains == False:
-        # Keep only tiles intersecting labels 
-        if FP_SHPFILE:
-            tiles_4326_aoi_gdf = pd.concat([tiles_4326_gt_gdf, tiles_4326_fp_gdf]) 
-        else:
-            tiles_4326_aoi_gdf = tiles_4326_gt_gdf.copy()
 
     # Get all the tiles in one gdf 
-    if EPT_SHPFILE and aoi_bbox_contains == False:
+    if EPT_SHPFILE:
         logger.info("- Concatenate label tiles and empty AoI tiles") 
-        tiles_4326_all_gdf = pd.concat([tiles_4326_aoi_gdf, empty_tiles_4326_aoi_gdf])
+        tiles_4326_all_gdf = pd.concat([tiles_4326_labels_gdf, empty_tiles_4326_aoi_gdf])
     else: 
-        tiles_4326_all_gdf = tiles_4326_aoi_gdf.copy()
-  
-    # - Remove duplicated tiles
-    if nb_labels > 1:
-        if 'year' in tiles_4326_all_gdf.keys():
-            tiles_4326_all_gdf['year'] = tiles_4326_all_gdf.year.astype(int)
-            tiles_4326_all_gdf.drop_duplicates(['title', 'year'], inplace=True)
-        else: 
-            tiles_4326_all_gdf.drop_duplicates(['title'], inplace=True)
+        tiles_4326_all_gdf = tiles_4326_labels_gdf.copy()
 
     # - Remove useless columns, reset feature id and redefine it according to xyz format  
-    logger.info('- Add tile IDs and reorganise data set')
+    logger.info('- Add tile IDs and reorganise the data set')
     tiles_4326_all_gdf = tiles_4326_all_gdf[['geometry', 'title', 'year'] if 'year' in tiles_4326_all_gdf.keys() else ['geometry', 'title']].copy()
     tiles_4326_all_gdf.reset_index(drop=True, inplace=True)
     tiles_4326_all_gdf = tiles_4326_all_gdf.apply(add_tile_id, axis=1)
+
+    # - Remove duplicated tiles
+    if nb_labels > 1:
+        tiles_4326_all_gdf.drop_duplicates(['id'], inplace=True)
+
     nb_tiles = len(tiles_4326_all_gdf)
     logger.info(f"There were {nb_tiles} tiles created")
 
+    # Get the number of tiles intersecting labels
+    tiles_4326_gt_gdf = gpd.sjoin(tiles_4326_all_gdf, gt_labels_4326_gdf, how='inner', predicate='intersects')
+    tiles_4326_gt_gdf.drop_duplicates(['id'], inplace=True)
+    logger.info(f"- Number of tiles intersecting GT labels = {len(tiles_4326_gt_gdf)}")
+
+    if FP_SHPFILE:
+        tiles_4326_fp_gdf = gpd.sjoin(tiles_4326_all_gdf, fp_labels_4326_gdf, how='inner', predicate='intersects')
+        tiles_4326_fp_gdf.drop_duplicates(['id'], inplace=True)
+        logger.info(f"- Number of tiles intersecting FP labels = {len(tiles_4326_fp_gdf)}")
+
     # Save tile shapefile
     logger.info("Export tiles to GeoJSON (EPSG:4326)...")  
     tile_filename = 'tiles.geojson'

examples/road-surface-classification/config_rs.yaml

@@ -92,4 +92,5 @@ assess_detections.py:
       tst: tst_detections_at_0dot05_threshold.gpkg
       oth: oth_detections_at_0dot05_threshold.gpkg
   output_folder: .
-  iou_threshold: 0.1
+  iou_threshold: 0.1
+  metrics_method: macro-average   # 1: macro-average ; 3: macro-weighted-average ; 2: micro-average

examples/swimming-pool-detection/GE/config_GE.yaml

@@ -79,3 +79,4 @@ assess_detections.py:
       tst: tst_detections_at_0dot05_threshold.gpkg
       oth: oth_detections_at_0dot05_threshold.gpkg
   output_folder: .
+  metrics_method: micro-average   # 1: macro-average ; 3: macro-weighted-average ; 2: micro-average

examples/swimming-pool-detection/NE/config_NE.yaml

@@ -84,3 +84,4 @@ assess_detections.py:
       tst: tst_detections_at_0dot05_threshold.gpkg
       oth: oth_detections_at_0dot05_threshold.gpkg
   output_folder: .
+    metrics_method: micro-average   # 1: macro-average ; 3: macro-weighted-average ; 2: micro-average

helpers/metrics.py

@@ -120,7 +120,7 @@ def get_fractional_sets(dets_gdf, labels_gdf, iou_threshold=0.25, area_threshold
     return tp_gdf, fp_gdf, fn_gdf, mismatched_classes_gdf, small_poly_gdf
 
 
-def get_metrics(tp_gdf, fp_gdf, fn_gdf, mismatch_gdf, id_classes=0):
+def get_metrics(tp_gdf, fp_gdf, fn_gdf, mismatch_gdf, id_classes=0, method='macro-average'):
     """Determine the metrics based on the TP, FP and FN
 
     Args:
@@ -129,6 +129,7 @@ def get_metrics(tp_gdf, fp_gdf, fn_gdf, mismatch_gdf, id_classes=0):
         fn_gdf (geodataframe): false negative labels
         mismatch_gdf (geodataframe): labels and detections intersecting with a mismatched class id
         id_classes (list): list of the possible class ids. Defaults to 0.
+        method (str): method used to compute multi-class metrics. Default to macro-average
     
     Returns:
         tuple: 
@@ -139,16 +140,28 @@ def get_metrics(tp_gdf, fp_gdf, fn_gdf, mismatch_gdf, id_classes=0):
             - float: f1 score.
     """
 
+    by_class_dict = {key: 0 for key in id_classes}
+    tp_k = by_class_dict.copy()
+    fp_k = by_class_dict.copy()
+    fn_k = by_class_dict.copy()
+    p_k = by_class_dict.copy()
+    r_k = by_class_dict.copy()
+    count_k = by_class_dict.copy()
+    pw_k = by_class_dict.copy()
+    rw_k = by_class_dict.copy()
+
     by_class_dict = {key: None for key in id_classes}
     tp_k = by_class_dict.copy()
     fp_k = by_class_dict.copy()
     fn_k = by_class_dict.copy()
     p_k = by_class_dict.copy()
     r_k = by_class_dict.copy()
+    count_k = by_class_dict.copy()
+    pw_k = by_class_dict.copy()
+    rw_k = by_class_dict.copy()
 
     for id_cl in id_classes:
 
-        tp_count = 0 if tp_gdf.empty else len(tp_gdf[tp_gdf.det_class==id_cl])
         pure_fp_count = 0 if fp_gdf.empty else len(fp_gdf[fp_gdf.det_class==id_cl])
         pure_fn_count = 0 if fn_gdf.empty else len(fn_gdf[fn_gdf.label_class==id_cl+1])  # label class starting at 1 and id class at 0
 
@@ -161,21 +174,33 @@ def get_metrics(tp_gdf, fp_gdf, fn_gdf, mismatch_gdf, id_classes=0):
 
         fp_count = pure_fp_count + mismatched_fp_count
         fn_count = pure_fn_count + mismatched_fn_count
+        tp_count = 0 if tp_gdf.empty else len(tp_gdf[tp_gdf.det_class==id_cl])
 
         tp_k[id_cl] = tp_count
         fp_k[id_cl] = fp_count
         fn_k[id_cl] = fn_count
 
-        if tp_count == 0:
-            p_k[id_cl] = 0
-            r_k[id_cl] = 0
-        else:            
-            p_k[id_cl] = tp_count / (tp_count + fp_count)
-            r_k[id_cl] = tp_count / (tp_count + fn_count)
-
-    precision = sum(p_k.values()) / len(id_classes)
-    recall = sum(r_k.values()) / len(id_classes)
-    
+        p_k[id_cl] = 0 if tp_count == 0 else tp_count / (tp_count + fp_count)
+        r_k[id_cl] = 0 if tp_count == 0 else tp_count / (tp_count + fn_count)
+        count_k[id_cl] = 0 if tp_count == 0 else tp_count + fn_count 
+
+    if method == 'macro-average':   
+        precision = sum(p_k.values()) / len(id_classes)
+        recall = sum(r_k.values()) / len(id_classes)
+    elif method == 'macro-weighted-average': 
+        for id_cl in id_classes:
+            pw_k[id_cl] = 0 if sum(count_k.values()) == 0 else (count_k[id_cl] / sum(count_k.values())) * p_k[id_cl]
+            rw_k[id_cl] = 0 if sum(count_k.values()) == 0 else (count_k[id_cl] / sum(count_k.values())) * r_k[id_cl] 
+        precision = sum(pw_k.values()) / len(id_classes)
+        recall = sum(rw_k.values()) / len(id_classes)
+    elif method == 'micro-average':  
+        if sum(tp_k.values()) == 0 and sum(fp_k.values()) == 0:
+            precision = 0
+            recall = 0
+        else:
+            precision = sum(tp_k.values()) / (sum(tp_k.values()) + sum(fp_k.values()))
+            recall = sum(tp_k.values()) / (sum(tp_k.values()) + sum(fn_k.values()))
+
     if precision==0 and recall==0:
         return tp_k, fp_k, fn_k, p_k, r_k, 0, 0, 0
 

requirements.in

@@ -5,16 +5,18 @@
 GDAL==3.0.4
 certifi>=2022.12.07
 future>=0.18.3
-geopandas==0.11.1
+fiona==1.9.6
+geopandas
 joblib
 loguru
 morecantile
-network
+networkx
 numpy==1.23.3
 oauthlib>=3.2.2
 opencv-python
 pillow==9.5.0
 plotly
+protobuf==4.25
 pygeohash
 pyyaml
 rasterio