Remove old functions from Kai Tak, HK notebook

Author: JoostGevaert

examples/hk_kaitak_ags3/hk_kaitak_ags3_to_brgi_geodb.py

@@ -45,89 +45,38 @@ def _():
     from pyproj.crs import CompoundCRS
     from shapely import wkt, Point
 
-    from bedrock_ge.gi.ags.read import ags_to_dfs
-    from bedrock_ge.gi.ags.transform import ags3_db_to_no_gis_brgi_db
-    from bedrock_ge.gi.concatenate import concatenate_databases
-    from bedrock_ge.gi.gis_geometry import calculate_gis_geometry
+    # Old functions that need to be removed from the Bedrock codebase
     from bedrock_ge.gi.validate import check_brgi_database, check_no_gis_brgi_database
-    from bedrock_ge.gi.write import write_gi_db_to_gpkg
+
+    from bedrock_ge.gi.ags_parser import ags_to_brgi_db_mapping
+    from bedrock_ge.gi.db_operations import merge_databases
+    from bedrock_ge.gi.mapper import map_to_brgi_db
+    from bedrock_ge.gi.geospatial import create_brgi_geospatial_database
+    from bedrock_ge.gi.schemas import InSituTestSchema
+    from bedrock_ge.gi.io_utils import geodf_to_df
+    from bedrock_ge.gi.write import write_brgi_db_to_file
 
     print(platform.system())
     print(sys.version)
     # print(sys.executable)
     return (
         CRS,
         Point,
-        ags3_db_to_no_gis_brgi_db,
-        ags_to_dfs,
-        calculate_gis_geometry,
-        check_brgi_database,
-        check_no_gis_brgi_database,
-        concatenate_databases,
+        ags_to_brgi_db_mapping,
+        create_brgi_geospatial_database,
+        geodf_to_df,
         gpd,
         io,
+        map_to_brgi_db,
+        merge_databases,
         mo,
-        pd,
         platform,
         requests,
+        write_brgi_db_to_file,
         zipfile,
     )
 
 
-@app.cell
-def _(
-    ags3_db_to_no_gis_brgi_db,
-    ags_to_dfs,
-    check_no_gis_brgi_database,
-    concatenate_databases,
-    zipfile,
-):
-    def zip_of_ags3s_to_bedrock_gi_database(zip, crs):
-        """Read AGS 3 files from a ZIP archive and convert them to a dictionary of pandas dataframes."""
-        brgi_db = {}
-        with zipfile.ZipFile(zip) as zip_ref:
-            # Iterate over files and directories in the .zip archive
-            for file_name in zip_ref.namelist():
-                # Only process files that have an .ags or .AGS extension
-                if file_name.lower().endswith(".ags"):
-                    print(f"\n🖥️ Processing {file_name} ...")
-                    with zip_ref.open(file_name) as ags3_file:
-                        # Convert content of a single AGS 3 file to a Dictionary of pandas dataframes (a database)
-                        ags3_db = ags_to_dfs(ags3_file)
-                    report_no = file_name.split("/")[0]
-                    ags3_db["PROJ"]["REPORT_NO"] = int(report_no)
-                    project_uid = f"{ags3_db['PROJ']['PROJ_ID'].iloc[0]}_{file_name}"
-                    ags3_db["PROJ"]["project_uid"] = project_uid
-                    # Remove (Static) CPT AGS 3 group 'STCN' from brgi_db, because CPT data processing needs to be reviewed.
-                    # Not efficient to create a GIS point for every point where a CPT measures a value.
-                    if "STCN" in ags3_db.keys():
-                        del ags3_db["STCN"]
-                    # Create GI data tables with bedrock-ge names and add columns (project_uid, location_uid, sample_uid),
-                    # such that data from multiple AGS files can be combined
-                    brgi_db_from_1_ags3_file = ags3_db_to_no_gis_brgi_db(ags3_db, crs)
-                    print(
-                        f"🧐 Validating the Bedrock GI database from AGS file {file_name}..."
-                    )
-                    check_no_gis_brgi_database(brgi_db_from_1_ags3_file)
-                    print(
-                        f"\n✅ Successfully converted {file_name} to Bedrock GI database and validated!\n"
-                    )
-                    print(
-                        f"🧵 Concatenating Bedrock GI database for {file_name} to existing Bedrock GI database...\n"
-                    )
-                    brgi_db = concatenate_databases(brgi_db, brgi_db_from_1_ags3_file)
-
-                    # Drop all rows that have completely duplicate rows in the Project table
-                    brgi_db["Project"] = brgi_db["Project"].drop_duplicates()
-                    # Then drop all that unfortunately still have a duplicate project_uid
-                    brgi_db["Project"] = brgi_db["Project"].drop_duplicates(
-                        subset="project_uid", keep="first"
-                    )
-        return brgi_db
-
-    return (zip_of_ags3s_to_bedrock_gi_database,)
-
-
 @app.cell(hide_code=True)
 def _(mo):
     mo.md(
@@ -227,26 +176,6 @@ def _(mo):
     return
 
 
-@app.cell
-def _():
-    from bedrock_ge.gi.ags3 import ags3_to_brgi_db_mapping
-    from bedrock_ge.gi.ags_parser import ags_to_brgi_db_mapping
-    from bedrock_ge.gi.db_operations import merge_databases
-    from bedrock_ge.gi.mapper import map_to_brgi_db
-    from bedrock_ge.gi.geospatial import create_brgi_geospatial_database
-    from bedrock_ge.gi.schemas import InSituTestSchema
-    from bedrock_ge.gi.io_utils import geodf_to_df
-    from bedrock_ge.gi.write import write_brgi_db_to_file
-    return (
-        ags_to_brgi_db_mapping,
-        create_brgi_geospatial_database,
-        geodf_to_df,
-        map_to_brgi_db,
-        merge_databases,
-        write_brgi_db_to_file,
-    )
-
-
 @app.cell
 def _(
     CRS,
@@ -258,6 +187,7 @@ def _(
 ):
     projected_crs = CRS("EPSG:2326")
     vertrical_crs = CRS("EPSG:5738")
+
     ags3_file_brgi_dbs = []
     with zipfile.ZipFile(zip) as zip_ref:
         # Iterate over files and directories in the .zip archive
@@ -288,31 +218,6 @@ def _(brgi_db):
     return
 
 
-@app.cell
-def _(CRS, pd, zip, zip_of_ags3s_to_bedrock_gi_database):
-    brgi_db_old = zip_of_ags3s_to_bedrock_gi_database(zip, CRS("EPSG:2326"))
-
-    # Some ISPT_NVAL (SPT count) are not numeric, e.g. "100/0.29"
-    # When converting to numeric, these non-numeric values are converted to NaN
-    brgi_db_old["InSitu_ISPT"]["ISPT_NVAL"] = pd.to_numeric(
-        brgi_db_old["InSitu_ISPT"]["ISPT_NVAL"], errors="coerce"
-    )
-    return (brgi_db_old,)
-
-
-@app.cell(hide_code=True)
-def _(brgi_db_old, mo):
-    sel_brgi_table = mo.ui.dropdown(brgi_db_old, value="Project")
-    mo.md(f"Select the Bedrock GI table you want to explore: {sel_brgi_table}")
-    return (sel_brgi_table,)
-
-
-@app.cell(hide_code=True)
-def _(sel_brgi_table):
-    sel_brgi_table.value
-    return
-
-
 @app.cell(hide_code=True)
 def _(mo):
     mo.md(
@@ -362,13 +267,6 @@ def _(brgi_geodb):
     return
 
 
-@app.cell
-def _(brgi_db_old, calculate_gis_geometry, check_brgi_database):
-    brgi_geodb_old = calculate_gis_geometry(brgi_db_old)
-    check_brgi_database(brgi_geodb_old)
-    return (brgi_geodb_old,)
-
-
 @app.cell(hide_code=True)
 def _(mo):
     mo.md(
@@ -383,12 +281,6 @@ def _(mo):
     return
 
 
-@app.cell
-def _(brgi_geodb_old):
-    brgi_geodb_old["LonLatHeight"].explore()
-    return
-
-
 @app.cell
 def _(Point, brgi_geodb):
     gdf = brgi_geodb.InSituTests["GEOL"]
@@ -415,9 +307,9 @@ def _(mo):
     return
 
 
-@app.cell(hide_code=True)
-def _(brgi_db_old, mo):
-    explore_brgi_table = mo.ui.dropdown(brgi_db_old, value="InSitu_ISPT")
+@app.cell
+def _(brgi_geodb, mo):
+    explore_brgi_table = mo.ui.dropdown(brgi_geodb.InSituTests, value="ISPT")
     mo.md(f"Select the GI table you want to explore: {explore_brgi_table}")
     return (explore_brgi_table,)
 
@@ -429,25 +321,29 @@ def _(explore_brgi_table, mo):
     return (filtered_table,)
 
 
+@app.cell
+def _(filtered_df, filtered_table):
+    print(f"filtered_table.value type: {type(filtered_table.value)}")
+    print(f"filtered_df.value type: {type(filtered_df.value)}")
+    return
+
+
 @app.cell(hide_code=True)
-def _(brgi_geodb_old, filtered_table, gpd, mo):
+def _(Point, filtered_table, gpd, mo):
     def gi_exploration_map(filtered_brgi_table):
         if "location_uid" not in filtered_brgi_table.value.columns:
             output = mo.md(
                 "No interactive map with the data selected in the table above can be shown, because the you're exploring isn't linked to the `LonLatHeight` table with a `location_uid` column, i.e. doesn't have `location_uid` as a foreign key."
             ).callout("warn")
         else:
-            filtered_df = filtered_brgi_table.value.merge(
-                brgi_geodb_old["LonLatHeight"], on="location_uid", how="inner"
+            fltrd_gdf = gpd.GeoDataFrame(filtered_brgi_table.value.copy())
+            fltrd_gdf["geometry"] = fltrd_gdf["geometry"].apply(
+                lambda geom: Point(geom.coords[0])
             )
-            filtered_gdf = gpd.GeoDataFrame(
-                filtered_df,
-                geometry=filtered_df["geometry"],
-                crs="EPSG:4326",  # 4326 is the WGS84 (lon, lat) EPSG code
-            )
-            output = filtered_gdf.explore()
+            output = fltrd_gdf.explore()
         return output
 
+
     gi_exploration_map(filtered_table)
     return (gi_exploration_map,)
 
@@ -472,8 +368,8 @@ def _(mo):
 
 
 @app.cell(hide_code=True)
-def _(brgi_db_old, mo):
-    explore_brgi_df = mo.ui.dropdown(brgi_db_old, value="InSitu_WETH")
+def _(brgi_geodb, mo):
+    explore_brgi_df = mo.ui.dropdown(brgi_geodb.InSituTests, value="WETH")
     mo.md(f"Select the GI table you want to explore: {explore_brgi_df}")
     return (explore_brgi_df,)