feat: shorten cache file path hashes (#189)

* feat: shorten cache file paths hashes

* chore: add licensecheck exception

* chore: change docstring examples

* chore: add exception to licensecheck

Author: RaczeQ

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Changed
+
+- Shortened the cache file paths hashes from default 64 characters to 8 [#188](https://github.com/kraina-ai/quackosm/issues/188)
+
 ## [0.12.1] - 2025-01-03
 
 ### Added

pyproject.toml

@@ -202,7 +202,9 @@ using = "requirements"
 zero = false
 ignore_licenses = ["UNKNOWN"]
 ignore_packages = [
-    'docformatter',   # uses MIT license, has mismatched license in analysis
-    'mkdocs-jupyter', # uses Apache-2.0 license, has mismatched license in analysis
-    'python-geohash', # uses both MIT and Apache-2.0 licenses, has mismatched license in analysis
+    'docformatter',     # uses MIT license, has mismatched license in analysis
+    'mkdocs-jupyter',   # uses Apache-2.0 license, has mismatched license in analysis
+    'python-geohash',   # uses both MIT and Apache-2.0 licenses, has mismatched license in analysis
+    'mkdocs-autorefs',  # uses ISC License, has mismatched license in analysis
+    'referencing',    # uses MIT license, has mismatched license in analysis
 ]

quackosm/functions.py

@@ -29,6 +29,7 @@
     "convert_osm_extract_to_geodataframe",
 ]
 
+
 def convert_pbf_to_duckdb(
     pbf_path: Union[str, Path, Iterable[Union[str, Path]]],
     tags_filter: Optional[Union[OsmTagsFilter, GroupedOsmTagsFilter]] = None,
@@ -154,7 +155,7 @@ def convert_pbf_to_duckdb(
         ...     monaco_pbf_path, tags_filter={"building": True, "amenity": True, "highway": True}
         ... ) # doctest: +IGNORE_RESULT
         >>> ddb_path.as_posix()
-        'files/monaco_6593ca69098459d039054bc5fe0a87c56681e29a5f59d38ce3485c03cb0e9374_noclip_compact.duckdb'
+        'files/monaco_6593ca69_noclip_compact.duckdb'
 
         Get features for Malé - the capital city of Maldives
 
@@ -204,7 +205,7 @@ def convert_pbf_to_duckdb(
         ...     )
         ... ) # doctest: +IGNORE_RESULT
         >>> ddb_path.as_posix()
-        'files/maldives_nofilter_4eeabb20ccd8aefeaa80b9a46a202ab985fd454760823b7012cc7778498a085b_compact.duckdb'
+        'files/maldives_nofilter_4eeabb20_compact.duckdb'
 
         >>> with duckdb.connect(str(ddb_path)) as con:
         ...     con.load_extension('spatial')
@@ -259,6 +260,7 @@ def convert_pbf_to_duckdb(
         duckdb_table_name=duckdb_table_name,
     )
 
+
 def convert_geometry_to_duckdb(
     geometry_filter: BaseGeometry = None,
     osm_extract_source: Union[OsmExtractSource, str] = OsmExtractSource.any,
@@ -356,7 +358,7 @@ def convert_geometry_to_duckdb(
         ... )
         >>> ddb_path = qosm.convert_geometry_to_duckdb(from_wkt(wkt)) # doctest: +IGNORE_RESULT
         >>> ddb_path.as_posix()
-        'files/bf4b33debfd6d3e605555340606df6ce7eea934958c1f3477aca0ccf79e7929f_nofilter_compact.duckdb'
+        'files/bf4b33de_nofilter_compact.duckdb'
 
         Inspect the file with duckdb
         >>> import duckdb
@@ -401,7 +403,7 @@ def convert_geometry_to_duckdb(
         ...     osm_extract_source='Geofabrik',
         ... ) # doctest: +IGNORE_RESULT
         >>> ddb_path.as_posix()
-        'files/bf4b33debfd6d3e605555340606df6ce7eea934958c1f3477aca0ccf79e7929f_nofilter_compact.duckdb'
+        'files/bf4b33de_nofilter_compact.duckdb'
 
         Inspect the file with duckdb
         >>> with duckdb.connect(str(ddb_path)) as con:
@@ -459,6 +461,7 @@ def convert_geometry_to_duckdb(
         duckdb_table_name=duckdb_table_name,
     )
 
+
 def convert_osm_extract_to_duckdb(
     osm_extract_query: str,
     osm_extract_source: Union[OsmExtractSource, str] = OsmExtractSource.any,
@@ -612,6 +615,7 @@ def convert_osm_extract_to_duckdb(
         duckdb_table_name=duckdb_table_name,
     )
 
+
 def convert_pbf_to_parquet(
     pbf_path: Union[str, Path, Iterable[Union[str, Path]]],
     tags_filter: Optional[Union[OsmTagsFilter, GroupedOsmTagsFilter]] = None,
@@ -739,7 +743,7 @@ def convert_pbf_to_parquet(
         ...     tags_filter={"building": True, "amenity": True, "highway": True}
         ... ) # doctest: +IGNORE_RESULT
         >>> gpq_path.as_posix()
-        'files/monaco_6593ca69098459d039054bc5fe0a87c56681e29a5f59d38ce3485c03cb0e9374_noclip_exploded.parquet'
+        'files/monaco_6593ca69_noclip_exploded.parquet'
 
         Inspect the file with duckdb
         >>> duckdb.read_parquet(str(gpq_path)).order("feature_id") # doctest: +SKIP
@@ -788,7 +792,7 @@ def convert_pbf_to_parquet(
         ...     )
         ... ) # doctest: +IGNORE_RESULT
         >>> gpq_path.as_posix()
-        'files/maldives_nofilter_4eeabb20ccd8aefeaa80b9a46a202ab985fd454760823b7012cc7778498a085b_compact.parquet'
+        'files/maldives_nofilter_4eeabb20_compact.parquet'
 
         Inspect the file with duckdb
         >>> duckdb.read_parquet(str(gpq_path)).order("feature_id") # doctest: +SKIP
@@ -942,7 +946,7 @@ def convert_geometry_to_parquet(
         ... )
         >>> gpq_path = qosm.convert_geometry_to_parquet(from_wkt(wkt)) # doctest: +IGNORE_RESULT
         >>> gpq_path.as_posix()
-        'files/bf4b33debfd6d3e605555340606df6ce7eea934958c1f3477aca0ccf79e7929f_nofilter_compact.parquet'
+        'files/bf4b33de_nofilter_compact.parquet'
 
         Inspect the file with duckdb
         >>> import duckdb
@@ -986,7 +990,7 @@ def convert_geometry_to_parquet(
         ...     osm_extract_source='Geofabrik',
         ... ) # doctest: +IGNORE_RESULT
         >>> gpq_path.as_posix()
-        'files/bf4b33debfd6d3e605555340606df6ce7eea934958c1f3477aca0ccf79e7929f_nofilter_compact.parquet'
+        'files/bf4b33de_nofilter_compact.parquet'
 
         Inspect the file with duckdb
         >>> duckdb.read_parquet(str(gpq_path)).order("feature_id") # doctest: +SKIP

quackosm/pbf_file_reader.py

@@ -1117,7 +1117,7 @@ def _generate_result_file_path(
             h.update(
                 (json.dumps(self.tags_filter or {}) + str(self.custom_sql_filter or "")).encode()
             )
-            osm_filter_tags_hash_part = f"{h.hexdigest()}{keep_all_tags_part}"
+            osm_filter_tags_hash_part = f"{h.hexdigest()[:8]}{keep_all_tags_part}"
 
         clipping_geometry_hash_part = self._generate_geometry_hash()
 
@@ -1127,7 +1127,7 @@ def _generate_result_file_path(
         if filter_osm_ids:
             h = hashlib.new("sha256")
             h.update(json.dumps(sorted(set(filter_osm_ids))).encode())
-            filter_osm_ids_hash_part = f"_{h.hexdigest()}"
+            filter_osm_ids_hash_part = f"_{h.hexdigest()[:8]}"
 
         if save_as_wkt:
             result_file_name = (
@@ -1151,7 +1151,7 @@ def _generate_result_file_path_from_geometry(
             h.update(
                 (json.dumps(self.tags_filter or {}) + str(self.custom_sql_filter or "")).encode()
             )
-            osm_filter_tags_hash_part = f"{h.hexdigest()}{keep_all_tags_part}"
+            osm_filter_tags_hash_part = f"{h.hexdigest()[:8]}{keep_all_tags_part}"
 
         clipping_geometry_hash_part = self._generate_geometry_hash()
 
@@ -1161,7 +1161,7 @@ def _generate_result_file_path_from_geometry(
         if filter_osm_ids:
             h = hashlib.new("sha256")
             h.update(json.dumps(sorted(set(filter_osm_ids))).encode())
-            filter_osm_ids_hash_part = f"_{h.hexdigest()}"
+            filter_osm_ids_hash_part = f"_{h.hexdigest()[:8]}"
 
         if save_as_wkt:
             result_file_name = (
@@ -1200,7 +1200,7 @@ def _generate_geometry_hash(self) -> str:
         if oriented_geometry is not None:
             h = hashlib.new("sha256")
             h.update(wktlib.dumps(oriented_geometry).encode())
-            clipping_geometry_hash_part = h.hexdigest()
+            clipping_geometry_hash_part = h.hexdigest()[:8]
 
         return clipping_geometry_hash_part
 
@@ -2989,11 +2989,7 @@ def _group_ways_with_polars(current_ways_group_path: Path, current_destination_p
         hive_partitioning=False,
     ).group_by("id").agg(pl.col("point").sort_by(pl.col("ref_idx"))).rename(
         {"point": "linestring"}
-    ).collect(
-        streaming=True
-    ).write_parquet(
-        current_destination_path
-    )
+    ).collect(streaming=True).write_parquet(current_destination_path)
 
 
 def _drop_duplicates_in_pyarrow_table(