Increase evaluation chunk size and consider input size in chunking computations. Previously, the chunking was solely based on the grid size. Now, both the grid size and the size of the input data (weights) are taken into consideration when determining the number of chunks and the size of each chunk. This change improves the efficiency of the program and avoids potential memory issues with large input data.

Author: Leguark

gempy_engine/core/data/options/evaluation_options.py

@@ -22,7 +22,7 @@ class EvaluationOptions:
     mesh_extraction_masking_options: MeshExtractionMaskingOptions = MeshExtractionMaskingOptions.INTERSECT
     mesh_extraction_fancy: bool = True
 
-    evaluation_chunk_size: int = 50_000
+    evaluation_chunk_size: int = 5_000_000
 
     compute_scalar_gradient: bool = False
 

gempy_engine/modules/evaluator/generic_evaluator.py

@@ -10,6 +10,7 @@
 
 def generic_evaluator(solver_input: SolverInput, weights: np.ndarray, options: InterpolationOptions) -> ExportedFields:
     grid_size = solver_input.xyz_to_interpolate.shape[0]
+    matrix_size = grid_size * weights.shape[0]
     scalar_field: np.ndarray = BackendTensor.t.zeros(grid_size, dtype=weights.dtype)
     gx_field: Optional[np.ndarray] = None
     gy_field: Optional[np.ndarray] = None
@@ -18,8 +19,8 @@ def generic_evaluator(solver_input: SolverInput, weights: np.ndarray, options: I
 
     # * Chunking the evaluation
     max_size = options.evaluation_chunk_size
-    n_chunks = int(np.ceil(grid_size / max_size))
-    chunk_size = int(np.ceil(grid_size / n_chunks))
+    n_chunks = int(np.ceil(matrix_size / max_size))
+    chunk_size = int(np.ceil(matrix_size / n_chunks))
     for i in range(n_chunks):
         slice_array = slice(i * chunk_size, (i + 1) * chunk_size)
         scalar_field_chunk, gx_field_chunk, gy_field_chunk, gz_field_chunk = _eval_on(